Update metadata.json

Implemented <parse>, <storage> etc sections within fluentd::source template

.travis.yml

@@ -6,8 +6,6 @@ script: bundle exec rake validate && bundle exec rake lint && bundle exec rake s
 matrix:
   fast_finish: true
   include:
-  - rvm: 2.1.10
-    env: PUPPET_VERSION="~> 3.0"
   - rvm: 2.1.10
     env: PUPPET_VERSION="~> 4.0" STRICT_VARIABLES="yes"
   - rvm: 2.2.7

CHANGELOG.md

@@ -1,4 +1,12 @@
 # Changelog
+
+## 1.0.0
+
+**Breaking changes:** This version drops puppet 3 support!
+
+* Fix hard coded user/group names in config manifest (#23)
+* migrate from stdlib validation to puppet datatypes (#24)
+
 ## 0.6.0
 * Updated README
 * fixed broken links, due to github organisation rename

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 4.0' : puppetversion = ENV['PUPPET_VERSION'].to_s
+ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 4.6' : puppetversion = ENV['PUPPET_VERSION'].to_s
 gem 'puppet', puppetversion
 gem 'puppetlabs_spec_helper', '>= 0.8.2'
 gem 'puppet-lint', '>= 1.0.0'
@@ -17,6 +17,6 @@ gem 'beaker-rspec', beakerrspec
 
 platforms :ruby_19 do
   gem 'public_suffix', '< 1.5.0', :platforms => :ruby_19
-  gem 'nokogiri', '< 1.7.0', :platforms => :ruby_19
+  gem 'nokogiri', '~> 1.8.1', :platforms => :ruby_19
   gem 'mime-types', '< 3.0', :platforms => :ruby_19
 end

README.md

@@ -1,6 +1,6 @@
-# wywy/fluentd Puppet Module
+# pilchkinstein-gtv/fluentd Puppet Module
 
-[![GitHub version](https://badge.fury.io/gh/wywy%2Fpuppet-fluentd.svg)](https://github.com/wywy/puppet-fluentd)
+[![GitHub version](https://badge.fury.io/gh/wywy%2Fpuppet-fluentd.svg)](https://github.com/pilchkinstein-gtv/puppet-fluentd)
 [![Build Status](https://travis-ci.org/wywy/puppet-fluentd.svg?branch=master)](https://travis-ci.org/wywy/puppet-fluentd)
 
 
@@ -131,6 +131,14 @@ include '::fluentd'
     'type'   => 'tail',
     'format' => 'json',
     'path'   => '/var/log/test-application/*.json',
+    'parse'  => {
+      'type'       => 'regexp',
+      'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+    },
+    'storage'=> {
+      'type'   => 'local',
+      'path'   => 'test.pos'
+    },
     'tag'    => 'application.test'
   }
 }
@@ -142,6 +150,14 @@ include '::fluentd'
   type tail
   format json
   path /var/log/test-application/*.json
+  <parse>
+    type regexp
+    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
+  </parse>
+  <storage>
+    type local
+    path test.pos
+  </storage>
   tag application.test
 </source>
 ```
@@ -263,14 +279,14 @@ This module gives you the possibility to install plugins as gem or files.
 ### Requirements
 
 Modules:
-* puppetlabs/apt >= 1.8.0 < 3.0.0
+* puppetlabs/apt >= 4.0.0
 
 ## Limitations
-This module has been built on and tested against Puppet 3.7.5 and higher.
+This module has been built on and tested against Puppet 4.6 and higher.
 
 The module has been tested on:
 
-* Ubuntu 12.04
+* Ubuntu 16.04
 
 Testing on other platforms has been light and cannot be guaranteed.
 
@@ -285,8 +301,8 @@ To run the tests install the ruby dependencies with `bundler` and execute
 bundle install --path vendor/bundle
 bundle exec rake spec
 bundle exec rake lint
-bundle exec rake beaker BEAKER_set=ubuntu-server-1204-x64
 bundle exec rake beaker BEAKER_set=ubuntu-server-1404-x64
+bundle exec rake beaker BEAKER_set=ubuntu-server-1604-x64
 bundle exec rake beaker BEAKER_set=debian-60-x64
 bundle exec rake beaker BEAKER_set=debian-78-x64
 bundle exec rake beaker BEAKER_set=debian-82-x64

examples/source.pp

@@ -2,9 +2,17 @@
 ::fluentd::source { 'test':
   priority => 10,
   config   => {
-    'type'   => 'tail',
+    'type'    => 'tail',
-    'format' => 'json',
+    'format'  => 'json',
-    'path'   => '/var/log/test-application/*.json',
+    'path'    => '/var/log/test-application/*.json',
-    'tag'    => 'application.test'
+    'parse'   => {
+        'type'       => 'regexp',
+        'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+    },
+    'storage' => {
+      'type' => 'local',
+      'path' => 'test.pos'
+    },
+    'tag'     => 'application.test'
   }
 }

manifests/config/file.pp

@@ -28,8 +28,8 @@ define fluentd::config::file (
   file { $config_path:
     ensure  => $ensure,
     content => $content,
-    owner   => 'td-agent',
+    owner   => $::fluentd::user_name,
-    group   => 'td-agent',
+    group   => $::fluentd::user_group,
     mode    => '0644',
     notify  => Class['Fluentd::Service'],
   }

manifests/filter.pp

@@ -38,23 +38,16 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::filter (
-  $ensure   = present,
+  Enum['present', 'absent'] $ensure   = present,
-  $priority = 20,
+  Integer                   $priority = 20,
-  $pattern  = '**',
+  String                    $pattern  = '**',
-  $config   = {},
+  Hash                      $config   = {},
 ) {
 
-  # parameter validation
-  if ! ($ensure in [ 'present', 'absent' ]) {
-    fail('ensure parameter must be present or absent')
-  }
-  validate_integer($priority)
-  validate_string($pattern)
-  validate_hash($config)
-
   fluentd::config::file { "filter-${title}":
     ensure   => $ensure,
     priority => $priority,
     content  => template( 'fluentd/filter.erb'),
   }
+
 }

manifests/init.pp

@@ -65,37 +65,24 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 class fluentd (
-  $repo_manage             = $::fluentd::params::repo_manage,
+  Boolean                     $repo_manage             = $::fluentd::params::repo_manage,
-  $package_ensure          = $::fluentd::params::package_ensure,
+  String                      $package_ensure          = $::fluentd::params::package_ensure,
-  $package_name            = $::fluentd::params::package_name,
+  String                      $package_name            = $::fluentd::params::package_name,
-  $package_install_options = $::fluentd::params::package_install_options,
+  Array                       $package_install_options = $::fluentd::params::package_install_options,
-  $service_manage          = $::fluentd::params::service_manage,
+  Boolean                     $service_manage          = $::fluentd::params::service_manage,
-  $service_name            = $::fluentd::params::service_name,
+  String                      $service_name            = $::fluentd::params::service_name,
-  $service_ensure          = $::fluentd::params::service_ensure,
+  Enum['running', 'stopped']  $service_ensure          = $::fluentd::params::service_ensure,
-  $service_enable          = $::fluentd::params::service_enable,
+  Boolean                     $service_enable          = $::fluentd::params::service_enable,
-  $config_path             = $::fluentd::params::config_path,
+  String                      $config_path             = $::fluentd::params::config_path,
-  $conf_dir                = $::fluentd::params::conf_dir,
+  String                      $conf_dir                = $::fluentd::params::conf_dir,
-  $config_file             = $::fluentd::params::config_file,
+  String                      $config_file             = $::fluentd::params::config_file,
-  $conf_dir_manage         = $::fluentd::params::conf_dir_manage,
+  Boolean                     $conf_dir_manage         = $::fluentd::params::conf_dir_manage,
-  $user_manage             = $::fluentd::params::user_manage,
+  Boolean                     $user_manage             = $::fluentd::params::user_manage,
-  $user_name               = $::fluentd::params::user_name,
+  String                      $user_name               = $::fluentd::params::user_name,
-  $user_group              = $::fluentd::params::user_group,
+  String                      $user_group              = $::fluentd::params::user_group,
-  $user_groups             = $::fluentd::params::user_groups,
+  Array                       $user_groups             = $::fluentd::params::user_groups,
 ) inherits fluentd::params {
 
-  # parameter validation
-  validate_bool($repo_manage)
-  validate_string($package_ensure)
-  validate_string($package_name)
-  validate_array($package_install_options)
-  validate_bool($service_manage)
-  validate_string($service_name)
-  validate_bool($service_enable)
-
-  if ! ($service_ensure in [ 'running', 'stopped' ]) {
-    fail('service_ensure parameter must be running or stopped')
-  }
-
   # class calls
   include '::fluentd::repo'
   include '::fluentd::install'

manifests/match.pp

@@ -43,23 +43,16 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::match (
-  $ensure   = present,
+  Enum['present', 'absent'] $ensure   = present,
-  $priority = 30,
+  Integer                   $priority = 30,
-  $pattern  = '**',
+  String                    $pattern  = '**',
-  $config   = {},
+  Hash                      $config   = {},
 ) {
 
-  # parameter validation
-  if ! ($ensure in [ 'present', 'absent' ]) {
-    fail('ensure parameter must be present or absent')
-  }
-  validate_integer($priority)
-  validate_string($pattern)
-  validate_hash($config)
-
   fluentd::config::file { "match-${title}":
     ensure   => $ensure,
     priority => $priority,
     content  => template( 'fluentd/match.erb'),
   }
+
 }

manifests/plugin.pp

@@ -26,9 +26,9 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::plugin (
-  $ensure = present,
+  String           $ensure = present,
-  $type   = 'gem',
+  String           $type   = 'gem',
-  $source = undef,
+  Optional[String] $source = undef,
 ) {
 
   case $type {
@@ -43,7 +43,6 @@ define fluentd::plugin (
       if ! ($ensure in [ 'present', 'absent' ]) {
         fail('ensure parameter must be present or absent')
       }
-      validate_string($source)
 
       fluentd::plugin::file { $name:
         ensure  => $ensure,

manifests/source.pp

@@ -34,17 +34,15 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::source (
-  $ensure   = present,
+  String  $ensure   = present,
-  $priority = 10,
+  Integer $priority = 10,
-  $config   = {},
+  Hash    $config   = {},
 ) {
 
   # parameter validation
   if ! ($ensure in [ 'present', 'absent' ]) {
     fail('ensure parameter must be present or absent')
   }
-  validate_integer($priority)
-  validate_hash($config)
 
   fluentd::config::file { "source-${title}":
     ensure   => $ensure,

metadata.json

@@ -1,17 +1,17 @@
 {
-  "name": "wywygmbh-fluentd",
+  "name": "pilchkinstein-fluentd",
-  "version": "0.6.0",
+  "version": "1.0.0",
-  "author": "wywy",
+  "author": "pilchkinstein-gtv",
   "summary": "Generic module for fluentd (td-agent).",
   "license": "Apache-2.0",
-  "source": "https://github.com/wywy/puppet-fluentd",
+  "source": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
-  "project_page": "https://github.com/wywy/puppet-fluentd",
+  "project_page": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
-  "issues_url": "https://github.com/wywy/puppet-fluentd/issues",
+  "issues_url": "https://github.com/pilchkinstein-gtv/puppet-fluentd/issues",
   "tags": ["fluentd", "td-agent"],
   "operatingsystem_support": [
     {
       "operatingsystem": "Ubuntu",
-      "operatingsystemrelease": [ "14.04", "12.04" ]
+      "operatingsystemrelease": [ "16.04", "14.04" ]
     },
     {
       "operatingsystem": "Debian",
@@ -24,7 +24,7 @@
   ],
   "dependencies": [
     { "name": "puppetlabs/stdlib", "version_requirement": ">= 4.6.0 < 5.0.0" },
-    { "name": "puppetlabs/apt", "version_requirement": ">= 2.0.0 < 3.0.0" }
+    { "name": "puppetlabs/apt", "version_requirement": ">= 4.0.0" }
   ],
   "data_provider": null
 }

spec/defines/filter_spec.rb

@@ -13,7 +13,7 @@ describe 'fluentd::filter' do
         let(:title) { 'test' }
         let(:params) do
           {
-            priority: '20',
+            priority: 20,
             pattern: '*.test',
             config: {
               'type'   => 'record_transformer',

spec/defines/match_spec.rb

@@ -13,7 +13,7 @@ describe 'fluentd::match' do
         let(:title) { 'test' }
         let(:params) do
           {
-              priority: '30',
+              priority: 30,
               pattern: '*.test',
               config: {
                   'flush_interval' => '30s',
@@ -41,7 +41,7 @@ describe 'fluentd::match' do
         let(:title) { 'test' }
         let(:params) do
           {
-              priority: '30',
+              priority: 30,
               pattern: '*.test',
               config: {
                   'type'  => 'copy',

spec/defines/source_spec.rb

@@ -13,11 +13,19 @@ describe 'fluentd::source' do
         let(:title) { 'test' }
         let(:params) do
           {
-              priority: '10',
+              priority: 10,
               config: {
                 'type'   => 'tail',
                 'format' => 'json',
                 'path'   => '/var/log/test-application/*.json',
+                'parse'  => {
+                    'type'       => 'regexp',
+                    'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+                },
+                'storage'=> {
+                    'type'   => 'local',
+                    'path'   => 'test.pos'
+                },
                 'tag'    => 'application.test'
               }
           }

spec/fixtures/files/source_tail.conf

@@ -2,5 +2,13 @@
   type tail
   format json
   path /var/log/test-application/*.json
+  <parse>
+    type regexp
+    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
+  </parse>
+  <storage>
+    type local
+    path test.pos
+  </storage>
   tag application.test
 </source>

templates/source.erb

@@ -1,5 +1,13 @@
 <source>
 <% @config.each do |key, val| -%>
+  <%- if val.is_a?( Hash ) -%>
+  <<%= key -%>>
+  <%- val.each do |key2, val2| -%>
+    <%= key2 -%> <%= val2 %>
+  <%- end -%>
+  </<%= key %>>
+  <%- else -%>
   <%= key -%> <%= val %>
+  <%- end -%>
 <% end -%>
 </source>