Upgrade to td-agent v4 on older Puppet installs

The fluentd_gem provider for installing fluentd plugins uses
Puppet's built-in gem provider. Unfortunately, that gem provider
on our Puppet version is not compatible with td-agent's new embedded
Ruby since it includes the no-rdoc and no-ri flags deprecated and
replaced by the no-document flag. This install function override
replaces the flags and might be removable after a Puppet upgrade.

.travis.yml

@@ -6,8 +6,6 @@ script: bundle exec rake validate && bundle exec rake lint && bundle exec rake s
 matrix:
   fast_finish: true
   include:
-  - rvm: 2.1.10
-    env: PUPPET_VERSION="~> 3.0"
   - rvm: 2.1.10
     env: PUPPET_VERSION="~> 4.0" STRICT_VARIABLES="yes"
   - rvm: 2.2.7

CHANGELOG.md

@@ -1,4 +1,18 @@
 # Changelog
+
+## 1.0.0
+
+**Breaking changes:** This version drops puppet 3 support!
+
+* Fix hard coded user/group names in config manifest (#23)
+* migrate from stdlib validation to puppet datatypes (#24)
+
+## 0.6.0
+* Updated README
+* fixed broken links, due to github organisation rename
+* fixed indentation for config files (filter, match, source)
+* use rspec-puppet-facts for unit testing
+
 ## 0.5.6
 * Fixed `conf_dir_manage` parameter call
 

Gemfile

@@ -1,12 +1,13 @@
 source 'https://rubygems.org'
 
-ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 4.0' : puppetversion = ENV['PUPPET_VERSION'].to_s
+ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 4.6' : puppetversion = ENV['PUPPET_VERSION'].to_s
 gem 'puppet', puppetversion
 gem 'puppetlabs_spec_helper', '>= 0.8.2'
 gem 'puppet-lint', '>= 1.0.0'
 gem 'facter', '>= 1.7.0'
 gem 'beaker-puppet_install_helper'
 gem 'metadata-json-lint'
+gem 'rspec-puppet-facts', '~> 1.7'
 
 beakerrspec = '> 0'
 platforms [:ruby_19, :ruby_21] do
@@ -16,6 +17,6 @@ gem 'beaker-rspec', beakerrspec
 
 platforms :ruby_19 do
   gem 'public_suffix', '< 1.5.0', :platforms => :ruby_19
-  gem 'nokogiri', '< 1.7.0', :platforms => :ruby_19
+  gem 'nokogiri', '~> 1.8.1', :platforms => :ruby_19
   gem 'mime-types', '< 3.0', :platforms => :ruby_19
-end
+end

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2015 wywy GmbH
+   Copyright 2017 ATVAG GmbH
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,7 +1,7 @@
-# wywygmbh/fluentd Puppet Module
+# pilchkinstein-gtv/fluentd Puppet Module
 
-[![GitHub version](https://badge.fury.io/gh/wywygmbh%2Fpuppet-fluentd.svg)](https://github.com/wywygmbh/puppet-fluentd)
-[![Build Status](https://travis-ci.org/wywygmbh/puppet-fluentd.svg?branch=master)](https://travis-ci.org/wywygmbh/puppet-fluentd)
+[![GitHub version](https://badge.fury.io/gh/wywy%2Fpuppet-fluentd.svg)](https://github.com/pilchkinstein-gtv/puppet-fluentd)
+[![Build Status](https://travis-ci.org/wywy/puppet-fluentd.svg?branch=master)](https://travis-ci.org/wywy/puppet-fluentd)
 
 
 
@@ -15,6 +15,7 @@
     * [Source](#source)
     * [Filter](#filter)
     * [Match](#match)
+    * [Match with buffer and chunk_keys](#match-with-buffer-and-chunk_keys)
     * [Match Store](#match-store)
   * [Plugin Installation](#plugin-installation)
   * [Requirements](#requirements)
@@ -48,42 +49,62 @@ include '::fluentd'
 
   Include repository to install recent fluentd (td-agent) from.
   **Default:** 'true'
- 
+
 `package_ensure`
 
   Package ensure.
   **Default:** 'installed'
- 
+
 `package_name`
 
   Package name.
   **Default:** 'td-agent'
- 
+
 `package_install_options`
 
   Package install options.
   **Default:** '[]'
- 
+
 `service_manage`
 
   Defines if the service should be managed by puppet.
   **Default:** 'true'
- 
+
 `service_name`
 
   Name of the service.
   **Default:** 'td-agent'
- 
+
 `service_ensure`
 
   Service ensure.
   **Default:** 'running'
- 
+
 `service_enable`
 
   Defines if the service should be enabled.
   **Default:** 'true'
- 
+
+`config_path`
+
+  Path to configuration files
+  **Default:** '/etc/td-agent'
+
+`config_dir`
+
+  Configuration directory name
+  **Default:** '${config_path}/conf.d'
+
+`config_file`
+
+  Default configuration file name
+  **Default:** '${config_path}/td-agent.conf'
+
+`conf_dir_manage`
+
+  Exclusively handle config files into fluentd::conf_dir. Other files not created by puppet, will be deleted
+  **Default:** 'false'
+
 `user_manage`
 
   Defines if the user should be manage, which will add the user
@@ -94,11 +115,11 @@ include '::fluentd'
 `user_name`
 
   **Default:** 'td-agent'
- 
+
 `user_group`
 
   **Default:** 'td-agent'
- 
+
 `user_groups`
 
   **Default:** '["adm"]'
@@ -111,7 +132,18 @@ include '::fluentd'
     'type'   => 'tail',
     'format' => 'json',
     'path'   => '/var/log/test-application/*.json',
-    'tag'    => 'application.test'
+    'parse'  => {
+      'type'       => 'regexp',
+      'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+    },
+    'storage'=> {
+      'type'   => 'local',
+      'path'   => 'test.pos'
+    },
+    'tag'    => 'application.test',
+    'parse'  => {
+      'message_format' => 'auto'
+    }
   }
 }
 ```
@@ -122,7 +154,18 @@ include '::fluentd'
   type tail
   format json
   path /var/log/test-application/*.json
+  <parse>
+    type regexp
+    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
+  </parse>
+  <storage>
+    type local
+    path test.pos
+  </storage>
   tag application.test
+  <parse>
+    message_format auto
+  </parse>
 </source>
 ```
 #### Filter
@@ -181,7 +224,41 @@ include '::fluentd'
   </server>
 </match>
 ```
-### Match Store
+#### Match with buffer and chunk_keys
+
+buffer_chunk_keys will not be included in the template. They will be filtered and only be used as a "pattern" for the buffer section.
+
+```
+::fluentd::match { 'test':
+  priority => 98,
+  pattern  => 'test.**',
+  config   => {
+    type              => 'file',
+    path              => '/var/log/td-agent/test/${host}',
+    append            => true,
+    buffer_chunk_keys => 'host',
+    buffer            => {
+      type       => file,
+      path       => /var/log/td-agent/test/buffer/,
+      flush_mode => immediate,
+    },
+  },
+}
+```
+**creates:**
+```
+<match test.**>
+  type file
+  path /var/log/td-agent/test/${host}
+  append true
+  <buffer host>
+    type file
+    path /var/log/td-agent/test/buffer/
+    flush_mode immediate
+  </buffer>
+</match>
+```
+#### Match Store
 ```puppet
 ::fluentd::match { 'test':
   priority => 30,
@@ -197,7 +274,13 @@ include '::fluentd'
       },
       {
         'type' => 'file',
-        'path' => '/tmp/td-agent-debug.log',
+        'path' => '/tmp/td-agent-debug-${host}.log',
+        'buffer_chunk_keys' => 'host',
+        'buffer'         => [{
+          '@type'    => 'file',
+          path       => '/var/log/td-agent/buffer/',
+          flush_mode => 'immediate',
+          }]
       }
     ]
   }
@@ -216,7 +299,11 @@ include '::fluentd'
   </store>
   <store>
     type file
-    path /tmp/crs
+    path /tmp/td-agent-debug-${host}.log
+    <buffer host>
+      @type file
+      path /var/log/td-agent/buffer/
+      flush_mode immediate
   </store>
 </match>
 ```
@@ -243,14 +330,14 @@ This module gives you the possibility to install plugins as gem or files.
 ### Requirements
 
 Modules:
-* puppetlabs/apt >= 1.8.0 < 3.0.0
+* puppetlabs/apt >= 4.0.0
 
 ## Limitations
-This module has been built on and tested against Puppet 3.7.5 and higher.
+This module has been built on and tested against Puppet 4.6 and higher.
 
 The module has been tested on:
 
-* Ubuntu 12.04
+* Ubuntu 16.04
 
 Testing on other platforms has been light and cannot be guaranteed.
 
@@ -265,8 +352,8 @@ To run the tests install the ruby dependencies with `bundler` and execute
 bundle install --path vendor/bundle
 bundle exec rake spec
 bundle exec rake lint
-bundle exec rake beaker BEAKER_set=ubuntu-server-1204-x64
 bundle exec rake beaker BEAKER_set=ubuntu-server-1404-x64
+bundle exec rake beaker BEAKER_set=ubuntu-server-1604-x64
 bundle exec rake beaker BEAKER_set=debian-60-x64
 bundle exec rake beaker BEAKER_set=debian-78-x64
 bundle exec rake beaker BEAKER_set=debian-82-x64
@@ -276,7 +363,7 @@ bundle exec rake beaker BEAKER_set=centos-70-x64
 
 ## License
 
-Copyright 2015 wywy GmbH
+Copyright 2017 ATVAG GmbH
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -290,4 +377,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-This Puppet module is being actively maintained by some fellow puppeteers at [wywy GmbH](http://wywy.com/).
+This Puppet module is being actively maintained by some fellow puppeteers at [wywy](http://wywy.com/).

Rakefile

@@ -4,8 +4,9 @@ require 'puppet-lint/tasks/puppet-lint'
 PuppetLint.configuration.log_format = '%{path}:%{line}:%{check}:%{KIND}:%{message}'
 PuppetLint.configuration.fail_on_warnings = true
 PuppetLint.configuration.send('relative')
-PuppetLint.configuration.send('disable_140chars')
+PuppetLint.configuration.send('disable_80chars')
 PuppetLint.configuration.send('disable_class_inherits_from_params_class')
+PuppetLint.configuration.send('disable_class_parameter_defaults')
 PuppetLint.configuration.send('disable_documentation')
 PuppetLint.configuration.send('disable_single_quote_string_with_variables')
 

examples/source.pp

@@ -2,9 +2,17 @@
 ::fluentd::source { 'test':
   priority => 10,
   config   => {
-    'type'   => 'tail',
-    'format' => 'json',
-    'path'   => '/var/log/test-application/*.json',
-    'tag'    => 'application.test'
+    'type'    => 'tail',
+    'format'  => 'json',
+    'path'    => '/var/log/test-application/*.json',
+    'parse'   => {
+        'type'       => 'regexp',
+        'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+    },
+    'storage' => {
+      'type' => 'local',
+      'path' => 'test.pos'
+    },
+    'tag'     => 'application.test'
   }
 }

files/GPG-KEY-td-agent

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: Encryption subkey has been removed due to apt-key parsing bug
+
+mQINBFhiI8wBEADThWLNd8IKPRw7Ygu3DHS4Sb/Yc6vSZSaMGJ6Wkj245jScvI+C
+nG4C4rtO/8ObUj5cUpb4CyfYZX8W4tp9x+W68c4paXevG4s+X4EE3uUsgdwTnFXi
+GMa57QDzR4p/JvjUjfGJ2UAr4Bfj8Q2S54LmIu6UAe82ce2B4tEHCeYSxkmVUDAZ
+utfmgKoVTbnceTemU0m5ANS6IC1/53KEhgB1sKm5G/FjRJGslHWb3mf+bLrhmlkP
+pA4BOKF2w3eFYH3LhWskxMS0SPM7J6aq+6LyNNqtlKL6lUS7qVjRQ6PlgFcmtG4J
+tijsZI62bDn1f44DmeLY+LMS/nM0xyIx94lYumGH5EYmjUECagqMool98/+Wx79A
+Thtg/1pYNzo8Z76qr0i3xLSRtsQ2Om2Rfal7VGadOrx4sqlkSaUaGI+hBc1r4tNy
+tERvBEMGSf78bWDbdzxSNEW4LUDUpniNQb0DrURfWkqRa3q4WcTJr8lpQM/NmAru
+owayAXQwKob+OIZ09/O69EaqVJ9MqsM3keQouSHShKvzNrppuo3D3z+Dpy05FsYw
+MAiIN7auXxy+XQwCVsKF083YaDHcC0I22GReEgt43yZXQ/b/J9QNrm5nJ+3Cpso3
+jJnMzubuniSOOdd3mXQ6MwgZvWgtH/nPF8oUX9VSGwqNohiKWcxQDxW7qQARAQAB
+tFRUcmVhc3VyZSBEYXRhLCBJbmMgKFRyZWFzdXJlIEFnZW50IE9mZmljaWFsIFNp
+Z25pbmcga2V5KSA8c3VwcG9ydEB0cmVhc3VyZS1kYXRhLmNvbT6JAjcEEwEIACEF
+AlhiI8wCGwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQkB+Rd6uXrL5GrhAA
+nh82+caSu9Qu/LW256gN5UjPUFhph66ElT1OVyAR2FoOmz2pJH3t8YYD5cUV2W6/
+xqJDmjl+vnL2HBgxjHKRCo2K3hrq6z4LoU7SpWDI1cZ03lkjh1yNx13S+9JvZNlp
+jit0WRIspke0n0vWSpNo4nh19Yg3EA1c+vGeHnmlYo6xwRHu6XOhhCwywtFRGC3a
+iMJzAV4N69ZU6P5VZZkC6LjYYQtF4aI10COLZ4AcObH2htGAZTj2KlZfdJHmr+Oa
+wY57giUYz7OF45LLCuqe+VwpGp2d3UK/MtCnXRLi5InMVJKDvyt18MzRDFuyA27e
+WSt+JumVqhEjawh3hmdzIS1cHKmv19gdeE8On2i2Lf8lyek8fsB/YPgADAmp2oSe
+cjLu0ocGbgxRjuCR29+6IG+DiUDFCkqFZNdLiGVqzjpjpYHaPhVe77ciwA8TCPru
+3dh5t/qv2HglSd7lj95IApZBtny5AK8NS4qtaOeZbBbbDRuOPL0c7fU3bqyIPy57
+zvdYi3KdjWZVCawcAmk3ILP83eFSivCRPRoyCqO+HX8U647BBWvlFuEbPa+Y1sgE
+12MEF/Y6VVJh3Ptw+h/qKRbra4LdA+5Y30q/9l6WGgbO/4h3NKmGeVCrAFvS3h92
+fS0ABYD1nAP7fSNS9RfYIqfBXtJem+tJ14YKJwWiAYU=
+=TPyg
+-----END PGP PUBLIC KEY BLOCK-----

lib/puppet/provider/package/fluentd-gem.rb

@@ -8,11 +8,20 @@ Puppet::Type.type(:package).provide :fluentd_gem, :parent => :gem do
     not a valid URL, it will be interpreted as the path to a local gem file.  If
     source is not present at all, the gem will be installed from the default gem
     repositories."
-  confine :exists => "/opt/td-agent/embedded/bin/gem"
+  confine :exists => "/opt/td-agent/bin/gem"
 
   has_feature :versionable, :install_options
 
-  commands :gemcmd => "/opt/td-agent/embedded/bin/gem"
+  commands :gemcmd => "/opt/td-agent/bin/gem"
+
+  def install(useversion = true)
+    command = [command(:gemcmd), "install"]
+    command << "--no-document" << resource[:name]
+    output = execute(command)
+
+    # Apparently some stupid gem versions don't exit non-0 on failure
+    self.fail "Could not install: #{output.chomp}" if output.include?("ERROR")
+  end
 
   def uninstall
     command = [command(:gemcmd), "uninstall"]

manifests/config/file.pp

@@ -28,8 +28,8 @@ define fluentd::config::file (
   file { $config_path:
     ensure  => $ensure,
     content => $content,
-    owner   => 'td-agent',
-    group   => 'td-agent',
+    owner   => $::fluentd::user_name,
+    group   => $::fluentd::user_group,
     mode    => '0644',
     notify  => Class['Fluentd::Service'],
   }

manifests/filter.pp

@@ -35,26 +35,19 @@
 # Copyright
 # ---------
 #
-# Copyright 2015 wywy GmbH, unless otherwise noted.
+# Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::filter (
-  $ensure   = present,
-  $priority = 20,
-  $pattern  = '**',
-  $config   = {},
+  Enum['present', 'absent'] $ensure   = present,
+  Integer                   $priority = 20,
+  String                    $pattern  = '**',
+  Hash                      $config   = {},
 ) {
 
-  # parameter validation
-  if ! ($ensure in [ 'present', 'absent' ]) {
-    fail('ensure parameter must be present or absent')
-  }
-  validate_integer($priority)
-  validate_string($pattern)
-  validate_hash($config)
-
   fluentd::config::file { "filter-${title}":
     ensure   => $ensure,
     priority => $priority,
     content  => template( 'fluentd/filter.erb'),
   }
+
 }

manifests/init.pp

@@ -8,9 +8,9 @@
 # [*repo_manage*]
 #   Include repository to install recent fluentd (td-agent) from
 #   Default: 'true'
-# [*conf_dir_manage*]
-#   Exclusively handle config files into fluentd::conf_dir. Other files not created by puppet, will be deleted
-#   Default: 'false'
+# [*repo_version*]
+#   The major version of td-agent to install
+#   Default: 3
 # [*package_ensure*]
 #   Package ensure
 #   Default: 'installed'
@@ -32,6 +32,18 @@
 # [*service_enable*]
 #   Defines if the service should be enabled
 #   Default: 'true'
+# [*config_path*]
+#   Path to configuration files
+#   Default: '/etc/td-agent'
+# [*config_dir*]
+#   Configuration directory name
+#   Default: '${config_path}/conf.d'
+# [*config_file*]
+#   Default configuration file name
+#   Default: '${config_path}/td-agent.conf'
+# [*conf_dir_manage*]
+#   Exclusively handle config files into fluentd::conf_dir. Other files not created by puppet, will be deleted
+#   Default: 'false'
 # [*user_manage*]
 #   Defines if the user should be manage, which will add the user
 #   to groups defined in $user_groups.
@@ -53,40 +65,28 @@
 # Copyright
 # ---------
 #
-# Copyright 2015 wywy GmbH, unless otherwise noted.
+# Copyright 2015 wywy, unless otherwise noted.
 #
 class fluentd (
-  $repo_manage             = $::fluentd::params::repo_manage,
-  $package_ensure          = $::fluentd::params::package_ensure,
-  $package_name            = $::fluentd::params::package_name,
-  $package_install_options = $::fluentd::params::package_install_options,
-  $service_manage          = $::fluentd::params::service_manage,
-  $service_name            = $::fluentd::params::service_name,
-  $service_ensure          = $::fluentd::params::service_ensure,
-  $service_enable          = $::fluentd::params::service_enable,
-  $config_path             = $::fluentd::params::config_path,
-  $conf_dir                = $::fluentd::params::conf_dir,
-  $conf_dir_manage         = $::fluentd::params::conf_dir_manage,
-  $config_file             = $::fluentd::params::config_file,
-  $user_manage             = $::fluentd::params::user_manage,
-  $user_name               = $::fluentd::params::user_name,
-  $user_group              = $::fluentd::params::user_group,
-  $user_groups             = $::fluentd::params::user_groups,
+  Boolean                     $repo_manage             = $::fluentd::params::repo_manage,
+  Integer                     $repo_version            = $::fluentd::params::repo_version,
+  String                      $package_ensure          = $::fluentd::params::package_ensure,
+  String                      $package_name            = $::fluentd::params::package_name,
+  Array                       $package_install_options = $::fluentd::params::package_install_options,
+  Boolean                     $service_manage          = $::fluentd::params::service_manage,
+  String                      $service_name            = $::fluentd::params::service_name,
+  Enum['running', 'stopped']  $service_ensure          = $::fluentd::params::service_ensure,
+  Boolean                     $service_enable          = $::fluentd::params::service_enable,
+  String                      $config_path             = $::fluentd::params::config_path,
+  String                      $conf_dir                = $::fluentd::params::conf_dir,
+  String                      $config_file             = $::fluentd::params::config_file,
+  Boolean                     $conf_dir_manage         = $::fluentd::params::conf_dir_manage,
+  Boolean                     $user_manage             = $::fluentd::params::user_manage,
+  String                      $user_name               = $::fluentd::params::user_name,
+  String                      $user_group              = $::fluentd::params::user_group,
+  Array                       $user_groups             = $::fluentd::params::user_groups,
 ) inherits fluentd::params {
 
-  # parameter validation
-  validate_bool($repo_manage)
-  validate_string($package_ensure)
-  validate_string($package_name)
-  validate_array($package_install_options)
-  validate_bool($service_manage)
-  validate_string($service_name)
-  validate_bool($service_enable)
-
-  if ! ($service_ensure in [ 'running', 'stopped' ]) {
-    fail('service_ensure parameter must be running or stopped')
-  }
-
   # class calls
   include '::fluentd::repo'
   include '::fluentd::install'
@@ -94,6 +94,12 @@ class fluentd (
   include '::fluentd::config'
   include '::fluentd::service'
 
+  # Realize any virtual configurations
+  Fluentd::Plugin <| |>
+  Fluentd::Source <| |>
+  Fluentd::Filter <| |>
+  Fluentd::Match  <| |>
+
   # dependencies
   Class['::Fluentd::Repo']
   -> Class['::Fluentd::Install']

manifests/match.pp

@@ -40,26 +40,19 @@
 # Copyright
 # ---------
 #
-# Copyright 2015 wywy GmbH, unless otherwise noted.
+# Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::match (
-  $ensure   = present,
-  $priority = 30,
-  $pattern  = '**',
-  $config   = {},
+  Enum['present', 'absent'] $ensure   = present,
+  Integer                   $priority = 30,
+  String                    $pattern  = '**',
+  Hash                      $config   = {},
 ) {
 
-  # parameter validation
-  if ! ($ensure in [ 'present', 'absent' ]) {
-    fail('ensure parameter must be present or absent')
-  }
-  validate_integer($priority)
-  validate_string($pattern)
-  validate_hash($config)
-
   fluentd::config::file { "match-${title}":
     ensure   => $ensure,
     priority => $priority,
     content  => template( 'fluentd/match.erb'),
   }
+
 }

manifests/params.pp

@@ -18,6 +18,8 @@ class fluentd::params {
   $user_name   = 'td-agent'
   $user_group  = 'td-agent'
   $user_groups = ['adm']
+  # repo params
+  $repo_version = 4
 
   case $::osfamily {
     'Debian': {

manifests/plugin.pp

@@ -23,12 +23,12 @@
 # Copyright
 # ---------
 #
-# Copyright 2015 wywy GmbH, unless otherwise noted.
+# Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::plugin (
-  $ensure = present,
-  $type   = 'gem',
-  $source = undef,
+  String           $ensure = present,
+  String           $type   = 'gem',
+  Optional[String] $source = undef,
 ) {
 
   case $type {
@@ -43,7 +43,6 @@ define fluentd::plugin (
       if ! ($ensure in [ 'present', 'absent' ]) {
         fail('ensure parameter must be present or absent')
       }
-      validate_string($source)
 
       fluentd::plugin::file { $name:
         ensure  => $ensure,

manifests/repo.pp

@@ -4,7 +4,7 @@ class fluentd::repo inherits fluentd {
   if $::fluentd::repo_manage {
     case $::osfamily {
       'Debian': {
-        include ::fluentd::repo::apt
+        contain ::fluentd::repo::apt
       }
       'RedHat': {
         include ::fluentd::repo::yum

manifests/repo/apt.pp

@@ -2,13 +2,13 @@
 #
 class fluentd::repo::apt (
   $ensure       = 'present',
-  $location     = downcase("http://packages.treasuredata.com/2/${::lsbdistid}/${::lsbdistcodename}"),
+  $location     = downcase("http://packages.treasuredata.com/${::fluentd::repo_version}/${::lsbdistid}/${::lsbdistcodename}"),
   $release      = $::lsbdistcodename,
   $repos        = 'contrib',
   $architecture = $::architecture,
   $key          = {
-    'id'     => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
-    'source' => 'http://packages.treasuredata.com/GPG-KEY-td-agent'
+    'id'      => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
+    'content' => file('fluentd/GPG-KEY-td-agent'),
   },
   $include      = {
     'src' => false,
@@ -27,4 +27,6 @@ class fluentd::repo::apt (
     key          => $key,
     include      => $include,
   }
+
+  Class['apt::update'] -> Package['fluentd']
 }

manifests/repo/yum.pp

@@ -3,7 +3,7 @@
 class fluentd::repo::yum (
   $ensure   = 'present',
   $descr    = 'TreasureData',
-  $baseurl  = 'https://packages.treasuredata.com/2/redhat/$releasever/$basearch',
+  $baseurl  = "https://packages.treasuredata.com/${::fluentd::repo_version}/redhat/\$releasever/\$basearch",
   $enabled  = '1',
   $gpgcheck = '1',
   $gpgkey   = 'https://packages.treasuredata.com/GPG-KEY-td-agent',

manifests/source.pp

@@ -31,20 +31,18 @@
 # Copyright
 # ---------
 #
-# Copyright 2015 wywy GmbH, unless otherwise noted.
+# Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::source (
-  $ensure   = present,
-  $priority = 10,
-  $config   = {},
+  String  $ensure   = present,
+  Integer $priority = 10,
+  Hash    $config   = {},
 ) {
 
   # parameter validation
   if ! ($ensure in [ 'present', 'absent' ]) {
     fail('ensure parameter must be present or absent')
   }
-  validate_integer($priority)
-  validate_hash($config)
 
   fluentd::config::file { "source-${title}":
     ensure   => $ensure,

metadata.json

@@ -1,17 +1,17 @@
 {
-  "name": "wywygmbh-fluentd",
-  "version": "0.5.6",
-  "author": "wywy GmbH",
+  "name": "pilchkinstein-fluentd",
+  "version": "1.0.0",
+  "author": "pilchkinstein-gtv",
   "summary": "Generic module for fluentd (td-agent).",
   "license": "Apache-2.0",
-  "source": "https://github.com/wywygmbh/puppet-fluentd",
-  "project_page": "https://github.com/wywygmbh/puppet-fluentd",
-  "issues_url": "https://github.com/wywygmbh/puppet-fluentd/issues",
+  "source": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
+  "project_page": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
+  "issues_url": "https://github.com/pilchkinstein-gtv/puppet-fluentd/issues",
   "tags": ["fluentd", "td-agent"],
   "operatingsystem_support": [
     {
       "operatingsystem": "Ubuntu",
-      "operatingsystemrelease": [ "14.04", "12.04" ]
+      "operatingsystemrelease": [ "16.04", "14.04" ]
     },
     {
       "operatingsystem": "Debian",
@@ -24,7 +24,7 @@
   ],
   "dependencies": [
     { "name": "puppetlabs/stdlib", "version_requirement": ">= 4.6.0 < 5.0.0" },
-    { "name": "puppetlabs/apt", "version_requirement": ">= 2.0.0 < 3.0.0" }
+    { "name": "puppetlabs/apt", "version_requirement": ">= 4.0.0" }
   ],
   "data_provider": null
 }

spec/classes/fluentd_config_spec.rb

@@ -1,46 +1,31 @@
 require 'spec_helper'
 
 describe 'fluentd::config', :type => :class do
-  shared_examples 'when called with no parameters' do
-    it {
-      should contain_file('/etc/td-agent/td-agent.conf').with({
-        'ensure' => 'file',
-        'owner'  => 'root',
-        'group'  => 'root',
-        'source' => 'puppet:///modules/fluentd/td-agent.conf',
-        'notify' => 'Class[Fluentd::Service]'
-      })
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-      should contain_file('/etc/td-agent/conf.d').with({
-        'ensure' => 'directory',
-        'owner'  => 'td-agent',
-        'group'  => 'td-agent',
-        'mode'   => '0750'
-      })
-    }
-  end
+      describe 'when called with no parameters' do
 
-  context 'when osfamily is Debian' do
-    let(:facts) {
-      {
-        :osfamily        => 'Debian',
-        :lsbdistid       => 'Ubuntu',
-        :operatingsystem => 'Ubuntu',
-        :lsbdistcodename => 'precise',
-        :architecture    => 'amd64',
-      }
-    }
+        it {
+          should contain_file('/etc/td-agent/td-agent.conf').with({
+            'ensure' => 'file',
+            'owner'  => 'root',
+            'group'  => 'root',
+            'source' => 'puppet:///modules/fluentd/td-agent.conf',
+            'notify' => 'Class[Fluentd::Service]'
+          })
 
-    include_examples 'when called with no parameters'
-  end
-
-  context 'when osfamily is RedHat' do
-    let(:facts) {
-      {
-        :osfamily => 'RedHat',
-      }
-    }
-
-    include_examples 'when called with no parameters'
+          should contain_file('/etc/td-agent/conf.d').with({
+            'ensure' => 'directory',
+            'owner'  => 'td-agent',
+            'group'  => 'td-agent',
+            'mode'   => '0750'
+          })
+        }
+      end
+    end
   end
 end

spec/classes/fluentd_install_spec.rb

@@ -1,38 +1,21 @@
 require 'spec_helper'
 
 describe 'fluentd::install', :type => :class do
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  shared_examples 'when called with no parameters' do
-    it {
-      should contain_package('fluentd').with({
-        'ensure'          => 'installed',
-        'name'            => 'td-agent',
-        'install_options' => []
-      })
-    }
+      describe 'when called with no parameters' do
+        it {
+          should contain_package('fluentd').with({
+            'ensure'          => 'installed',
+            'name'            => 'td-agent',
+            'install_options' => []
+          })
+        }
+      end
+    end
   end
-
-  context 'when osfamily is Debian' do
-    let(:facts) {
-      {
-        :osfamily        => 'Debian',
-        :lsbdistid       => 'Ubuntu',
-        :operatingsystem => 'Ubuntu',
-        :lsbdistcodename => 'precise',
-        :architecture    => 'amd64',
-      }
-    }
-    include_examples 'when called with no parameters'
-  end
-
-  context 'when osfamily is RedHat' do
-    let(:facts) {
-      {
-        :osfamily => 'RedHat',
-      }
-    }
-
-    include_examples 'when called with no parameters'
-  end
-
 end

spec/classes/fluentd_repo_apt_spec.rb

@@ -1,34 +1,81 @@
 require 'spec_helper'
 
 describe 'fluentd::repo::apt', :type => :class do
-  let(:facts) {
-    {
-      :osfamily        => 'Debian',
-      :lsbdistid       => 'Ubuntu',
-      :operatingsystem => 'Ubuntu',
-      :lsbdistcodename => 'precise',
-      :architecture    => 'amd64',
-    }
-  }
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  describe 'when called with no parameters on Ubuntu' do
-    it {
-      should contain_class('apt')
-      should contain_apt__source('treasure-data').with({
-        'ensure'       => 'present',
-        'location'     => 'http://packages.treasuredata.com/2/ubuntu/precise',
-        'release'      => 'precise',
-        'repos'        => 'contrib',
-        'architecture' => 'amd64',
-        'key'          => {
-          'id'     => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
-          'source' => 'http://packages.treasuredata.com/GPG-KEY-td-agent'
-        },
-        'include'      => {
-          'src' => false,
-          'deb' => true
+      case facts[:osfamily]
+      when 'Debian'
+        it {
+          should contain_class('apt')
+          should contain_apt__source('treasure-data').with({
+            'ensure'       => 'present',
+            'repos'        => 'contrib',
+            'architecture' => 'amd64',
+            'key'          => {
+                'id'     => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
+                'source' => 'http://packages.treasuredata.com/GPG-KEY-td-agent'
+            },
+            'include'      => {
+                'src' => false,
+                'deb' => true
+            }
+          })
         }
-      })
-    }
+        case facts[:operatingsystem]
+        when 'Ubuntu'
+          case facts[:lsbdistcodename]
+          when 'precise'
+            it {
+              should contain_apt__source('treasure-data').with({
+                'location'     => 'http://packages.treasuredata.com/2/ubuntu/precise',
+                'release'      => 'precise'
+              })
+            }
+          when 'trusty'
+            it {
+              should contain_apt__source('treasure-data').with({
+                'location'     => 'http://packages.treasuredata.com/2/ubuntu/trusty',
+                'release'      => 'trusty'
+              })
+            }
+          when 'xenial'
+            it {
+              should contain_apt__source('treasure-data').with({
+                'location'     => 'http://packages.treasuredata.com/2/ubuntu/xenial',
+                'release'      => 'xenial'
+              })
+            }
+          end
+        when 'Debian'
+          case facts[:lsbdistcodename]
+          when 'jessie'
+            it {
+              should contain_apt__source('treasure-data').with({
+                'location'     => 'http://packages.treasuredata.com/2/debian/jessie',
+                'release'      => 'jessie'
+              })
+          }
+          when 'squeeze'
+            it {
+              should contain_apt__source('treasure-data').with({
+                'location'     => 'http://packages.treasuredata.com/2/debian/squeeze',
+                'release'      => 'squeeze'
+              })
+            }
+          when 'wheezy'
+            it {
+              should contain_apt__source('treasure-data').with({
+                'location'     => 'http://packages.treasuredata.com/2/debian/wheezy',
+                'release'      => 'wheezy'
+              })
+            }
+          end
+        end
+      end
+    end
   end
 end

spec/classes/fluentd_repo_spec.rb

@@ -1,30 +1,21 @@
 require 'spec_helper'
 
 describe 'fluentd::repo', :type => :class do
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  context 'when called with no parameters on osfamily Debian' do
-    let(:facts) {
-      {
-        :osfamily        => 'Debian',
-        :lsbdistid       => 'Ubuntu',
-        :operatingsystem => 'Ubuntu',
-        :lsbdistcodename => 'precise',
-        :architecture    => 'amd64',
-      }
-    }
-    it {
-      should contain_class('fluentd::repo::apt')
-    }
-  end
-
-  context 'when called with no parameters on osfamily RedHat' do
-    let(:facts) {
-      {
-        :osfamily => 'RedHat',
-      }
-    }
-    it {
-      should contain_class('fluentd::repo::yum')
-    }
+      describe 'when called with no parameters' do
+        case facts[:osfamily]
+        when 'Debian'
+          it { is_expected.to contain_class('fluentd::repo::apt') }
+        when 'RedHat'
+          it { is_expected.to contain_class('fluentd::repo::yum') }
+        end
+      end
+    end
   end
 end
+

spec/classes/fluentd_repo_yum_spec.rb

@@ -1,27 +1,32 @@
 require 'spec_helper'
 
 describe 'fluentd::repo::yum', :type => :class do
-  let(:facts) {
-    {
-      :osfamily => 'RedHat',
-    }
-  }
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  describe 'when called with no parameters on RedHat' do
-    it {
-      should contain_yumrepo('treasure-data').with({
-        'ensure'   => 'present',
-        'baseurl'  => 'https://packages.treasuredata.com/2/redhat/$releasever/$basearch',
-        'descr'    => 'TreasureData',
-        'enabled'  => '1',
-        'gpgcheck' => '1'
-      }).that_notifies('Exec[add GPG key]')
+      describe 'when called with no parameters' do
+        case facts[:osfamily]
+        when 'RedHat'
+          it {
+            should contain_yumrepo('treasure-data').with({
+              'ensure'   => 'present',
+              'baseurl'  => 'https://packages.treasuredata.com/2/redhat/$releasever/$basearch',
+              'descr'    => 'TreasureData',
+              'enabled'  => '1',
+              'gpgcheck' => '1'
+            }).that_notifies('Exec[add GPG key]')
 
-      should contain_exec('add GPG key').with({
-        'command'     => 'rpm --import https://packages.treasuredata.com/GPG-KEY-td-agent',
-        'path'        => '/bin:/usr/bin/',
-        'refreshonly' => 'true'
-      })
-    }
+            should contain_exec('add GPG key').with({
+              'command'     => 'rpm --import https://packages.treasuredata.com/GPG-KEY-td-agent',
+              'path'        => '/bin:/usr/bin/',
+              'refreshonly' => 'true'
+            })
+          }
+        end
+      end
+    end
   end
 end

spec/classes/fluentd_service_spec.rb

@@ -1,39 +1,23 @@
 require 'spec_helper'
 
 describe 'fluentd::config', :type => :class do
-  shared_examples 'when called with no parameters' do
-    it {
-      should contain_service('fluentd').with({
-        'name'       => 'td-agent',
-        'ensure'     => 'running',
-        'enable'     => 'true',
-        'hasstatus'  => 'true',
-        'hasrestart' => 'true'
-      })
-    }
-  end
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  context 'when osfamily is Debian' do
-    let(:facts) {
-      {
-        :osfamily        => 'Debian',
-        :lsbdistid       => 'Ubuntu',
-        :operatingsystem => 'Ubuntu',
-        :lsbdistcodename => 'precise',
-        :architecture    => 'amd64',
-      }
-    }
-
-    include_examples 'when called with no parameters'
-  end
-
-  context 'when osfamily is RedHat' do
-    let(:facts) {
-      {
-        :osfamily => 'RedHat',
-      }
-    }
-
-    include_examples 'when called with no parameters'
+      describe 'when called with no parameters' do
+        it {
+          should contain_service('fluentd').with({
+            'name'       => 'td-agent',
+            'ensure'     => 'running',
+            'enable'     => 'true',
+            'hasstatus'  => 'true',
+            'hasrestart' => 'true'
+          })
+        }
+      end
+    end
   end
 end

spec/classes/fluentd_spec.rb

@@ -1,37 +1,20 @@
 require 'spec_helper'
 
 describe 'fluentd', :type => :class do
-  shared_examples 'when called with no parameters' do
-    it {
-      should contain_class('fluentd::repo')
-      should contain_class('fluentd::install')
-      should contain_class('fluentd::config')
-      should contain_class('fluentd::service')
-    }
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
+
+      describe 'when called with no parameters' do
+        it {
+          should contain_class('fluentd::repo')
+          should contain_class('fluentd::install')
+          should contain_class('fluentd::config')
+          should contain_class('fluentd::service')
+        }
+      end
+    end
   end
-
-  context 'when osfamily is Debian' do
-    let(:facts) {
-      {
-        :osfamily        => 'Debian',
-        :lsbdistid       => 'Ubuntu',
-        :operatingsystem => 'Ubuntu',
-        :lsbdistcodename => 'precise',
-        :architecture    => 'amd64',
-      }
-    }
-
-    include_examples 'when called with no parameters'
-  end
-
-  context 'when osfamily is RedHat' do
-    let(:facts) {
-      {
-        :osfamily => 'RedHat',
-      }
-    }
-
-    include_examples 'when called with no parameters'
-  end
-
 end

spec/classes/fluentd_user_spec.rb

@@ -1,37 +1,21 @@
 require 'spec_helper'
 
 describe 'fluentd::user', :type => :class do
-  shared_examples 'when called with no parameters' do
-    it {
-      should contain_user('fluentd').with({
-        'name'   => 'td-agent',
-        'gid'    => 'td-agent',
-        'groups' => ['adm']
-      })
-    }
-  end
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  context 'when osfamily is Debian' do
-    let(:facts) {
-      {
-        :osfamily        => 'Debian',
-        :lsbdistid       => 'Ubuntu',
-        :operatingsystem => 'Ubuntu',
-        :lsbdistcodename => 'precise',
-        :architecture    => 'amd64',
-      }
-    }
-
-    include_examples 'when called with no parameters'
-  end
-
-  context 'when osfamily is RedHat' do
-    let(:facts) {
-      {
-        :osfamily => 'RedHat',
-      }
-    }
-
-    include_examples 'when called with no parameters'
+      describe 'when called with no parameters' do
+        it {
+          should contain_user('fluentd').with({
+            'name'   => 'td-agent',
+            'gid'    => 'td-agent',
+            'groups' => ['adm']
+          })
+        }
+      end
+    end
   end
 end

spec/defines/filter_spec.rb

@@ -1,40 +1,34 @@
 require 'spec_helper'
 
 describe 'fluentd::filter' do
-  let(:facts) {
-    {
-      :osfamily        => 'Debian',
-      :lsbdistid       => 'Ubuntu',
-      :operatingsystem => 'Ubuntu',
-      :lsbdistcodename => 'precise',
-      :architecture    => 'amd64',
-    }
-  }
-  let(:pre_condition) { 'include fluentd' }
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  context 'basic filter' do
-    let(:title) { 'test' }
-    let(:params) do
-      {
-          priority: '20',
-          pattern: '*.test',
-          config: {
+      let(:pre_condition) { 'include fluentd' }
+
+      describe 'basic filter' do
+        let(:title) { 'test' }
+        let(:params) do
+          {
+            priority: 20,
+            pattern: '*.test',
+            config: {
               'type'   => 'record_transformer',
               'record' => {
                 'hostname' => '${hostname}'
               }
+            }
           }
-      }
-    end
+        end
 
-    it do
-      is_expected.to contain_file('/etc/td-agent/conf.d/20-filter-test.conf').
-        with_content(/<filter \*.test>/).
-        with_content(/type record_transformer/).
-        with_content(/<record>/).
-        with_content(/hostname \$\{hostname\}/).
-        with_content(/<\/record>/).
-        with_content(/<\/filter>/)
+        it do
+          is_expected.to contain_file('/etc/td-agent/conf.d/20-filter-test.conf').
+          with_content(IO.read(File.join(File.dirname(__FILE__), '../fixtures/files/filter_record_transformer.conf')))
+        end
+      end
     end
   end
-end
+end

spec/defines/match_spec.rb

@@ -1,94 +1,71 @@
 require 'spec_helper'
 
 describe 'fluentd::match' do
-  let(:facts) {
-    {
-      :osfamily        => 'Debian',
-      :lsbdistid       => 'Ubuntu',
-      :operatingsystem => 'Ubuntu',
-      :lsbdistcodename => 'precise',
-      :architecture    => 'amd64',
-    }
-  }
-  let(:pre_condition) { 'include fluentd' }
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  context 'basic match' do
-    let(:title) { 'test' }
-    let(:params) do
-      {
-          priority: '30',
-          pattern: '*.test',
-          config: {
-              'flush_interval' => '30s',
-              'type'           => 'secure_forward',
-              'secure'         => 'yes',
-              'shared_key'     => 'my_shared_key',
-              'self_hostname'  => 'instance.test.com',
-              'ca_cert_path'   => '/path/to/ca.cert',
-              'server'         => [
-                  {
-                    'host' => 'test.server.com',
-                  }
-              ]
+      let(:pre_condition) { 'include fluentd' }
+
+      describe 'basic match' do
+        let(:title) { 'test' }
+        let(:params) do
+          {
+              priority: 30,
+              pattern: '*.test',
+              config: {
+                  'flush_interval' => '30s',
+                  'type'           => 'secure_forward',
+                  'secure'         => 'yes',
+                  'shared_key'     => 'my_shared_key',
+                  'self_hostname'  => 'instance.test.com',
+                  'ca_cert_path'   => '/path/to/ca.cert',
+                  'server'         => [
+                      {
+                        'host' => 'test.server.com',
+                      }
+                  ]
+              }
           }
-      }
-    end
+        end
 
-    it do
-      is_expected.to contain_file('/etc/td-agent/conf.d/30-match-test.conf').
-        with_content(/<match \*.test>/).
-        with_content(/flush_interval 30s/).
-        with_content(/type secure_forward/).
-        with_content(/secure yes/).
-        with_content(/shared_key my_shared_key/).
-        with_content(/self_hostname instance.test.com/).
-        with_content(/ca_cert_path \/path\/to\/ca.cert/).
-        with_content(/<server>/).
-        with_content(/host test.server.com/).
-        with_content(/<\/server>/).
-        with_content(/<\/match>/)
-    end
-  end
+        it do
+          is_expected.to contain_file('/etc/td-agent/conf.d/30-match-test.conf').
+              with_content(IO.read(File.join(File.dirname(__FILE__), '../fixtures/files/match_secure_forward.conf')))
+        end
+      end
 
-  context 'multiple store' do
-    let(:title) { 'test' }
-    let(:params) do
-      {
-          priority: '30',
-          pattern: '*.test',
-          config: {
-              'type'  => 'copy',
-              'store' => [
-                  {
-                    'type'           => 'elasticsearch',
-                    'logstashformat' => true,
-                    'hosts'          => '172.20.10.17:9200',
-                    'flush_interval' => '30s',
-                  },
-                  {
-                    'type' => 'file',
-                    'path' => '/tmp/td-agent-debug.log',
-                }
-              ]
+      describe 'multiple store' do
+        let(:title) { 'test' }
+        let(:params) do
+          {
+              priority: 30,
+              pattern: '*.test',
+              config: {
+                  'type'  => 'copy',
+                  'store' => [
+                      {
+                        'type'           => 'elasticsearch',
+                        'logstashformat' => true,
+                        'hosts'          => '172.20.10.17:9200',
+                        'flush_interval' => '30s',
+                      },
+                      {
+                        'type' => 'file',
+                        'path' => '/tmp/td-agent-debug.log',
+                    }
+                  ]
+              }
           }
-      }
-    end
+        end
 
-    it do
-      is_expected.to contain_file('/etc/td-agent/conf.d/30-match-test.conf').
-          with_content(/<match \*.test>/).
-          with_content(/type copy/).
-          with_content(/<store>/).
-          with_content(/type elasticsearch/).
-          with_content(/logstashformat true/).
-          with_content(/hosts 172.20.10.17:9200/).
-          with_content(/flush_interval 30s/).
-          with_content(/<\/store>/).
-          with_content(/<store>/).
-          with_content(/type file/).
-          with_content(/path \/tmp\/td-agent-debug.log/).
-          with_content(/<\/store>/).
-          with_content(/<\/match>/)
+        it do
+          is_expected.to contain_file('/etc/td-agent/conf.d/30-match-test.conf').
+              with_content(IO.read(File.join(File.dirname(__FILE__), '../fixtures/files/match_copy.conf')))
+        end
+      end
     end
   end
 end

spec/defines/plugin_spec.rb

@@ -1,52 +1,51 @@
 require 'spec_helper'
 
 describe 'fluentd::plugin' do
-  let(:facts) {
-    {
-      :osfamily        => 'Debian',
-      :lsbdistid       => 'Ubuntu',
-      :operatingsystem => 'Ubuntu',
-      :lsbdistcodename => 'precise',
-      :architecture    => 'amd64',
-    }
-  }
-  let(:pre_condition) { 'include fluentd' }
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  context 'gem plugin' do
-    let(:title) { 'fluent-plugin-elasticsearch' }
-    let(:params) do
-      {
-          type: 'gem'
-      }
-    end
+      let(:pre_condition) { 'include fluentd' }
 
-    it do
-      should contain_package('fluent-plugin-elasticsearch').with({
-        'ensure'   => 'present',
-        'provider' => 'fluentd_gem',
-        'notify'   => 'Class[Fluentd::Service]'
-      })
+      describe 'gem plugin' do
+        let(:title) { 'fluent-plugin-elasticsearch' }
+        let(:params) do
+          {
+              type: 'gem'
+          }
+        end
+
+        it do
+          should contain_package('fluent-plugin-elasticsearch').with({
+            'ensure'   => 'present',
+            'provider' => 'fluentd_gem',
+            'notify'   => 'Class[Fluentd::Service]'
+          })
+        end
+      end
+
+      describe 'file plugin' do
+        let(:title) { 'fluent-plugin-test' }
+        let(:params) do
+          {
+              type: 'file',
+              source: 'puppet:///path/to/source'
+          }
+        end
+
+        it do
+          should contain_file('/etc/td-agent/plugin/fluent-plugin-test').with({
+            'ensure' => 'present',
+            'owner'  => 'td-agent',
+            'group'  => 'td-agent',
+            'mode'   => '0640',
+            'source' => 'puppet:///path/to/source',
+            'notify' => 'Class[Fluentd::Service]'
+          })
+        end
+      end
     end
   end
-
-  context 'file plugin' do
-    let(:title) { 'fluent-plugin-test' }
-    let(:params) do
-      {
-          type: 'file',
-          source: 'puppet:///path/to/source'
-      }
-    end
-
-    it do
-      should contain_file('/etc/td-agent/plugin/fluent-plugin-test').with({
-        'ensure' => 'present',
-        'owner'  => 'td-agent',
-        'group'  => 'td-agent',
-        'mode'   => '0640',
-        'source' => 'puppet:///path/to/source',
-        'notify' => 'Class[Fluentd::Service]'
-      })
-    end
-  end
-end
+end

spec/defines/source_spec.rb

@@ -1,39 +1,41 @@
 require 'spec_helper'
 
 describe 'fluentd::source' do
-  let(:facts) {
-    {
-      :osfamily        => 'Debian',
-      :lsbdistid       => 'Ubuntu',
-      :operatingsystem => 'Ubuntu',
-      :lsbdistcodename => 'precise',
-      :architecture    => 'amd64',
-    }
-  }
-  let(:pre_condition) { 'include fluentd' }
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let :facts do
+        facts
+      end
 
-  context 'basic source' do
-    let(:title) { 'test' }
-    let(:params) do
-      {
-          priority: '10',
-          config: {
-            'type'   => 'tail',
-            'format' => 'json',
-            'path'   => '/var/log/test-application/*.json',
-            'tag'    => 'application.test'
+      let(:pre_condition) { 'include fluentd' }
+
+      describe 'basic source' do
+        let(:title) { 'test' }
+        let(:params) do
+          {
+              priority: 10,
+              config: {
+                'type'   => 'tail',
+                'format' => 'json',
+                'path'   => '/var/log/test-application/*.json',
+                'parse'  => {
+                    'type'       => 'regexp',
+                    'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+                },
+                'storage'=> {
+                    'type'   => 'local',
+                    'path'   => 'test.pos'
+                },
+                'tag'    => 'application.test'
+              }
           }
-      }
-    end
+        end
 
-    it do
-      is_expected.to contain_file('/etc/td-agent/conf.d/10-source-test.conf').
-        with_content(/<source>/).
-        with_content(/type tail/).
-        with_content(/format json/).
-        with_content(/path \/var\/log\/test-application\/\*.json/).
-        with_content(/tag application.test/).
-        with_content(/<\/source>/)
+        it do
+          is_expected.to contain_file('/etc/td-agent/conf.d/10-source-test.conf').
+              with_content(IO.read(File.join(File.dirname(__FILE__), '../fixtures/files/source_tail.conf')))
+        end
+      end
     end
   end
-end
+end

spec/fixtures/files/filter_record_transformer.conf

@@ -0,0 +1,6 @@
+<filter *.test>
+  type record_transformer
+  <record>
+    hostname ${hostname}
+  </record>
+</filter>

spec/fixtures/files/match_copy.conf

@@ -0,0 +1,13 @@
+<match *.test>
+  type copy
+  <store>
+    type elasticsearch
+    logstashformat true
+    hosts 172.20.10.17:9200
+    flush_interval 30s
+  </store>
+  <store>
+    type file
+    path /tmp/td-agent-debug.log
+  </store>
+</match>

spec/fixtures/files/match_secure_forward.conf

@@ -0,0 +1,11 @@
+<match *.test>
+  flush_interval 30s
+  type secure_forward
+  secure yes
+  shared_key my_shared_key
+  self_hostname instance.test.com
+  ca_cert_path /path/to/ca.cert
+  <server>
+    host test.server.com
+  </server>
+</match>

spec/fixtures/files/source_tail.conf

@@ -0,0 +1,14 @@
+<source>
+  type tail
+  format json
+  path /var/log/test-application/*.json
+  <parse>
+    type regexp
+    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
+  </parse>
+  <storage>
+    type local
+    path test.pos
+  </storage>
+  tag application.test
+</source>

spec/spec_helper.rb

@@ -1 +1,3 @@
 require 'puppetlabs_spec_helper/module_spec_helper'
+require 'rspec-puppet-facts'
+include RspecPuppetFacts

templates/filter.erb

@@ -1,13 +1,13 @@
 <filter <%= @pattern %>>
 <% @config.each do |key, val| -%>
-    <%- if val.is_a?( Hash ) -%>
-        <<%= key -%>>
-        <%- val.each do |key2, val2| -%>
-            <%= key2 -%> <%= val2 %>
-        <%- end -%>
-        </<%= key %>>
-    <%- else -%>
-        <%= key -%> <%= val %>
-    <%- end -%>
+  <%- if val.is_a?( Hash ) -%>
+  <<%= key -%>>
+  <%- val.each do |key2, val2| -%>
+    <%= key2 -%> <%= val2 %>
+  <%- end -%>
+  </<%= key %>>
+  <%- else -%>
+  <%= key -%> <%= val %>
+  <%- end -%>
 <% end -%>
 </filter>

templates/match.erb

@@ -1,39 +1,41 @@
 <match <%= @pattern %>>
 <% @config.each do |key, val| -%>
-    <%- if val.is_a?( Hash ) -%>
-        <<%= key -%>>
-        <%- val.each do |key, val| -%>
-            <%= key -%> <%= val %>
-        <%- end -%>
-        </<%= key %>>
-    <%- elsif val.is_a?( Array ) -%>
-        <%- val.each do |k, v| -%>
-            <<%= key -%>>
-            <%- if k.is_a?( Hash ) -%>
-                <%- k.each do |x, y| -%>
-                    <%- if y.is_a?( Array ) -%>
-                        <%- y.each do |a, b| -%>
-                            <<%= x -%>>
-                            <%- if a.is_a?( Hash ) -%>
-                                <%- a.each do |c, d| -%>
-                                    <%= c -%> <%= d %>
-                                <%- end -%>
-                            <%- else -%>
-                                <%= a -%> <%= b %>
-                            <%- end -%>
-                            </<%= x %>>
-                        <%- end -%>
-                    <%- else -%>
-                        <%= x -%> <%= y %>
-                    <%- end -%>
-                <%- end -%>
-            <%- else -%>
-                <%= k -%> <%= v %>
-            <%- end -%>
-            </<%= key %>>
-        <%- end -%>
-    <%- else -%>
-        <%= key -%> <%= val %>
+  <%- if key == 'buffer_chunk_keys'; next; end-%>
+  <%- if val.is_a?( Hash ) -%>
+  <<%= key -%><%- if key == 'buffer' && !@config['buffer_chunk_keys'].nil? -%> <%= @config['buffer_chunk_keys'] -%><%- end -%>>
+  <%- val.each do |key, val| -%>
+    <%= key -%> <%= val %>
+  <%- end -%>
+  </<%= key %>>
+  <%- elsif val.is_a?( Array ) -%>
+  <%- val.each do |k, v| -%>
+  <<%= key -%>>
+    <%- if k.is_a?( Hash ) -%>
+    <%- k.each do |x, y| -%>
+    <%- if y.is_a?( Array ) -%>
+    <%- y.each do |a, b| -%>
+    <<%= x -%><%- if x == 'buffer' && !k['buffer_chunk_keys'].nil? -%> <%= k['buffer_chunk_keys'] -%><%- end -%>>
+      <%- if a.is_a?( Hash ) -%>
+      <%- a.each do |c, d| -%>
+      <%= c -%> <%= d %>
+      <%- end -%>
+      <%- else -%>
+      <%= a -%> <%= b %>
+      <%- end -%>
+    </<%= x %>>
     <%- end -%>
+    <%- else -%>
+    <%- if x == 'buffer_chunk_keys'; next; end -%>
+    <%= x -%> <%= y %>
+    <%- end -%>
+    <%- end -%>
+    <%- else -%>
+    <%= k -%> <%= v %>
+    <%- end -%>
+  </<%= key %>>
+  <%- end -%>
+  <%- else -%>
+  <%= key -%> <%= val %>
+  <%- end -%>
 <% end -%>
 </match>

templates/source.erb

@@ -1,5 +1,13 @@
 <source>
 <% @config.each do |key, val| -%>
+  <%- if val.is_a?( Hash ) -%>
+  <<%= key -%>>
+  <%- val.each do |key, val| -%>
     <%= key -%> <%= val %>
+  <%- end -%>
+  </<%= key %>>
+  <%- else -%>
+  <%= key -%> <%= val %>
+  <%- end -%>
 <% end -%>
 </source>