Upgrade to td-agent v4 on older Puppet installs

The fluentd_gem provider for installing fluentd plugins uses
Puppet's built-in gem provider. Unfortunately, that gem provider
on our Puppet version is not compatible with td-agent's new embedded
Ruby since it includes the no-rdoc and no-ri flags deprecated and
replaced by the no-document flag. This install function override
replaces the flags and might be removable after a Puppet upgrade.

.travis.yml

@@ -6,8 +6,6 @@ script: bundle exec rake validate && bundle exec rake lint && bundle exec rake s
 matrix:
   fast_finish: true
   include:
-  - rvm: 2.1.10
-    env: PUPPET_VERSION="~> 3.0"
   - rvm: 2.1.10
     env: PUPPET_VERSION="~> 4.0" STRICT_VARIABLES="yes"
   - rvm: 2.2.7

CHANGELOG.md

@@ -1,4 +1,12 @@
 # Changelog
+
+## 1.0.0
+
+**Breaking changes:** This version drops puppet 3 support!
+
+* Fix hard coded user/group names in config manifest (#23)
+* migrate from stdlib validation to puppet datatypes (#24)
+
 ## 0.6.0
 * Updated README
 * fixed broken links, due to github organisation rename

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 4.0' : puppetversion = ENV['PUPPET_VERSION'].to_s
+ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 4.6' : puppetversion = ENV['PUPPET_VERSION'].to_s
 gem 'puppet', puppetversion
 gem 'puppetlabs_spec_helper', '>= 0.8.2'
 gem 'puppet-lint', '>= 1.0.0'
@@ -17,6 +17,6 @@ gem 'beaker-rspec', beakerrspec
 
 platforms :ruby_19 do
   gem 'public_suffix', '< 1.5.0', :platforms => :ruby_19
-  gem 'nokogiri', '< 1.7.0', :platforms => :ruby_19
+  gem 'nokogiri', '~> 1.8.1', :platforms => :ruby_19
   gem 'mime-types', '< 3.0', :platforms => :ruby_19
-end
+end

README.md

@@ -1,6 +1,6 @@
-# wywy/fluentd Puppet Module
+# pilchkinstein-gtv/fluentd Puppet Module
 
-[![GitHub version](https://badge.fury.io/gh/wywy%2Fpuppet-fluentd.svg)](https://github.com/wywy/puppet-fluentd)
+[![GitHub version](https://badge.fury.io/gh/wywy%2Fpuppet-fluentd.svg)](https://github.com/pilchkinstein-gtv/puppet-fluentd)
 [![Build Status](https://travis-ci.org/wywy/puppet-fluentd.svg?branch=master)](https://travis-ci.org/wywy/puppet-fluentd)
 
 
@@ -15,6 +15,7 @@
     * [Source](#source)
     * [Filter](#filter)
     * [Match](#match)
+    * [Match with buffer and chunk_keys](#match-with-buffer-and-chunk_keys)
     * [Match Store](#match-store)
   * [Plugin Installation](#plugin-installation)
   * [Requirements](#requirements)
@@ -88,7 +89,7 @@ include '::fluentd'
 
   Path to configuration files
   **Default:** '/etc/td-agent'
-  
+
 `config_dir`
 
   Configuration directory name
@@ -131,7 +132,18 @@ include '::fluentd'
     'type'   => 'tail',
     'format' => 'json',
     'path'   => '/var/log/test-application/*.json',
-    'tag'    => 'application.test'
+    'parse'  => {
+      'type'       => 'regexp',
+      'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+    },
+    'storage'=> {
+      'type'   => 'local',
+      'path'   => 'test.pos'
+    },
+    'tag'    => 'application.test',
+    'parse'  => {
+      'message_format' => 'auto'
+    }
   }
 }
 ```
@@ -142,7 +154,18 @@ include '::fluentd'
   type tail
   format json
   path /var/log/test-application/*.json
+  <parse>
+    type regexp
+    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
+  </parse>
+  <storage>
+    type local
+    path test.pos
+  </storage>
   tag application.test
+  <parse>
+    message_format auto
+  </parse>
 </source>
 ```
 #### Filter
@@ -201,7 +224,41 @@ include '::fluentd'
   </server>
 </match>
 ```
-### Match Store
+#### Match with buffer and chunk_keys
+
+buffer_chunk_keys will not be included in the template. They will be filtered and only be used as a "pattern" for the buffer section.
+
+```
+::fluentd::match { 'test':
+  priority => 98,
+  pattern  => 'test.**',
+  config   => {
+    type              => 'file',
+    path              => '/var/log/td-agent/test/${host}',
+    append            => true,
+    buffer_chunk_keys => 'host',
+    buffer            => {
+      type       => file,
+      path       => /var/log/td-agent/test/buffer/,
+      flush_mode => immediate,
+    },
+  },
+}
+```
+**creates:**
+```
+<match test.**>
+  type file
+  path /var/log/td-agent/test/${host}
+  append true
+  <buffer host>
+    type file
+    path /var/log/td-agent/test/buffer/
+    flush_mode immediate
+  </buffer>
+</match>
+```
+#### Match Store
 ```puppet
 ::fluentd::match { 'test':
   priority => 30,
@@ -217,7 +274,13 @@ include '::fluentd'
       },
       {
         'type' => 'file',
-        'path' => '/tmp/td-agent-debug.log',
+        'path' => '/tmp/td-agent-debug-${host}.log',
+        'buffer_chunk_keys' => 'host',
+        'buffer'         => [{
+          '@type'    => 'file',
+          path       => '/var/log/td-agent/buffer/',
+          flush_mode => 'immediate',
+          }]
       }
     ]
   }
@@ -236,7 +299,11 @@ include '::fluentd'
   </store>
   <store>
     type file
-    path /tmp/crs
+    path /tmp/td-agent-debug-${host}.log
+    <buffer host>
+      @type file
+      path /var/log/td-agent/buffer/
+      flush_mode immediate
   </store>
 </match>
 ```
@@ -263,14 +330,14 @@ This module gives you the possibility to install plugins as gem or files.
 ### Requirements
 
 Modules:
-* puppetlabs/apt >= 1.8.0 < 3.0.0
+* puppetlabs/apt >= 4.0.0
 
 ## Limitations
-This module has been built on and tested against Puppet 3.7.5 and higher.
+This module has been built on and tested against Puppet 4.6 and higher.
 
 The module has been tested on:
 
-* Ubuntu 12.04
+* Ubuntu 16.04
 
 Testing on other platforms has been light and cannot be guaranteed.
 
@@ -285,8 +352,8 @@ To run the tests install the ruby dependencies with `bundler` and execute
 bundle install --path vendor/bundle
 bundle exec rake spec
 bundle exec rake lint
-bundle exec rake beaker BEAKER_set=ubuntu-server-1204-x64
 bundle exec rake beaker BEAKER_set=ubuntu-server-1404-x64
+bundle exec rake beaker BEAKER_set=ubuntu-server-1604-x64
 bundle exec rake beaker BEAKER_set=debian-60-x64
 bundle exec rake beaker BEAKER_set=debian-78-x64
 bundle exec rake beaker BEAKER_set=debian-82-x64

examples/source.pp

@@ -2,9 +2,17 @@
 ::fluentd::source { 'test':
   priority => 10,
   config   => {
-    'type'   => 'tail',
-    'format' => 'json',
-    'path'   => '/var/log/test-application/*.json',
-    'tag'    => 'application.test'
+    'type'    => 'tail',
+    'format'  => 'json',
+    'path'    => '/var/log/test-application/*.json',
+    'parse'   => {
+        'type'       => 'regexp',
+        'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+    },
+    'storage' => {
+      'type' => 'local',
+      'path' => 'test.pos'
+    },
+    'tag'     => 'application.test'
   }
 }

files/GPG-KEY-td-agent

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: Encryption subkey has been removed due to apt-key parsing bug
+
+mQINBFhiI8wBEADThWLNd8IKPRw7Ygu3DHS4Sb/Yc6vSZSaMGJ6Wkj245jScvI+C
+nG4C4rtO/8ObUj5cUpb4CyfYZX8W4tp9x+W68c4paXevG4s+X4EE3uUsgdwTnFXi
+GMa57QDzR4p/JvjUjfGJ2UAr4Bfj8Q2S54LmIu6UAe82ce2B4tEHCeYSxkmVUDAZ
+utfmgKoVTbnceTemU0m5ANS6IC1/53KEhgB1sKm5G/FjRJGslHWb3mf+bLrhmlkP
+pA4BOKF2w3eFYH3LhWskxMS0SPM7J6aq+6LyNNqtlKL6lUS7qVjRQ6PlgFcmtG4J
+tijsZI62bDn1f44DmeLY+LMS/nM0xyIx94lYumGH5EYmjUECagqMool98/+Wx79A
+Thtg/1pYNzo8Z76qr0i3xLSRtsQ2Om2Rfal7VGadOrx4sqlkSaUaGI+hBc1r4tNy
+tERvBEMGSf78bWDbdzxSNEW4LUDUpniNQb0DrURfWkqRa3q4WcTJr8lpQM/NmAru
+owayAXQwKob+OIZ09/O69EaqVJ9MqsM3keQouSHShKvzNrppuo3D3z+Dpy05FsYw
+MAiIN7auXxy+XQwCVsKF083YaDHcC0I22GReEgt43yZXQ/b/J9QNrm5nJ+3Cpso3
+jJnMzubuniSOOdd3mXQ6MwgZvWgtH/nPF8oUX9VSGwqNohiKWcxQDxW7qQARAQAB
+tFRUcmVhc3VyZSBEYXRhLCBJbmMgKFRyZWFzdXJlIEFnZW50IE9mZmljaWFsIFNp
+Z25pbmcga2V5KSA8c3VwcG9ydEB0cmVhc3VyZS1kYXRhLmNvbT6JAjcEEwEIACEF
+AlhiI8wCGwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQkB+Rd6uXrL5GrhAA
+nh82+caSu9Qu/LW256gN5UjPUFhph66ElT1OVyAR2FoOmz2pJH3t8YYD5cUV2W6/
+xqJDmjl+vnL2HBgxjHKRCo2K3hrq6z4LoU7SpWDI1cZ03lkjh1yNx13S+9JvZNlp
+jit0WRIspke0n0vWSpNo4nh19Yg3EA1c+vGeHnmlYo6xwRHu6XOhhCwywtFRGC3a
+iMJzAV4N69ZU6P5VZZkC6LjYYQtF4aI10COLZ4AcObH2htGAZTj2KlZfdJHmr+Oa
+wY57giUYz7OF45LLCuqe+VwpGp2d3UK/MtCnXRLi5InMVJKDvyt18MzRDFuyA27e
+WSt+JumVqhEjawh3hmdzIS1cHKmv19gdeE8On2i2Lf8lyek8fsB/YPgADAmp2oSe
+cjLu0ocGbgxRjuCR29+6IG+DiUDFCkqFZNdLiGVqzjpjpYHaPhVe77ciwA8TCPru
+3dh5t/qv2HglSd7lj95IApZBtny5AK8NS4qtaOeZbBbbDRuOPL0c7fU3bqyIPy57
+zvdYi3KdjWZVCawcAmk3ILP83eFSivCRPRoyCqO+HX8U647BBWvlFuEbPa+Y1sgE
+12MEF/Y6VVJh3Ptw+h/qKRbra4LdA+5Y30q/9l6WGgbO/4h3NKmGeVCrAFvS3h92
+fS0ABYD1nAP7fSNS9RfYIqfBXtJem+tJ14YKJwWiAYU=
+=TPyg
+-----END PGP PUBLIC KEY BLOCK-----

lib/puppet/provider/package/fluentd-gem.rb

@@ -8,11 +8,20 @@ Puppet::Type.type(:package).provide :fluentd_gem, :parent => :gem do
     not a valid URL, it will be interpreted as the path to a local gem file.  If
     source is not present at all, the gem will be installed from the default gem
     repositories."
-  confine :exists => "/opt/td-agent/embedded/bin/gem"
+  confine :exists => "/opt/td-agent/bin/gem"
 
   has_feature :versionable, :install_options
 
-  commands :gemcmd => "/opt/td-agent/embedded/bin/gem"
+  commands :gemcmd => "/opt/td-agent/bin/gem"
+
+  def install(useversion = true)
+    command = [command(:gemcmd), "install"]
+    command << "--no-document" << resource[:name]
+    output = execute(command)
+
+    # Apparently some stupid gem versions don't exit non-0 on failure
+    self.fail "Could not install: #{output.chomp}" if output.include?("ERROR")
+  end
 
   def uninstall
     command = [command(:gemcmd), "uninstall"]

manifests/config/file.pp

@@ -28,8 +28,8 @@ define fluentd::config::file (
   file { $config_path:
     ensure  => $ensure,
     content => $content,
-    owner   => 'td-agent',
-    group   => 'td-agent',
+    owner   => $::fluentd::user_name,
+    group   => $::fluentd::user_group,
     mode    => '0644',
     notify  => Class['Fluentd::Service'],
   }

manifests/filter.pp

@@ -38,23 +38,16 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::filter (
-  $ensure   = present,
-  $priority = 20,
-  $pattern  = '**',
-  $config   = {},
+  Enum['present', 'absent'] $ensure   = present,
+  Integer                   $priority = 20,
+  String                    $pattern  = '**',
+  Hash                      $config   = {},
 ) {
 
-  # parameter validation
-  if ! ($ensure in [ 'present', 'absent' ]) {
-    fail('ensure parameter must be present or absent')
-  }
-  validate_integer($priority)
-  validate_string($pattern)
-  validate_hash($config)
-
   fluentd::config::file { "filter-${title}":
     ensure   => $ensure,
     priority => $priority,
     content  => template( 'fluentd/filter.erb'),
   }
+
 }

manifests/init.pp

@@ -8,6 +8,9 @@
 # [*repo_manage*]
 #   Include repository to install recent fluentd (td-agent) from
 #   Default: 'true'
+# [*repo_version*]
+#   The major version of td-agent to install
+#   Default: 3
 # [*package_ensure*]
 #   Package ensure
 #   Default: 'installed'
@@ -65,37 +68,25 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 class fluentd (
-  $repo_manage             = $::fluentd::params::repo_manage,
-  $package_ensure          = $::fluentd::params::package_ensure,
-  $package_name            = $::fluentd::params::package_name,
-  $package_install_options = $::fluentd::params::package_install_options,
-  $service_manage          = $::fluentd::params::service_manage,
-  $service_name            = $::fluentd::params::service_name,
-  $service_ensure          = $::fluentd::params::service_ensure,
-  $service_enable          = $::fluentd::params::service_enable,
-  $config_path             = $::fluentd::params::config_path,
-  $conf_dir                = $::fluentd::params::conf_dir,
-  $config_file             = $::fluentd::params::config_file,
-  $conf_dir_manage         = $::fluentd::params::conf_dir_manage,
-  $user_manage             = $::fluentd::params::user_manage,
-  $user_name               = $::fluentd::params::user_name,
-  $user_group              = $::fluentd::params::user_group,
-  $user_groups             = $::fluentd::params::user_groups,
+  Boolean                     $repo_manage             = $::fluentd::params::repo_manage,
+  Integer                     $repo_version            = $::fluentd::params::repo_version,
+  String                      $package_ensure          = $::fluentd::params::package_ensure,
+  String                      $package_name            = $::fluentd::params::package_name,
+  Array                       $package_install_options = $::fluentd::params::package_install_options,
+  Boolean                     $service_manage          = $::fluentd::params::service_manage,
+  String                      $service_name            = $::fluentd::params::service_name,
+  Enum['running', 'stopped']  $service_ensure          = $::fluentd::params::service_ensure,
+  Boolean                     $service_enable          = $::fluentd::params::service_enable,
+  String                      $config_path             = $::fluentd::params::config_path,
+  String                      $conf_dir                = $::fluentd::params::conf_dir,
+  String                      $config_file             = $::fluentd::params::config_file,
+  Boolean                     $conf_dir_manage         = $::fluentd::params::conf_dir_manage,
+  Boolean                     $user_manage             = $::fluentd::params::user_manage,
+  String                      $user_name               = $::fluentd::params::user_name,
+  String                      $user_group              = $::fluentd::params::user_group,
+  Array                       $user_groups             = $::fluentd::params::user_groups,
 ) inherits fluentd::params {
 
-  # parameter validation
-  validate_bool($repo_manage)
-  validate_string($package_ensure)
-  validate_string($package_name)
-  validate_array($package_install_options)
-  validate_bool($service_manage)
-  validate_string($service_name)
-  validate_bool($service_enable)
-
-  if ! ($service_ensure in [ 'running', 'stopped' ]) {
-    fail('service_ensure parameter must be running or stopped')
-  }
-
   # class calls
   include '::fluentd::repo'
   include '::fluentd::install'
@@ -103,6 +94,12 @@ class fluentd (
   include '::fluentd::config'
   include '::fluentd::service'
 
+  # Realize any virtual configurations
+  Fluentd::Plugin <| |>
+  Fluentd::Source <| |>
+  Fluentd::Filter <| |>
+  Fluentd::Match  <| |>
+
   # dependencies
   Class['::Fluentd::Repo']
   -> Class['::Fluentd::Install']

manifests/match.pp

@@ -43,23 +43,16 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::match (
-  $ensure   = present,
-  $priority = 30,
-  $pattern  = '**',
-  $config   = {},
+  Enum['present', 'absent'] $ensure   = present,
+  Integer                   $priority = 30,
+  String                    $pattern  = '**',
+  Hash                      $config   = {},
 ) {
 
-  # parameter validation
-  if ! ($ensure in [ 'present', 'absent' ]) {
-    fail('ensure parameter must be present or absent')
-  }
-  validate_integer($priority)
-  validate_string($pattern)
-  validate_hash($config)
-
   fluentd::config::file { "match-${title}":
     ensure   => $ensure,
     priority => $priority,
     content  => template( 'fluentd/match.erb'),
   }
+
 }

manifests/params.pp

@@ -18,6 +18,8 @@ class fluentd::params {
   $user_name   = 'td-agent'
   $user_group  = 'td-agent'
   $user_groups = ['adm']
+  # repo params
+  $repo_version = 4
 
   case $::osfamily {
     'Debian': {

manifests/plugin.pp

@@ -26,9 +26,9 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::plugin (
-  $ensure = present,
-  $type   = 'gem',
-  $source = undef,
+  String           $ensure = present,
+  String           $type   = 'gem',
+  Optional[String] $source = undef,
 ) {
 
   case $type {
@@ -43,7 +43,6 @@ define fluentd::plugin (
       if ! ($ensure in [ 'present', 'absent' ]) {
         fail('ensure parameter must be present or absent')
       }
-      validate_string($source)
 
       fluentd::plugin::file { $name:
         ensure  => $ensure,

manifests/repo.pp

@@ -4,7 +4,7 @@ class fluentd::repo inherits fluentd {
   if $::fluentd::repo_manage {
     case $::osfamily {
       'Debian': {
-        include ::fluentd::repo::apt
+        contain ::fluentd::repo::apt
       }
       'RedHat': {
         include ::fluentd::repo::yum

manifests/repo/apt.pp

@@ -2,13 +2,13 @@
 #
 class fluentd::repo::apt (
   $ensure       = 'present',
-  $location     = downcase("http://packages.treasuredata.com/2/${::lsbdistid}/${::lsbdistcodename}"),
+  $location     = downcase("http://packages.treasuredata.com/${::fluentd::repo_version}/${::lsbdistid}/${::lsbdistcodename}"),
   $release      = $::lsbdistcodename,
   $repos        = 'contrib',
   $architecture = $::architecture,
   $key          = {
-    'id'     => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
-    'source' => 'http://packages.treasuredata.com/GPG-KEY-td-agent'
+    'id'      => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
+    'content' => file('fluentd/GPG-KEY-td-agent'),
   },
   $include      = {
     'src' => false,
@@ -27,4 +27,6 @@ class fluentd::repo::apt (
     key          => $key,
     include      => $include,
   }
+
+  Class['apt::update'] -> Package['fluentd']
 }

manifests/repo/yum.pp

@@ -3,7 +3,7 @@
 class fluentd::repo::yum (
   $ensure   = 'present',
   $descr    = 'TreasureData',
-  $baseurl  = 'https://packages.treasuredata.com/2/redhat/$releasever/$basearch',
+  $baseurl  = "https://packages.treasuredata.com/${::fluentd::repo_version}/redhat/\$releasever/\$basearch",
   $enabled  = '1',
   $gpgcheck = '1',
   $gpgkey   = 'https://packages.treasuredata.com/GPG-KEY-td-agent',

manifests/source.pp

@@ -34,17 +34,15 @@
 # Copyright 2015 wywy, unless otherwise noted.
 #
 define fluentd::source (
-  $ensure   = present,
-  $priority = 10,
-  $config   = {},
+  String  $ensure   = present,
+  Integer $priority = 10,
+  Hash    $config   = {},
 ) {
 
   # parameter validation
   if ! ($ensure in [ 'present', 'absent' ]) {
     fail('ensure parameter must be present or absent')
   }
-  validate_integer($priority)
-  validate_hash($config)
 
   fluentd::config::file { "source-${title}":
     ensure   => $ensure,

metadata.json

@@ -1,17 +1,17 @@
 {
-  "name": "wywygmbh-fluentd",
-  "version": "0.6.0",
-  "author": "wywy",
+  "name": "pilchkinstein-fluentd",
+  "version": "1.0.0",
+  "author": "pilchkinstein-gtv",
   "summary": "Generic module for fluentd (td-agent).",
   "license": "Apache-2.0",
-  "source": "https://github.com/wywy/puppet-fluentd",
-  "project_page": "https://github.com/wywy/puppet-fluentd",
-  "issues_url": "https://github.com/wywy/puppet-fluentd/issues",
+  "source": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
+  "project_page": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
+  "issues_url": "https://github.com/pilchkinstein-gtv/puppet-fluentd/issues",
   "tags": ["fluentd", "td-agent"],
   "operatingsystem_support": [
     {
       "operatingsystem": "Ubuntu",
-      "operatingsystemrelease": [ "14.04", "12.04" ]
+      "operatingsystemrelease": [ "16.04", "14.04" ]
     },
     {
       "operatingsystem": "Debian",
@@ -24,7 +24,7 @@
   ],
   "dependencies": [
     { "name": "puppetlabs/stdlib", "version_requirement": ">= 4.6.0 < 5.0.0" },
-    { "name": "puppetlabs/apt", "version_requirement": ">= 2.0.0 < 3.0.0" }
+    { "name": "puppetlabs/apt", "version_requirement": ">= 4.0.0" }
   ],
   "data_provider": null
 }

spec/defines/filter_spec.rb

@@ -13,7 +13,7 @@ describe 'fluentd::filter' do
         let(:title) { 'test' }
         let(:params) do
           {
-            priority: '20',
+            priority: 20,
             pattern: '*.test',
             config: {
               'type'   => 'record_transformer',

spec/defines/match_spec.rb

@@ -13,7 +13,7 @@ describe 'fluentd::match' do
         let(:title) { 'test' }
         let(:params) do
           {
-              priority: '30',
+              priority: 30,
               pattern: '*.test',
               config: {
                   'flush_interval' => '30s',
@@ -41,7 +41,7 @@ describe 'fluentd::match' do
         let(:title) { 'test' }
         let(:params) do
           {
-              priority: '30',
+              priority: 30,
               pattern: '*.test',
               config: {
                   'type'  => 'copy',

spec/defines/source_spec.rb

@@ -13,11 +13,19 @@ describe 'fluentd::source' do
         let(:title) { 'test' }
         let(:params) do
           {
-              priority: '10',
+              priority: 10,
               config: {
                 'type'   => 'tail',
                 'format' => 'json',
                 'path'   => '/var/log/test-application/*.json',
+                'parse'  => {
+                    'type'       => 'regexp',
+                    'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
+                },
+                'storage'=> {
+                    'type'   => 'local',
+                    'path'   => 'test.pos'
+                },
                 'tag'    => 'application.test'
               }
           }

spec/fixtures/files/source_tail.conf

@@ -2,5 +2,13 @@
   type tail
   format json
   path /var/log/test-application/*.json
+  <parse>
+    type regexp
+    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
+  </parse>
+  <storage>
+    type local
+    path test.pos
+  </storage>
   tag application.test
 </source>

templates/match.erb

@@ -1,7 +1,8 @@
 <match <%= @pattern %>>
 <% @config.each do |key, val| -%>
+  <%- if key == 'buffer_chunk_keys'; next; end-%>
   <%- if val.is_a?( Hash ) -%>
-  <<%= key -%>>
+  <<%= key -%><%- if key == 'buffer' && !@config['buffer_chunk_keys'].nil? -%> <%= @config['buffer_chunk_keys'] -%><%- end -%>>
   <%- val.each do |key, val| -%>
     <%= key -%> <%= val %>
   <%- end -%>
@@ -13,7 +14,7 @@
     <%- k.each do |x, y| -%>
     <%- if y.is_a?( Array ) -%>
     <%- y.each do |a, b| -%>
-    <<%= x -%>>
+    <<%= x -%><%- if x == 'buffer' && !k['buffer_chunk_keys'].nil? -%> <%= k['buffer_chunk_keys'] -%><%- end -%>>
       <%- if a.is_a?( Hash ) -%>
       <%- a.each do |c, d| -%>
       <%= c -%> <%= d %>
@@ -24,6 +25,7 @@
     </<%= x %>>
     <%- end -%>
     <%- else -%>
+    <%- if x == 'buffer_chunk_keys'; next; end -%>
     <%= x -%> <%= y %>
     <%- end -%>
     <%- end -%>

templates/source.erb

@@ -1,5 +1,13 @@
 <source>
 <% @config.each do |key, val| -%>
+  <%- if val.is_a?( Hash ) -%>
+  <<%= key -%>>
+  <%- val.each do |key, val| -%>
+    <%= key -%> <%= val %>
+  <%- end -%>
+  </<%= key %>>
+  <%- else -%>
   <%= key -%> <%= val %>
+  <%- end -%>
 <% end -%>
 </source>