Upgrade to td-agent v4 on older Puppet installs

The fluentd_gem provider for installing fluentd plugins uses
Puppet's built-in gem provider. Unfortunately, that gem provider
on our Puppet version is not compatible with td-agent's new embedded
Ruby since it includes the no-rdoc and no-ri flags deprecated and
replaced by the no-document flag. This install function override
replaces the flags and might be removable after a Puppet upgrade.

README.md

@@ -15,6 +15,7 @@
     * [Source](#source)
     * [Filter](#filter)
     * [Match](#match)
+    * [Match with buffer and chunk_keys](#match-with-buffer-and-chunk_keys)
     * [Match Store](#match-store)
   * [Plugin Installation](#plugin-installation)
   * [Requirements](#requirements)
@@ -88,7 +89,7 @@ include '::fluentd'
 
   Path to configuration files
   **Default:** '/etc/td-agent'
-  
+
 `config_dir`
 
   Configuration directory name
@@ -139,7 +140,10 @@ include '::fluentd'
       'type'   => 'local',
       'path'   => 'test.pos'
     },
-    'tag'    => 'application.test'
+    'tag'    => 'application.test',
+    'parse'  => {
+      'message_format' => 'auto'
+    }
   }
 }
 ```
@@ -159,6 +163,9 @@ include '::fluentd'
     path test.pos
   </storage>
   tag application.test
+  <parse>
+    message_format auto
+  </parse>
 </source>
 ```
 #### Filter
@@ -217,7 +224,41 @@ include '::fluentd'
   </server>
 </match>
 ```
-### Match Store
+#### Match with buffer and chunk_keys
+
+buffer_chunk_keys will not be included in the template. They will be filtered and only be used as a "pattern" for the buffer section.
+
+```
+::fluentd::match { 'test':
+  priority => 98,
+  pattern  => 'test.**',
+  config   => {
+    type              => 'file',
+    path              => '/var/log/td-agent/test/${host}',
+    append            => true,
+    buffer_chunk_keys => 'host',
+    buffer            => {
+      type       => file,
+      path       => /var/log/td-agent/test/buffer/,
+      flush_mode => immediate,
+    },
+  },
+}
+```
+**creates:**
+```
+<match test.**>
+  type file
+  path /var/log/td-agent/test/${host}
+  append true
+  <buffer host>
+    type file
+    path /var/log/td-agent/test/buffer/
+    flush_mode immediate
+  </buffer>
+</match>
+```
+#### Match Store
 ```puppet
 ::fluentd::match { 'test':
   priority => 30,
@@ -233,7 +274,13 @@ include '::fluentd'
       },
       {
         'type' => 'file',
-        'path' => '/tmp/td-agent-debug.log',
+        'path' => '/tmp/td-agent-debug-${host}.log',
+        'buffer_chunk_keys' => 'host',
+        'buffer'         => [{
+          '@type'    => 'file',
+          path       => '/var/log/td-agent/buffer/',
+          flush_mode => 'immediate',
+          }]
       }
     ]
   }
@@ -252,7 +299,11 @@ include '::fluentd'
   </store>
   <store>
     type file
-    path /tmp/crs
+    path /tmp/td-agent-debug-${host}.log
+    <buffer host>
+      @type file
+      path /var/log/td-agent/buffer/
+      flush_mode immediate
   </store>
 </match>
 ```

files/GPG-KEY-td-agent

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: Encryption subkey has been removed due to apt-key parsing bug
+
+mQINBFhiI8wBEADThWLNd8IKPRw7Ygu3DHS4Sb/Yc6vSZSaMGJ6Wkj245jScvI+C
+nG4C4rtO/8ObUj5cUpb4CyfYZX8W4tp9x+W68c4paXevG4s+X4EE3uUsgdwTnFXi
+GMa57QDzR4p/JvjUjfGJ2UAr4Bfj8Q2S54LmIu6UAe82ce2B4tEHCeYSxkmVUDAZ
+utfmgKoVTbnceTemU0m5ANS6IC1/53KEhgB1sKm5G/FjRJGslHWb3mf+bLrhmlkP
+pA4BOKF2w3eFYH3LhWskxMS0SPM7J6aq+6LyNNqtlKL6lUS7qVjRQ6PlgFcmtG4J
+tijsZI62bDn1f44DmeLY+LMS/nM0xyIx94lYumGH5EYmjUECagqMool98/+Wx79A
+Thtg/1pYNzo8Z76qr0i3xLSRtsQ2Om2Rfal7VGadOrx4sqlkSaUaGI+hBc1r4tNy
+tERvBEMGSf78bWDbdzxSNEW4LUDUpniNQb0DrURfWkqRa3q4WcTJr8lpQM/NmAru
+owayAXQwKob+OIZ09/O69EaqVJ9MqsM3keQouSHShKvzNrppuo3D3z+Dpy05FsYw
+MAiIN7auXxy+XQwCVsKF083YaDHcC0I22GReEgt43yZXQ/b/J9QNrm5nJ+3Cpso3
+jJnMzubuniSOOdd3mXQ6MwgZvWgtH/nPF8oUX9VSGwqNohiKWcxQDxW7qQARAQAB
+tFRUcmVhc3VyZSBEYXRhLCBJbmMgKFRyZWFzdXJlIEFnZW50IE9mZmljaWFsIFNp
+Z25pbmcga2V5KSA8c3VwcG9ydEB0cmVhc3VyZS1kYXRhLmNvbT6JAjcEEwEIACEF
+AlhiI8wCGwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQkB+Rd6uXrL5GrhAA
+nh82+caSu9Qu/LW256gN5UjPUFhph66ElT1OVyAR2FoOmz2pJH3t8YYD5cUV2W6/
+xqJDmjl+vnL2HBgxjHKRCo2K3hrq6z4LoU7SpWDI1cZ03lkjh1yNx13S+9JvZNlp
+jit0WRIspke0n0vWSpNo4nh19Yg3EA1c+vGeHnmlYo6xwRHu6XOhhCwywtFRGC3a
+iMJzAV4N69ZU6P5VZZkC6LjYYQtF4aI10COLZ4AcObH2htGAZTj2KlZfdJHmr+Oa
+wY57giUYz7OF45LLCuqe+VwpGp2d3UK/MtCnXRLi5InMVJKDvyt18MzRDFuyA27e
+WSt+JumVqhEjawh3hmdzIS1cHKmv19gdeE8On2i2Lf8lyek8fsB/YPgADAmp2oSe
+cjLu0ocGbgxRjuCR29+6IG+DiUDFCkqFZNdLiGVqzjpjpYHaPhVe77ciwA8TCPru
+3dh5t/qv2HglSd7lj95IApZBtny5AK8NS4qtaOeZbBbbDRuOPL0c7fU3bqyIPy57
+zvdYi3KdjWZVCawcAmk3ILP83eFSivCRPRoyCqO+HX8U647BBWvlFuEbPa+Y1sgE
+12MEF/Y6VVJh3Ptw+h/qKRbra4LdA+5Y30q/9l6WGgbO/4h3NKmGeVCrAFvS3h92
+fS0ABYD1nAP7fSNS9RfYIqfBXtJem+tJ14YKJwWiAYU=
+=TPyg
+-----END PGP PUBLIC KEY BLOCK-----

lib/puppet/provider/package/fluentd-gem.rb

@@ -8,11 +8,20 @@ Puppet::Type.type(:package).provide :fluentd_gem, :parent => :gem do
     not a valid URL, it will be interpreted as the path to a local gem file.  If
     source is not present at all, the gem will be installed from the default gem
     repositories."
-  confine :exists => "/opt/td-agent/embedded/bin/gem"
+  confine :exists => "/opt/td-agent/bin/gem"
 
   has_feature :versionable, :install_options
 
-  commands :gemcmd => "/opt/td-agent/embedded/bin/gem"
+  commands :gemcmd => "/opt/td-agent/bin/gem"
+
+  def install(useversion = true)
+    command = [command(:gemcmd), "install"]
+    command << "--no-document" << resource[:name]
+    output = execute(command)
+
+    # Apparently some stupid gem versions don't exit non-0 on failure
+    self.fail "Could not install: #{output.chomp}" if output.include?("ERROR")
+  end
 
   def uninstall
     command = [command(:gemcmd), "uninstall"]

manifests/init.pp

@@ -8,6 +8,9 @@
 # [*repo_manage*]
 #   Include repository to install recent fluentd (td-agent) from
 #   Default: 'true'
+# [*repo_version*]
+#   The major version of td-agent to install
+#   Default: 3
 # [*package_ensure*]
 #   Package ensure
 #   Default: 'installed'
@@ -66,6 +69,7 @@
 #
 class fluentd (
   Boolean                     $repo_manage             = $::fluentd::params::repo_manage,
+  Integer                     $repo_version            = $::fluentd::params::repo_version,
   String                      $package_ensure          = $::fluentd::params::package_ensure,
   String                      $package_name            = $::fluentd::params::package_name,
   Array                       $package_install_options = $::fluentd::params::package_install_options,
@@ -90,6 +94,12 @@ class fluentd (
   include '::fluentd::config'
   include '::fluentd::service'
 
+  # Realize any virtual configurations
+  Fluentd::Plugin <| |>
+  Fluentd::Source <| |>
+  Fluentd::Filter <| |>
+  Fluentd::Match  <| |>
+
   # dependencies
   Class['::Fluentd::Repo']
   -> Class['::Fluentd::Install']

manifests/params.pp

@@ -18,6 +18,8 @@ class fluentd::params {
   $user_name   = 'td-agent'
   $user_group  = 'td-agent'
   $user_groups = ['adm']
+  # repo params
+  $repo_version = 4
 
   case $::osfamily {
     'Debian': {

manifests/repo.pp

@@ -4,7 +4,7 @@ class fluentd::repo inherits fluentd {
   if $::fluentd::repo_manage {
     case $::osfamily {
       'Debian': {
-        include ::fluentd::repo::apt
+        contain ::fluentd::repo::apt
       }
       'RedHat': {
         include ::fluentd::repo::yum

manifests/repo/apt.pp

@@ -2,13 +2,13 @@
 #
 class fluentd::repo::apt (
   $ensure       = 'present',
-  $location     = downcase("http://packages.treasuredata.com/2/${::lsbdistid}/${::lsbdistcodename}"),
+  $location     = downcase("http://packages.treasuredata.com/${::fluentd::repo_version}/${::lsbdistid}/${::lsbdistcodename}"),
   $release      = $::lsbdistcodename,
   $repos        = 'contrib',
   $architecture = $::architecture,
   $key          = {
-    'id'     => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
+    'id'      => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
-    'source' => 'http://packages.treasuredata.com/GPG-KEY-td-agent'
+    'content' => file('fluentd/GPG-KEY-td-agent'),
   },
   $include      = {
     'src' => false,
@@ -27,4 +27,6 @@ class fluentd::repo::apt (
     key          => $key,
     include      => $include,
   }
+
+  Class['apt::update'] -> Package['fluentd']
 }

manifests/repo/yum.pp

@@ -3,7 +3,7 @@
 class fluentd::repo::yum (
   $ensure   = 'present',
   $descr    = 'TreasureData',
-  $baseurl  = 'https://packages.treasuredata.com/2/redhat/$releasever/$basearch',
+  $baseurl  = "https://packages.treasuredata.com/${::fluentd::repo_version}/redhat/\$releasever/\$basearch",
   $enabled  = '1',
   $gpgcheck = '1',
   $gpgkey   = 'https://packages.treasuredata.com/GPG-KEY-td-agent',

templates/match.erb

@@ -1,7 +1,8 @@
 <match <%= @pattern %>>
 <% @config.each do |key, val| -%>
+  <%- if key == 'buffer_chunk_keys'; next; end-%>
   <%- if val.is_a?( Hash ) -%>
-  <<%= key -%>>
+  <<%= key -%><%- if key == 'buffer' && !@config['buffer_chunk_keys'].nil? -%> <%= @config['buffer_chunk_keys'] -%><%- end -%>>
   <%- val.each do |key, val| -%>
     <%= key -%> <%= val %>
   <%- end -%>
@@ -13,7 +14,7 @@
     <%- k.each do |x, y| -%>
     <%- if y.is_a?( Array ) -%>
     <%- y.each do |a, b| -%>
-    <<%= x -%>>
+    <<%= x -%><%- if x == 'buffer' && !k['buffer_chunk_keys'].nil? -%> <%= k['buffer_chunk_keys'] -%><%- end -%>>
       <%- if a.is_a?( Hash ) -%>
       <%- a.each do |c, d| -%>
       <%= c -%> <%= d %>
@@ -24,6 +25,7 @@
     </<%= x %>>
     <%- end -%>
     <%- else -%>
+    <%- if x == 'buffer_chunk_keys'; next; end -%>
     <%= x -%> <%= y %>
     <%- end -%>
     <%- end -%>

templates/source.erb

@@ -2,8 +2,8 @@
 <% @config.each do |key, val| -%>
   <%- if val.is_a?( Hash ) -%>
   <<%= key -%>>
-  <%- val.each do |key2, val2| -%>
+  <%- val.each do |key, val| -%>
-    <%= key2 -%> <%= val2 %>
+    <%= key -%> <%= val %>
   <%- end -%>
   </<%= key %>>
   <%- else -%>