README.md

@@ -1,6 +1,6 @@
-# pilchkinstein-gtv/fluentd Puppet Module
+# wywy/fluentd Puppet Module
 
-[![GitHub version](https://badge.fury.io/gh/wywy%2Fpuppet-fluentd.svg)](https://github.com/pilchkinstein-gtv/puppet-fluentd)
+[![GitHub version](https://badge.fury.io/gh/wywy%2Fpuppet-fluentd.svg)](https://github.com/wywy/puppet-fluentd)
 [![Build Status](https://travis-ci.org/wywy/puppet-fluentd.svg?branch=master)](https://travis-ci.org/wywy/puppet-fluentd)
 
 
@@ -15,7 +15,6 @@
     * [Source](#source)
     * [Filter](#filter)
     * [Match](#match)
-    * [Match with buffer and chunk_keys](#match-with-buffer-and-chunk_keys)
     * [Match Store](#match-store)
   * [Plugin Installation](#plugin-installation)
   * [Requirements](#requirements)
@@ -132,18 +131,7 @@ include '::fluentd'
     'type'   => 'tail',
     'format' => 'json',
     'path'   => '/var/log/test-application/*.json',
-    'parse'  => {
-      'type'       => 'regexp',
-      'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
-    },
-    'storage'=> {
-      'type'   => 'local',
-      'path'   => 'test.pos'
-    },
-    'tag'    => 'application.test',
-    'parse'  => {
-      'message_format' => 'auto'
-    }
+    'tag'    => 'application.test'
   }
 }
 ```
@@ -154,18 +142,7 @@ include '::fluentd'
   type tail
   format json
   path /var/log/test-application/*.json
-  <parse>
-    type regexp
-    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
-  </parse>
-  <storage>
-    type local
-    path test.pos
-  </storage>
   tag application.test
-  <parse>
-    message_format auto
-  </parse>
 </source>
 ```
 #### Filter
@@ -224,41 +201,7 @@ include '::fluentd'
   </server>
 </match>
 ```
-#### Match with buffer and chunk_keys
-
-buffer_chunk_keys will not be included in the template. They will be filtered and only be used as a "pattern" for the buffer section.
-
-```
-::fluentd::match { 'test':
-  priority => 98,
-  pattern  => 'test.**',
-  config   => {
-    type              => 'file',
-    path              => '/var/log/td-agent/test/${host}',
-    append            => true,
-    buffer_chunk_keys => 'host',
-    buffer            => {
-      type       => file,
-      path       => /var/log/td-agent/test/buffer/,
-      flush_mode => immediate,
-    },
-  },
-}
-```
-**creates:**
-```
-<match test.**>
-  type file
-  path /var/log/td-agent/test/${host}
-  append true
-  <buffer host>
-    type file
-    path /var/log/td-agent/test/buffer/
-    flush_mode immediate
-  </buffer>
-</match>
-```
-#### Match Store
+### Match Store
 ```puppet
 ::fluentd::match { 'test':
   priority => 30,
@@ -274,13 +217,7 @@ buffer_chunk_keys will not be included in the template. They will be filtered an
       },
       {
         'type' => 'file',
-        'path' => '/tmp/td-agent-debug-${host}.log',
-        'buffer_chunk_keys' => 'host',
-        'buffer'         => [{
-          '@type'    => 'file',
-          path       => '/var/log/td-agent/buffer/',
-          flush_mode => 'immediate',
-          }]
+        'path' => '/tmp/td-agent-debug.log',
       }
     ]
   }
@@ -299,11 +236,7 @@ buffer_chunk_keys will not be included in the template. They will be filtered an
   </store>
   <store>
     type file
-    path /tmp/td-agent-debug-${host}.log
-    <buffer host>
-      @type file
-      path /var/log/td-agent/buffer/
-      flush_mode immediate
+    path /tmp/crs
   </store>
 </match>
 ```

examples/source.pp

@@ -5,14 +5,6 @@
     'type'   => 'tail',
     'format' => 'json',
     'path'   => '/var/log/test-application/*.json',
-    'parse'   => {
-        'type'       => 'regexp',
-        'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
-    },
-    'storage' => {
-      'type' => 'local',
-      'path' => 'test.pos'
-    },
     'tag'    => 'application.test'
   }
 }

files/GPG-KEY-td-agent

@@ -1,30 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-Comment: Encryption subkey has been removed due to apt-key parsing bug
-
-mQINBFhiI8wBEADThWLNd8IKPRw7Ygu3DHS4Sb/Yc6vSZSaMGJ6Wkj245jScvI+C
-nG4C4rtO/8ObUj5cUpb4CyfYZX8W4tp9x+W68c4paXevG4s+X4EE3uUsgdwTnFXi
-GMa57QDzR4p/JvjUjfGJ2UAr4Bfj8Q2S54LmIu6UAe82ce2B4tEHCeYSxkmVUDAZ
-utfmgKoVTbnceTemU0m5ANS6IC1/53KEhgB1sKm5G/FjRJGslHWb3mf+bLrhmlkP
-pA4BOKF2w3eFYH3LhWskxMS0SPM7J6aq+6LyNNqtlKL6lUS7qVjRQ6PlgFcmtG4J
-tijsZI62bDn1f44DmeLY+LMS/nM0xyIx94lYumGH5EYmjUECagqMool98/+Wx79A
-Thtg/1pYNzo8Z76qr0i3xLSRtsQ2Om2Rfal7VGadOrx4sqlkSaUaGI+hBc1r4tNy
-tERvBEMGSf78bWDbdzxSNEW4LUDUpniNQb0DrURfWkqRa3q4WcTJr8lpQM/NmAru
-owayAXQwKob+OIZ09/O69EaqVJ9MqsM3keQouSHShKvzNrppuo3D3z+Dpy05FsYw
-MAiIN7auXxy+XQwCVsKF083YaDHcC0I22GReEgt43yZXQ/b/J9QNrm5nJ+3Cpso3
-jJnMzubuniSOOdd3mXQ6MwgZvWgtH/nPF8oUX9VSGwqNohiKWcxQDxW7qQARAQAB
-tFRUcmVhc3VyZSBEYXRhLCBJbmMgKFRyZWFzdXJlIEFnZW50IE9mZmljaWFsIFNp
-Z25pbmcga2V5KSA8c3VwcG9ydEB0cmVhc3VyZS1kYXRhLmNvbT6JAjcEEwEIACEF
-AlhiI8wCGwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQkB+Rd6uXrL5GrhAA
-nh82+caSu9Qu/LW256gN5UjPUFhph66ElT1OVyAR2FoOmz2pJH3t8YYD5cUV2W6/
-xqJDmjl+vnL2HBgxjHKRCo2K3hrq6z4LoU7SpWDI1cZ03lkjh1yNx13S+9JvZNlp
-jit0WRIspke0n0vWSpNo4nh19Yg3EA1c+vGeHnmlYo6xwRHu6XOhhCwywtFRGC3a
-iMJzAV4N69ZU6P5VZZkC6LjYYQtF4aI10COLZ4AcObH2htGAZTj2KlZfdJHmr+Oa
-wY57giUYz7OF45LLCuqe+VwpGp2d3UK/MtCnXRLi5InMVJKDvyt18MzRDFuyA27e
-WSt+JumVqhEjawh3hmdzIS1cHKmv19gdeE8On2i2Lf8lyek8fsB/YPgADAmp2oSe
-cjLu0ocGbgxRjuCR29+6IG+DiUDFCkqFZNdLiGVqzjpjpYHaPhVe77ciwA8TCPru
-3dh5t/qv2HglSd7lj95IApZBtny5AK8NS4qtaOeZbBbbDRuOPL0c7fU3bqyIPy57
-zvdYi3KdjWZVCawcAmk3ILP83eFSivCRPRoyCqO+HX8U647BBWvlFuEbPa+Y1sgE
-12MEF/Y6VVJh3Ptw+h/qKRbra4LdA+5Y30q/9l6WGgbO/4h3NKmGeVCrAFvS3h92
-fS0ABYD1nAP7fSNS9RfYIqfBXtJem+tJ14YKJwWiAYU=
-=TPyg
------END PGP PUBLIC KEY BLOCK-----

lib/puppet/provider/package/fluentd-gem.rb

@@ -8,20 +8,11 @@ Puppet::Type.type(:package).provide :fluentd_gem, :parent => :gem do
     not a valid URL, it will be interpreted as the path to a local gem file.  If
     source is not present at all, the gem will be installed from the default gem
     repositories."
-  confine :exists => "/opt/td-agent/bin/gem"
+  confine :exists => "/opt/td-agent/embedded/bin/gem"
 
   has_feature :versionable, :install_options
 
-  commands :gemcmd => "/opt/td-agent/bin/gem"
-
-  def install(useversion = true)
-    command = [command(:gemcmd), "install"]
-    command << "--no-document" << resource[:name]
-    output = execute(command)
-
-    # Apparently some stupid gem versions don't exit non-0 on failure
-    self.fail "Could not install: #{output.chomp}" if output.include?("ERROR")
-  end
+  commands :gemcmd => "/opt/td-agent/embedded/bin/gem"
 
   def uninstall
     command = [command(:gemcmd), "uninstall"]

manifests/init.pp

@@ -8,9 +8,6 @@
 # [*repo_manage*]
 #   Include repository to install recent fluentd (td-agent) from
 #   Default: 'true'
-# [*repo_version*]
-#   The major version of td-agent to install
-#   Default: 3
 # [*package_ensure*]
 #   Package ensure
 #   Default: 'installed'
@@ -69,7 +66,6 @@
 #
 class fluentd (
   Boolean                     $repo_manage             = $::fluentd::params::repo_manage,
-  Integer                     $repo_version            = $::fluentd::params::repo_version,
   String                      $package_ensure          = $::fluentd::params::package_ensure,
   String                      $package_name            = $::fluentd::params::package_name,
   Array                       $package_install_options = $::fluentd::params::package_install_options,
@@ -94,12 +90,6 @@ class fluentd (
   include '::fluentd::config'
   include '::fluentd::service'
 
-  # Realize any virtual configurations
-  Fluentd::Plugin <| |>
-  Fluentd::Source <| |>
-  Fluentd::Filter <| |>
-  Fluentd::Match  <| |>
-
   # dependencies
   Class['::Fluentd::Repo']
   -> Class['::Fluentd::Install']

manifests/params.pp

@@ -18,8 +18,6 @@ class fluentd::params {
   $user_name   = 'td-agent'
   $user_group  = 'td-agent'
   $user_groups = ['adm']
-  # repo params
-  $repo_version = 4
 
   case $::osfamily {
     'Debian': {

manifests/repo.pp

@@ -4,7 +4,7 @@ class fluentd::repo inherits fluentd {
   if $::fluentd::repo_manage {
     case $::osfamily {
       'Debian': {
-        contain ::fluentd::repo::apt
+        include ::fluentd::repo::apt
       }
       'RedHat': {
         include ::fluentd::repo::yum

manifests/repo/apt.pp

@@ -2,13 +2,13 @@
 #
 class fluentd::repo::apt (
   $ensure       = 'present',
-  $location     = downcase("http://packages.treasuredata.com/${::fluentd::repo_version}/${::lsbdistid}/${::lsbdistcodename}"),
+  $location     = downcase("http://packages.treasuredata.com/2/${::lsbdistid}/${::lsbdistcodename}"),
   $release      = $::lsbdistcodename,
   $repos        = 'contrib',
   $architecture = $::architecture,
   $key          = {
     'id'     => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE',
-    'content' => file('fluentd/GPG-KEY-td-agent'),
+    'source' => 'http://packages.treasuredata.com/GPG-KEY-td-agent'
   },
   $include      = {
     'src' => false,
@@ -27,6 +27,4 @@ class fluentd::repo::apt (
     key          => $key,
     include      => $include,
   }
-
-  Class['apt::update'] -> Package['fluentd']
 }

manifests/repo/yum.pp

@@ -3,7 +3,7 @@
 class fluentd::repo::yum (
   $ensure   = 'present',
   $descr    = 'TreasureData',
-  $baseurl  = "https://packages.treasuredata.com/${::fluentd::repo_version}/redhat/\$releasever/\$basearch",
+  $baseurl  = 'https://packages.treasuredata.com/2/redhat/$releasever/$basearch',
   $enabled  = '1',
   $gpgcheck = '1',
   $gpgkey   = 'https://packages.treasuredata.com/GPG-KEY-td-agent',

metadata.json

@@ -1,12 +1,12 @@
 {
-  "name": "pilchkinstein-fluentd",
+  "name": "wycore-fluentd",
   "version": "1.0.0",
-  "author": "pilchkinstein-gtv",
+  "author": "wywy",
   "summary": "Generic module for fluentd (td-agent).",
   "license": "Apache-2.0",
-  "source": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
-  "project_page": "https://github.com/pilchkinstein-gtv/puppet-fluentd",
-  "issues_url": "https://github.com/pilchkinstein-gtv/puppet-fluentd/issues",
+  "source": "https://github.com/wycore/puppet-fluentd",
+  "project_page": "https://github.com/wycore/puppet-fluentd",
+  "issues_url": "https://github.com/wycore/puppet-fluentd/issues",
   "tags": ["fluentd", "td-agent"],
   "operatingsystem_support": [
     {

spec/defines/source_spec.rb

@@ -18,14 +18,6 @@ describe 'fluentd::source' do
                 'type'   => 'tail',
                 'format' => 'json',
                 'path'   => '/var/log/test-application/*.json',
-                'parse'  => {
-                    'type'       => 'regexp',
-                    'expression' => '^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$'
-                },
-                'storage'=> {
-                    'type'   => 'local',
-                    'path'   => 'test.pos'
-                },
                 'tag'    => 'application.test'
               }
           }

spec/fixtures/files/source_tail.conf

@@ -2,13 +2,5 @@
   type tail
   format json
   path /var/log/test-application/*.json
-  <parse>
-    type regexp
-    expression ^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$
-  </parse>
-  <storage>
-    type local
-    path test.pos
-  </storage>
   tag application.test
 </source>

templates/match.erb

@@ -1,8 +1,7 @@
 <match <%= @pattern %>>
 <% @config.each do |key, val| -%>
-  <%- if key == 'buffer_chunk_keys'; next; end-%>
   <%- if val.is_a?( Hash ) -%>
-  <<%= key -%><%- if key == 'buffer' && !@config['buffer_chunk_keys'].nil? -%> <%= @config['buffer_chunk_keys'] -%><%- end -%>>
+  <<%= key -%>>
   <%- val.each do |key, val| -%>
     <%= key -%> <%= val %>
   <%- end -%>
@@ -14,7 +13,7 @@
     <%- k.each do |x, y| -%>
     <%- if y.is_a?( Array ) -%>
     <%- y.each do |a, b| -%>
-    <<%= x -%><%- if x == 'buffer' && !k['buffer_chunk_keys'].nil? -%> <%= k['buffer_chunk_keys'] -%><%- end -%>>
+    <<%= x -%>>
       <%- if a.is_a?( Hash ) -%>
       <%- a.each do |c, d| -%>
       <%= c -%> <%= d %>
@@ -25,7 +24,6 @@
     </<%= x %>>
     <%- end -%>
     <%- else -%>
-    <%- if x == 'buffer_chunk_keys'; next; end -%>
     <%= x -%> <%= y %>
     <%- end -%>
     <%- end -%>

templates/source.erb

@@ -1,13 +1,5 @@
 <source>
 <% @config.each do |key, val| -%>
-  <%- if val.is_a?( Hash ) -%>
-  <<%= key -%>>
-  <%- val.each do |key, val| -%>
   <%= key -%> <%= val %>
-  <%- end -%>
-  </<%= key %>>
-  <%- else -%>
-  <%= key -%> <%= val %>
-  <%- end -%>
 <% end -%>
 </source>