From 45449583d9e6466e8e34531f79d05f2bce4250d6 Mon Sep 17 00:00:00 2001
From: Robert Waffen <robert.waffen@mybet.com>
Date: Sat, 11 Nov 2017 12:40:33 +0100
Subject: [PATCH] Extend templates for fluentd 0.14 new sections

- extend source to support format and parse section
- extend match to support buffers with chunk keys
- update readme
---
 README.md            | 61 ++++++++++++++++++++++++++++++++++++++++----
 templates/match.erb  |  6 +++--
 templates/source.erb |  8 ++++++
 3 files changed, 68 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index eb5d093..535973e 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,7 @@
     * [Source](#source)
     * [Filter](#filter)
     * [Match](#match)
+    * [Match with buffer and chunk_keys](#match-with-buffer-and-chunk_keys)
     * [Match Store](#match-store)
   * [Plugin Installation](#plugin-installation)
   * [Requirements](#requirements)
@@ -88,7 +89,7 @@ include '::fluentd'
 
   Path to configuration files
   **Default:** '/etc/td-agent'
-  
+
 `config_dir`
 
   Configuration directory name
@@ -131,7 +132,10 @@ include '::fluentd'
     'type'   => 'tail',
     'format' => 'json',
     'path'   => '/var/log/test-application/*.json',
-    'tag'    => 'application.test'
+    'tag'    => 'application.test',
+    'parse'  => {
+      'message_format' => 'auto'
+    }
   }
 }
 ```
@@ -143,6 +147,9 @@ include '::fluentd'
   format json
   path /var/log/test-application/*.json
   tag application.test
+  <parse>
+    message_format auto
+  </parse>
 </source>
 ```
 #### Filter
@@ -201,7 +208,41 @@ include '::fluentd'
   </server>
 </match>
 ```
-### Match Store
+#### Match with buffer and chunk_keys
+
+buffer_chunk_keys will not be included in the template. They will be filtered and only be used as a "pattern" for the buffer section.
+
+```
+::fluentd::match { 'test':
+  priority => 98,
+  pattern  => 'test.**',
+  config   => {
+    type              => 'file',
+    path              => '/var/log/td-agent/test/${host}',
+    append            => true,
+    buffer_chunk_keys => 'host',
+    buffer            => {
+      type       => file,
+      path       => /var/log/td-agent/test/buffer/,
+      flush_mode => immediate,
+    },
+  },
+}
+```
+**creates:**
+```
+<match test.**>
+  type file
+  path /var/log/td-agent/test/${host}
+  append true
+  <buffer host>
+    type file
+    path /var/log/td-agent/test/buffer/
+    flush_mode immediate
+  </buffer>
+</match>
+```
+#### Match Store
 ```puppet
 ::fluentd::match { 'test':
   priority => 30,
@@ -217,7 +258,13 @@ include '::fluentd'
       },
       {
         'type' => 'file',
-        'path' => '/tmp/td-agent-debug.log',
+        'path' => '/tmp/td-agent-debug-${host}.log',
+        'buffer_chunk_keys' => 'host',
+        'buffer'         => [{
+          '@type'    => 'file',
+          path       => '/var/log/td-agent/buffer/',
+          flush_mode => 'immediate',
+          }]
       }
     ]
   }
@@ -236,7 +283,11 @@ include '::fluentd'
   </store>
   <store>
     type file
-    path /tmp/crs
+    path /tmp/td-agent-debug-${host}.log
+    <buffer host>
+      @type file
+      path /var/log/td-agent/buffer/
+      flush_mode immediate
   </store>
 </match>
 ```
diff --git a/templates/match.erb b/templates/match.erb
index f623894..7fc487a 100644
--- a/templates/match.erb
+++ b/templates/match.erb
@@ -1,7 +1,8 @@
 <match <%= @pattern %>>
 <% @config.each do |key, val| -%>
+  <%- if key == 'buffer_chunk_keys'; next; end-%>
   <%- if val.is_a?( Hash ) -%>
-  <<%= key -%>>
+  <<%= key -%><%- if key == 'buffer' && !@config['buffer_chunk_keys'].nil? -%> <%= @config['buffer_chunk_keys'] -%><%- end -%>>
   <%- val.each do |key, val| -%>
     <%= key -%> <%= val %>
   <%- end -%>
@@ -13,7 +14,7 @@
     <%- k.each do |x, y| -%>
     <%- if y.is_a?( Array ) -%>
     <%- y.each do |a, b| -%>
-    <<%= x -%>>
+    <<%= x -%><%- if x == 'buffer' && !k['buffer_chunk_keys'].nil? -%> <%= k['buffer_chunk_keys'] -%><%- end -%>>
       <%- if a.is_a?( Hash ) -%>
       <%- a.each do |c, d| -%>
       <%= c -%> <%= d %>
@@ -24,6 +25,7 @@
     </<%= x %>>
     <%- end -%>
     <%- else -%>
+    <%- if x == 'buffer_chunk_keys'; next; end -%>
     <%= x -%> <%= y %>
     <%- end -%>
     <%- end -%>
diff --git a/templates/source.erb b/templates/source.erb
index 04f119a..82478b9 100644
--- a/templates/source.erb
+++ b/templates/source.erb
@@ -1,5 +1,13 @@
 <source>
 <% @config.each do |key, val| -%>
+  <%- if val.is_a?( Hash ) -%>
+  <<%= key -%>>
+  <%- val.each do |key, val| -%>
+    <%= key -%> <%= val %>
+  <%- end -%>
+  </<%= key %>>
+  <%- else -%>
   <%= key -%> <%= val %>
+  <%- end -%>
 <% end -%>
 </source>