Add fault-tolerant file buffers for logs

Vagrantfile

@@ -56,11 +56,25 @@ Vagrant.configure("2") do |config|
           systemctl restart rsyslog
         fi
 
+        # Setup TLS
+        if [ ! -f /vagrant/tmp/ca_key.pem ]; then
+          echo "Generating TLS certificates..."
+          cd /vagrant/tmp
+          openssl req -newkey rsa:4096 \
+                      -x509 \
+                      -sha256 \
+                      -days 3650 \
+                      -nodes \
+                      -out ca_cert.pem \
+                      -keyout ca_key.pem \
+                      -subj "/C=US/ST=Local/L=Local/O=Org/OU=IT/CN=example.com" \
+                      2> /dev/null
+        fi
+
         # Install td-agent
         cp /vagrant/td-agent.repo /etc/yum.repos.d/
         yum check-update
         yum install -y td-agent
-        td-agent-gem install fluent-plugin-secure-forward
         td-agent-gem install fluent-plugin-gelf-hs gelf
         systemctl -q enable td-agent
 
@@ -74,6 +88,8 @@ Vagrant.configure("2") do |config|
         node.vm.provision "shell", inline: <<-SHELL
 
           cp /vagrant/td-agent-server.conf /etc/td-agent/td-agent.conf
+          mkdir -p /var/log/graylog_buffer
+          chown -R td-agent:td-agent /var/log/graylog_buffer
           systemctl restart td-agent
 
           # Install jq
@@ -145,8 +161,8 @@ Vagrant.configure("2") do |config|
 
           # Configure td-agent
           cp /vagrant/td-agent.conf /etc/td-agent/td-agent.conf
-          mkdir -p /var/log/containers
-          chown -R td-agent:td-agent /var/log/containers
+          mkdir -p /var/log/containers /var/log/fluentd_buffer
+          chown -R td-agent:td-agent /var/log/containers /var/log/fluentd_buffer
           chmod -R 755 /var/log
           systemctl restart td-agent
 

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.7'
 services:
 
   traefik:
-    image: traefik:2.1.4
+    image: traefik:2.2.1
     restart: always
     networks:
       - traefik-net
@@ -32,7 +32,7 @@ services:
        tag: traefik
 
   mongo:
-    image: mongo:4.2.2
+    image: mongo:4.2.8
     restart: always
     networks:
       - graylog
@@ -44,7 +44,7 @@ services:
        tag: graylog.db
 
   elasticsearch:
-    image: elasticsearch:6.8.6
+    image: elasticsearch:6.8.10
     restart: always
     environment:
       - http:host=0.0.0.0
@@ -65,7 +65,7 @@ services:
        tag: graylog.elasticsearch
 
   graylog:
-    image: graylog/graylog:3.2.2
+    image: graylog/graylog:3.3.2
     restart: always
     environment:
       - GRAYLOG_PASSWORD_SECRET=LongerPassword01

td-agent-server.conf

@@ -1,12 +1,12 @@
 <source>
-  @type secure_forward
-  shared_key test
-  self_hostname 172.28.128.30
+  @type forward
   port 2514
-  secure true
-  ca_cert_path /vagrant/tmp/ca_cert.pem
-  ca_private_key_path /vagrant/tmp/ca_key.pem
-  ca_private_key_passphrase test
+  <transport tls>
+    version TLSv1_2
+    insecure true
+    cert_path /vagrant/tmp/ca_cert.pem
+    private_key_path /vagrant/tmp/ca_key.pem
+  </transport>
 </source>
 
 <source>
@@ -34,5 +34,9 @@
   protocol tcp
   host localhost
   port 12201
-  flush_interval 5s
+  <buffer>
+    @type file
+    path /var/log/graylog_buffer
+    flush_interval 0s
+  </buffer>
 </match>

td-agent.conf

@@ -26,9 +26,14 @@
     path /var/log/containers/${tag}
     append true
     <buffer tag>
-      timekey 5s
-      flush_mode immediate
+      @type file
+      path /var/log/containers/buffer
+      flush_interval 0s
     </buffer>
+    <format>
+      @type single_value
+      message_key log
+    </format>
   </store>
   <store>
     @type rewrite_tag_filter
@@ -41,14 +46,17 @@
 </match>
 
 <match **>
-  @type secure_forward
-  shared_key test
-  self_hostname 172.28.128.31
-  secure true
-  ca_cert_path /vagrant/tmp/ca_cert.pem
-
+  @type forward
+  transport tls
+  tls_cert_path /vagrant/tmp/ca_cert.pem
   <server>
+    name example.com
     host 172.28.128.30
     port 2514
   </server>
+  <buffer>
+    @type file
+    path /var/log/fluentd_buffer
+    flush_interval 0s
+  </buffer>
 </match>