Add fault-tolerant file buffers for logs

Vagrantfile

@@ -56,11 +56,25 @@ Vagrant.configure("2") do |config|
           systemctl restart rsyslog
         fi
 
+        # Setup TLS
+        if [ ! -f /vagrant/tmp/ca_key.pem ]; then
+          echo "Generating TLS certificates..."
+          cd /vagrant/tmp
+          openssl req -newkey rsa:4096 \
+                      -x509 \
+                      -sha256 \
+                      -days 3650 \
+                      -nodes \
+                      -out ca_cert.pem \
+                      -keyout ca_key.pem \
+                      -subj "/C=US/ST=Local/L=Local/O=Org/OU=IT/CN=example.com" \
+                      2> /dev/null
+        fi
+
         # Install td-agent
         cp /vagrant/td-agent.repo /etc/yum.repos.d/
         yum check-update
         yum install -y td-agent
-        td-agent-gem install fluent-plugin-secure-forward
         td-agent-gem install fluent-plugin-gelf-hs gelf
         systemctl -q enable td-agent
 
@@ -74,6 +88,8 @@ Vagrant.configure("2") do |config|
         node.vm.provision "shell", inline: <<-SHELL
 
           cp /vagrant/td-agent-server.conf /etc/td-agent/td-agent.conf
+          mkdir -p /var/log/graylog_buffer
+          chown -R td-agent:td-agent /var/log/graylog_buffer
           systemctl restart td-agent
 
           # Install jq
@@ -85,53 +101,53 @@ Vagrant.configure("2") do |config|
           /usr/local/bin/docker-compose up -d 2> /dev/null
 
           # Wait 120 seconds for Graylog to come online
-           SECONDS=0
-           while true
-           do
-             GRAYLOG_STATE=$(
-               docker inspect vagrant_graylog_1 \
-                 | jq --raw-output '.[] | .State.Health.Status')
+          SECONDS=0
+          while true
+          do
+            GRAYLOG_STATE=$(
+              docker inspect vagrant_graylog_1 \
+                | jq --raw-output '.[] | .State.Health.Status')
 
-             if [[ "$GRAYLOG_STATE" == "healthy" ]]; then
-               echo "Graylog is available."
-               sleep 5
-               break
-             elif [[ "$GRAYLOG_STATE" != "starting" ]]; then
-               echo "Something is wrong with Graylog. Aborting."
-               exit 1
-             elif [[ $SECONDS -le 120 ]]; then
-               echo "Waiting for Graylog ($SECONDS/120 seconds)"
-               sleep 10
-             else
-               echo "Waiting on Graylog timed out. Aborting."
-               exit 1
-             fi
-           done
+            if [[ "$GRAYLOG_STATE" == "healthy" ]]; then
+              echo "Graylog is available."
+              sleep 5
+              break
+            elif [[ "$GRAYLOG_STATE" != "starting" ]]; then
+              echo "Something is wrong with Graylog. Aborting."
+              exit 1
+            elif [[ $SECONDS -le 120 ]]; then
+              echo "Waiting for Graylog ($SECONDS/120 seconds)"
+              sleep 10
+            else
+              echo "Waiting on Graylog timed out. Aborting."
+              exit 1
+            fi
+          done
 
-           # Check for existing GELF TCP Input
-           INPUTSTATE=$(
-             curl -s -X GET \
-                 -H "Content-Type: application/json" \
-                 -H "X-Requested-By: cli" \
-                 -u admin:admin \
-                 "http://graylog.172.28.128.30.xip.io:8080/api/system/inputstates")
+          # Check for existing GELF TCP Input
+          INPUTSTATE=$(
+            curl -s -X GET \
+                -H "Content-Type: application/json" \
+                -H "X-Requested-By: cli" \
+                -u admin:admin \
+                "http://graylog.172.28.128.30.xip.io:8080/api/system/inputstates")
 
-           INPUT_TYPES=$(echo $INPUTSTATE | jq --raw-output '.states | .[] | .message_input.type')
+          INPUT_TYPES=$(echo $INPUTSTATE | jq --raw-output '.states | .[] | .message_input.type')
 
-           for TYPE in $INPUT_TYPES; do
-             if [[ "$TYPE" == "org.graylog2.inputs.gelf.tcp.GELFTCPInput" ]]; then
-               echo "Found GELF TCP input in Graylog, aborting input installation."
-               exit
-             fi
-           done
+          for TYPE in $INPUT_TYPES; do
+            if [[ "$TYPE" == "org.graylog2.inputs.gelf.tcp.GELFTCPInput" ]]; then
+              echo "Found GELF TCP input in Graylog, aborting input installation."
+              exit
+            fi
+          done
 
-           # Install GELF TCP Input
-           curl -i -s -X POST \
-               -H "Content-Type: application/json" \
-               -H "X-Requested-By: cli" \
-               -u admin:admin \
-               "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
-               -d @GELFTCPInput.json
+          # Install GELF TCP Input
+          curl -i -s -X POST \
+              -H "Content-Type: application/json" \
+              -H "X-Requested-By: cli" \
+              -u admin:admin \
+              "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
+              -d @GELFTCPInput.json
 
         SHELL
 
@@ -145,8 +161,8 @@ Vagrant.configure("2") do |config|
 
           # Configure td-agent
           cp /vagrant/td-agent.conf /etc/td-agent/td-agent.conf
-          mkdir -p /var/log/containers
-          chown -R td-agent:td-agent /var/log/containers
+          mkdir -p /var/log/containers /var/log/fluentd_buffer
+          chown -R td-agent:td-agent /var/log/containers /var/log/fluentd_buffer
           chmod -R 755 /var/log
           systemctl restart td-agent
 

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.7'
 services:
 
   traefik:
-    image: traefik:2.1.4
+    image: traefik:2.2.1
     restart: always
     networks:
       - traefik-net
@@ -32,7 +32,7 @@ services:
        tag: traefik
 
   mongo:
-    image: mongo:4.2.2
+    image: mongo:4.2.8
     restart: always
     networks:
       - graylog
@@ -44,7 +44,7 @@ services:
        tag: graylog.db
 
   elasticsearch:
-    image: elasticsearch:6.8.6
+    image: elasticsearch:6.8.10
     restart: always
     environment:
       - http:host=0.0.0.0
@@ -65,7 +65,7 @@ services:
        tag: graylog.elasticsearch
 
   graylog:
-    image: graylog/graylog:3.2.2
+    image: graylog/graylog:3.3.2
     restart: always
     environment:
       - GRAYLOG_PASSWORD_SECRET=LongerPassword01

td-agent-server.conf

@@ -1,12 +1,12 @@
 <source>
-  @type secure_forward
-  shared_key test
-  self_hostname 172.28.128.30
+  @type forward
   port 2514
-  secure true
-  ca_cert_path /vagrant/tmp/ca_cert.pem
-  ca_private_key_path /vagrant/tmp/ca_key.pem
-  ca_private_key_passphrase test
+  <transport tls>
+    version TLSv1_2
+    insecure true
+    cert_path /vagrant/tmp/ca_cert.pem
+    private_key_path /vagrant/tmp/ca_key.pem
+  </transport>
 </source>
 
 <source>
@@ -34,5 +34,9 @@
   protocol tcp
   host localhost
   port 12201
-  flush_interval 5s
+  <buffer>
+    @type file
+    path /var/log/graylog_buffer
+    flush_interval 0s
+  </buffer>
 </match>

td-agent.conf

@@ -26,9 +26,14 @@
     path /var/log/containers/${tag}
     append true
     <buffer tag>
-      timekey 5s
-      flush_mode immediate
+      @type file
+      path /var/log/containers/buffer
+      flush_interval 0s
     </buffer>
+    <format>
+      @type single_value
+      message_key log
+    </format>
   </store>
   <store>
     @type rewrite_tag_filter
@@ -41,14 +46,17 @@
 </match>
 
 <match **>
-  @type secure_forward
-  shared_key test
-  self_hostname 172.28.128.31
-  secure true
-  ca_cert_path /vagrant/tmp/ca_cert.pem
-
+  @type forward
+  transport tls
+  tls_cert_path /vagrant/tmp/ca_cert.pem
   <server>
+    name example.com
     host 172.28.128.30
     port 2514
   </server>
+  <buffer>
+    @type file
+    path /var/log/fluentd_buffer
+    flush_interval 0s
+  </buffer>
 </match>