Update input config and mount Graylog cert data

.gitignore

@@ -1,2 +1 @@
 .vagrant
-pki

GELFTCPInput.json

@@ -1,13 +1,20 @@
 {
-  "title": "Fluentd",
+  "title": "td-agent",
   "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput",
   "global": true,
   "configuration": {
     "bind_address": "0.0.0.0",
     "decompress_size_limit": 8388608,
-    "recv_buffer_size": 1048576,
+    "max_message_size": 2097152,
     "number_worker_threads": 4,
-    "port": 12201
+    "port": 12201,
+    "recv_buffer_size": 1048576,
+    "tcp_keepalive": false,
+    "tls_cert_file": "/usr/share/graylog/certs/rootCA.crt",
+    "tls_client_auth": "required",
+    "tls_enable": true,
+    "tls_key_file": "/usr/share/graylog/certs/rootCA.key",
+    "user_null_delimiter": true
         },
   "node": null
 }

Vagrantfile

@@ -19,10 +19,11 @@ Vagrant.configure("2") do |config|
     sed -i "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config
 
     # Import GPG keys
-    curl -s https://download.docker.com/linux/centos/gpg -o docker-key
+    rpm --import \
-    rpm --import docker-key \
       /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 \
-      http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+      https://download.docker.com/linux/centos/gpg \
+      http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 \
+      https://packages.treasuredata.com/GPG-KEY-td-agent
 
     # Install Docker Community Edition
     yum-config-manager --add-repo \
@@ -49,6 +50,18 @@ Vagrant.configure("2") do |config|
     systemctl start rsyslog
     systemctl -q enable rsyslog
 
+    # Install td-agent
+    cp /vagrant/td-agent.repo /etc/yum.repos.d/
+    yum check-update
+    yum install -y td-agent
+    td-agent-gem install fluent-plugin-gelf-hs gelf
+    cp /vagrant/td-agent.conf /etc/td-agent/td-agent.conf
+    mkdir -p /var/log/containers
+    chown -R td-agent:td-agent /var/log/containers
+    chmod -R 755 /var/log
+    systemctl restart td-agent
+    systemctl -q enable td-agent
+
     # Add rsyslog forwarding option if it does not exist
     if ! grep -q "127.0.0.1:5140" /etc/rsyslog.conf; then
       echo "*.* @127.0.0.1:5140" >> /etc/rsyslog.conf
@@ -62,41 +75,41 @@ Vagrant.configure("2") do |config|
 
   # Start compose services and add default input
   config.vm.provision "shell", inline: <<-SHELL
-
-    # Remove old keys and create directories
-    mkdir -p /vagrant/pki
-    rm -r /vagrant/pki/*
-    mkdir -p /vagrant/pki/{fluentd,graylog}
-
-    # Generate and install TLS keys
-    cd /vagrant/pki
-
-    # Generate Graylog's CA
-    openssl genrsa -out rootCA.key 4096 2> /dev/null
-    openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 \
-        -out rootCA.crt -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" 2> /dev/null
-
-    # Generate Fluentd's keys
-    openssl genrsa -out fluentd.key 4096 2> /dev/null
-    openssl req -new -sha256 -key fluentd.key \
-        -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" -out fluentd.csr 2> /dev/null
-
-    # Sign Fluentd's certificate
-    openssl x509 -req -in fluentd.csr -CA rootCA.crt -CAkey rootCA.key \
-        -CAcreateserial -out fluentd-signed.crt -days 500 -sha256 2> /dev/null
-
-    mv fluentd*.* fluentd/
-    mv root*.* graylog/
-
     # Bring up containers
     cd /vagrant
     /usr/local/bin/docker-compose up -d 2> /dev/null
     cd /vagrant/wordpress
     /usr/local/bin/docker-compose up -d 2> /dev/null
-    cd /vagrant
+
+    # Create directories and ensure they are empty
+    mkdir -p /home/vagrant/certs/
+    rm -r /home/vagrant/certs/
+    mkdir -p /home/vagrant/certs/{td-agent,graylog}
+
+    # Generate Graylog's CA
+    cd /home/vagrant/certs
+    openssl genrsa -out graylog/rootCA.key 4096 2> /dev/null
+    openssl req -x509 -new -nodes -key graylog/rootCA.key -sha256 -days 1024 \
+        -out graylog/rootCA.crt -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" \
+        2> /dev/null
+
+    # Generate td-agent's keys
+    openssl genrsa -out td-agent/td-agent.key 4096 2> /dev/null
+    openssl req -new -sha256 -key td-agent/td-agent.key \
+        -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" -out td-agent/td-agent.csr \
+        2> /dev/null
+
+    # Sign td-agent's keys
+    openssl x509 -req -in td-agent/td-agent.csr -CA graylog/rootCA.crt \
+        -CAkey graylog/rootCA.key -CAcreateserial -days 1024 -sha256 \
+        -out td-agent/td-agent-signed.crt 2> /dev/null
+
+    # Fix permissions
+    chown -R vagrant:vagrant /home/vagrant/
+    chown -R 1100:1100 /home/vagrant/certs/graylog
 
     # Wait 120 seconds for Graylog to come online
-    INSTALL_INPUT=0
+    cd /vagrant
     SECONDS=0
     while true
     do
@@ -106,18 +119,17 @@ Vagrant.configure("2") do |config|
 
       if [[ "$GRAYLOG_STATE" == "healthy" ]]; then
         echo "Graylog is available."
-        INSTALL_INPUT=1
         sleep 5
         break
       elif [[ "$GRAYLOG_STATE" != "starting" ]]; then
         echo "Something is wrong with Graylog. Aborting."
-        break
+        exit 1
       elif [[ $SECONDS -le 120 ]]; then
         echo "Waiting for Graylog ($SECONDS/120 seconds)"
         sleep 10
       else
         echo "Waiting on Graylog timed out. Aborting."
-        break
+        exit 1
       fi
     done
 
@@ -134,22 +146,17 @@ Vagrant.configure("2") do |config|
     for TYPE in $INPUT_TYPES; do
       if [[ "$TYPE" == "org.graylog2.inputs.gelf.tcp.GELFTCPInput" ]]; then
         echo "Found GELF TCP input in Graylog, aborting input installation."
-        INPUT_INSTALL=1
+        exit
-        break
       fi
     done
 
     # Install GELF TCP Input
-    if [[ $INSTALL_INPUT -eq 1 ]]; then
+    curl -i -s -X POST \
-      echo "Installing GELF TCP input"
+        -H "Content-Type: application/json" \
-      curl -i -s -X POST \
+        -H "X-Requested-By: cli" \
-          -H "Content-Type: application/json" \
+        -u admin:admin \
-          -H "X-Requested-By: cli" \
+        "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
-          -u admin:admin \
+        -d @GELFTCPInput.json
-          "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
-          -d @GELFTCPInput.json
-    fi
-
   SHELL
 
 end

docker-compose.yml

@@ -25,19 +25,6 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
 
-  fluentd:
-    build: ./fluentd
-    restart: always
-    volumes:
-      - /var/log:/var/log/hostlogs
-      - /vagrant/pki/fluentd:/fluentd/etc/pki
-    networks:
-      - graylog
-    ports:
-      - 24224:24224
-      - 24224:24224/udp
-      - 5140:5140/udp
-
   mongo:
     image: mongo:4.2.2
     restart: always
@@ -72,6 +59,8 @@ services:
       - "traefik.http.services.graylog.loadbalancer.server.port=9000"
       - "traefik.docker.network=vagrant_traefik-net"
       - "traefik.enable=true"
+    volumes:
+      - /home/vagrant/certs/graylog:/usr/share/graylog/certs
     networks:
       - graylog
       - traefik-net
@@ -89,8 +78,6 @@ services:
       - 12201:12201
       # GELF UDP
       - 12201:12201/udp
-    volumes:
-      - /vagrant/pki/graylog:/usr/share/graylog/pki
 
 networks:
   traefik-net:

fluentd/Dockerfile

@@ -1,10 +0,0 @@
-FROM fluent/fluentd:v1.9.0-debian-1.0
-USER root
-RUN apt-get update \
- && apt-get -y install --no-install-recommends wget \
- && rm -rf /var/lib/apt/lists/*
-RUN gem uninstall gelf -v 3.1.0
-RUN gem install gelf -v 3.0.0
-RUN gem install fluent-plugin-rewrite-tag-filter
-RUN gem install fluent-plugin-gelf-hs
-COPY fluent.conf /fluentd/etc/

td-agent.conf

@@ -11,8 +11,8 @@
 
 <source>
   @type tail
-  path /var/log/hostlogs/httpd/access_log
+  path /var/log/httpd/access_log
-  pos_file /var/log/hostlogs/httpd/access_log.pos
+  pos_file /var/log/td-agent/access_log.pos
   tag httpd.access
   <parse>
     @type apache2
@@ -23,7 +23,7 @@
   @type copy
   <store>
     @type file
-    path /var/log/hostlogs/containers/${tag}
+    path /var/log/containers/${tag}
     append true
     <buffer tag>
       timekey 5s
@@ -42,11 +42,11 @@
 
 <match **>
   @type gelf
-  host vagrant_graylog_1
-  port 12201
   protocol tcp
+  host localhost
+  port 12201
   tls true
-  tls_options {"cert":"/fluentd/etc/pki/fluentd-signed.crt",
+  tls_options {"cert":"/home/vagrant/certs/td-agent/td-agent-signed.crt",
-               "key":"/fluentd/etc/pki/fluentd.key"}
+               "key":"/home/vagrant/certs/td-agent/td-agent.key"}
   flush_interval 5s
 </match>

td-agent.repo

@@ -0,0 +1,5 @@
+[treasuredata]
+name=TreasureData
+baseurl=http://packages.treasuredata.com/3/redhat/$releasever/$basearch
+gpgcheck=1
+gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent