Update input config and mount Graylog cert data

GELFTCPInput.json

@@ -1,13 +1,20 @@
 {
-  "title": "Fluentd",
+  "title": "td-agent",
   "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput",
   "global": true,
   "configuration": {
     "bind_address": "0.0.0.0",
     "decompress_size_limit": 8388608,
-    "recv_buffer_size": 1048576,
+    "max_message_size": 2097152,
     "number_worker_threads": 4,
-    "port": 12201
+    "port": 12201,
+    "recv_buffer_size": 1048576,
+    "tcp_keepalive": false,
+    "tls_cert_file": "/usr/share/graylog/certs/rootCA.crt",
+    "tls_client_auth": "required",
+    "tls_enable": true,
+    "tls_key_file": "/usr/share/graylog/certs/rootCA.key",
+    "user_null_delimiter": true
         },
   "node": null
 }

Vagrantfile

@@ -19,10 +19,11 @@ Vagrant.configure("2") do |config|
     sed -i "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config
 
     # Import GPG keys
-    curl -s https://download.docker.com/linux/centos/gpg -o docker-key
+    rpm --import \
-    rpm --import docker-key \
       /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 \
-      http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+      https://download.docker.com/linux/centos/gpg \
+      http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 \
+      https://packages.treasuredata.com/GPG-KEY-td-agent
 
     # Install Docker Community Edition
     yum-config-manager --add-repo \
@@ -49,6 +50,18 @@ Vagrant.configure("2") do |config|
     systemctl start rsyslog
     systemctl -q enable rsyslog
 
+    # Install td-agent
+    cp /vagrant/td-agent.repo /etc/yum.repos.d/
+    yum check-update
+    yum install -y td-agent
+    td-agent-gem install fluent-plugin-gelf-hs gelf
+    cp /vagrant/td-agent.conf /etc/td-agent/td-agent.conf
+    mkdir -p /var/log/containers
+    chown -R td-agent:td-agent /var/log/containers
+    chmod -R 755 /var/log
+    systemctl restart td-agent
+    systemctl -q enable td-agent
+
     # Add rsyslog forwarding option if it does not exist
     if ! grep -q "127.0.0.1:5140" /etc/rsyslog.conf; then
       echo "*.* @127.0.0.1:5140" >> /etc/rsyslog.conf
@@ -67,9 +80,36 @@ Vagrant.configure("2") do |config|
     /usr/local/bin/docker-compose up -d 2> /dev/null
     cd /vagrant/wordpress
     /usr/local/bin/docker-compose up -d 2> /dev/null
-    cd /vagrant
+
+    # Create directories and ensure they are empty
+    mkdir -p /home/vagrant/certs/
+    rm -r /home/vagrant/certs/
+    mkdir -p /home/vagrant/certs/{td-agent,graylog}
+
+    # Generate Graylog's CA
+    cd /home/vagrant/certs
+    openssl genrsa -out graylog/rootCA.key 4096 2> /dev/null
+    openssl req -x509 -new -nodes -key graylog/rootCA.key -sha256 -days 1024 \
+        -out graylog/rootCA.crt -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" \
+        2> /dev/null
+
+    # Generate td-agent's keys
+    openssl genrsa -out td-agent/td-agent.key 4096 2> /dev/null
+    openssl req -new -sha256 -key td-agent/td-agent.key \
+        -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" -out td-agent/td-agent.csr \
+        2> /dev/null
+
+    # Sign td-agent's keys
+    openssl x509 -req -in td-agent/td-agent.csr -CA graylog/rootCA.crt \
+        -CAkey graylog/rootCA.key -CAcreateserial -days 1024 -sha256 \
+        -out td-agent/td-agent-signed.crt 2> /dev/null
+
+    # Fix permissions
+    chown -R vagrant:vagrant /home/vagrant/
+    chown -R 1100:1100 /home/vagrant/certs/graylog
 
     # Wait 120 seconds for Graylog to come online
+    cd /vagrant
     SECONDS=0
     while true
     do

docker-compose.yml

@@ -25,18 +25,6 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
 
-  fluentd:
-    build: ./fluentd
-    restart: always
-    volumes:
-      - /var/log:/var/log/hostlogs
-    networks:
-      - graylog
-    ports:
-      - 24224:24224
-      - 24224:24224/udp
-      - 5140:5140/udp
-
   mongo:
     image: mongo:4.2.2
     restart: always
@@ -71,6 +59,8 @@ services:
       - "traefik.http.services.graylog.loadbalancer.server.port=9000"
       - "traefik.docker.network=vagrant_traefik-net"
       - "traefik.enable=true"
+    volumes:
+      - /home/vagrant/certs/graylog:/usr/share/graylog/certs
     networks:
       - graylog
       - traefik-net

fluentd/Dockerfile

@@ -1,10 +0,0 @@
-FROM fluent/fluentd:v1.9.0-debian-1.0
-USER root
-RUN apt-get update \
- && apt-get -y install --no-install-recommends wget \
- && rm -rf /var/lib/apt/lists/*
-RUN gem install fluent-plugin-rewrite-tag-filter
-RUN gem install gelf
-RUN cd /fluentd/plugins \
- && wget -q https://raw.githubusercontent.com/emsearcy/fluent-plugin-gelf/master/lib/fluent/plugin/out_gelf.rb
-COPY fluent.conf /fluentd/etc/

td-agent.conf

@@ -11,8 +11,8 @@
 
 <source>
   @type tail
-  path /var/log/hostlogs/httpd/access_log
+  path /var/log/httpd/access_log
-  pos_file /var/log/hostlogs/httpd/access_log.pos
+  pos_file /var/log/td-agent/access_log.pos
   tag httpd.access
   <parse>
     @type apache2
@@ -23,7 +23,7 @@
   @type copy
   <store>
     @type file
-    path /var/log/hostlogs/containers/${tag}
+    path /var/log/containers/${tag}
     append true
     <buffer tag>
       timekey 5s
@@ -43,7 +43,10 @@
 <match **>
   @type gelf
   protocol tcp
-  host vagrant_graylog_1
+  host localhost
   port 12201
+  tls true
+  tls_options {"cert":"/home/vagrant/certs/td-agent/td-agent-signed.crt",
+               "key":"/home/vagrant/certs/td-agent/td-agent.key"}
   flush_interval 5s
 </match>

td-agent.repo

@@ -0,0 +1,5 @@
+[treasuredata]
+name=TreasureData
+baseurl=http://packages.treasuredata.com/3/redhat/$releasever/$basearch
+gpgcheck=1
+gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent