eh

.gitignore

@@ -1 +1,2 @@
 .vagrant
+pki

GELFTCPInput.json

@@ -1,20 +1,13 @@
 {
-  "title": "td-agent",
+  "title": "Fluentd",
   "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput",
   "global": true,
   "configuration": {
     "bind_address": "0.0.0.0",
     "decompress_size_limit": 8388608,
-    "max_message_size": 2097152,
-    "number_worker_threads": 4,
-    "port": 12201,
     "recv_buffer_size": 1048576,
-    "tcp_keepalive": false,
-    "tls_cert_file": "/usr/share/graylog/certs/rootCA.crt",
-    "tls_client_auth": "required",
-    "tls_enable": true,
-    "tls_key_file": "/usr/share/graylog/certs/rootCA.key",
-    "user_null_delimiter": true
+    "number_worker_threads": 4,
+    "port": 12201
         },
   "node": null
 }

Vagrantfile

@@ -19,11 +19,10 @@ Vagrant.configure("2") do |config|
     sed -i "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config
 
     # Import GPG keys
-    rpm --import \
+    curl -s https://download.docker.com/linux/centos/gpg -o docker-key
+    rpm --import docker-key \
       /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 \
-      https://download.docker.com/linux/centos/gpg \
-      http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 \
-      https://packages.treasuredata.com/GPG-KEY-td-agent
+      http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
 
     # Install Docker Community Edition
     yum-config-manager --add-repo \
@@ -50,18 +49,6 @@ Vagrant.configure("2") do |config|
     systemctl start rsyslog
     systemctl -q enable rsyslog
 
-    # Install td-agent
-    cp /vagrant/td-agent.repo /etc/yum.repos.d/
-    yum check-update
-    yum install -y td-agent
-    td-agent-gem install fluent-plugin-gelf-hs gelf
-    cp /vagrant/td-agent.conf /etc/td-agent/td-agent.conf
-    mkdir -p /var/log/containers
-    chown -R td-agent:td-agent /var/log/containers
-    chmod -R 755 /var/log
-    systemctl restart td-agent
-    systemctl -q enable td-agent
-
     # Add rsyslog forwarding option if it does not exist
     if ! grep -q "127.0.0.1:5140" /etc/rsyslog.conf; then
       echo "*.* @127.0.0.1:5140" >> /etc/rsyslog.conf
@@ -75,41 +62,41 @@ Vagrant.configure("2") do |config|
 
   # Start compose services and add default input
   config.vm.provision "shell", inline: <<-SHELL
+
+    # Remove old keys and create directories
+    mkdir -p /vagrant/pki
+    rm -r /vagrant/pki/*
+    mkdir -p /vagrant/pki/{fluentd,graylog}
+
+    # Generate and install TLS keys
+    cd /vagrant/pki
+
+    # Generate Graylog's CA
+    openssl genrsa -out rootCA.key 4096 2> /dev/null
+    openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 \
+        -out rootCA.crt -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" 2> /dev/null
+
+    # Generate Fluentd's keys
+    openssl genrsa -out fluentd.key 4096 2> /dev/null
+    openssl req -new -sha256 -key fluentd.key \
+        -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" -out fluentd.csr 2> /dev/null
+
+    # Sign Fluentd's certificate
+    openssl x509 -req -in fluentd.csr -CA rootCA.crt -CAkey rootCA.key \
+        -CAcreateserial -out fluentd-signed.crt -days 500 -sha256 2> /dev/null
+
+    mv fluentd*.* fluentd/
+    mv root*.* graylog/
+
     # Bring up containers
     cd /vagrant
     /usr/local/bin/docker-compose up -d 2> /dev/null
     cd /vagrant/wordpress
     /usr/local/bin/docker-compose up -d 2> /dev/null
-
-    # Create directories and ensure they are empty
-    mkdir -p /home/vagrant/certs/
-    rm -r /home/vagrant/certs/
-    mkdir -p /home/vagrant/certs/{td-agent,graylog}
-
-    # Generate Graylog's CA
-    cd /home/vagrant/certs
-    openssl genrsa -out graylog/rootCA.key 4096 2> /dev/null
-    openssl req -x509 -new -nodes -key graylog/rootCA.key -sha256 -days 1024 \
-        -out graylog/rootCA.crt -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" \
-        2> /dev/null
-
-    # Generate td-agent's keys
-    openssl genrsa -out td-agent/td-agent.key 4096 2> /dev/null
-    openssl req -new -sha256 -key td-agent/td-agent.key \
-        -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" -out td-agent/td-agent.csr \
-        2> /dev/null
-
-    # Sign td-agent's keys
-    openssl x509 -req -in td-agent/td-agent.csr -CA graylog/rootCA.crt \
-        -CAkey graylog/rootCA.key -CAcreateserial -days 1024 -sha256 \
-        -out td-agent/td-agent-signed.crt 2> /dev/null
-
-    # Fix permissions
-    chown -R vagrant:vagrant /home/vagrant/
-    chown -R 1100:1100 /home/vagrant/certs/graylog
+    cd /vagrant
 
     # Wait 120 seconds for Graylog to come online
-    cd /vagrant
+    INSTALL_INPUT=0
     SECONDS=0
     while true
     do
@@ -119,17 +106,18 @@ Vagrant.configure("2") do |config|
 
       if [[ "$GRAYLOG_STATE" == "healthy" ]]; then
         echo "Graylog is available."
+        INSTALL_INPUT=1
         sleep 5
         break
       elif [[ "$GRAYLOG_STATE" != "starting" ]]; then
         echo "Something is wrong with Graylog. Aborting."
-        exit 1
+        break
       elif [[ $SECONDS -le 120 ]]; then
         echo "Waiting for Graylog ($SECONDS/120 seconds)"
         sleep 10
       else
         echo "Waiting on Graylog timed out. Aborting."
-        exit 1
+        break
       fi
     done
 
@@ -146,17 +134,22 @@ Vagrant.configure("2") do |config|
     for TYPE in $INPUT_TYPES; do
       if [[ "$TYPE" == "org.graylog2.inputs.gelf.tcp.GELFTCPInput" ]]; then
         echo "Found GELF TCP input in Graylog, aborting input installation."
-        exit
+        INPUT_INSTALL=1
+        break
       fi
     done
 
     # Install GELF TCP Input
-    curl -i -s -X POST \
-        -H "Content-Type: application/json" \
-        -H "X-Requested-By: cli" \
-        -u admin:admin \
-        "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
-        -d @GELFTCPInput.json
+    if [[ $INSTALL_INPUT -eq 1 ]]; then
+      echo "Installing GELF TCP input"
+      curl -i -s -X POST \
+          -H "Content-Type: application/json" \
+          -H "X-Requested-By: cli" \
+          -u admin:admin \
+          "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
+          -d @GELFTCPInput.json
+    fi
+
   SHELL
 
 end

docker-compose.yml

@@ -25,6 +25,19 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
 
+  fluentd:
+    build: ./fluentd
+    restart: always
+    volumes:
+      - /var/log:/var/log/hostlogs
+      - /vagrant/pki/fluentd:/fluentd/etc/pki
+    networks:
+      - graylog
+    ports:
+      - 24224:24224
+      - 24224:24224/udp
+      - 5140:5140/udp
+
   mongo:
     image: mongo:4.2.2
     restart: always
@@ -59,8 +72,6 @@ services:
       - "traefik.http.services.graylog.loadbalancer.server.port=9000"
       - "traefik.docker.network=vagrant_traefik-net"
       - "traefik.enable=true"
-    volumes:
-      - /home/vagrant/certs/graylog:/usr/share/graylog/certs
     networks:
       - graylog
       - traefik-net
@@ -78,6 +89,8 @@ services:
       - 12201:12201
       # GELF UDP
       - 12201:12201/udp
+    volumes:
+      - /vagrant/pki/graylog:/usr/share/graylog/pki
 
 networks:
   traefik-net:

fluentd/Dockerfile

@@ -0,0 +1,10 @@
+FROM fluent/fluentd:v1.9.0-debian-1.0
+USER root
+RUN apt-get update \
+ && apt-get -y install --no-install-recommends wget \
+ && rm -rf /var/lib/apt/lists/*
+RUN gem uninstall gelf -v 3.1.0
+RUN gem install gelf -v 3.0.0
+RUN gem install fluent-plugin-rewrite-tag-filter
+RUN gem install fluent-plugin-gelf-hs
+COPY fluent.conf /fluentd/etc/

fluentd/fluent.conf

@@ -11,8 +11,8 @@
 
 <source>
   @type tail
-  path /var/log/httpd/access_log
-  pos_file /var/log/td-agent/access_log.pos
+  path /var/log/hostlogs/httpd/access_log
+  pos_file /var/log/hostlogs/httpd/access_log.pos
   tag httpd.access
   <parse>
     @type apache2
@@ -23,7 +23,7 @@
   @type copy
   <store>
     @type file
-    path /var/log/containers/${tag}
+    path /var/log/hostlogs/containers/${tag}
     append true
     <buffer tag>
       timekey 5s
@@ -42,11 +42,11 @@
 
 <match **>
   @type gelf
-  protocol tcp
-  host localhost
+  host vagrant_graylog_1
   port 12201
+  protocol tcp
   tls true
-  tls_options {"cert":"/home/vagrant/certs/td-agent/td-agent-signed.crt",
-               "key":"/home/vagrant/certs/td-agent/td-agent.key"}
+  tls_options {"cert":"/fluentd/etc/pki/fluentd-signed.crt",
+               "key":"/fluentd/etc/pki/fluentd.key"}
   flush_interval 5s
 </match>

td-agent.repo

@@ -1,5 +0,0 @@
-[treasuredata]
-name=TreasureData
-baseurl=http://packages.treasuredata.com/3/redhat/$releasever/$basearch
-gpgcheck=1
-gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent