Update README to reflect the multi-machine setup

.gitignore

@@ -1 +1,2 @@
 .vagrant
+tmp

GELFTCPInput.json

@@ -1,20 +1,13 @@
 {
-  "title": "td-agent",
+  "title": "Fluentd",
   "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput",
   "global": true,
   "configuration": {
     "bind_address": "0.0.0.0",
     "decompress_size_limit": 8388608,
-    "max_message_size": 2097152,
-    "number_worker_threads": 4,
-    "port": 12201,
     "recv_buffer_size": 1048576,
-    "tcp_keepalive": false,
-    "tls_cert_file": "/usr/share/graylog/certs/rootCA.crt",
-    "tls_client_auth": "required",
-    "tls_enable": true,
-    "tls_key_file": "/usr/share/graylog/certs/rootCA.key",
-    "user_null_delimiter": true
+    "number_worker_threads": 4,
+    "port": 12201
         },
   "node": null
 }

README.md

@@ -1,22 +1,21 @@
 # Graylog Demo
 
 
-This is a demonstration of Graylog, a centralized log management system featuring a shell provisioned CentOS 7 Vagrant box. To illustrate various log collection methods `httpd`, `rsyslog` and `docker` are installed and a simple WordPress instance is deployed via Docker Compose. Log collection incorporates Fluentd to ship logs into a Graylog instance from containers, the syslog, and arbitrary filesystem logs.
+This is a demonstration of Graylog, a centralized log management system featuring a shell provisioned CentOS 7 Vagrant box. To illustrate various log collection methods `httpd`, `rsyslog` and `docker` are installed and a simple WordPress instance is deployed via Docker Compose. Log collection incorporates td-agent (a version of Fluentd) to ship logs into a Graylog instance from containers, the syslog, and arbitrary filesystem logs.
 
 This demonstration assumes you are familiar with using Vagrant + VirtualBox to automate the installation of virtual machines, although you can reference the Vagrantfile's shell provisioning sections to manually set up a system if you so desire. Please install these prerequisites before attempting the quick start below.
 
 #### Notes about setup
-- This demonstration uses Traefik for routing and the [xip.io](http://xip.io/) wildcard DNS service. If DNS fails to resolve for whatever reason you may want to set the domains to the IP inside your operating system's hosts file, e.g.
+- This demonstration uses Traefik for some routing and the [xip.io](http://xip.io/) wildcard DNS service. If DNS fails to resolve for whatever reason you may want to set the domains to the IP inside your operating system's hosts file, e.g.
 
 ```
 172.28.128.30   traefik.172.28.128.30.xip.io
 172.28.128.30   graylog.172.28.128.30.xip.io
-172.28.128.30   wordpress.172.28.128.30.xip.io
 ```
 
-- Vagrant will provision a virtual machine with a static private Class B address (specifically `172.28.128.30`). If you would like to change this IP address to something different you will need to change the `PRIVATE_NET_IP` variable and the scripted API calls in the `Vagrantfile`. You'll also need to modify the few wildcard DNS references to it in the two `docker-compose.yml` files.
+- Vagrant will provision two virtual machines with two consecutive private Class B addresses (starting from `172.28.128.30`). If you would like to change this base IP address to something different you will need to look through the project and find various references. Unfortunately, this is not a simple variable you can set for the entire project.
 
-- Vagrant is set to allocate 4 cores and 4 GB of RAM, you may need to adjust this for your machine if necessary.
+- Vagrant is set to allocate 4 cores and 4 GB of RAM per machine (this is 8 cores / 8 GB of memory total) you may need to adjust this for your machine if necessary.
 
 - After deploying, Graylog takes the longest to become available and it may take 30 seconds to a few minutes to bring it up depending on your machine.
 
@@ -26,7 +25,7 @@ This demonstration assumes you are familiar with using Vagrant + VirtualBox to a
 
 
 ## Quick Start
-_This section assumes you will be using the default `172.28.128.30` IP address_
+_This section assumes you will be using the default `172.28.128.30` and `172.28.128.31` IP addresses_
 1. Clone the repository and navigate inside its directory
 2. Create and provision the VM using `vagrant up`
 3. Navigate to [http://graylog.172.28.128.30.xip.io:8080/](http://graylog.172.28.128.30.xip.io:8080/)
@@ -36,13 +35,13 @@ _This section assumes you will be using the default `172.28.128.30` IP address_
 7. Press the start button on the top right to start updating the feed every second
 
 #### Docker Test
-- Generate Docker logs by simply navigating to the WordPress install page [http://wordpress.172.28.128.30.xip.io:8080/](http://wordpress.172.28.128.30.xip.io:8080/)
+- Generate Docker logs by simply navigating to the WordPress install page: [http://172.28.128.31:8080](http://172.28.128.31:8080/wp-admin/install.php)
 
 #### File Test
-- Collect logs from Apache's `access_log` file by going to [http://172.28.128.30/](http://172.28.128.30/)
+- Collect logs from Apache's `access_log` file by going to [http://172.28.128.31/](http://172.28.128.31/)
 
 #### Syslog Test
-1. Go back to the terminal inside the project's directory and type `vagrant ssh`
+1. Go back to the terminal inside the project's directory and type `vagrant ssh systems` or `vagrant ssh graylog`
 2. You can test Syslog collection with `logger` e.g. `logger -t test Hello world` (or just wait for some to appear)
 
 ### Copyrights and Licenses

Vagrantfile

@@ -1,162 +1,162 @@
 # vi: set ft=ruby :
 
-PRIVATE_NET_IP = "172.28.128.30"
+PRIVATE_NET_IP = "172.28.128."
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "centos/7"
-  config.vm.network "private_network", ip: PRIVATE_NET_IP
-  config.vm.synced_folder ".", "/vagrant", type: "nfs"
 
-  config.vm.provider "virtualbox" do |vbox|
-    vbox.memory = 4096
-    vbox.cpus = 4
+  vmservers = ["graylog", "systems"]
+  last_octet = 30
+
+  vmservers.each do |server|
+    config.vm.define "#{server}" do |node|
+      node.vm.box = "centos/7"
+      node.vm.hostname = "#{server}"
+      node.vm.network "private_network", ip: PRIVATE_NET_IP + last_octet.to_s
+      node.vm.synced_folder ".", "/vagrant", type: "nfs"
+      last_octet = last_octet + 1
+
+      node.vm.provider "virtualbox" do |vbox|
+        vbox.memory = 4096
+        vbox.cpus = 4
+      end
+
+      # Common provision
+      node.vm.provision "shell", inline: <<-SHELL
+
+        # Set SELinux to permissive
+        setenforce 0
+        sed -i "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config
+
+        # Import GPG keys
+        rpm --import \
+          /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 \
+          https://download.docker.com/linux/centos/gpg \
+          http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 \
+          https://packages.treasuredata.com/GPG-KEY-td-agent
+
+        # Install Docker Community Edition
+        yum-config-manager --add-repo \
+            https://download.docker.com/linux/centos/docker-ce.repo
+        yum install -y docker-ce docker-ce-cli containerd.io
+        systemctl start docker
+        systemctl -q enable docker
+        usermod -aG docker vagrant
+
+        # Convenience
+        yum install -y vim
+
+        # Install rsyslog
+        yum install -y rsyslog
+        systemctl start rsyslog
+        systemctl -q enable rsyslog
+
+        # Add rsyslog forwarding option if it does not exist
+        if ! grep -q "127.0.0.1:5140" /etc/rsyslog.conf; then
+          echo "*.* @127.0.0.1:5140" >> /etc/rsyslog.conf
+          systemctl restart rsyslog
+        fi
+
+        # Install td-agent
+        cp /vagrant/td-agent.repo /etc/yum.repos.d/
+        yum check-update
+        yum install -y td-agent
+        td-agent-gem install fluent-plugin-secure-forward
+        td-agent-gem install fluent-plugin-gelf-hs gelf
+        systemctl -q enable td-agent
+
+      SHELL
+
+      # Commmon provision: install docker-compose
+      node.vm.provision "shell", path: "install-compose.sh"
+
+      # Graylog specific provision
+      if server == "graylog"
+        node.vm.provision "shell", inline: <<-SHELL
+
+          cp /vagrant/td-agent-server.conf /etc/td-agent/td-agent.conf
+          systemctl restart td-agent
+
+          # Install jq
+          yum install -y epel-release
+          yum install -y jq
+
+          # Start Graylog
+          cd /vagrant
+          /usr/local/bin/docker-compose up -d 2> /dev/null
+
+          # Wait 120 seconds for Graylog to come online
+           SECONDS=0
+           while true
+           do
+             GRAYLOG_STATE=$(
+               docker inspect vagrant_graylog_1 \
+                 | jq --raw-output '.[] | .State.Health.Status')
+
+             if [[ "$GRAYLOG_STATE" == "healthy" ]]; then
+               echo "Graylog is available."
+               sleep 5
+               break
+             elif [[ "$GRAYLOG_STATE" != "starting" ]]; then
+               echo "Something is wrong with Graylog. Aborting."
+               exit 1
+             elif [[ $SECONDS -le 120 ]]; then
+               echo "Waiting for Graylog ($SECONDS/120 seconds)"
+               sleep 10
+             else
+               echo "Waiting on Graylog timed out. Aborting."
+               exit 1
+             fi
+           done
+
+           # Check for existing GELF TCP Input
+           INPUTSTATE=$(
+             curl -s -X GET \
+                 -H "Content-Type: application/json" \
+                 -H "X-Requested-By: cli" \
+                 -u admin:admin \
+                 "http://graylog.172.28.128.30.xip.io:8080/api/system/inputstates")
+
+           INPUT_TYPES=$(echo $INPUTSTATE | jq --raw-output '.states | .[] | .message_input.type')
+
+           for TYPE in $INPUT_TYPES; do
+             if [[ "$TYPE" == "org.graylog2.inputs.gelf.tcp.GELFTCPInput" ]]; then
+               echo "Found GELF TCP input in Graylog, aborting input installation."
+               exit
+             fi
+           done
+
+           # Install GELF TCP Input
+           curl -i -s -X POST \
+               -H "Content-Type: application/json" \
+               -H "X-Requested-By: cli" \
+               -u admin:admin \
+               "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
+               -d @GELFTCPInput.json
+
+        SHELL
+
+      elsif server == "systems"
+        node.vm.provision "shell", inline: <<-SHELL
+
+          # Install apache
+          yum install -y httpd
+          systemctl start httpd
+          systemctl -q enable httpd
+
+          # Configure td-agent
+          cp /vagrant/td-agent.conf /etc/td-agent/td-agent.conf
+          mkdir -p /var/log/containers
+          chown -R td-agent:td-agent /var/log/containers
+          chmod -R 755 /var/log
+          systemctl restart td-agent
+
+          # Bring up WordPress test containers
+          cd /vagrant/wordpress
+          /usr/local/bin/docker-compose up -d 2> /dev/null
+
+        SHELL
+
+      end
+    end
   end
-
-  config.vm.provision "shell", inline: <<-SHELL
-
-    # Set SELinux to permissive
-    setenforce 0
-    sed -i "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config
-
-    # Import GPG keys
-    rpm --import \
-      /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 \
-      https://download.docker.com/linux/centos/gpg \
-      http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 \
-      https://packages.treasuredata.com/GPG-KEY-td-agent
-
-    # Install Docker Community Edition
-    yum-config-manager --add-repo \
-        https://download.docker.com/linux/centos/docker-ce.repo
-    yum install -y docker-ce docker-ce-cli containerd.io
-    systemctl start docker
-    systemctl -q enable docker
-    usermod -aG docker vagrant
-
-    # Convenience
-    yum install -y vim
-
-    # Install jq
-    yum install -y epel-release
-    yum install -y jq
-
-    # Install apache
-    yum install -y httpd
-    systemctl start httpd
-    systemctl -q enable httpd
-
-    # Install rsyslog
-    yum install -y rsyslog
-    systemctl start rsyslog
-    systemctl -q enable rsyslog
-
-    # Install td-agent
-    cp /vagrant/td-agent.repo /etc/yum.repos.d/
-    yum check-update
-    yum install -y td-agent
-    td-agent-gem install fluent-plugin-gelf-hs gelf
-    cp /vagrant/td-agent.conf /etc/td-agent/td-agent.conf
-    mkdir -p /var/log/containers
-    chown -R td-agent:td-agent /var/log/containers
-    chmod -R 755 /var/log
-    systemctl restart td-agent
-    systemctl -q enable td-agent
-
-    # Add rsyslog forwarding option if it does not exist
-    if ! grep -q "127.0.0.1:5140" /etc/rsyslog.conf; then
-      echo "*.* @127.0.0.1:5140" >> /etc/rsyslog.conf
-      systemctl restart rsyslog
-    fi
-
-  SHELL
-
-  # Install newest docker-compose
-  config.vm.provision "shell", path: "install-compose.sh"
-
-  # Start compose services and add default input
-  config.vm.provision "shell", inline: <<-SHELL
-    # Bring up containers
-    cd /vagrant
-    /usr/local/bin/docker-compose up -d 2> /dev/null
-    cd /vagrant/wordpress
-    /usr/local/bin/docker-compose up -d 2> /dev/null
-
-    # Create directories and ensure they are empty
-    mkdir -p /home/vagrant/certs/
-    rm -r /home/vagrant/certs/
-    mkdir -p /home/vagrant/certs/{td-agent,graylog}
-
-    # Generate Graylog's CA
-    cd /home/vagrant/certs
-    openssl genrsa -out graylog/rootCA.key 4096 2> /dev/null
-    openssl req -x509 -new -nodes -key graylog/rootCA.key -sha256 -days 1024 \
-        -out graylog/rootCA.crt -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" \
-        2> /dev/null
-
-    # Generate td-agent's keys
-    openssl genrsa -out td-agent/td-agent.key 4096 2> /dev/null
-    openssl req -new -sha256 -key td-agent/td-agent.key \
-        -subj "/C=US/ST=GA/O=MyOrg/CN=localhost" -out td-agent/td-agent.csr \
-        2> /dev/null
-
-    # Sign td-agent's keys
-    openssl x509 -req -in td-agent/td-agent.csr -CA graylog/rootCA.crt \
-        -CAkey graylog/rootCA.key -CAcreateserial -days 1024 -sha256 \
-        -out td-agent/td-agent-signed.crt 2> /dev/null
-
-    # Fix permissions
-    chown -R vagrant:vagrant /home/vagrant/
-    chown -R 1100:1100 /home/vagrant/certs/graylog
-
-    # Wait 120 seconds for Graylog to come online
-    cd /vagrant
-    SECONDS=0
-    while true
-    do
-      GRAYLOG_STATE=$(
-        docker inspect vagrant_graylog_1 \
-          | jq --raw-output '.[] | .State.Health.Status')
-
-      if [[ "$GRAYLOG_STATE" == "healthy" ]]; then
-        echo "Graylog is available."
-        sleep 5
-        break
-      elif [[ "$GRAYLOG_STATE" != "starting" ]]; then
-        echo "Something is wrong with Graylog. Aborting."
-        exit 1
-      elif [[ $SECONDS -le 120 ]]; then
-        echo "Waiting for Graylog ($SECONDS/120 seconds)"
-        sleep 10
-      else
-        echo "Waiting on Graylog timed out. Aborting."
-        exit 1
-      fi
-    done
-
-    # Check for existing GELF TCP Input
-    INPUTSTATE=$(
-      curl -s -X GET \
-          -H "Content-Type: application/json" \
-          -H "X-Requested-By: cli" \
-          -u admin:admin \
-          "http://graylog.172.28.128.30.xip.io:8080/api/system/inputstates")
-
-    INPUT_TYPES=$(echo $INPUTSTATE | jq --raw-output '.states | .[] | .message_input.type')
-
-    for TYPE in $INPUT_TYPES; do
-      if [[ "$TYPE" == "org.graylog2.inputs.gelf.tcp.GELFTCPInput" ]]; then
-        echo "Found GELF TCP input in Graylog, aborting input installation."
-        exit
-      fi
-    done
-
-    # Install GELF TCP Input
-    curl -i -s -X POST \
-        -H "Content-Type: application/json" \
-        -H "X-Requested-By: cli" \
-        -u admin:admin \
-        "http://graylog.172.28.128.30.xip.io:8080/api/system/inputs" \
-        -d @GELFTCPInput.json
-  SHELL
-
 end

docker-compose.yml

@@ -24,12 +24,24 @@ services:
       - "traefik.enable=true"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
+    logging:
+      driver: "fluentd"
+      options:
+       fluentd-address: "tcp://127.0.0.1:24224"
+       fluentd-async-connect: "true"
+       tag: traefik
 
   mongo:
     image: mongo:4.2.2
     restart: always
     networks:
       - graylog
+    logging:
+      driver: "fluentd"
+      options:
+       fluentd-address: "tcp://127.0.0.1:24224"
+       fluentd-async-connect: "true"
+       tag: graylog.db
 
   elasticsearch:
     image: elasticsearch:6.8.6
@@ -45,6 +57,12 @@ services:
         hard: -1
     networks:
       - graylog
+    logging:
+      driver: "fluentd"
+      options:
+       fluentd-address: "tcp://127.0.0.1:24224"
+       fluentd-async-connect: "true"
+       tag: graylog.elasticsearch
 
   graylog:
     image: graylog/graylog:3.2.2
@@ -59,8 +77,6 @@ services:
       - "traefik.http.services.graylog.loadbalancer.server.port=9000"
       - "traefik.docker.network=vagrant_traefik-net"
       - "traefik.enable=true"
-    volumes:
-      - /home/vagrant/certs/graylog:/usr/share/graylog/certs
     networks:
       - graylog
       - traefik-net
@@ -78,6 +94,12 @@ services:
       - 12201:12201
       # GELF UDP
       - 12201:12201/udp
+    logging:
+      driver: "fluentd"
+      options:
+       fluentd-address: "tcp://127.0.0.1:24224"
+       fluentd-async-connect: "true"
+       tag: graylog
 
 networks:
   traefik-net:

td-agent-server.conf

@@ -0,0 +1,38 @@
+<source>
+  @type secure_forward
+  shared_key test
+  self_hostname 172.28.128.30
+  port 2514
+  secure true
+  ca_cert_path /vagrant/tmp/ca_cert.pem
+  ca_private_key_path /vagrant/tmp/ca_key.pem
+  ca_private_key_passphrase test
+</source>
+
+<source>
+  @type forward
+  port 24224
+</source>
+
+<source>
+  @type syslog
+  port 5140
+  tag system.local
+</source>
+
+<filter httpd.access>
+  @type parser
+  key_name message
+  reserve_data true
+  <parse>
+    @type apache2
+  </parse>
+</filter>
+
+<match **>
+  @type gelf
+  protocol tcp
+  host localhost
+  port 12201
+  flush_interval 5s
+</match>

td-agent.conf

@@ -15,7 +15,7 @@
   pos_file /var/log/td-agent/access_log.pos
   tag httpd.access
   <parse>
-    @type apache2
+    @type none
   </parse>
 </source>
 
@@ -41,12 +41,14 @@
 </match>
 
 <match **>
-  @type gelf
-  protocol tcp
-  host localhost
-  port 12201
-  tls true
-  tls_options {"cert":"/home/vagrant/certs/td-agent/td-agent-signed.crt",
-               "key":"/home/vagrant/certs/td-agent/td-agent.key"}
-  flush_interval 5s
+  @type secure_forward
+  shared_key test
+  self_hostname 172.28.128.31
+  secure true
+  ca_cert_path /vagrant/tmp/ca_cert.pem
+
+  <server>
+    host 172.28.128.30
+    port 2514
+  </server>
 </match>

wordpress/docker-compose.yml

@@ -11,26 +11,25 @@ services:
       MYSQL_USER: wordpress
       MYSQL_PASSWORD: Password1
       MYSQL_RANDOM_ROOT_PASSWORD: '1'
-    networks:
-      - default
+    logging:
+      driver: "fluentd"
+      options:
+       fluentd-address: "tcp://127.0.0.1:24224"
+       fluentd-async-connect: "true"
+       tag: devel.kris.db
 
   wordpress:
     depends_on:
       - db
     image: wordpress:latest
     restart: always
+    ports:
+      - 8080:80
     environment:
       WORDPRESS_DB_HOST: db:3306
       WORDPRESS_DB_USER: wordpress
       WORDPRESS_DB_PASSWORD: Password1
       WORDPRESS_DB_NAME: wordpress
-    networks:
-      - default
-      - traefik-net
-    labels:
-      - "traefik.http.routers.wordpress.rule=Host(`wordpress.172.28.128.30.xip.io`)"
-      - "traefik.docker.network=vagrant_traefik-net"
-      - "traefik.enable=true"
     logging:
       driver: "fluentd"
       options:
@@ -38,10 +37,5 @@ services:
        fluentd-async-connect: "true"
        tag: devel.kris
 
-networks:
-  traefik-net:
-    external:
-      name: vagrant_traefik-net
-
 volumes:
   db_data: {}