shipping logs to cloudwatch

Dockerfile

@@ -1,29 +1,33 @@
 FROM ubuntu:14.04
-MAINTAINER Michal Raczka me@michaloo.net
+MAINTAINER Nelson Hernandez nelson@meetearnest.com
 
-# install curl and fluentd deps
-RUN apt-get update \
-    && apt-get install -y curl libcurl4-openssl-dev ruby ruby-dev make
+#install apt tools for adding repositories
+RUN apt-get update
+RUN apt-get install -y --force-yes curl software-properties-common python-software-properties
+
+#install apt repository for easily installing native ruby's
+RUN apt-add-repository ppa:brightbox/ruby-ng
+RUN apt-get update
+
+#fluent plugsins below depened on ruby2.0 and ruby2.0-dev specifically.
+RUN apt-get -y install ruby2.0 ruby2.0-dev ruby-switch make g++
+RUN ruby-switch --set ruby2.0
 
 # install fluentd with plugins
-RUN gem install fluentd --no-ri --no-rdoc \
-    && fluent-gem install fluent-plugin-elasticsearch \
-    fluent-plugin-record-modifier fluent-plugin-exclude-filter \
+RUN gem install fluentd fluent-plugin-cloudwatch-logs fluent-plugin-kubernetes_metadata_filter --no-ri --no-rdoc \
     && mkdir /etc/fluentd/
 
-# install docker-gen
-RUN cd /usr/local/bin \
-    && curl -L https://github.com/jwilder/docker-gen/releases/download/0.4.0/docker-gen-linux-amd64-0.4.0.tar.gz \
-    | tar -xzv
-
 # add startup scripts and config files
 ADD ./bin    /app/bin
 ADD ./config /app/config
 
-ADD config/fluentd-custom.tmpl /app/config/fluentd.tmpl
-
 WORKDIR /app
 
+
+RUN cd /usr/local/bin \
+   && curl -L https://github.com/jwilder/docker-gen/releases/download/0.4.0/docker-gen-linux-amd64-0.4.0.tar.gz \
+    | tar -xzv
+
 ENV ES_HOST localhost
 ENV ES_PORT 9200
 ENV LOG_ENV production

README.md

@@ -1,5 +1,17 @@
 # Docker fluentd
 
+## servers require the following files to deploy with fleetctl
+```
+/etc/sysconfig/aws_credentials
+/home/core/.docker/config.json
+```
+
+## to deploy using Fleet, the FLEETCTL_TUNNEL must be set locally.
+```
+export FLEETCLT_TUNNEL=<ip_of_1_server_in_cluster>
+```
+
+
 Docker image with:
 
 - [docker-gen](https://github.com/jwilder/docker-gen)

config/fluentd.tmpl

@@ -1,42 +1,21 @@
-## File input
-## read docker logs with tag=docker.container
-
-{{range $key, $value := .}}{{ $logs := $value.Env.LOG }}{{ if $logs }}
-
 <source>
   type tail
   format json
   time_key time
-  time_format %Y-%m-%dT%H:%M:%S.%N%Z
-  path /var/lib/docker/containers/{{ $value.ID }}/{{ $value.ID }}-json.log
-  pos_file /var/lib/docker/containers/{{ $value.ID }}/{{ $value.ID }}-json.log.pos
-  tag docker.container.{{printf "%.*s" 12 $value.ID}}
-  rotate_wait 5
+  path /var/lib/docker/containers/*/*-json.log
+  pos_file /var/lib/docker/containers/containers.log.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.container.*
 </source>
 
-<match docker.container.{{printf "%.*s" 12 $value.ID}}>
-    type record_modifier
-    tag docker.filtered.{{printf "%.*s" 12 $value.ID}}
-    image {{ $value.Image }}
-    host ${hostname}
-    name {{ $logs }}
-    env {{LOG_ENV}}
-</match>
-{{end}}{{ end }}
-
-<match docker.filtered.**>
-  type exclude_filter
-  key stream
-  value stdout
-  regexp false # default false, string comparison
-  add_tag_prefix stderr
-</match>
+<filter docker.container.** >
+  @type kubernetes_metadata
+</filter>
 
-<match stderr.docker.filtered.**>
-    type elasticsearch
-    host {{ .Env.ES_HOST }}
-    port {{ .Env.ES_PORT }}
-    index_name fluentd
-    type_name fluentd
-    logstash_format true
+<match docker.container.** >
+  type cloudwatch_logs
+  log_group_name kubernetes
+  log_stream_name "#{ENV['HOSTNAME']}"
+  auto_create_stream true
+  include_time_key true
 </match>

fluentd.service

@@ -0,0 +1,30 @@
+## config file used for fleetctl
+[Unit]
+Description=Ship container logs to elasticsearch. 
+After=docker.service
+Requires=docker.service
+
+[Service]
+TimeoutStartSec=10m
+User=core
+EnvironmentFile=/etc/sysconfig/aws_credentials
+
+ExecStartPre=-/usr/bin/docker kill fluentd
+ExecStartPre=-/usr/bin/docker rm fluentd
+ExecStartPre=/usr/bin/docker pull earnest/fluentd:latest
+
+ExecStart=/usr/bin/docker run --name fluentd \
+  -e AWS_REGION=${AWS_REGION} \
+  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+  -v /var/lib/docker:/var/lib/docker \
+  -v /var/run/docker.sock:/tmp/docker.sock \
+  earnest/fluentd
+
+ExecStop=/usr/bin/docker stop -t 30 fluentd
+
+[Install]
+WantedBy=multi-user.target
+
+[X-Fleet]
+Global=true