initial dev release 0.0.1 that includes filebeat 5.2.2

.final_builds/jobs/filebeat/index.yml

@@ -0,0 +1,7 @@
+---
+builds:
+  3dccd6daa81455c9ed8ed8e152db3af78447619d:
+    version: 3dccd6daa81455c9ed8ed8e152db3af78447619d
+    sha1: 3f60fa69d01832de0721c84f920a67215ddbba32
+    blobstore_id: cecfebf6-c070-45ee-8871-ead3f3c6de07
+format-version: '2'

.final_builds/license/index.yml

@@ -0,0 +1,7 @@
+---
+builds:
+  37cb05ddf3167b3a7eeb030b21b27a71622c65cd:
+    version: 37cb05ddf3167b3a7eeb030b21b27a71622c65cd
+    sha1: 50526d2486d2a78cb51993ca704376af4a321a03
+    blobstore_id: 19ba05d9-8ae3-422f-87f0-dd9be86f45ec
+format-version: '2'

.final_builds/packages/filebeat/index.yml

@@ -0,0 +1,7 @@
+---
+builds:
+  b40b69d412a0d138962db0564b1a08214465b2c5:
+    version: b40b69d412a0d138962db0564b1a08214465b2c5
+    sha1: e4f8d6d9d692806bf42ef741f85e8ace52290422
+    blobstore_id: 05f80b3a-e1dc-45f7-ac2d-bdef761b8938
+format-version: '2'

.gitignore

@@ -0,0 +1,16 @@
+config/dev.yml
+config/private.yml
+releases/*.tgz
+releases/**/*.tgz
+dev_releases
+.blobs
+blobs
+.dev_builds
+.idea
+.DS_Store
+.final_builds/jobs/**/*.tgz
+.final_builds/packages/**/*.tgz
+*.swp
+*~
+*#
+#*

README.md

@@ -1,2 +1,19 @@
 # filebeat-boshrelease
-BOSH release for filebeat log shipper
+BOSH release for filebeat log shipper.
+
+filebeat is a logshipper from elastic (elastic.co).
+
+### Creating a development bosh release
+```
+bosh create release --force
+bosh upload release
+```
+
+### Deploying to *bosh-lite*.
+a sample cloud-config and deployment manifest is provided in the `templates` directory.
+
+```
+bosh update cloud-config templates/bosh-lite-cloud-config.yml
+bosh deployment templates/bosh-lite-deployment.yml
+bosh deploy
+```

config/blobs.yml

@@ -0,0 +1,5 @@
+---
+filebeat/filebeat-5.2.2-linux-x86_64.tar.gz:
+  object_id: 12cfb7fa-924d-48d2-b5d5-e49e8d557fc1
+  sha: 0f8e2f5f1145051435352b0e6a8b776040ea36e4
+  size: 8444096

config/final.yml

@@ -0,0 +1,8 @@
+---
+blobstore:
+  provider: s3
+  options:
+    region: ap-southeast-2
+    endpoint: https://s3-ap-southeast-2.amazonaws.com
+    bucket_name: dta-filebeat-boshrelease
+final_name: filebeat

jobs/filebeat/monit

@@ -0,0 +1,5 @@
+check process filebeat
+  with pidfile /var/vcap/sys/run/filebeat/filebeat.pid
+  start program "/var/vcap/jobs/filebeat/bin/monit_debugger filebeat_ctl '/var/vcap/jobs/filebeat/bin/filebeat_ctl start'" with timeout 120 seconds
+  stop program "/var/vcap/jobs/filebeat/bin/monit_debugger filebeat_ctl '/var/vcap/jobs/filebeat/bin/filebeat_ctl stop'"
+  group vcap

jobs/filebeat/spec

@@ -0,0 +1,24 @@
+---
+name: filebeat
+templates:
+  bin/filebeat_ctl: bin/filebeat_ctl
+  bin/monit_debugger: bin/monit_debugger
+  config/filebeat.yml.erb: config/filebeat.yml
+  data/properties.sh.erb: data/properties.sh
+  helpers/ctl_setup.sh: helpers/ctl_setup.sh
+  helpers/ctl_utils.sh: helpers/ctl_utils.sh
+
+
+packages:
+  - filebeat
+
+properties:
+  filebeat.configuration:
+    description: filebeat configuration in yaml format
+    default: |
+      filebeat.prospectors:
+      - input_type: log
+        paths:
+          - /var/vcap/sys/log/*/*.log
+      output.logstash:
+        hosts: ["10.244.0.25:5044"]

jobs/filebeat/templates/bin/filebeat_ctl

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Setup env vars and folders for the webapp_ctl script
+source /var/vcap/jobs/filebeat/helpers/ctl_setup.sh 'filebeat'
+
+case $1 in
+
+  start)
+    pid_guard $PIDFILE $JOB_NAME
+    pid_guard $PIDFILE-init $JOB_NAME-init
+
+    chown -R vcap:vcap $STORE_DIR $LOG_DIR $RUN_DIR
+
+    echo $$ > $PIDFILE
+    chown vcap:vcap $PIDFILE
+
+    chpst -u vcap:vcap /var/vcap/packages/filebeat/bin/filebeat \
+        -c /var/vcap/jobs/filebeat/config/filebeat.yml \
+        -path.data $STORE_DIR \
+        -path.logs $LOG_DIR \
+         >>$LOG_DIR/$JOB_NAME.stdout.log \
+         2>>$LOG_DIR/$JOB_NAME.stderr.log 
+
+    ;;
+
+  stop)
+    kill_and_wait $PIDFILE
+
+    ;;
+  *)
+    echo "Usage: filebeat_ctl {start|stop}"
+
+    ;;
+
+esac
+exit 0

jobs/filebeat/templates/bin/monit_debugger

@@ -0,0 +1,13 @@
+#!/bin/sh
+# USAGE monit_debugger <label> command to run
+mkdir -p /var/vcap/sys/log/monit
+{
+  echo "MONIT-DEBUG date"
+  date
+  echo "MONIT-DEBUG env"
+  env
+  echo "MONIT-DEBUG $@"
+  $2 $3 $4 $5 $6 $7
+  R=$?
+  echo "MONIT-DEBUG exit code $R"
+} >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1

jobs/filebeat/templates/config/filebeat.yml.erb

@@ -0,0 +1,12 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+#=========================== Filebeat prospectors =============================
+
+<%= p("filebeat.configuration") %>

jobs/filebeat/templates/data/properties.sh.erb

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# job template binding variables
+
+# job name & index of this VM within cluster
+# e.g. JOB_NAME=redis, JOB_INDEX=0
+export NAME='<%= name %>'
+export JOB_INDEX=<%= index %>
+# full job name, like redis/0 or webapp/3
+export JOB_FULL="$NAME/$JOB_INDEX"

jobs/filebeat/templates/helpers/ctl_setup.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Setup env vars and folders for the ctl script
+# This helps keep the ctl script as readable
+# as possible
+
+# Usage options:
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+JOB_NAME=$1
+output_label=${1:-JOB_NAME}
+
+export JOB_DIR=/var/vcap/jobs/$JOB_NAME
+chmod 755 $JOB_DIR # to access file via symlink
+
+# Load some bosh deployment properties into env vars
+# Try to put all ERb into data/properties.sh.erb
+# incl $NAME, $JOB_INDEX, $WEBAPP_DIR
+source $JOB_DIR/data/properties.sh
+
+source $JOB_DIR/helpers/ctl_utils.sh
+redirect_output ${output_label}
+
+export HOME=${HOME:-/home/vcap}
+
+# Add all packages' /bin & /sbin into $PATH
+for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
+do
+  export PATH=${package_bin_dir}:$PATH
+done
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-''} # default to empty
+for package_bin_dir in $(ls -d /var/vcap/packages/*/lib)
+do
+  export LD_LIBRARY_PATH=${package_bin_dir}:$LD_LIBRARY_PATH
+done
+
+# Setup log, run and tmp folders
+
+export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
+export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
+export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
+export STORE_DIR=/var/vcap/store/$JOB_NAME
+for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR
+do
+  mkdir -p ${dir}
+  chown vcap:vcap ${dir}
+  chmod 775 ${dir}
+done
+export TMPDIR=$TMP_DIR
+
+export C_INCLUDE_PATH=/var/vcap/packages/mysqlclient/include/mysql:/var/vcap/packages/sqlite/include:/var/vcap/packages/libpq/include
+export LIBRARY_PATH=/var/vcap/packages/mysqlclient/lib/mysql:/var/vcap/packages/sqlite/lib:/var/vcap/packages/libpq/lib
+
+# consistent place for vendoring python libraries within package
+if [[ -d ${WEBAPP_DIR:-/xxxx} ]]
+then
+  export PYTHONPATH=$WEBAPP_DIR/vendor/lib/python
+fi
+
+if [[ -d /var/vcap/packages/java8 ]]
+then
+  export JAVA_HOME="/var/vcap/packages/java8"
+fi
+
+export PIDFILE=$RUN_DIR/$JOB_NAME.pid
+
+echo '$PATH' $PATH