flume.conf

# Simple flume-elasticsearch deployment with logger enabled for debugging
#
#             _____________________________________________________
#             |      ---{es_channel}----{es_sink}-----------------|-- http://123.123.123.100:9300
#             |     /                                             |
#             |    /                                              |
#  apache-----|--{tail -f /var/log/apache2/access.logexec source} |
#             |    \                                              |
#             |     \                                             |
#             |      ---{log_channel}----{log_sink}---------------|-- /var/log/flume.log
#             |___________________________________________________|

#Source, Channels, and Sinks
agent.sources = tail
agent.channels = log_channel, es_channel
agent.sinks = log_sink, es_sink

#Tail of source
agent.sources.tail.type = exec
agent.sources.tail.command = tail -f /var/log/apache2/access.log
agent.sources.tail.channels = log_channel, es_channel

#Log sink for Debugging
agent.sinks.log_sink.channel = log_channel
agent.sinks.log_sink.type = logger

#ES sink for reporting
agent.sinks.es_sink.channel=es_channel
agent.sinks.es_sink.type=org.apache.flume.sink.elasticsearch.ElasticSearchSink
agent.sinks.es_sink.hostNames=123.123.123.100:9300
agent.sinks.es_sink.batchSize = 1
agent.sinks.es_sink.serializer = org.apache.flume.sink.elasticsearch.ElasticSearchLogStashEventSerializer
#Log Channel for debugging
agent.channels.log_channel.type = memory
agent.channels.log_channel.capacity = 10000

#ES Channel for reporting
agent.channels.es_channel.type = memory
agent.channels.es_channel.capacity = 10000