td-agent.conf

# This Fluentd configuration file enables the collection of log files
# that can be specified at the time of its creation in an environment
# variable, assuming that the config_generator.sh script runs to generate
# a configuration file for each log file to collect.
# Logs collected will be sent to the cluster's Elasticsearch service.
#
# Currently the collector uses a text format rather than allowing the user
# to specify how to parse each file.

# Pick up all the auto-generated input config files, one for each file
# specified in the FILES_TO_COLLECT environment variable.
@include files/*

<filter file.**>
  @type record_transformer
  enable_ruby
  <record>
    filename "${tag_parts[1..-1].join('.')}"
    tag ${tag}
  </record>
</filter>

<filter file.**>
  @type kubernetes_metadata
</filter>

# All the auto-generated files should use the tag "file.<filename>".
<match file.**>
   type elasticsearch
   log_level info
   include_tag_key true
   host elasticsearch-logging.kube-system
   port 9200
   logstash_format true
   # Set the chunk limit the same as for fluentd-gcp.
   buffer_chunk_limit 16M
   # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB
   buffer_queue_limit 32
   flush_interval 5s
   # Never wait longer than 5 minutes between retries.
   max_retry_wait 300
   # Disable the limit on the number of retries (retry forever).
   disable_retry_limit
   message log
</match>