diff --git a/.gitignore b/.gitignore index 89227d2a..9731a0ee 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ cantabular-import/analysis/count-log-events/*.txt cantabular-import/analysis/tmp/*.txt cantabular-import/analysis/simple-plot/*.svg + +cantabular-import/elk/elasticsearch_data/* diff --git a/cantabular-import/.env b/cantabular-import/.env index 727dbfa5..7c0699a7 100644 --- a/cantabular-import/.env +++ b/cantabular-import/.env @@ -1,3 +1,3 @@ -COMPOSE_FILE=deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:dp-filter-api.yml:zebedee.yml:dp-download-service.yml:dp-frontend-dataset-controller.yml:dp-api-router.yml:dp-frontend-router.yml:dp-publishing-dataset-controller.yml:florence.yml:the-train.yml:babbage.yml:dp-frontend-filter-flex-dataset.yml:dp-population-types-api.yml +COMPOSE_FILE=elk.yml:deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:dp-filter-api.yml:zebedee.yml:dp-download-service.yml:dp-frontend-dataset-controller.yml:dp-api-router.yml:dp-frontend-router.yml:dp-publishing-dataset-controller.yml:florence.yml:the-train.yml:babbage.yml:dp-frontend-filter-flex-dataset.yml:dp-population-types-api.yml COMPOSE_PATH_SEPARATOR=: COMPOSE_PROJECT_NAME=cantabular-import-journey diff --git a/cantabular-import/.env.backend b/cantabular-import/.env.backend index 590a15b7..9b9806f0 100644 --- a/cantabular-import/.env.backend +++ b/cantabular-import/.env.backend @@ -1,3 +1,3 @@ -COMPOSE_FILE=deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-filter-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:zebedee.yml:dp-download-service.yml:dp-population-types-api.yml +COMPOSE_FILE=elk.yml:deps.yml:dp-import-api.yml:dp-import-cantabular-dataset.yml:dp-import-cantabular-dimension-options.yml:dp-cantabular-server.yml:dp-cantabular-api-ext.yml:dp-dataset-api.yml:dp-cantabular-csv-exporter.yml:dp-cantabular-dimension-api.yml:dp-filter-api.yml:dp-cantabular-filter-flex-api.yml:dp-cantabular-metadata-service.yml:dp-cantabular-metadata-exporter.yml:dp-cantabular-xlsx-exporter.yml:dp-recipe-api.yml:zebedee.yml:dp-download-service.yml:dp-population-types-api.yml COMPOSE_PATH_SEPARATOR=: COMPOSE_PROJECT_NAME=cantabular-import-journey diff --git a/cantabular-import/elk.yml b/cantabular-import/elk.yml new file mode 100644 index 00000000..c405114a --- /dev/null +++ b/cantabular-import/elk.yml @@ -0,0 +1,40 @@ +version: '3.3' +services: + logstash: + image: logstash:7.16.2 + container_name: logstash + restart: always + volumes: + - ./elk/logstash/logstash.conf:/etc/logstash/logstash.conf + command: logstash -f /etc/logstash/logstash.conf + depends_on: + - elasticsearch + ports: + - '5044:5044' + - '5044:5044/udp' + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + + elasticsearch: + image: elasticsearch:7.16.2 + container_name: elasticsearch + restart: always + volumes: + - ./elk/elasticsearch_data:/usr/share/elasticsearch/data/ + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + discovery.type: single-node + ports: + - '9200:9200' + - '9300:9300' + + kibana: + image: kibana:7.16.2 + container_name: kibana + restart: always + ports: + - '5601:5601' + environment: + - ELASTICSEARCH_URL=http://elasticsearch:9200 + depends_on: + - elasticsearch diff --git a/cantabular-import/elk/logstash/logstash.conf b/cantabular-import/elk/logstash/logstash.conf new file mode 100644 index 00000000..825aaa41 --- /dev/null +++ b/cantabular-import/elk/logstash/logstash.conf @@ -0,0 +1,13 @@ +input { + gelf { + type => docker + port => 5044 + } +} + +output { + elasticsearch { + hosts => ["http://elasticsearch:9200"] + index => "containers-%{+YYYY.MM.dd}" + } +} \ No newline at end of file diff --git a/cantabular-import/florence.yml b/cantabular-import/florence.yml index 19277f4e..c0fdc293 100644 --- a/cantabular-import/florence.yml +++ b/cantabular-import/florence.yml @@ -25,3 +25,8 @@ services: API_ROUTER_URL: "http://dp-api-router:23200" ROUTER_URL: "http://dp-frontend-router:20000" DATASET_CONTROLLER_URL: "http://dp-publishing-dataset-controller:24000" + logging: + driver: gelf + options: + gelf-address: "udp://127.0.0.1:5044" + tag: "florence"