From c156f093cad9c411fa014d19093822c6ce219847 Mon Sep 17 00:00:00 2001 From: "valentin.kozlov" Date: Sat, 20 Apr 2024 22:26:19 +0000 Subject: [PATCH 1/3] feat: add config for goaccess web log analyser --- .env.sample | 7 +++++++ compose.prod.yml | 16 ++++++++++++++++ compose.yml | 2 +- goaccess-conf/goaccess-skim.sh | 7 +++++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100755 goaccess-conf/goaccess-skim.sh diff --git a/.env.sample b/.env.sample index 14cb82f..7f353cd 100644 --- a/.env.sample +++ b/.env.sample @@ -29,3 +29,10 @@ TERMS_OF_USE_URL=https://website.with.policy/acceptable-use-policy/ # can be left blank, but prefer putting one LETSENCRYPT_EMAIL= CORS_ORIGINS=http://localhost,http://foo.bar + +# Monitoring based on goaccess +GOACCESS_LOCAL=./goaccess-conf/ +GOACCESS_WEB_ROUTE=goaccess # no "/" leading or trailing +GOACCESS_USER=operator +GOACCESS_PASSWORD=$$bcrypt$$encrypted # bcrypt encrypted password, use "htpasswd -B -n operator" to generate. use $$ instead of $ in GOACCESS_PASSWORD value +GOACCESS_ACCESSLOG_FORMAT=TRAEFIKCLF diff --git a/compose.prod.yml b/compose.prod.yml index a7fa9ab..cd6d068 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -36,3 +36,19 @@ services: labels: traefik.http.routers.frontend.tls: true traefik.http.routers.frontend.tls.certResolver: letsEncrypt + + goaccess: + restart: always + image: mteamkit/goaccess-service + volumes: + - ${MLFLOW_BASE_PATH_LOCAL}/traefik/log/:/var/log/srv-access/ + - ${GOACCESS_LOCAL}:/opt/goaccess/ + env_file: .env + networks: + - backend + labels: + traefik.http.routers.goaccess.rule: Host(`${DOMAIN}`) && PathPrefix(`/${GOACCESS_WEB_ROUTE}`) + traefik.http.routers.goaccess.tls: true + traefik.http.routers.goaccess.tls.certResolver: letsEncrypt + traefik.http.routers.goaccess.middlewares: auth + traefik.http.middlewares.auth.basicauth.users: ${GOACCESS_USER}:${GOACCESS_PASSWORD} diff --git a/compose.yml b/compose.yml index f41d652..2d9d7eb 100644 --- a/compose.yml +++ b/compose.yml @@ -32,7 +32,7 @@ services: expose: - 5000 labels: - - "traefik.http.routers.backend.rule=Host(`${DOMAIN}`) && !PathPrefix(`/signup`)" + - "traefik.http.routers.backend.rule=Host(`${DOMAIN}`) && !PathPrefix(`/signup`) && !PathPrefix(`/${GOACCESS_WEB_ROUTE}`)" - "traefik.http.middlewares.limit.buffering.maxRequestBodyBytes=12884901888" - "traefik.http.middlewares.limit.buffering.memRequestBodyBytes=10485760" - "traefik.http.middlewares.limit.buffering.maxResponseBodyBytes=12884901888" diff --git a/goaccess-conf/goaccess-skim.sh b/goaccess-conf/goaccess-skim.sh new file mode 100755 index 0000000..e007b3f --- /dev/null +++ b/goaccess-conf/goaccess-skim.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Script to skim access log file +# Only one input is expected: one access log file +# Output of the filtering is redirected to file in the goaccess-exec.sh script + +cat $1 |grep -e "api/2.0" -e "/signup" From 6d9b2931904cf4904bbf65e3ef6a53b2e0dd58fa Mon Sep 17 00:00:00 2001 From: "valentin.kozlov" Date: Mon, 22 Apr 2024 17:22:04 +0000 Subject: [PATCH 2/3] feat: update input and output paths for goaccess --- .env.sample | 5 +++-- compose.prod.yml | 3 ++- goaccess-conf/goaccess-skim.sh | 7 ------- 3 files changed, 5 insertions(+), 10 deletions(-) delete mode 100755 goaccess-conf/goaccess-skim.sh diff --git a/.env.sample b/.env.sample index 7f353cd..4e8222f 100644 --- a/.env.sample +++ b/.env.sample @@ -31,8 +31,9 @@ LETSENCRYPT_EMAIL= CORS_ORIGINS=http://localhost,http://foo.bar # Monitoring based on goaccess -GOACCESS_LOCAL=./goaccess-conf/ -GOACCESS_WEB_ROUTE=goaccess # no "/" leading or trailing +GOACCESS_ETC_PATH_LOCAL=./goaccess-etc +GOACCESS_OUT_PATH_LOCAL=$MLFLOW_BASE_PATH_LOCAL/goaccess-out +GOACCESS_WEB_ROUTE=goaccess # [!] no "/" leading or trailing [!] GOACCESS_USER=operator GOACCESS_PASSWORD=$$bcrypt$$encrypted # bcrypt encrypted password, use "htpasswd -B -n operator" to generate. use $$ instead of $ in GOACCESS_PASSWORD value GOACCESS_ACCESSLOG_FORMAT=TRAEFIKCLF diff --git a/compose.prod.yml b/compose.prod.yml index cd6d068..da76cd3 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -42,7 +42,8 @@ services: image: mteamkit/goaccess-service volumes: - ${MLFLOW_BASE_PATH_LOCAL}/traefik/log/:/var/log/srv-access/ - - ${GOACCESS_LOCAL}:/opt/goaccess/ + - ${GOACCESS_ETC_PATH_LOCAL}:/opt/goaccess-etc/ + - ${GOACCESS_OUT_PATH_LOCAL}:/opt/goaccess-out/ env_file: .env networks: - backend diff --git a/goaccess-conf/goaccess-skim.sh b/goaccess-conf/goaccess-skim.sh deleted file mode 100755 index e007b3f..0000000 --- a/goaccess-conf/goaccess-skim.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# Script to skim access log file -# Only one input is expected: one access log file -# Output of the filtering is redirected to file in the goaccess-exec.sh script - -cat $1 |grep -e "api/2.0" -e "/signup" From ce2dce7d7127efde784a8f45fc334fb57fef51b5 Mon Sep 17 00:00:00 2001 From: "valentin.kozlov" Date: Mon, 22 Apr 2024 17:27:26 +0000 Subject: [PATCH 3/3] feat: add script for goaccess access.log skimming --- goaccess-etc/goaccess-skim.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 goaccess-etc/goaccess-skim.sh diff --git a/goaccess-etc/goaccess-skim.sh b/goaccess-etc/goaccess-skim.sh new file mode 100755 index 0000000..e007b3f --- /dev/null +++ b/goaccess-etc/goaccess-skim.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Script to skim access log file +# Only one input is expected: one access log file +# Output of the filtering is redirected to file in the goaccess-exec.sh script + +cat $1 |grep -e "api/2.0" -e "/signup"