Implement log shipping to Graylog via GELF

opencti-platform/opencti-graphql/config/default.json

@@ -26,14 +26,26 @@
       "logs_console": true,
       "logs_max_files": 7,
       "logs_directory": "./logs",
+      "logs_shipping": true,
+      "logs_shipping_level": "info",
+      "logs_shipping_env_var_prefix": "APP__",
+      "logs_graylog_host": "127.0.0.1",
+      "logs_graylog_port": 12201,
+      "logs_graylog_adapter": "udp",
       "logs_redacted_inputs": ["password", "secret", "token"],
       "extended_error_message": false
     },
     "audit_logs": {
       "logs_files": true,
       "logs_console": true,
       "logs_max_files": 7,
-      "logs_directory": "./logs"
+      "logs_directory": "./logs",
+      "logs_shipping": true,
+      "logs_shipping_level": "info",
+      "logs_shipping_env_var_prefix": "APP_AUDIT_",
+      "logs_graylog_host": "127.0.0.1",
+      "logs_graylog_port": 12201,
+      "logs_graylog_adapter": "udp"
     },
     "event_loop_logs": {
       "enabled": false,

opencti-platform/opencti-graphql/package.json

@@ -158,6 +158,7 @@
     "validator": "13.11.0",
     "winston": "3.14.2",
     "winston-daily-rotate-file": "5.0.0",
+    "winston-gelf": "1.7.0",
     "ws": "8.18.0",
     "xml2js": "0.6.2"
   },

opencti-platform/opencti-graphql/src/boot.js

@@ -1,4 +1,4 @@
-import { environment, getStoppingState, logApp, setStoppingState } from './config/conf';
+import { environment, getStoppingState, logApp, setStoppingState, shutdownLoggers } from './config/conf';
 import platformInit, { checkDeactivatedFeatureFlags, checkSystemDependencies } from './initialization';
 import cacheManager from './manager/cacheManager';
 import { shutdownRedisClients } from './database/redis';
@@ -40,6 +40,18 @@ export const platformStart = async () => {
     }
   } catch (mainError) {
     logApp.error(mainError);
+
+    try {
+      await shutdownLoggers();
+    } catch (e) {
+      /*
+        errors when shutting down the loggers can't be logged to them, so we just try using the standard console as a
+        "best effort" to give them some visibility
+      */
+      // eslint-disable-next-line no-console
+      console.error(e);
+    }
+
     process.exit(1);
   }
 };

opencti-platform/opencti-graphql/src/config/conf.js

@@ -34,6 +34,7 @@ import { UnknownError, UnsupportedError } from './errors';
 import { ENTITY_TYPE_PUBLIC_DASHBOARD } from '../modules/publicDashboard/publicDashboard-types';
 import { AI_BUS } from '../modules/ai/ai-types';
 import { SUPPORT_BUS } from '../modules/support/support-types';
+import { createLogShippingTransport } from './log-shipping';
 
 // https://golang.org/src/crypto/x509/root_linux.go
 const LINUX_CERTFILES = [
@@ -96,6 +97,7 @@ nconf.file('default', resolveEnvFile('default'));
 const appLogLevel = nconf.get('app:app_logs:logs_level');
 const appLogFileTransport = booleanConf('app:app_logs:logs_files', true);
 const appLogConsoleTransport = booleanConf('app:app_logs:logs_console', true);
+const appLogShippingTransport = booleanConf('app:app_logs:logs_shipping', false);
 export const appLogExtendedErrors = booleanConf('app:app_logs:extended_error_message', false);
 export const extendedErrors = (metaExtension) => {
   if (appLogExtendedErrors) {
@@ -127,6 +129,10 @@ if (appLogFileTransport) {
 if (appLogConsoleTransport) {
   appLogTransports.push(new winston.transports.Console());
 }
+if (appLogShippingTransport) {
+  const conf = nconf.get('app:app_logs');
+  appLogTransports.push(createLogShippingTransport(conf));
+}
 
 const appLogger = winston.createLogger({
   level: appLogLevel,
@@ -137,6 +143,7 @@ const appLogger = winston.createLogger({
 // Setup audit log logApp
 const auditLogFileTransport = booleanConf('app:audit_logs:logs_files', true);
 const auditLogConsoleTransport = booleanConf('app:audit_logs:logs_console', true);
+const auditLogShippingTransport = booleanConf('app:audit_logs:logs_shipping', false);
 const auditLogTransports = [];
 if (auditLogFileTransport) {
   const dirname = nconf.get('app:audit_logs:logs_directory');
@@ -152,6 +159,10 @@ if (auditLogFileTransport) {
 if (auditLogConsoleTransport) {
   auditLogTransports.push(new winston.transports.Console());
 }
+if (auditLogShippingTransport) {
+  const conf = nconf.get('app:audit_logs');
+  auditLogTransports.push(createLogShippingTransport(conf));
+}
 const auditLogger = winston.createLogger({
   level: 'info',
   format: format.combine(timestamp(), format.errors({ stack: true }), format.json()),
@@ -276,6 +287,20 @@ export const logTelemetry = {
   }
 };
 
+export function shutdownLoggers() {
+  const shutdownPromises = [appLogger, auditLogger, supportLogger].map(
+    (logger) => new Promise(
+      (resolve) => {
+        logger
+          .end()
+          .on('finish', resolve);
+      }
+    )
+  );
+
+  return Promise.all(shutdownPromises);
+}
+
 const BasePathConfig = nconf.get('app:base_path')?.trim() ?? '';
 const AppBasePath = BasePathConfig.endsWith('/') ? BasePathConfig.slice(0, -1) : BasePathConfig;
 export const basePath = isEmpty(AppBasePath) || AppBasePath.startsWith('/') ? AppBasePath : `/${AppBasePath}`;

opencti-platform/opencti-graphql/src/config/log-shipping.js

@@ -0,0 +1,50 @@
+import { format } from 'winston';
+import GelfTransport from 'winston-gelf';
+
+/**
+ * Create a new log shipping transport.
+ * @param {Object} conf The transport configuration
+ * @param {string} conf.logs_shipping_level The minimum log level of messages to send to ship
+ * @param {string} conf.logs_shipping_env_var_prefix The prefix used to match environment variables. Matching
+ *     variables will be added as meta info to the log data. The value of this property will be stripped from the name
+ *     of the environment variable.
+ * @param {string} conf.logs_graylog_host The Graylog host to connect to
+ * @param {number} conf.logs_graylog_port The port to use when connecting to the Graylog host
+ * @param {'tcp'|'udp'} conf.logs_graylog_adapter The adapter (udp/tcp) to use when connecting to the Graylog host
+ * @returns {import('winston-gelf')} The newly created log shipping transport
+ */
+export function createLogShippingTransport(conf) {
+  return new GelfTransport({
+    level: conf.logs_shipping_level,
+    format: format.combine(
+      envVarsFormat(conf.logs_shipping_env_var_prefix)(),
+      format.json(),
+    ),
+    gelfPro: {
+      adapterName: `${conf.logs_graylog_adapter}.js`, // append '.js', as a workaround for https://github.com/evanw/esbuild/issues/3328
+      adapterOptions: {
+        host: conf.logs_graylog_host,
+        port: conf.logs_graylog_port,
+      },
+    },
+  });
+}
+
+function envVarsFormat(prefix) {
+  const envVars = findPrefixedEnvVars(prefix);
+
+  return format(
+    (info) => ({ ...info, ...envVars })
+  );
+}
+
+function findPrefixedEnvVars(prefix) {
+  return Object.fromEntries(
+    Object.entries(process.env)
+      .flatMap(([key, value]) => {
+        return key.startsWith(prefix)
+          ? [[key.substring(prefix.length), value]]
+          : [];
+      })
+  );
+}

opencti-platform/opencti-graphql/yarn.lock

@@ -8309,6 +8309,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"gelf-pro@npm:^1.2.2":
+  version: 1.3.11
+  resolution: "gelf-pro@npm:1.3.11"
+  dependencies:
+    lodash: "npm:~4.17.21"
+  checksum: 10/a1b06e5238fdfe0f963fe79422e6018910c91ce9fb2bfb327a30a55c74c0bee1e03b0a2c243636c2e7e6e3cb670dd2baf5691bac79d77fe7abc293f0580c59bf
+  languageName: node
+  linkType: hard
+
 "gensync@npm:^1.0.0-beta.2":
   version: 1.0.0-beta.2
   resolution: "gensync@npm:1.0.0-beta.2"
@@ -10968,6 +10977,7 @@ __metadata:
     vitest: "npm:2.0.5"
     winston: "npm:3.14.2"
     winston-daily-rotate-file: "npm:5.0.0"
+    winston-gelf: "npm:1.7.0"
     ws: "npm:8.18.0"
     xml2js: "npm:0.6.2"
   languageName: unknown
@@ -13866,6 +13876,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"winston-gelf@npm:1.7.0":
+  version: 1.7.0
+  resolution: "winston-gelf@npm:1.7.0"
+  dependencies:
+    gelf-pro: "npm:^1.2.2"
+  checksum: 10/a9242bc1455baecadb59e367f8432d5f8ac3613838789b953b032f9e147eafd393d83aa065f041a8c001c166706f0071c296f760e6853c75c6817c4c9c796352
+  languageName: node
+  linkType: hard
+
 "winston-transport@npm:^4.7.0":
   version: 4.7.1
   resolution: "winston-transport@npm:4.7.1"