[FEATURE] Création de l'API Maddo (Mise à Disposition de DOnnées) (PIX-16420).

api/.ls-lint.yml

@@ -1,4 +1,5 @@
 ls:
+  .maddo.js: kebab-case
   .js: kebab-case
   _test.js: kebab-case
 

api/Procfile-maddo

@@ -0,0 +1,7 @@
+postdeploy: npm run postdeploy:maddo
+# Do not call npm start directly
+# npm does not forward process signals (e.g. SIGINT / SIGKILL ...)
+# see https://github.com/1024pix/pix/pull/796
+# and https://github.com/npm/npm/issues/4603
+# for more information
+web: exec node maddo.js

api/index.js

@@ -1,16 +1,15 @@
 import 'dotenv/config';
 
-import { validateEnvironmentVariables } from './src/shared/infrastructure/validate-environment-variables.js';
-
-validateEnvironmentVariables();
-
 import { disconnect, prepareDatabaseConnection } from './db/knex-database-connection.js';
 import { createServer } from './server.js';
-import { config } from './src/shared/config.js';
+import { config, schema as configSchema } from './src/shared/config.js';
 import { learningContentCache } from './src/shared/infrastructure/caches/learning-content-cache.js';
 import { quitAllStorages } from './src/shared/infrastructure/key-value-storages/index.js';
 import { logger } from './src/shared/infrastructure/utils/logger.js';
 import { redisMonitor } from './src/shared/infrastructure/utils/redis-monitor.js';
+import { validateEnvironmentVariables } from './src/shared/infrastructure/validate-environment-variables.js';
+
+validateEnvironmentVariables(configSchema);
 
 let server;
 

api/maddo.js

@@ -0,0 +1,64 @@
+import 'dotenv/config';
+
+import { disconnect, prepareDatabaseConnection } from './db/knex-database-connection.js';
+import { createMaddoServer } from './server.maddo.js';
+import { config, schema as configSchema } from './src/shared/config.maddo.js';
+import { quitAllStorages } from './src/shared/infrastructure/key-value-storages/index.js';
+import { logger } from './src/shared/infrastructure/utils/logger.js';
+import { redisMonitor } from './src/shared/infrastructure/utils/redis-monitor.js';
+import { validateEnvironmentVariables } from './src/shared/infrastructure/validate-environment-variables.js';
+
+validateEnvironmentVariables(configSchema);
+
+let server;
+
+async function _setupEcosystem() {
+  /*
+    First connection with Knex requires infrastructure operations such as
+    DNS resolution. So we execute one harmless query to our database
+    so those matters are resolved before starting the server.
+  */
+  await prepareDatabaseConnection();
+}
+
+const start = async function () {
+  if (config.featureToggles.setupEcosystemBeforeStart) {
+    await _setupEcosystem();
+  }
+  server = await createMaddoServer();
+  await server.start();
+};
+
+async function _exitOnSignal(signal) {
+  logger.info(`Received signal: ${signal}.`);
+  logger.info('Stopping HAPI server...');
+  await server.stop({ timeout: 30000 });
+  await server.directMetrics?.clearMetrics();
+  if (server.oppsy) {
+    logger.info('Stopping HAPI Oppsy server...');
+    await server.oppsy.stop();
+  }
+  logger.info('Closing connections to database...');
+  await disconnect();
+  logger.info('Closing connections to cache...');
+  await quitAllStorages();
+  logger.info('Closing connections to redis monitor...');
+  await redisMonitor.quit();
+  logger.info('Exiting process...');
+}
+
+process.on('SIGTERM', () => {
+  _exitOnSignal('SIGTERM');
+});
+process.on('SIGINT', () => {
+  _exitOnSignal('SIGINT');
+});
+
+(async () => {
+  try {
+    await start();
+  } catch (error) {
+    logger.error(error);
+    throw error;
+  }
+})();

api/package.json

@@ -157,11 +157,13 @@
     "lint:js:uncached": "eslint .",
     "lint:translations": "eslint --format node_modules/eslint-plugin-i18n-json/formatter.js translations",
     "lint:translations:fix": "npm run lint:translations -- --fix",
-    "postdeploy": "DEBUG=knex:* npm run db:migrate",
+    "postdeploy": "DEBUG=knex:query npm run db:migrate",
+    "postdeploy:maddo": "node scripts/wait-for-api-deployment",
     "preinstall": "npx check-engine",
-    "scalingo-postbuild": "node scripts/generate-cron > cron.json",
+    "scalingo-postbuild": "node scripts/generate-cron > cron.json && node scripts/generate-procfile",
     "dev": "nodemon index.js",
     "start": "node index.js",
+    "start:maddo": "node maddo.js",
     "start:watch": "npm run dev",
     "start:job": "node worker.js",
     "start:job:fast": "node worker.js fast",
@@ -185,4 +187,4 @@
     "monitoring:metrics": "node scripts/arborescence-monitoring/add-metrics-to-gist.js",
     "modulix:test": "npm run test:api:path -- 'tests/devcomp/unit/infrastructure/datasources/learning-content/module-datasource_test.js' 'tests/devcomp/acceptance/module-instantiation_test.js' 'tests/devcomp/unit/infrastructure/datasources/learning-content/validation/module-validation_test.js'"
   }
-}
+}

api/scripts/generate-procfile.js

@@ -0,0 +1,7 @@
+import { copyFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { exit } from 'node:process';
+
+if (!process.env.MADDO) exit(0);
+
+await copyFile(resolve(import.meta.dirname, '../Procfile-maddo'), resolve(import.meta.dirname, '../Procfile'));

api/scripts/scalingo-post-ra-creation.sh

@@ -1,4 +1,16 @@
 #!/bin/bash
+set -ex
+
+install-scalingo-cli
+
+if [[ -z "$MADDO" ]]; then
+  npm run postdeploy
+  scalingo -a pix-api-maddo-review-pr$PR_NUMBER set-env DATABASE_URL="$SCALINGO_POSTGRESQL_URL"
+  scalingo -a pix-api-maddo-review-pr$PR_NUMBER restart
+  npm run db:seed
+else
+  npm run postdeploy:maddo
+  scalingo -a pix-api-review-pr$PR_NUMBER set-env DATAMART_URL="$SCALINGO_POSTGRESQL_URL"
+  scalingo -a pix-api-review-pr$PR_NUMBER restart
+fi
 
-npm run postdeploy
-npm run db:seed

api/scripts/wait-for-api-deployment.js

@@ -0,0 +1,35 @@
+import { exit } from 'node:process';
+import { setTimeout } from 'node:timers/promises';
+
+import packageJson from '../package.json' with { type: 'json' };
+
+const apiDomain = process.env.DOMAIN_PIX_API;
+const expectedVersion = packageJson.version;
+
+const apiInfoURL = new URL('/api', apiDomain);
+
+while (true) {
+  const apiInfo = await fetchApiInfoURL();
+
+  if (apiInfo?.version === expectedVersion) {
+    exit(0);
+  }
+
+  await setTimeout(10_000);
+}
+
+async function fetchApiInfoURL() {
+  console.info('Fetching', apiInfoURL.href);
+
+  // eslint-disable-next-line n/no-unsupported-features/node-builtins
+  const res = await fetch(apiInfoURL);
+  if (!res.ok) {
+    if (res.status === 404) {
+      return null;
+    }
+
+    throw new Error(`Pix API responded with ${res.statusText}`);
+  }
+
+  return res.json();
+}

api/server.maddo.js

@@ -0,0 +1,191 @@
+import Oppsy from '@1024pix/oppsy';
+import Hapi from '@hapi/hapi';
+import { parse } from 'neoqs';
+
+import { setupErrorHandling } from './config/server-setup-error-handling.js';
+import { knex } from './db/knex-database-connection.js';
+import { authentication } from './lib/infrastructure/authentication.js';
+import { Metrics } from './src/monitoring/infrastructure/metrics.js';
+import * as healthcheckRoutes from './src/shared/application/healthcheck/index.js';
+import { config } from './src/shared/config.js';
+import { monitoringTools } from './src/shared/infrastructure/monitoring-tools.js';
+import { plugins } from './src/shared/infrastructure/plugins/index.js';
+import { deserializer } from './src/shared/infrastructure/serializers/jsonapi/deserializer.js';
+import { swaggers } from './src/shared/swaggers.js';
+import { handleFailAction } from './src/shared/validate.js';
+
+monitoringTools.installHapiHook();
+
+const { logOpsMetrics, port, logging } = config;
+const createMaddoServer = async () => {
+  const server = createBareServer();
+
+  // initialisation of Datadog link for metrics publication
+  const metrics = new Metrics({ config });
+  server.directMetrics = metrics;
+
+  if (logOpsMetrics) {
+    // OPS metrics via direct metrics
+    if (config.featureToggles.isDirectMetricsEnabled) await enableOpsMetrics(server, metrics);
+    // OPS metrics via Oppsy
+    if (!config.featureToggles.isOppsyDisabled) await enableLegacyOpsMetrics(server);
+  }
+
+  setupErrorHandling(server);
+
+  setupAuthentication(server);
+
+  await setupRoutesAndPlugins(server);
+
+  await setupOpenApiSpecification(server);
+
+  setupDeserialization(server);
+
+  return server;
+};
+
+const createBareServer = function () {
+  const serverConfiguration = {
+    compression: false,
+    debug: { request: false, log: false },
+    routes: {
+      validate: {
+        failAction: handleFailAction,
+      },
+      cors: {
+        origin: ['*'],
+        additionalHeaders: ['X-Requested-With'],
+      },
+      response: {
+        emptyStatusCode: 204,
+      },
+    },
+    port,
+    query: {
+      parser: (query) => parse(query),
+    },
+    router: {
+      isCaseSensitive: false,
+      stripTrailingSlash: true,
+    },
+  };
+
+  // Force https on non-dev environments
+  if (config.environment !== 'development') {
+    serverConfiguration.routes.security = {
+      hsts: {
+        includeSubDomains: true,
+        preload: true,
+        maxAge: 31536000,
+      },
+    };
+  }
+
+  return new Hapi.server(serverConfiguration);
+};
+
+const enableOpsMetrics = async function (server, metrics) {
+  metrics.addRecurrentMetrics(
+    [
+      { type: 'gauge', name: 'captain.api.memory.rss', value: 'rss' },
+      { type: 'gauge', name: 'captain.api.memory.heapTotal', value: 'heapTotal' },
+      { type: 'gauge', name: 'captain.api.memory.heapUsed', value: 'heapUsed' },
+      { type: 'gauge', name: 'captain.api.conteneur', constValue: 1 },
+    ],
+    5000,
+    process.memoryUsage,
+  );
+
+  server.pixCustomIntervals = metrics.intervals;
+
+  const gaugeConnections = (pool) => () => {
+    metrics.addMetricPoint({ type: 'gauge', name: 'captain.api.knex.db_connections_used', value: pool.numUsed() });
+    metrics.addMetricPoint({ type: 'gauge', name: 'captain.api.knex.db_connections_free', value: pool.numFree() });
+    metrics.addMetricPoint({
+      type: 'gauge',
+      name: 'captain.api.knex.db_connections_pending_creation',
+      value: pool.numPendingCreates(),
+    });
+    metrics.addMetricPoint({
+      type: 'gauge',
+      name: 'captain.api.knex.db_connections_pending_destroy',
+      value: pool.pendingDestroys.length,
+    });
+  };
+
+  const client = knex.client;
+  gaugeConnections(client.pool)();
+
+  client.pool.on('createSuccess', gaugeConnections(client.pool));
+  client.pool.on('acquireSuccess', gaugeConnections(client.pool));
+  client.pool.on('release', gaugeConnections(client.pool));
+  client.pool.on('destroySuccess', gaugeConnections(client.pool));
+
+  server.events.on('response', (request) => {
+    const info = request.info;
+
+    const statusCode = request.raw.res.statusCode;
+    const responseTime = (info.completed !== undefined ? info.completed : info.responded) - info.received;
+
+    metrics.addMetricPoint({
+      type: 'histogram',
+      name: 'captain.api.duration',
+      tags: [`method:${request.route.method}`, `route:${request.route.path}`, `statusCode:${statusCode}`],
+      value: responseTime,
+    });
+  });
+};
+
+const enableLegacyOpsMetrics = async function (server) {
+  const oppsy = new Oppsy(server);
+
+  oppsy.on('ops', (data) => {
+    const knexPool = knex.client.pool;
+    server.log(['ops'], {
+      ...data,
+      knexPool: {
+        used: knexPool.numUsed(),
+        free: knexPool.numFree(),
+        pendingAcquires: knexPool.numPendingAcquires(),
+        pendingCreates: knexPool.numPendingCreates(),
+      },
+    });
+  });
+
+  oppsy.start(logging.opsEventIntervalInSeconds * 1000);
+  server.oppsy = oppsy;
+};
+
+const setupDeserialization = function (server) {
+  server.ext('onPreHandler', async (request, h) => {
+    if (request.payload?.data) {
+      request.deserializedPayload = await deserializer.deserialize(request.payload);
+    }
+    return h.continue;
+  });
+};
+
+const setupAuthentication = function (server) {
+  server.auth.scheme(authentication.schemeName, authentication.scheme);
+  authentication.strategies.forEach((strategy) => {
+    server.auth.strategy(strategy.name, authentication.schemeName, strategy.configuration);
+  });
+  server.auth.default(authentication.defaultStrategy);
+};
+
+const setupRoutesAndPlugins = async function (server) {
+  const routes = [healthcheckRoutes].map((route) => ({
+    plugin: route,
+    options: { tags: ['maddo'] },
+  }));
+
+  await server.register([...plugins, ...routes]);
+};
+
+const setupOpenApiSpecification = async function (server) {
+  for (const swaggerRegisterArgs of swaggers) {
+    await server.register(...swaggerRegisterArgs);
+  }
+};
+
+export { createMaddoServer };