Merge pull request #3309 from incubateur-ademe/poc_snbc_rebase

Poc snbc rebase

.github/workflows/cd-backend.yml

@@ -43,3 +43,32 @@ jobs:
           --platform=linux/amd64 
           +backend-build
           --ENV_NAME=${{ github.event.inputs.target }}
+
+  deploy-backend:
+    name: Déploie le backend
+    needs: build-backend
+    environment: ${{ github.event.inputs.target }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - uses: earthly/actions/setup-earthly@v1
+        with:
+          version: v0.7.20
+      - name: Login into registry
+        run: earthly --use-inline-cache +docker-dev-login --GH_USER=${{ secrets.GH_USER }} --GH_TOKEN=${{ secrets.GH_TOKEN }}
+
+      - name: Deploy app on Koyeb
+        run: >
+          earthly 
+          --use-inline-cache 
+          +backend-deploy
+          --ENV_NAME=${{ github.event.inputs.target }}
+          --KOYEB_API_KEY=${{ secrets.KOYEB_API_KEY }}
+          --TRAJECTOIRE_SNBC_SHEET_ID=${{ vars.TRAJECTOIRE_SNBC_SHEET_ID }}
+          --TRAJECTOIRE_SNBC_XLSX_ID=${{ vars.TRAJECTOIRE_SNBC_XLSX_ID }}
+          --TRAJECTOIRE_SNBC_RESULT_FOLDER_ID=${{ vars.TRAJECTOIRE_SNBC_RESULT_FOLDER_ID }}

.gitignore

@@ -12,3 +12,4 @@ docker-data
 .arg
 
 node_modules
+.DS_Store

Earthfile

@@ -2,6 +2,7 @@ VERSION 0.7
 LOCALLY
 # chemins vers les modules front
 ARG --global APP_DIR='./app.territoiresentransitions.react'
+ARG --global BACKEND_DIR='./backend'
 ARG --global SITE_DIR='./packages/site'
 ARG --global AUTH_DIR='./packages/auth'
 ARG --global PANIER_DIR='./packages/panier'
@@ -18,6 +19,8 @@ ARG --global FRONT_DEPS_TAG=$(openssl dgst -sha256 -r ./package-lock.json | head
 ARG --global FRONT_DEPS_IMG_NAME=$REG_TARGET/front-deps:$FRONT_DEPS_TAG
 ARG --global APP_TAG=$ENV_NAME-$FRONT_DEPS_TAG-$(sh ./subdirs_hash.sh $APP_DIR,$UI_DIR,$API_DIR)
 ARG --global APP_IMG_NAME=$REG_TARGET/app:$APP_TAG
+# TODO changer le tag
+ARG --global BACKEND_IMG_NAME=$REG_TARGET/backend:$APP_TAG 
 ARG --global SITE_IMG_NAME=$REG_TARGET/site:$ENV_NAME-$FRONT_DEPS_TAG-$(sh ./subdirs_hash.sh $SITE_DIR,$UI_DIR,$API_DIR)
 ARG --global AUTH_IMG_NAME=$REG_TARGET/auth:$ENV_NAME-$FRONT_DEPS_TAG-$(sh ./subdirs_hash.sh $AUTH_DIR,$UI_DIR,$API_DIR)
 ARG --global PANIER_IMG_NAME=$REG_TARGET/panier:$ENV_NAME-$FRONT_DEPS_TAG-$(sh ./subdirs_hash.sh $PANIER_DIR,$UI_DIR,$API_DIR)
@@ -28,6 +31,9 @@ ARG --global DL_TAG=$ENV_NAME-$(sh ./subdirs_hash.sh data_layer)
 ARG --global DB_SAVE_IMG_NAME=$REG_TARGET/db-save:$DL_TAG
 ARG --global DB_VOLUME_NAME=supabase_db_tet
 ARG --global GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+ARG --global GIT_COMMIT_SHORT_SHA=$(git rev-parse --short HEAD)
+ARG --global GIT_COMMIT_TIMESTAMP=$(git show -s --format=%cI HEAD)
+ARG --global APPLICATION_VERSION=$(git describe --tags --always)
 
 postgres:
     FROM postgres:15
@@ -269,11 +275,14 @@ node-fr: ## construit l'image de base pour les images utilisant node
 
 front-deps: ## construit l'image contenant les dépendances des modules front
     FROM +node-fr
+    # tsconfig global
+    COPY ./tsconfig.json ./
     # dépendances globales
     COPY ./package.json ./
     COPY ./package-lock.json ./
     # dépendances des modules
     COPY $APP_DIR/package.json ./$APP_DIR/
+    COPY $BACKEND_DIR/package.json ./$BACKEND_DIR/
     COPY $AUTH_DIR/package.json ./$AUTH_DIR/
     COPY $SITE_DIR/package.json ./$SITE_DIR/
     COPY $PANIER_DIR/package.json ./$PANIER_DIR/
@@ -287,6 +296,56 @@ front-deps-builder:
     LOCALLY
     DO +BUILD_IF_NO_IMG --IMG_NAME=front-deps --IMG_TAG=$FRONT_DEPS_TAG --BUILD_TARGET=front-deps
 
+backend-pre-build:
+    # Build stage: todo: check if we use the same image as frontends
+    FROM +front-deps
+
+    COPY --chown=node:node $BACKEND_DIR/. $BACKEND_DIR/
+    COPY --chown=node:node $API_DIR/. $API_DIR
+    RUN npm run build:backend && npm prune --omit=dev
+
+    SAVE ARTIFACT . /app
+
+backend-build:
+    # TODO: why not us an alpine node:lts-alpine ?
+    FROM +node-fr
+
+    ENV NODE_ENV production
+    ENV PORT 3000
+    ENV GIT_COMMIT_SHORT_SHA=$GIT_COMMIT_SHORT_SHA
+    ENV GIT_COMMIT_TIMESTAMP=$GIT_COMMIT_TIMESTAMP
+    ENV APPLICATION_VERSION=$APPLICATION_VERSION
+
+    COPY --chown=node:node +backend-pre-build/app/package*.json .
+    COPY --chown=node:node +backend-pre-build/app/node_modules ./node_modules
+    COPY --chown=node:node +backend-pre-build/app/backend/package.json ./backend/package.json
+    COPY --chown=node:node +backend-pre-build/app/backend/node_modules ./backend/node_modules
+    COPY --chown=node:node +backend-pre-build/app/backend/dist ./backend/dist
+
+    EXPOSE ${PORT}
+
+    CMD ["node", "backend/dist/backend/src/main.js"]
+    SAVE IMAGE --cache-from=$BACKEND_IMG_NAME --push $BACKEND_IMG_NAME
+
+backend-deploy: ## Déploie le backend dans une app Koyeb existante
+    ARG --required KOYEB_API_KEY
+    ARG --required TRAJECTOIRE_SNBC_SHEET_ID
+    ARG --required TRAJECTOIRE_SNBC_XLSX_ID
+    ARG --required TRAJECTOIRE_SNBC_RESULT_FOLDER_ID
+    FROM +koyeb
+    RUN ./koyeb services update $ENV_NAME-backend/backend \
+        --docker $BACKEND_IMG_NAME \
+        --env ENV_NAME=$ENV_NAME \
+        --env DEPLOYMENT_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
+        --env GCLOUD_SERVICE_ACCOUNT_KEY=@GCLOUD_SERVICE_ACCOUNT_KEY \
+        --env SUPABASE_DATABASE_URL=@SUPABASE_DATABASE_URL_$ENV_NAME \
+        --env SUPABASE_URL=@SUPABASE_URL_$ENV_NAME \
+        --env SUPABASE_SERVICE_ROLE_KEY=@SUPABASE_SERVICE_ROLE_KEY_$ENV_NAME \
+        --env SUPABASE_JWT_SECRET=@SUPABASE_JWT_SECRET_$ENV_NAME \
+        --env TRAJECTOIRE_SNBC_SHEET_ID=$TRAJECTOIRE_SNBC_SHEET_ID \
+        --env TRAJECTOIRE_SNBC_XLSX_ID=$TRAJECTOIRE_SNBC_XLSX_ID \
+        --env TRAJECTOIRE_SNBC_RESULT_FOLDER_ID=$TRAJECTOIRE_SNBC_RESULT_FOLDER_ID
+
 app-build: ## construit l'image de l'app
     ARG PLATFORM
     ARG --required ANON_KEY
@@ -545,6 +604,8 @@ storybook-test-run: # lance les tests du module `ui`
 
 curl-test-build:
     FROM curlimages/curl:8.1.0
+    USER root
+    RUN apk --update add jq
     COPY ./data_layer/scripts/curl_test.sh /curl_test.sh
     ENTRYPOINT sh ./curl_test.sh
     SAVE IMAGE curl-test:latest

backend/.env.default

@@ -0,0 +1,8 @@
+TRAJECTOIRE_SNBC_SHEET_ID=
+TRAJECTOIRE_SNBC_XLSX_ID=
+TRAJECTOIRE_SNBC_RESULT_FOLDER_ID=
+GCLOUD_SERVICE_ACCOUNT_KEY=
+SUPABASE_DATABASE_URL=postgresql://postgres:postgres@localhost:54322/postgres
+SUPABASE_JWT_SECRET=super-secret-jwt-token-with-at-least-32-characters-long
+SUPABASE_URL=http://127.0.0.1:54321
+SUPABASE_SERVICE_ROLE_KEY=

backend/.eslintrc.js

@@ -0,0 +1,25 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    tsconfigRootDir: __dirname,
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  ignorePatterns: ['.eslintrc.js'],
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+  },
+};

backend/.gitignore

@@ -0,0 +1,56 @@
+# compiled output
+/dist
+/node_modules
+/build
+
+# Logs
+logs
+*.log
+npm-debug.log*
+pnpm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
+.DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# temp directory
+.temp
+.tmp
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

backend/.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}

backend/README.md

@@ -0,0 +1,40 @@
+# Backend
+
+## Description
+
+Backend de territoires en transitions basé sur [Nest](https://github.com/nestjs/nest).
+
+## Installation
+
+```bash
+$ npm install
+```
+
+## Configuration
+
+Les variables d'environnement suivantes doivent être définies dans un fichier .env (voir le fichier [.env.default](.env.default)):
+
+- **TRAJECTOIRE_SNBC_SHEET_ID**: Identifiant du Google spreadsheet utilisé pour le calcul de la trajectoire.
+- **TRAJECTOIRE_SNBC_XLSX_ID**: Identifiant du Xlsx original utilisé pour le calcul de la trajectoire (stocké sur le drive)
+- **TRAJECTOIRE_SNBC_RESULT_FOLDER_ID**: Identifiant du dossier Google Drive dans lequel les spreadsheets des trajectoires calculées doivent être sauvés (un fichier par EPCI). A noter qu'il existe **un dossier par environnement** (dev, preprod, prod).
+- **GCLOUD_SERVICE_ACCOUNT_KEY**: contenu au format json du fichier de clé de compte de service permettant l'utilisation des api Google Drive et Google Spreadsheet.
+- **SUPABASE_JWT_SECRET**: clé de signature des tokens jwt. Utilisé pour vérifier la signature des tokens.
+
+Ces variables d'environnement sont définies:
+
+- pour **TRAJECTOIRE_SNBC_SHEET_ID**, **TRAJECTOIRE_SNBC_XLSX_ID** et **TRAJECTOIRE_SNBC_RESULT_FOLDER_ID** dans les [variables d'environnement de Github](https://github.com/incubateur-ademe/territoires-en-transitions/settings/environments/1431973268/edit) utilisées pour configurer le [déploiement Koyeb](https://app.koyeb.com/services/c7001069-ca11-4fd7-86c6-7feb45b9b68d/settings). Les identifiants peuvent également être récupérés à partir du drive de `territoiresentransitions`.
+- pour  **GCLOUD_SERVICE_ACCOUNT_KEY**, **SUPABASE_JWT_SECRET** et **SUPABASE_SERVICE_ROLE_KEY** dans le [gestionnaire de secret de Koyeb](https://app.koyeb.com/secrets)
+
+## Scripts disponibles
+
+```bash
+
+# Lance en développement avec rechargement automatique lorsqu'un fichier est modifié
+$ npm run start:dev
+
+# Lance avec le mode production
+$ npm run start:prod
+
+# Lance les tests
+$ npm run test
+```

backend/nest-cli.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://json.schemastore.org/nest-cli",
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src",
+  "entryFile": "backend/src/main",
+  "compilerOptions": {
+    "deleteOutDir": true
+  }
+}

backend/package.json

@@ -0,0 +1,98 @@
+{
+  "name": "@tet/backend",
+  "version": "0.0.1",
+  "description": "",
+  "author": "",
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "nest build",
+    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
+    "start": "node dist/backend/src/main",
+    "start:dev": "nest start --watch",
+    "start:debug": "nest start --debug --watch",
+    "start:prod": "node dist/backend/src/main",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:cov": "jest --coverage",
+    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
+    "test:e2e": "jest --config ./test/jest-e2e.json"
+  },
+  "dependencies": {
+    "@anatine/zod-nestjs": "^2.0.9",
+    "@anatine/zod-openapi": "^2.2.6",
+    "@kant2002/jszip": "^2.7.1",
+    "@nestjs/common": "^10.0.0",
+    "@nestjs/config": "^3.2.3",
+    "@nestjs/core": "^10.0.0",
+    "@nestjs/jwt": "^10.2.0",
+    "@nestjs/platform-express": "^10.0.0",
+    "@nestjs/swagger": "^7.4.0",
+    "@sentry/nestjs": "^8.19.0",
+    "@sentry/profiling-node": "^8.19.0",
+    "@supabase/supabase-js": "^2.40.0",
+    "@trpc/server": "^10.45.2",
+    "async-retry": "^1.3.3",
+    "class-transformer": "^0.5.1",
+    "class-validator": "^0.14.1",
+    "drizzle-orm": "^0.32.0",
+    "drizzle-zod": "^0.5.1",
+    "es-toolkit": "^1.16.0",
+    "gaxios": "^6.7.0",
+    "google-auth-library": "^9.11.0",
+    "googleapis": "^140.0.1",
+    "lodash": "^4.17.21",
+    "luxon": "^3.5.0",
+    "openapi3-ts": "^4.4.0",
+    "postgres": "^3.4.4",
+    "reflect-metadata": "^0.2.0",
+    "rxjs": "^7.8.1",
+    "slugify": "^1.6.6",
+    "xlsx-template": "^1.4.4",
+    "zod": "^3.20.0"
+  },
+  "devDependencies": {
+    "@nestjs/cli": "^10.0.0",
+    "@nestjs/schematics": "^10.0.0",
+    "@nestjs/testing": "^10.0.0",
+    "@types/async-retry": "^1.4.3",
+    "@types/express": "^4.17.17",
+    "@types/jest": "^29.5.2",
+    "@types/luxon": "^3.4.2",
+    "@types/node": "^20.3.1",
+    "@types/supertest": "^6.0.0",
+    "@typescript-eslint/eslint-plugin": "^7.0.0",
+    "@typescript-eslint/parser": "^7.0.0",
+    "drizzle-kit": "^0.23.0",
+    "eslint": "^8.42.0",
+    "eslint-config-prettier": "^9.0.0",
+    "eslint-plugin-prettier": "^5.0.0",
+    "jest": "^29.5.0",
+    "prettier": "^3.0.0",
+    "source-map-support": "^0.5.21",
+    "supertest": "^7.0.0",
+    "ts-jest": "^29.1.0",
+    "ts-loader": "^9.4.3",
+    "ts-node": "^10.9.1",
+    "tsconfig-paths": "^4.2.0",
+    "typescript": "^5.1.3"
+  },
+  "jest": {
+    "moduleFileExtensions": [
+      "js",
+      "json",
+      "ts"
+    ],
+    "rootDir": "src",
+    "testRegex": ".*\\.spec\\.ts$",
+    "transform": {
+      "^.+\\.(t|j)s$": "ts-jest"
+    },
+    "collectCoverageFrom": [
+      "**/*.(t|j)s"
+    ],
+    "coverageDirectory": "../coverage",
+    "testEnvironment": "node"
+  }
+}