diff --git a/backend/src/common/models/backend-configuration.models.ts b/backend/src/common/models/backend-configuration.models.ts index 06bc9816070..2b9eea69c3a 100644 --- a/backend/src/common/models/backend-configuration.models.ts +++ b/backend/src/common/models/backend-configuration.models.ts @@ -35,6 +35,10 @@ export const backendConfigurationSchema = z.object({ description: 'Identifiant du dossier Google Drive pour le stockage des résultats de calcul de la trajectoire SNBC', }), + REFERENTIEL_TE_SHEET_ID: z.string().min(1).openapi({ + description: + "Identifiant de la feuille de calcul Google Sheets pour l'import du nouveau référentiel", + }), }); export type BackendConfigurationType = z.infer< typeof backendConfigurationSchema diff --git a/backend/src/common/services/zod.helper.ts b/backend/src/common/services/zod.helper.ts new file mode 100644 index 00000000000..399902ecd70 --- /dev/null +++ b/backend/src/common/services/zod.helper.ts @@ -0,0 +1,29 @@ +import * as Zod from 'zod'; + +export const getPropertyPaths = (schema: Zod.ZodType): string[] => { + // Adjusted: Signature now uses Zod.ZodType to eliminate null& undefined check + // check if schema is nullable or optional + if (schema instanceof Zod.ZodNullable || schema instanceof Zod.ZodOptional) { + return getPropertyPaths(schema.unwrap()); + } + // check if schema is an array + if (schema instanceof Zod.ZodArray) { + return getPropertyPaths(schema.element); + } + // check if schema is an object + if (schema instanceof Zod.ZodObject) { + // get key/value pairs from schema + const entries = Object.entries(schema.shape); // Adjusted: Uses Zod.ZodType as generic to remove instanceof check. Since .shape returns ZodRawShape which has Zod.ZodType as type for each key. + // loop through key/value pairs + return entries.flatMap(([key, value]) => { + // get nested keys + const nested = getPropertyPaths(value).map( + (subKey) => `${key}.${subKey}` + ); + // return nested keys + return nested.length ? nested : key; + }); + } + // return empty array + return []; +}; diff --git a/backend/src/referentiels/controllers/referentiels.controller.ts b/backend/src/referentiels/controllers/referentiels.controller.ts index 4fd2ee3dcb3..3bc4212256e 100644 --- a/backend/src/referentiels/controllers/referentiels.controller.ts +++ b/backend/src/referentiels/controllers/referentiels.controller.ts @@ -1,5 +1,5 @@ import { createZodDto } from '@anatine/zod-nestjs'; -import { Controller, Get, Logger, Param } from '@nestjs/common'; +import { Controller, Get, Logger, Param, Post } from '@nestjs/common'; import { ApiResponse, ApiTags } from '@nestjs/swagger'; import { AllowPublicAccess } from '../../auth/decorators/allow-public-access.decorator'; import { TokenInfo } from '../../auth/decorators/token-info.decorators'; @@ -28,4 +28,14 @@ export class ReferentielsController { ): Promise { return this.referentielsService.getReferentiel(referentielId, true); } + + @AllowPublicAccess() + @Post(':referentiel_id/import') + @ApiResponse({ type: GetReferentielResponseClass }) + async importReferentiel( + @Param('referentiel_id') referentielId: ReferentielType, + @TokenInfo() tokenInfo: SupabaseJwtPayload + ): Promise { + return this.referentielsService.importReferentiel(referentielId); + } } diff --git a/backend/src/referentiels/models/action-definition.table.ts b/backend/src/referentiels/models/action-definition.table.ts index 44d2b22ed5e..d7f52d253d8 100644 --- a/backend/src/referentiels/models/action-definition.table.ts +++ b/backend/src/referentiels/models/action-definition.table.ts @@ -10,6 +10,7 @@ import { import { createInsertSchema, createSelectSchema } from 'drizzle-zod'; import { z } from 'zod'; import { ActionType } from './action-type.enum'; +import { referentielDefinitionTable } from './referentiel-definition.table'; import { referentielList } from './referentiel.enum'; // Todo: change it reference another table instead @@ -30,6 +31,9 @@ export const actionDefinitionTable = pgTable('action_definition', { .notNull(), action_id: actionIdVarchar.primaryKey().notNull(), referentiel: referentielEnum('referentiel').notNull(), + referentiel_id: varchar('referentiel_id', { length: 30 }) + .notNull() + .references(() => referentielDefinitionTable.id), identifiant: text('identifiant').notNull(), nom: text('nom').notNull(), description: text('description').notNull(), @@ -63,6 +67,22 @@ export const actionDefinitionMinimalWithTypeLevel = export const createActionDefinitionSchema = createInsertSchema( actionDefinitionTable ); +export const importActionDefinitionSchema = + createActionDefinitionSchema.partial({ + action_id: true, + description: true, + nom: true, + contexte: true, + exemples: true, + ressources: true, + referentiel: true, + referentiel_id: true, + reduction_potentiel: true, + perimetre_evaluation: true, + }); +export type ImportActionDefinitionType = z.infer< + typeof importActionDefinitionSchema +>; export type ActionDefinitionAvecParentType = Pick< ActionDefinitionType, diff --git a/backend/src/referentiels/models/action-type.enum.ts b/backend/src/referentiels/models/action-type.enum.ts index 33655de8eb0..a3ce870addd 100644 --- a/backend/src/referentiels/models/action-type.enum.ts +++ b/backend/src/referentiels/models/action-type.enum.ts @@ -1,3 +1,5 @@ +import { pgEnum } from 'drizzle-orm/pg-core'; + export enum ActionType { REFERENTIEL = 'referentiel', AXE = 'axe', @@ -6,3 +8,12 @@ export enum ActionType { SOUS_ACTION = 'sous-action', TACHE = 'tache', } +export const orderedActionType = [ + ActionType.REFERENTIEL, + ActionType.AXE, + ActionType.SOUS_AXE, + ActionType.ACTION, + ActionType.SOUS_ACTION, + ActionType.TACHE, +] as const; +export const actionTypeEnum = pgEnum('action_type', orderedActionType); diff --git a/backend/src/referentiels/models/get-referentiel.response.ts b/backend/src/referentiels/models/get-referentiel.response.ts index 6c931b6bc81..68d14989c7d 100644 --- a/backend/src/referentiels/models/get-referentiel.response.ts +++ b/backend/src/referentiels/models/get-referentiel.response.ts @@ -6,6 +6,7 @@ import { referentielActionDtoSchema } from './referentiel-action.dto'; export const getReferentielResponseSchema = extendApi( z.object({ + version: z.string(), ordered_item_types: z.array(z.nativeEnum(ActionType)), items_list: z.array(actionDefinitionMinimalWithTypeLevel).optional(), items_tree: referentielActionDtoSchema.optional(), diff --git a/backend/src/referentiels/models/referentiel-changelog.dto.ts b/backend/src/referentiels/models/referentiel-changelog.dto.ts new file mode 100644 index 00000000000..ce129dece98 --- /dev/null +++ b/backend/src/referentiels/models/referentiel-changelog.dto.ts @@ -0,0 +1,11 @@ +import { z } from 'zod'; + +export const referentielChangelogSchema = z.object({ + version: z.string().describe('Version du referentiel, ex: 1.0.0'), + date: z.string().describe('Date de publication de la version'), + description: z.string().describe('Description des changements de la version'), +}); + +export type ReferentielChangelogType = z.infer< + typeof referentielChangelogSchema +>; diff --git a/backend/src/referentiels/models/referentiel-definition.table.ts b/backend/src/referentiels/models/referentiel-definition.table.ts new file mode 100644 index 00000000000..fe00937d7ee --- /dev/null +++ b/backend/src/referentiels/models/referentiel-definition.table.ts @@ -0,0 +1,31 @@ +import { InferSelectModel, sql } from 'drizzle-orm'; +import { pgTable, timestamp, varchar } from 'drizzle-orm/pg-core'; +import { createInsertSchema, createSelectSchema } from 'drizzle-zod'; +import { actionTypeEnum } from './action-type.enum'; + +export const referentielDefinitionTable = pgTable('referentiel_definition', { + id: varchar('id', { length: 30 }).primaryKey().notNull(), + nom: varchar('nom', { length: 300 }).notNull(), + version: varchar('version', { length: 16 }).notNull().default('1.0.0'), + hierarchie: actionTypeEnum('hierarchie').array().notNull(), + created_at: timestamp('created_at', { withTimezone: true, mode: 'string' }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), + modified_at: timestamp('modified_at', { withTimezone: true, mode: 'string' }) + .default(sql`CURRENT_TIMESTAMP`) + .notNull(), +}); + +export type ReferentielDefinitionType = InferSelectModel< + typeof referentielDefinitionTable +>; +export type CreateRefentielDefinitionType = InferSelectModel< + typeof referentielDefinitionTable +>; + +export const referentielDefinitionSchema = createSelectSchema( + referentielDefinitionTable +); +export const createReferentielDefinitionSchema = createInsertSchema( + referentielDefinitionTable +); diff --git a/backend/src/referentiels/models/referentiel.enum.ts b/backend/src/referentiels/models/referentiel.enum.ts index 9f9d60f9c45..701be195d78 100644 --- a/backend/src/referentiels/models/referentiel.enum.ts +++ b/backend/src/referentiels/models/referentiel.enum.ts @@ -1,9 +1,13 @@ export enum ReferentielType { ECI = 'eci', CAE = 'cae', + TE = 'te', + TE_TEST = 'te-test', } // WARNING: not using Object.values to use it with pgTable export const referentielList = [ ReferentielType.CAE, ReferentielType.ECI, + ReferentielType.TE, + ReferentielType.TE_TEST, ] as const; diff --git a/backend/src/referentiels/referentiels.module.ts b/backend/src/referentiels/referentiels.module.ts index a672a8dbb9f..cf4ed8ec594 100644 --- a/backend/src/referentiels/referentiels.module.ts +++ b/backend/src/referentiels/referentiels.module.ts @@ -3,6 +3,7 @@ import { AuthModule } from '../auth/auth.module'; import { CollectivitesModule } from '../collectivites/collectivites.module'; import { CommonModule } from '../common/common.module'; import { PersonnalisationsModule } from '../personnalisations/personnalisations.module'; +import { SheetModule } from '../spreadsheets/sheet.module'; import { ReferentielsScoringController } from './controllers/referentiels-scoring.controller'; import { ReferentielsController } from './controllers/referentiels.controller'; import ReferentielsScoringService from './services/referentiels-scoring.service'; @@ -13,6 +14,7 @@ import ReferentielsService from './services/referentiels.service'; AuthModule, CollectivitesModule, CommonModule, + SheetModule, PersonnalisationsModule, ], providers: [ReferentielsService, ReferentielsScoringService], diff --git a/backend/src/referentiels/services/referentiels-scoring.service.ts b/backend/src/referentiels/services/referentiels-scoring.service.ts index a5746a1fe19..b68da03b253 100644 --- a/backend/src/referentiels/services/referentiels-scoring.service.ts +++ b/backend/src/referentiels/services/referentiels-scoring.service.ts @@ -61,7 +61,7 @@ export default class ReferentielsScoringService { niveauAccesMinimum ); } - await this.referentielsService.checkReferentielExists(referentielId); + await this.referentielsService.getReferentielDefinition(referentielId); return this.collectivitesService.getCollectiviteAvecType(collectiviteId); } diff --git a/backend/src/referentiels/services/referentiels.service.ts b/backend/src/referentiels/services/referentiels.service.ts index 0ce5f8d2b3a..c1a7cf24e0a 100644 --- a/backend/src/referentiels/services/referentiels.service.ts +++ b/backend/src/referentiels/services/referentiels.service.ts @@ -4,25 +4,50 @@ import { Injectable, Logger, NotFoundException, + UnprocessableEntityException, } from '@nestjs/common'; -import { asc, eq, getTableColumns } from 'drizzle-orm'; +import { asc, eq, getTableColumns, sql } from 'drizzle-orm'; import * as _ from 'lodash'; +import * as semver from 'semver'; +import BackendConfigurationService from '../../common/services/backend-configuration.service'; import DatabaseService from '../../common/services/database.service'; +import SheetService from '../../spreadsheets/services/sheet.service'; import { ActionDefinitionAvecParentType, actionDefinitionTable, + CreateActionDefinitionType, + importActionDefinitionSchema, + ImportActionDefinitionType, } from '../models/action-definition.table'; -import { actionRelationTable } from '../models/action-relation.table'; +import { + actionRelationTable, + CreateActionRelationType, +} from '../models/action-relation.table'; import { ActionType } from '../models/action-type.enum'; import { GetReferentielResponseType } from '../models/get-referentiel.response'; import { ReferentielActionType } from '../models/referentiel-action.dto'; -import { referentielList, ReferentielType } from '../models/referentiel.enum'; +import { + referentielChangelogSchema, + ReferentielChangelogType, +} from '../models/referentiel-changelog.dto'; +import { + referentielDefinitionTable, + ReferentielDefinitionType, +} from '../models/referentiel-definition.table'; +import { ReferentielType } from '../models/referentiel.enum'; @Injectable() export default class ReferentielsService { private readonly logger = new Logger(ReferentielsService.name); - constructor(private readonly databaseService: DatabaseService) {} + private readonly CHANGELOG_SPREADSHEET_RANGE = 'Versions!A:Z'; + private readonly REFERENTIEL_SPREADSHEET_RANGE = 'Referentiel!A:Z'; + + constructor( + private readonly backendConfigurationService: BackendConfigurationService, + private readonly databaseService: DatabaseService, + private readonly sheetService: SheetService + ) {} buildReferentielTree( actionDefinitions: ActionDefinitionAvecParentType[], @@ -155,12 +180,6 @@ export default class ReferentielsService { } } - async checkReferentielExists(referentielId: ReferentielType): Promise { - if (!referentielList.includes(referentielId)) { - throw new NotFoundException(`Referentiel ${referentielId} not found`); - } - } - async getReferentiel( referentielId: ReferentielType, uniquementPourScoring?: boolean @@ -169,7 +188,9 @@ export default class ReferentielsService { `Recherche des actions pour le referentiel ${referentielId}` ); - await this.checkReferentielExists(referentielId); + const referentielDefinition = await this.getReferentielDefinition( + referentielId + ); const colonnes = uniquementPourScoring ? { @@ -197,34 +218,254 @@ export default class ReferentielsService { `${actionDefinitions.length} actions trouvees pour le referentiel ${referentielId}` ); - // TODO: get it from referentiel table - let orderedActionTypes: ActionType[] = [ - ActionType.REFERENTIEL, - ActionType.AXE, - ActionType.SOUS_AXE, - ActionType.ACTION, - ActionType.SOUS_ACTION, - ActionType.TACHE, - ]; - if (referentielId === 'eci') { - // Pas de sous axes pour l'ECI - orderedActionTypes = [ - ActionType.REFERENTIEL, - ActionType.AXE, - ActionType.ACTION, - ActionType.SOUS_ACTION, - ActionType.TACHE, - ]; - } - const actionsTree = this.buildReferentielTree( actionDefinitions, - orderedActionTypes + referentielDefinition.hierarchie ); return { items_tree: actionsTree, - ordered_item_types: orderedActionTypes, + version: referentielDefinition.version, + ordered_item_types: referentielDefinition.hierarchie, }; } + + async getReferentielDefinition( + referentielId: ReferentielType + ): Promise { + this.logger.log( + `Recherche de la definition du referentiel ${referentielId}` + ); + + const referentielDefinitions = await this.databaseService.db + .select() + .from(referentielDefinitionTable) + .where(eq(referentielDefinitionTable.id, referentielId)) + .limit(1); + + if (!referentielDefinitions.length) { + throw new NotFoundException( + `Referentiel definition ${referentielId} not found` + ); + } + + return referentielDefinitions[0]; + } + + getReferentielSpreadsheetId(referentielId: ReferentielType): string { + if ( + referentielId === ReferentielType.TE || + referentielId === ReferentielType.TE_TEST + ) { + return this.backendConfigurationService.getBackendConfiguration() + .REFERENTIEL_TE_SHEET_ID; + } + throw new HttpException( + `Referentiel ${referentielId} cannot be imported, missing configuration`, + HttpStatus.INTERNAL_SERVER_ERROR + ); + } + + getActionParent(actionId: string): string | null { + const actionIdParts = actionId.split('.'); + if (actionIdParts.length <= 1) { + const firstPart = actionIdParts[0]; + const firstPartParts = firstPart.split('_'); + if (firstPartParts.length > 1) { + return firstPartParts[0]; + } else { + return null; + } + } else { + return actionIdParts.slice(0, -1).join('.'); + } + } + + async importReferentiel( + referentielId: ReferentielType + ): Promise { + const spreadsheetId = this.getReferentielSpreadsheetId(referentielId); + + const referentielDefinition = await this.getReferentielDefinition( + referentielId + ); + + let changeLogVersions: ReferentielChangelogType[] = []; + try { + const changelogData = + await this.sheetService.getDataFromSheet( + spreadsheetId, + referentielChangelogSchema, + this.CHANGELOG_SPREADSHEET_RANGE, + ['version'] + ); + changeLogVersions = changelogData.data; + } catch (e) { + throw new UnprocessableEntityException( + 'Impossible de lire le tableau de version, veuillez vérifier que tous les champs sont correctement remplis.' + ); + } + + if (!changeLogVersions.length) { + throw new UnprocessableEntityException(`No version found in changelog`); + } + let lastVersion = changeLogVersions[0].version; + changeLogVersions.forEach((version) => { + if (semver.gt(version.version, lastVersion)) { + lastVersion = version.version; + } + }); + this.logger.log(`Last version found in changelog: ${lastVersion}`); + if (!semver.gt(lastVersion, referentielDefinition.version)) { + throw new UnprocessableEntityException( + `Version ${lastVersion} is not greater than current version ${referentielDefinition.version}, please add a new version in the changelog` + ); + } else { + // Update the version (will be saved in the transaction at the end) + referentielDefinition.version = lastVersion; + } + + const importActionDefinitions = + await this.sheetService.getDataFromSheet( + spreadsheetId, + importActionDefinitionSchema, + this.REFERENTIEL_SPREADSHEET_RANGE, + ['identifiant'] + ); + + const actionDefinitions: CreateActionDefinitionType[] = []; + importActionDefinitions.data.forEach((action) => { + const actionId = `${referentielId}_${action.identifiant}`; + const alreadyExists = actionDefinitions.find( + (action) => action.action_id === actionId + ); + if (!alreadyExists) { + const createActionDefinition: CreateActionDefinitionType = { + identifiant: action.identifiant, + action_id: `${referentielId}_${action.identifiant}`, + nom: action.nom || '', + description: action.description || '', + contexte: action.contexte || '', + exemples: action.exemples || '', + ressources: action.ressources || '', + reduction_potentiel: action.reduction_potentiel || '', + perimetre_evaluation: action.perimetre_evaluation || '', + preuve: action.preuve || '', + points: action.points || null, + pourcentage: action.pourcentage || null, + categorie: action.categorie || null, + referentiel: referentielId, + referentiel_id: referentielDefinition.id, + }; + actionDefinitions.push(createActionDefinition); + } else { + throw new UnprocessableEntityException( + `Action ${actionId} is duplicated in the spreadsheet` + ); + } + }); + // Add root action + actionDefinitions.push({ + action_id: referentielId, + identifiant: '', + nom: referentielDefinition.nom, + description: '', + contexte: '', + exemples: '', + ressources: '', + reduction_potentiel: '', + perimetre_evaluation: '', + preuve: '', + referentiel: referentielId, + referentiel_id: referentielDefinition.id, + }); + + // Sort to create parent relations in the right order + actionDefinitions.sort((a, b) => { + return a.action_id.localeCompare(b.action_id); + }); + const actionRelations: CreateActionRelationType[] = []; + actionDefinitions.forEach((action) => { + const parent = this.getActionParent(action.action_id); + if (parent) { + const foundParent = actionDefinitions.find( + (action) => action.action_id === parent + ); + if (foundParent) { + const actionRelation: CreateActionRelationType = { + id: action.action_id, + parent: parent, + referentiel: referentielId, + }; + actionRelations.push(actionRelation); + } else { + throw new UnprocessableEntityException( + `Action parent ${parent} not found for action ${action.action_id}` + ); + } + } else { + // Root action + const actionRelation: CreateActionRelationType = { + id: action.action_id, + parent: null, + referentiel: referentielId, + }; + actionRelations.push(actionRelation); + } + }); + + await this.databaseService.db.transaction(async (tx) => { + await tx + .insert(actionRelationTable) + .values(actionRelations) + .onConflictDoNothing(); + + await tx + .insert(actionDefinitionTable) + .values(actionDefinitions) + .onConflictDoUpdate({ + target: [actionDefinitionTable.action_id], + set: { + nom: sql.raw(`excluded.${actionDefinitionTable.nom.name}`), + description: sql.raw( + `excluded.${actionDefinitionTable.description.name}` + ), + contexte: sql.raw( + `excluded.${actionDefinitionTable.contexte.name}` + ), + exemples: sql.raw( + `excluded.${actionDefinitionTable.exemples.name}` + ), + ressources: sql.raw( + `excluded.${actionDefinitionTable.ressources.name}` + ), + reduction_potentiel: sql.raw( + `excluded.${actionDefinitionTable.reduction_potentiel.name}` + ), + perimetre_evaluation: sql.raw( + `excluded.${actionDefinitionTable.perimetre_evaluation.name}` + ), + preuve: sql.raw(`excluded.${actionDefinitionTable.preuve.name}`), + points: sql.raw(`excluded.${actionDefinitionTable.points.name}`), + pourcentage: sql.raw( + `excluded.${actionDefinitionTable.pourcentage.name}` + ), + }, + }); + + await tx + .insert(referentielDefinitionTable) + .values(referentielDefinition) + .onConflictDoUpdate({ + target: [referentielDefinitionTable.id], + set: { + version: sql.raw( + `excluded.${referentielDefinitionTable.version.name}` + ), + }, + }); + }); + + return this.getReferentiel(referentielId); + } } diff --git a/backend/src/spreadsheets/services/sheet.service.ts b/backend/src/spreadsheets/services/sheet.service.ts index 6543a8338fd..11dea15f6af 100644 --- a/backend/src/spreadsheets/services/sheet.service.ts +++ b/backend/src/spreadsheets/services/sheet.service.ts @@ -1,10 +1,13 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { default as retry, Options } from 'async-retry'; +import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'; +import { Options, default as retry } from 'async-retry'; import { Response } from 'express'; import * as gaxios from 'gaxios'; import * as auth from 'google-auth-library'; import { drive_v3, google, sheets_v4 } from 'googleapis'; +import { default as _ } from 'lodash'; +import { z } from 'zod'; import { initApplicationCredentials } from '../../common/services/gcloud.helper'; +import { getPropertyPaths } from '../../common/services/zod.helper'; import { SheetValueInputOption, SheetValueRenderOption, @@ -20,6 +23,9 @@ export default class SheetService { minTimeout: 60000, // Wait for 1min due to sheet api quota limitation }; + readonly A_CODE_CHAR = 65; + readonly DEFAULT_RANGE = 'A:Z'; + private authClient: | auth.Compute | auth.JWT @@ -174,47 +180,52 @@ export default class SheetService { }> { const authClient = await this.getAuthClient(); - const sheetValues = await retry( + const sheetValues = await this.executeWithQuotaRetry( async ( bail: (e: Error) => void, _num: number ): Promise => { - try { - const getOptions: sheets_v4.Params$Resource$Spreadsheets$Values$Get = - { - auth: authClient, - spreadsheetId: spreadsheetId, - range: range, - valueRenderOption: valueRenderOption, - }; - this.logger.log( - `Get raw data from sheet ${spreadsheetId} with range ${range} (attempt ${_num})` - ); - const sheetResponse = await sheets.spreadsheets.values.get( - getOptions - ); - return sheetResponse.data; - } catch (error) { - this.logger.error(error); - if (error instanceof gaxios.GaxiosError) { - const gaxiosError = error as gaxios.GaxiosError; - this.logger.error( - `Error while retrieving sheet data: status ${gaxiosError.status}, code ${gaxiosError.code}, message: ${gaxiosError.message}` - ); - if (error.status === 429) { - this.logger.log(`Error due to api quota limitation, retrying`); - throw error; - } - } - bail(error as Error); - } - }, - {} + const getOptions: sheets_v4.Params$Resource$Spreadsheets$Values$Get = { + auth: authClient, + spreadsheetId: spreadsheetId, + range: range, + valueRenderOption: valueRenderOption, + }; + this.logger.log( + `Get raw data from sheet ${spreadsheetId} with range ${range} (attempt ${_num})` + ); + const sheetResponse = await sheets.spreadsheets.values.get(getOptions); + return sheetResponse.data; + } ); return { data: sheetValues?.values || null }; } + async executeWithQuotaRetry( + fn: (bail: (err: Error) => void, num: number) => Promise + ): Promise { + return retry(async (bail, num): Promise => { + try { + return await fn(bail, num); + } catch (error) { + this.logger.error(error); + if (error instanceof gaxios.GaxiosError) { + const gaxiosError = error as gaxios.GaxiosError; + this.logger.error( + `Error while executing function: status ${gaxiosError.status}, code ${gaxiosError.code}, message: ${gaxiosError.message}` + ); + if (error.status === 429) { + this.logger.log(`Quota limitation, retrying`); + throw error; + } + } + // No need to retry + bail(error as Error); + } + }, this.RETRY_STRATEGY) as Promise; + } + async overwriteRawDataToSheet( spreadsheetId: string, range: string, @@ -222,38 +233,164 @@ export default class SheetService { valueInputOption?: SheetValueInputOption ) { const authClient = await this.getAuthClient(); - await retry( + await this.executeWithQuotaRetry( async (bail: (e: Error) => void, num: number): Promise => { - try { - this.logger.log( - `Overwrite data to sheet ${spreadsheetId} in range ${range} (attempt ${num})` - ); - await sheets.spreadsheets.values.update({ - auth: authClient, - spreadsheetId, - range: range, - valueInputOption: valueInputOption || SheetValueInputOption.RAW, - requestBody: { - values: data, - }, - }); - } catch (error) { - this.logger.error(error); - if (error instanceof gaxios.GaxiosError) { - const gaxiosError = error as gaxios.GaxiosError; - this.logger.error( - `Error while overwriting sheet data: status ${gaxiosError.status}, code ${gaxiosError.code}, message: ${gaxiosError.message}` + this.logger.log( + `Overwrite data to sheet ${spreadsheetId} in range ${range} (attempt ${num})` + ); + await sheets.spreadsheets.values.update({ + auth: authClient, + spreadsheetId, + range: range, + valueInputOption: valueInputOption || SheetValueInputOption.RAW, + requestBody: { + values: data, + }, + }); + } + ); + } + + getDefaultRangeFromHeader(header: string[], sheetName?: string): string { + const numLetters = header.length; + let rangeEnd = ''; + const quotient = Math.floor(numLetters / 26); + const remainder = numLetters % 26; + if (quotient > 0) { + rangeEnd += String.fromCharCode(this.A_CODE_CHAR + quotient - 1); + } + rangeEnd += String.fromCharCode(this.A_CODE_CHAR + remainder - 1); + const rangeWithoutSheet = `A:${rangeEnd}`; + if (sheetName) { + return `${sheetName}!${rangeWithoutSheet}`; + } + return rangeWithoutSheet; + } + + async getDataFromSheet( + spreadsheetId: string, + schema: z.ZodObject, + range?: string, + idProperties?: string[] + ): Promise<{ data: T[]; header: string[] | null }> { + const expectedHeader = getPropertyPaths(schema); + this.logger.log( + `Found schema for sheet with expected header ${expectedHeader}` + ); + if (!range) { + range = this.getDefaultRangeFromHeader(expectedHeader); + this.logger.log(`Use default range ${range} from schema`); + } else { + this.logger.log(`Use range ${range} for schema`); + } + + const data: any[] = []; + let header: string[] | null = null; + const readDataResult = await this.getRawDataFromSheet( + spreadsheetId, + range || this.DEFAULT_RANGE + ); + if (readDataResult.data) { + header = readDataResult.data[0]; + for (let iRow = 1; iRow < readDataResult.data.length; iRow++) { + const dataRecord: any = {}; + const row = readDataResult.data[iRow]; + for (let iField = 0; iField < row.length; iField++) { + if (header[iField]) { + const fieldName = header[iField].trim(); + const fieldNameSchema = expectedHeader.find( + (header) => header.toLowerCase() === fieldName.toLowerCase() + ); + let fieldDef = + fieldNameSchema && schema.shape + ? schema.shape[fieldNameSchema] + : null; + if (fieldDef instanceof z.ZodOptional) { + fieldDef = fieldDef.unwrap(); + } + if (fieldDef instanceof z.ZodNullable) { + fieldDef = fieldDef.unwrap(); + } + // Skip empty fields, warning: 0 is not empty + if ( + fieldNameSchema && + row[iField] !== undefined && + row[iField] !== null && + row[iField] !== '' + ) { + let value: string | number | boolean = `${row[iField]}`.trim(); + // Always save original field name, to be able to keep it for debugging + _.set(dataRecord, fieldNameSchema, value); + + //logger.info(`Found field ${fieldName} with value ${row[iField]}`); + + // try to parse as float + if (fieldDef instanceof z.ZodNumber) { + // Remove spaces + const valueWithoutSpace = value.replace(/\s/g, ''); + //console.log(valueWithoutSpace); + let floatValue = parseFloat(valueWithoutSpace); + if (!isNaN(floatValue)) { + //console.log(`Parsed as float ${floatValue}`); + // Try to replace , by . + floatValue = parseFloat(valueWithoutSpace.replace(/,/g, '.')); + //console.log(`Parsed as float ${floatValue}`); + const floatValueStr = floatValue.toString(); + if ( + fieldNameSchema || + floatValueStr.length === value.length + ) { + value = floatValue; + } + } + } else if (fieldDef instanceof z.ZodBoolean) { + // Remove spaces + const valueWithoutSpace = value + .replace(/\s/g, '') + .toLowerCase(); + value = valueWithoutSpace === 'true' ? true : false; + } + _.set(dataRecord, fieldNameSchema, value); + } + } + } + + if (Object.keys(dataRecord).length > 0) { + // lines without id are ignored + let missingIdProperties = false; + if (idProperties) { + missingIdProperties = idProperties.some((idProperty) => + _.isNil(dataRecord[idProperty]) ); - if (error.status === 429) { - this.logger.log(`Quota error, retrying`); - throw error; + } + if (!missingIdProperties) { + try { + const parsedDataRecord = schema.parse(dataRecord); + data.push(parsedDataRecord); + } catch (e) { + let errorMessage = `Invalid sheet record on line ${iRow} ${JSON.stringify( + dataRecord + )}`; + this.logger.error(errorMessage); + this.logger.error(e); + if (e instanceof z.ZodError) { + const errorList = e.errors.map((error) => { + return `${error.path.join('.')} - ${error.message} (${ + error.code + })`; + }); + errorMessage += `: ${errorList.join(', ')}`; + } + throw new HttpException(errorMessage, HttpStatus.BAD_REQUEST); } + } else { + this.logger.debug(`Missing id properties on line ${iRow}`); } - // No need to trigger retry if it's not a quota error - bail(error as Error); + } else { + this.logger.warn(`Empty record on line ${iRow}`); } - }, - this.RETRY_STRATEGY - ); + } + } + return { data, header }; } } diff --git a/data_layer/sqitch/deploy/referentiel/referentiel.sql b/data_layer/sqitch/deploy/referentiel/referentiel.sql new file mode 100644 index 00000000000..9ea2d273740 --- /dev/null +++ b/data_layer/sqitch/deploy/referentiel/referentiel.sql @@ -0,0 +1,47 @@ +-- Deploy tet:referentiel/referentiel to pg + +BEGIN; + +ALTER TYPE referentiel ADD VALUE IF NOT EXISTS 'te'; +ALTER TYPE referentiel ADD VALUE IF NOT EXISTS 'te-test'; + +-- Create the referentiels table. use referentiels instead of referentiel to avoid conflict with enum +create table referentiel_definition +( + id varchar(30) primary key, + nom varchar(300) not null, + version varchar(16) not null default '1.0.0', -- Semver + hierarchie action_type ARRAY NOT NULL, + created_at timestamp with time zone default CURRENT_TIMESTAMP not null, + modified_at timestamp with time zone default CURRENT_TIMESTAMP not null +); +comment on table referentiel_definition is + 'Les référentiels de la plateforme'; + +insert into referentiel_definition (id, nom, version, hierarchie) values +('cae', 'Climat Air Energie', '1.0.0', '{"referentiel", "axe", "sous-axe", "action", "sous-action", "tache"}'), +('eci', 'Économie Circulaire', '1.0.0', '{"referentiel", "axe", "action", "sous-action", "tache"}'), +('te', 'Transition Écologique', '0.1.0', '{"referentiel", "axe", "sous-axe", "action", "sous-action", "tache"}'), +('te-test', 'Transition Écologique', '0.1.0', '{"referentiel", "axe", "sous-axe", "action", "sous-action", "tache"}'); + +create trigger set_modified_at + before update + on referentiel_definition + for each row +execute procedure update_modified_at(); + +alter table action_definition add column referentiel_id varchar(30); + +UPDATE action_definition +SET referentiel_id = referentiel::text; + +ALTER TABLE action_definition + ALTER COLUMN referentiel_id SET NOT NULL; + +alter table "public"."action_definition" +add constraint "referentiel_id_fkey" +foreign key ("referentiel_id") +references "public"."referentiel_definition" ("id") +on delete restrict; + +COMMIT; diff --git a/data_layer/sqitch/revert/referentiel/referentiel.sql b/data_layer/sqitch/revert/referentiel/referentiel.sql new file mode 100644 index 00000000000..d17e637e3a7 --- /dev/null +++ b/data_layer/sqitch/revert/referentiel/referentiel.sql @@ -0,0 +1,9 @@ +-- Revert tet:referentiel/referentiel from pg + +BEGIN; + +alter table action_definition drop column if exists referentiel_id; + +drop table referentiel_definition; + +COMMIT; diff --git a/data_layer/sqitch/sqitch.plan b/data_layer/sqitch/sqitch.plan index e9f7207439b..8d8231d10f4 100644 --- a/data_layer/sqitch/sqitch.plan +++ b/data_layer/sqitch/sqitch.plan @@ -716,3 +716,5 @@ indicateur/fix_json_upsert 2024-09-11T12:49:34Z System Administrator # ajoute les conditions pour exclure les données SNBC @v4.12.3 2024-10-01T16:24:17Z System Administrator # ajoute les conditions pour exclure les données SNBC + +referentiel/referentiel 2024-10-07T16:09:16Z System Administrator # Ajout de la table référentiel diff --git a/data_layer/sqitch/verify/referentiel/referentiel.sql b/data_layer/sqitch/verify/referentiel/referentiel.sql new file mode 100644 index 00000000000..1507a41f6a4 --- /dev/null +++ b/data_layer/sqitch/verify/referentiel/referentiel.sql @@ -0,0 +1,11 @@ +-- Verify tet:referentiel/referentiel on pg + +BEGIN; + +SELECT 1/COUNT(*) FROM referentiel_definition; + +SELECT 1/COUNT(*) FROM action_definition WHERE referentiel_id = 'cae'; + +SELECT 1/COUNT(*) FROM action_definition WHERE referentiel_id = 'eci'; + +ROLLBACK; diff --git a/package.json b/package.json index 5d0c0bda818..aedfc919bfd 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "remark-gfm": "^4.0.0", "remixicon": "^4.3.0", "rxjs": "^7.8.0", + "semver": "^7.6.3", "sharp": "^0.33.4", "slugify": "^1.6.6", "swr": "^2.2.0", @@ -163,6 +164,7 @@ "@types/react-router-dom": "^5.1.8", "@types/react-table": "^7.7.15", "@types/retry": "^0.12.5", + "@types/semver": "^7.5.8", "@types/supertest": "^6.0.2", "@types/uuid": "^8.3.1", "@types/zxcvbn": "^4.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f330f9db242..4209db08524 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -281,6 +281,9 @@ importers: rxjs: specifier: ^7.8.0 version: 7.8.1 + semver: + specifier: ^7.6.3 + version: 7.6.3 sharp: specifier: ^0.33.4 version: 0.33.5 @@ -477,6 +480,9 @@ importers: '@types/retry': specifier: ^0.12.5 version: 0.12.5 + '@types/semver': + specifier: ^7.5.8 + version: 7.5.8 '@types/supertest': specifier: ^6.0.2 version: 6.0.2