feat(spreadsheets): ajout de la capacité à lire/écrire dans un spread…

…sheet

Earthfile

@@ -322,7 +322,7 @@ backend-build:
 backend-deploy: ## Déploie le backend dans une app Koyeb existante
     ARG --required KOYEB_API_KEY
     FROM +koyeb
-    RUN ./koyeb services update $ENV_NAME-backend/backend --docker $BACKEND_IMG_NAME --env GIT_COMMIT_SHORT_SHA=$GIT_COMMIT_SHORT_SHA
+    RUN ./koyeb services update $ENV_NAME-backend/backend --docker $BACKEND_IMG_NAME --env GIT_COMMIT_SHORT_SHA=$GIT_COMMIT_SHORT_SHA --env GCLOUD_SERVICE_ACCOUNT_KEY=@GCLOUD_SERVICE_ACCOUNT_KEY
 
 app-build: ## construit l'image de l'app
     ARG PLATFORM

backend/package.json

@@ -25,13 +25,18 @@
     "@nestjs/platform-express": "^10.0.0",
     "@sentry/nestjs": "^8.19.0",
     "@sentry/profiling-node": "^8.19.0",
+    "async-retry": "^1.3.3",
+    "gaxios": "^6.7.0",
+    "google-auth-library": "^9.11.0",
+    "googleapis": "^140.0.1",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1"
   },
   "devDependencies": {
     "@nestjs/cli": "^10.0.0",
     "@nestjs/schematics": "^10.0.0",
     "@nestjs/testing": "^10.0.0",
+    "@types/async-retry": "^1.4.3",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.2",
     "@types/node": "^20.3.1",

backend/src/app.controller.ts

@@ -1,9 +1,13 @@
 import { Controller, Get, NotFoundException } from '@nestjs/common';
 import { AppService } from './app.service';
+import SheetService from './spreadsheets/services/sheet.service';
 
 @Controller()
 export class AppController {
-  constructor(private readonly appService: AppService) {}
+  constructor(
+    private readonly appService: AppService,
+    private readonly sheetService: SheetService,
+  ) {}
 
   @Get()
   getHello(): string {
@@ -14,4 +18,9 @@ export class AppController {
   getException(): string {
     throw new NotFoundException('User not found');
   }
+
+  @Get('test')
+  getTestResult() {
+    return this.sheetService.testDownload();
+  }
 }

backend/src/app.module.ts

@@ -1,9 +1,10 @@
 import { Module } from '@nestjs/common';
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
+import { SheetModule } from './spreadsheets/SheetModule';
 
 @Module({
-  imports: [],
+  imports: [SheetModule],
   controllers: [AppController],
   providers: [AppService],
 })

backend/src/main.ts

@@ -8,11 +8,25 @@ import {
 } from '@nestjs/core';
 import { AppModule } from './app.module';
 import { SENTRY_DSN } from './common/configs/SentryInit';
+import * as fs from 'fs';
 
 const port = process.env.PORT || 8080;
 console.log(`Launching NestJS app on port ${port}`);
 
 async function bootstrap() {
+  if (process.env.GCLOUD_SERVICE_ACCOUNT_KEY) {
+    const serviceAccountFile = `${__dirname}/keyfile.json`;
+    console.log(
+      'Writing Google Cloud credentials to file:',
+      serviceAccountFile,
+    );
+    fs.writeFileSync(
+      serviceAccountFile,
+      process.env.GCLOUD_SERVICE_ACCOUNT_KEY,
+    );
+    process.env.GOOGLE_APPLICATION_CREDENTIALS = serviceAccountFile;
+  }
+
   const app = await NestFactory.create(AppModule);
   const { httpAdapter } = app.get(HttpAdapterHost);
 

backend/src/spreadsheets/models/SheetValueInputOption.ts

@@ -0,0 +1,19 @@
+/**
+ * See https://developers.google.com/sheets/api/reference/rest/v4/ValueInputOption?hl=fr
+ */
+enum SheetValueInputOption {
+  /**
+   * Default input value. This value must not be used.
+   */
+  INPUT_VALUE_OPTION_UNSPECIFIED = 'INPUT_VALUE_OPTION_UNSPECIFIED',
+  /**
+   * The values the user has entered will not be parsed and will be stored as-is.
+   */
+  RAW = 'RAW',
+
+  /**
+   * The values will be parsed as if the user typed them into the UI. Numbers will stay as numbers, but strings may be converted to numbers, dates, etc. following the same rules that are applied when entering text into a cell via the Google Sheets UI.
+   */
+  USER_ENTERED = 'USER_ENTERED',
+}
+export default SheetValueInputOption;

backend/src/spreadsheets/models/SheetValueRenderOption.ts

@@ -0,0 +1,17 @@
+enum SheetValueRenderOption {
+  /**
+   * Values will be calculated & formatted in the response according to the cell's formatting. Formatting is based on the spreadsheet's locale, not the requesting user's locale. For example, if A1 is 1.23 and A2 is =A1 and formatted as currency, then A2 would return "$1.23".
+   */
+  FORMATTED_VALUE = 'FORMATTED_VALUE',
+
+  /**
+   * Values will be calculated, but not formatted in the reply. For example, if A1 is 1.23 and A2 is =A1 and formatted as currency, then A2 would return the number 1.23.
+   */
+  UNFORMATTED_VALUE = 'UNFORMATTED_VALUE',
+
+  /**
+   * Values will not be calculated. The reply will include the formulas. For example, if A1 is 1.23 and A2 is =A1 and formatted as currency, then A2 would return "=A1".
+   */
+  FORMULA = 'FORMULA',
+}
+export default SheetValueRenderOption;

backend/src/spreadsheets/services/sheet.service.ts

@@ -0,0 +1,162 @@
+import { Injectable } from '@nestjs/common';
+import * as auth from 'google-auth-library';
+import { google, sheets_v4, drive_v3 } from 'googleapis';
+import * as retry from 'async-retry';
+import * as gaxios from 'gaxios';
+import SheetValueRenderOption from '../models/SheetValueRenderOption';
+import SheetValueInputOption from '../models/SheetValueInputOption';
+const sheets = google.sheets({ version: 'v4' });
+const drive = google.drive({ version: 'v3' });
+
+@Injectable()
+export default class SheetService {
+  readonly RETRY_STRATEGY: retry.Options = {
+    minTimeout: 60000, // Wait for 1min due to sheet api quota limitation
+  };
+
+  private authClient:
+    | auth.Compute
+    | auth.JWT
+    | auth.UserRefreshClient
+    | auth.BaseExternalAccountClient
+    | auth.Impersonated
+    | null = null;
+
+  async getAuthClient(): Promise<
+    | auth.Compute
+    | auth.JWT
+    | auth.UserRefreshClient
+    | auth.BaseExternalAccountClient
+    | auth.Impersonated
+  > {
+    if (!this.authClient) {
+      this.authClient = await google.auth.getClient({
+        scopes: [
+          'https://www.googleapis.com/auth/spreadsheets',
+          'https://www.googleapis.com/auth/drive',
+        ],
+      });
+    }
+    return this.authClient;
+  }
+
+  async copyFile(fileId: string, copyTitle: string): Promise<string> {
+    const authClient = await this.getAuthClient();
+
+    const copyOptions: drive_v3.Params$Resource$Files$Copy = {
+      auth: authClient,
+      fileId: fileId,
+      requestBody: {
+        name: copyTitle,
+      },
+    };
+    const copyResponse = await drive.files.copy(copyOptions);
+    return copyResponse.data.id!;
+  }
+
+  async getRawDataFromSheet(
+    spreadsheetId: string,
+    range: string,
+    valueRenderOption: SheetValueRenderOption = SheetValueRenderOption.FORMATTED_VALUE,
+  ): Promise<{
+    data: any[][] | null;
+  }> {
+    const authClient = await this.getAuthClient();
+
+    const sheetValues = await retry(
+      async (bail, num): Promise<sheets_v4.Schema$ValueRange | undefined> => {
+        try {
+          const getOptions: sheets_v4.Params$Resource$Spreadsheets$Values$Get =
+            {
+              auth: authClient,
+              spreadsheetId: spreadsheetId,
+              range: range,
+              valueRenderOption: valueRenderOption,
+            };
+          //logger.info(`Get raw data from sheet ${spreadsheetId} with range ${range} (attempt ${num})`);
+          const sheetResponse =
+            await sheets.spreadsheets.values.get(getOptions);
+          return sheetResponse.data;
+        } catch (error) {
+          //logger.exception(error);
+          if (error instanceof gaxios.GaxiosError) {
+            //const gaxiosError = error as gaxios.GaxiosError;
+            /*logger.error(
+                      `Error while retrieving sheet data: status ${gaxiosError.status}, code ${gaxiosError.code}, message: ${gaxiosError.message}`
+                  );*/
+            if (error.status === 429) {
+              //logger.info(`Error due to api quota limitation, retrying`);
+              throw error;
+            }
+          }
+          bail(error as Error);
+        }
+      },
+      {},
+    );
+
+    return { data: sheetValues?.values || null };
+  }
+
+  async overwriteRawDataToSheet(
+    spreadsheetId: string,
+    range: string,
+    data: any[][],
+    valueInputOption?: SheetValueInputOption,
+  ) {
+    const authClient = await this.getAuthClient();
+    await retry(async (bail, num): Promise<void> => {
+      try {
+        //logger.info(`Overwrite data to sheet ${spreadsheetId} (attempt ${num})`);
+        await sheets.spreadsheets.values.update({
+          auth: authClient,
+          spreadsheetId,
+          range: range,
+          valueInputOption: valueInputOption || SheetValueInputOption.RAW,
+          requestBody: {
+            values: data,
+          },
+        });
+      } catch (error) {
+        //logger.exception(error);
+        if (error instanceof gaxios.GaxiosError) {
+          //const gaxiosError = error as gaxios.GaxiosError;
+          /*logger.error(
+                      `Error while overwriting sheet data: status ${gaxiosError.status}, code ${gaxiosError.code}, message: ${gaxiosError.message}`
+                  );*/
+          if (error.status === 429) {
+            //logger.info(`Quota error, retrying`);
+            throw error;
+          }
+        }
+        // No need to trigger retry if it's not a quota error
+        bail(error as Error);
+      }
+    }, this.RETRY_STRATEGY);
+  }
+
+  async testDownload() {
+    console.log('Copying file...');
+    const fileCopyId = await this.copyFile(
+      '1_l9NkgNIefC4BZLMhZ20GzrG3xlH2kxuH2vFzCgDFw8',
+      'Trajectoire SNBC Territorialisée - Compute',
+    );
+
+    console.log('File copied with id:', fileCopyId);
+
+    const epciCode = 200043495;
+    // Write the epci code
+    await this.overwriteRawDataToSheet(fileCopyId, 'Caract_territoire!F6', [
+      [epciCode],
+    ]);
+
+    // Getting computed data from spreadsheet
+    const data = await this.getRawDataFromSheet(
+      fileCopyId,
+      'Caract_territoire!F1239',
+      SheetValueRenderOption.UNFORMATTED_VALUE,
+    );
+
+    return data;
+  }
+}

backend/src/spreadsheets/sheet.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import SheetService from './services/sheet.service';
+
+@Module({
+  providers: [SheetService],
+  exports: [SheetService],
+  controllers: [],
+})
+export class SheetModule {}