feat(api): add a script to clean knowledge-element-snapshots

api/scripts/prod/clean-ke-snapshots.js

@@ -0,0 +1,100 @@
+import { knex } from '../../db/knex-database-connection.js';
+import { Script } from '../../src/shared/application/scripts/script.js';
+import { ScriptRunner } from '../../src/shared/application/scripts/script-runner.js';
+
+const DEFAULT_CHUNK_SIZE = 10000;
+const DEFAULT_PAUSE_DURATION = 2000;
+
+const pause = async (duration) => {
+  return new Promise((resolve) => {
+    setTimeout(resolve, duration);
+  });
+};
+
+function getKnowlegdeElementSnapshotsQuery() {
+  return knex('knowledge-element-snapshots').whereRaw("(snapshot->0->>'userId') is not null");
+}
+
+function getKnowlegdeElementSnapshotLimit(firstId, limit = DEFAULT_CHUNK_SIZE) {
+  return getKnowlegdeElementSnapshotsQuery()
+    .select('id', 'snapshot')
+    .limit(limit)
+    .where('id', '>=', firstId)
+    .orderBy('id', 'asc');
+}
+function getKnowlegdeElementSnapshotCount() {
+  return getKnowlegdeElementSnapshotsQuery().count().first();
+}
+
+// Définition du script
+export class CleanKeSnapshotScript extends Script {
+  constructor() {
+    super({
+      description: 'This script will remove unused properties from the column knowledge-element-snapshots.snapshot',
+      permanent: false,
+      options: {
+        chunkSize: {
+          type: 'number',
+          default: DEFAULT_CHUNK_SIZE,
+          description: 'number of records to update in one update',
+        },
+        pauseDuration: {
+          type: 'number',
+          default: DEFAULT_PAUSE_DURATION,
+          description: 'Time in ms between each chunk processing',
+        },
+      },
+    });
+  }
+
+  async handle({
+    options = { chunkSize: DEFAULT_CHUNK_SIZE, pauseDuration: DEFAULT_PAUSE_DURATION },
+    logger,
+    dependencies = { pause },
+  }) {
+    const logInfo = (message) => logger.info({ event: 'CleanKeSnapshotScript' }, message);
+
+    const snapshotToClean = await getKnowlegdeElementSnapshotCount();
+
+    const nbChunk = Math.ceil(snapshotToClean.count / (options.chunkSize || DEFAULT_CHUNK_SIZE));
+
+    logInfo(`Start cleaning ${snapshotToClean.count} (${nbChunk} batch) knowledge-element-snapshots to clean.`);
+
+    let snapshots = await getKnowlegdeElementSnapshotLimit(0, options.chunkSize);
+    let chunkDone = 0;
+
+    while (snapshots.length > 0) {
+      await knex.transaction(async (trx) => {
+        for (const { id, snapshot } of snapshots) {
+          await trx('knowledge-element-snapshots')
+            .where('id', id)
+            .update({
+              // we keep only these property from snapshot
+              snapshot: JSON.stringify(
+                snapshot.map(({ source, status, skillId, answerId, createdAt, earnedPix, competenceId }) => ({
+                  source,
+                  status,
+                  skillId,
+                  answerId,
+                  createdAt,
+                  earnedPix,
+                  competenceId,
+                })),
+              ),
+            });
+        }
+      });
+      const lastSnapshotRow = snapshots[snapshots.length - 1];
+      snapshots = await getKnowlegdeElementSnapshotLimit(lastSnapshotRow.id + 1, options.chunkSize);
+
+      if (snapshots.length > 0 && options.pauseDuration > 0) {
+        await dependencies.pause(options.pauseDuration);
+      }
+      chunkDone += 1;
+      logInfo(`${chunkDone}/${nbChunk} chunk done ! `);
+    }
+  }
+}
+
+// Exécution du script
+await ScriptRunner.execute(import.meta.url, CleanKeSnapshotScript);

api/src/shared/infrastructure/repositories/knowledge-element-repository.js

@@ -112,15 +112,6 @@ const findUniqByUserIdGroupedByCompetenceId = async function ({ userId, limitDat
   return _.groupBy(knowledgeElements, 'competenceId');
 };
 
-const findSnapshotGroupedByCompetencesForUsers = async function (userIdsAndDates) {
-  const knowledgeElementsGroupedByUser = await _findSnapshotsForUsers(userIdsAndDates);
-
-  for (const [userId, knowledgeElements] of Object.entries(knowledgeElementsGroupedByUser)) {
-    knowledgeElementsGroupedByUser[userId] = _.groupBy(knowledgeElements, 'competenceId');
-  }
-  return knowledgeElementsGroupedByUser;
-};
-
 const countValidatedByCompetencesForUsersWithinCampaign = async function (userIdsAndDates, campaignLearningContent) {
   return _countValidatedByCompetencesForUsersWithinCampaign(userIdsAndDates, campaignLearningContent);
 };
@@ -167,7 +158,6 @@ export {
   countValidatedByCompetencesForUsersWithinCampaign,
   findInvalidatedAndDirectByUserId,
   findSnapshotForUsers,
-  findSnapshotGroupedByCompetencesForUsers,
   findUniqByUserId,
   findUniqByUserIdAndAssessmentId,
   findUniqByUserIdAndCompetenceId,

api/tests/integration/scripts/prod/clean-ke-snapshots_test.js

@@ -0,0 +1,127 @@
+import { CleanKeSnapshotScript } from '../../../../scripts/prod/clean-ke-snapshots.js';
+import { databaseBuilder, expect, knex, sinon } from '../../../test-helper.js';
+
+describe('Script | Prod | Clean knowledge-element-snapshot snapshot (jsonb)', function () {
+  describe('Options', function () {
+    it('has the correct options', function () {
+      // when
+      const script = new CleanKeSnapshotScript();
+      const { options, description, permanent } = script.metaInfo;
+      expect(permanent).to.be.false;
+      expect(description).to.equal(
+        'This script will removed unused property in  the column knowledge-element-snapshots.snapshot',
+      );
+      // then
+      expect(options.chunkSize).to.deep.include({
+        type: 'number',
+        default: 10000,
+        description: 'number of records to update in one update',
+      });
+      expect(options.pauseDuration).to.deep.include({
+        type: 'number',
+        default: 2000,
+        description: 'Time in ms between each chunk processing',
+      });
+    });
+  });
+
+  describe('Handle', function () {
+    let script;
+    let logger;
+    let dependencies;
+    let user, campaign, learner;
+
+    beforeEach(async function () {
+      script = new CleanKeSnapshotScript();
+      logger = { info: sinon.spy(), error: sinon.spy(), debug: sinon.spy() };
+      dependencies = { pause: sinon.stub() };
+
+      campaign = databaseBuilder.factory.buildCampaign();
+      user = databaseBuilder.factory.buildUser();
+      learner = databaseBuilder.factory.buildOrganizationLearner({
+        userId: user.id,
+      });
+      const participation = databaseBuilder.factory.buildCampaignParticipation({
+        organizationLearnerId: learner.id,
+        userId: user.id,
+        campaign: campaign.id,
+        participantExternalId: null,
+        sharedAt: new Date('2025-01-01'),
+      });
+      const participation2 = databaseBuilder.factory.buildCampaignParticipation({
+        organizationLearnerId: learner.id,
+        userId: user.id,
+        campaign: campaign.id,
+        participantExternalId: null,
+        sharedAt: new Date('2025-01-02'),
+      });
+      const participation3 = databaseBuilder.factory.buildCampaignParticipation({
+        organizationLearnerId: learner.id,
+        userId: user.id,
+        campaign: campaign.id,
+        participantExternalId: null,
+        sharedAt: new Date('2025-01-03'),
+      });
+
+      databaseBuilder.factory.knowledgeElementSnapshotFactory.buildSnapshot({
+        id: 1,
+        userId: user.id,
+        snappedAt: participation.sharedAt,
+        knowledgeElementsAttributes: [{ skillId: 'skill_1', status: 'validated', earnedPix: 40, userId: user.id }],
+      });
+      databaseBuilder.factory.knowledgeElementSnapshotFactory.buildSnapshot({
+        id: 2,
+        userId: user.id,
+        snappedAt: participation2.sharedAt,
+        knowledgeElementsAttributes: [{ skillId: 'skill_2', status: 'validated', earnedPix: 40, userId: user.id }],
+      });
+      databaseBuilder.factory.knowledgeElementSnapshotFactory.buildSnapshot({
+        id: 3,
+        userId: user.id,
+        snappedAt: participation3.sharedAt,
+        knowledgeElementsAttributes: [
+          { skillId: 'skill_2', status: 'validated', earnedPix: 40, userId: null, assessmentId: null },
+        ],
+      });
+
+      await databaseBuilder.commit();
+    });
+
+    it('should log how many element it will clean', async function () {
+      // when
+      await script.handle({ options: { pauseDuration: 0 }, logger, dependencies });
+
+      // then
+      expect(logger.info).to.have.been.calledWithExactly(
+        { event: 'CleanKeSnapshotScript' },
+        'Start cleaning 2 (1 batch) knowledge-element-snapshots to clean.',
+      );
+    });
+
+    it('should clean snapshot one by one', async function () {
+      // given
+      const options = { chunkSize: 1, pauseDuration: 0 };
+
+      // when
+      await script.handle({ options, logger, dependencies });
+
+      // then
+      const snapshots = await knex('knowledge-element-snapshots').pluck('snapshot');
+
+      expect(snapshots.flat().every(({ userId, assessmentId }) => !userId && !assessmentId)).to.be.true;
+      expect(logger.info).to.have.been.calledWithExactly({ event: 'CleanKeSnapshotScript' }, '1/2 chunk done ! ');
+      expect(logger.info).to.have.been.calledWithExactly({ event: 'CleanKeSnapshotScript' }, '2/2 chunk done ! ');
+    });
+
+    it('should pause between chunks', async function () {
+      // given
+      const options = { chunkSize: 1, pauseDuration: 10 };
+
+      // when
+      await script.handle({ options, logger, dependencies });
+
+      // then
+      expect(dependencies.pause).to.have.been.calledOnceWithExactly(10);
+    });
+  });
+});