separate audit filters and allow array filters

locales/en/admin.json

@@ -106,6 +106,23 @@
     "Keep": "Keep",
     "Discard": "Discard"
   },
+  "auditLevels": {
+    "error": "Errors",
+    "warning": "Warnings",
+    "info": "Infos"
+  },
+  "auditObjectTypes": {
+    "interview": "Interview",
+    "household": "Household",
+    "home": "Home",
+    "person": "Person",
+    "visitedPlace": "Visited place",
+    "trip": "Trip",
+    "segment": "Segment",
+    "vehicle": "Vehicle",
+    "place": "Place"
+  },
   "InterviewId": "interview ID",
-  "HouseholdSize": "Household size"
+  "HouseholdSize": "Household size",
+  "Select": "Select"
 }

locales/fr/admin.json

@@ -106,6 +106,23 @@
     "Keep": "Garder",
     "Discard": "Écarter"
   },
+  "auditLevels": {
+    "error": "Erreurs",
+    "warning": "Avertissements",
+    "info": "Infos"
+  },
+  "auditObjectTypes": {
+    "interview": "Entrevue",
+    "household": "Ménage",
+    "home": "Domicile",
+    "person": "Personne",
+    "visitedPlace": "Lieu visité",
+    "trip": "Déplacement",
+    "segment": "Segment",
+    "vehicle": "Véhicule",
+    "place": "Lieu"
+  },
   "InterviewId": "ID d'entrevue",
-  "HouseholdSize": "Taille du ménage"
+  "HouseholdSize": "Taille du ménage",
+  "Select": "Sélectionner"
 }

packages/evolution-backend/src/api/survey.validation.routes.ts

@@ -196,7 +196,7 @@ router.get('/interviewSummary/:interviewUuid', async (req: Request, res: Respons
     }
 });
 
-router.post('/validation/errors', async (req: Request, res: Response) => {
+router.post('/validation/auditStats', async (req: Request, res: Response) => {
     try {
         const { ...filters } = req.body;
 
@@ -208,10 +208,10 @@ router.post('/validation/errors', async (req: Request, res: Response) => {
                 actualFilters[key] = filters[key];
             }
         });
-        const response = await Interviews.getValidationErrors({ filter: actualFilters });
+        const response = await Interviews.getValidationAuditStats({ filter: actualFilters });
         return res.status(200).json({
             status: 'success',
-            errors: response.errors
+            auditStats: response.auditStats
         });
     } catch (error) {
         console.log('error getting interview list:', error);

packages/evolution-backend/src/models/__tests__/interviews.db.test.ts

@@ -151,7 +151,7 @@ beforeAll(async () => {
     await dbQueries.create(googleUserInterviewAttributes);
 });
 
-afterAll(async() => {
+afterAll(async () => {
     await truncate(knex, 'sv_audits');
     await truncate(knex, 'sv_interviews');
     await truncate(knex, 'users');
@@ -356,7 +356,8 @@ describe('Update Interview', () => {
         const addAttributes = { responses: { foo: 'test' }, validations: { accessCode: true, other: 'data' } };
         const newAttributes = {
             responses: Object.assign({}, localUserInterviewAttributes.responses, addAttributes.responses),
-            validations: Object.assign({}, localUserInterviewAttributes.validations, addAttributes.validations) };
+            validations: Object.assign({}, localUserInterviewAttributes.validations, addAttributes.validations)
+        };
         const interview = await dbQueries.update(localUserInterviewAttributes.uuid, newAttributes, 'uuid');
         expect(interview.uuid).toEqual(localUserInterviewAttributes.uuid);
 
@@ -370,7 +371,8 @@ describe('Update Interview', () => {
         const addAttributes = { responses: { foo: 'test' }, validations: { accessCode: true, other: 'data' } };
         const newAttributes = {
             responses: Object.assign({}, localUserInterviewAttributes.responses, addAttributes.responses),
-            validations: Object.assign({}, localUserInterviewAttributes.validations, addAttributes.validations) };
+            validations: Object.assign({}, localUserInterviewAttributes.validations, addAttributes.validations)
+        };
         await expect(dbQueries.update('not a uuid', newAttributes, 'uuid'))
             .rejects
             .toThrow(expect.anything());
@@ -381,7 +383,7 @@ describe('Update Interview', () => {
         const newAttributes = {
             responses: Object.assign({}, localUserInterviewAttributes.responses, addAttributes.responses),
             validations: Object.assign({}, localUserInterviewAttributes.validations, addAttributes.validations),
-            logs: [ { timestamp: 1234, valuesByPath: { 'responses.some': 'foo' } } ]
+            logs: [{ timestamp: 1234, valuesByPath: { 'responses.some': 'foo' } }]
         };
         const interview = await dbQueries.update(localUserInterviewAttributes.uuid, newAttributes, 'uuid');
         expect(interview.uuid).toEqual(localUserInterviewAttributes.uuid);
@@ -631,7 +633,8 @@ describe('list interviews', () => {
             filters: {},
             pageIndex: 0,
             pageSize: -1,
-            sort: ['is_valid'] });
+            sort: ['is_valid']
+        });
         expect(totalCount).toEqual(nbActiveInterviews);
         expect(page.length).toEqual(nbActiveInterviews);
 
@@ -640,7 +643,8 @@ describe('list interviews', () => {
             filters: {},
             pageIndex: 0,
             pageSize: -1,
-            sort: ['responses.accessCode'] });
+            sort: ['responses.accessCode']
+        });
         expect(totalCountAsc).toEqual(nbActiveInterviews);
         expect(pageAsc.length).toEqual(nbActiveInterviews);
 
@@ -649,7 +653,8 @@ describe('list interviews', () => {
             filters: {},
             pageIndex: 0,
             pageSize: -1,
-            sort: [{ field: 'responses.accessCode', order: 'desc' }] });
+            sort: [{ field: 'responses.accessCode', order: 'desc' }]
+        });
         expect(totalCountDesc).toEqual(nbActiveInterviews);
         expect(pageDesc.length).toEqual(nbActiveInterviews);
         // Only the first 2 have values, first is now last
@@ -666,20 +671,21 @@ describe('list interviews', () => {
             filters: {},
             pageIndex: 0,
             pageSize: -1,
-            sort: ['is_valid', { field: 'responses.home.someField', order: 'desc' } ] });
+            sort: ['is_valid', { field: 'responses.home.someField', order: 'desc' }]
+        });
         expect(totalCount3).toEqual(nbActiveInterviews);
         expect(page3.length).toEqual(nbActiveInterviews);
 
     });
 
     // Parameters for list come from external, we cannot guarantee the types
-    test('inject bad data', async() => {
+    test('inject bad data', async () => {
         // Add invalid order by, should throw an error
         await expect(dbQueries.getList({
             filters: {},
             pageIndex: 0,
             pageSize: -1,
-            sort: [ { field: 'responses.accessCode', order: 'desc; select * from sv_interviews' as any } ]
+            sort: [{ field: 'responses.accessCode', order: 'desc; select * from sv_interviews' as any }]
         }))
             .rejects
             .toThrowError('Cannot get interview list in table sv_interviews database (knex error: Invalid sort order for interview query: desc; select * from sv_interviews (DBINTO0001))');
@@ -742,25 +748,61 @@ describe('Queries with audits', () => {
 
     });
 
-    afterAll(async() => {
+    afterAll(async () => {
         await truncate(knex, 'sv_audits');
     });
 
     test('Get the complete list of validation errors', async () => {
-        const { errors } = await dbQueries.getValidationErrors({ filters: {} });
-        expect(errors.length).toEqual(3);
-        expect(errors.find(error => error.key === 'errorOne' && error.cnt === '6')).toBeDefined();
-        expect(errors.find(error => error.key === 'errorTwo' && error.cnt === '1')).toBeDefined();
-        expect(errors.find(error => error.key === 'errorThree' && error.cnt === '1')).toBeDefined();
+        const { auditStats } = await dbQueries.getValidationAuditStats({ filters: {} });
+        expect(auditStats.error).toEqual({
+            person: [
+                {
+                    key: 'errorOne',
+                    cnt: '6',
+                    level: 'error',
+                    object_type: 'person'
+                }
+            ],
+            interview: [
+                {
+                    key: 'errorThree',
+                    cnt: '1',
+                    level: 'error',
+                    object_type: 'interview'
+                }
+            ],
+            household: [
+                {
+                    key: 'errorTwo',
+                    cnt: '1',
+                    level: 'error',
+                    object_type: 'household'
+                }
+            ]
+        });
     });
 
     test('Get validation errors with a validity filter', async () => {
-        const { errors } = await dbQueries.getValidationErrors({ filters: { is_valid: { value: true } } });
-        expect(errors.length).toEqual(2);
-        expect(errors).toEqual([
-            { key: 'errorOne', cnt: '3' },
-            { key: 'errorTwo', cnt: '1' }
-        ]);
+        const { auditStats } = await dbQueries.getValidationAuditStats({ filters: { is_valid: { value: true } } });
+        console.dir(auditStats, { depth: null });
+        expect(auditStats.error).toEqual({
+            person: [
+                {
+                    key: 'errorOne',
+                    cnt: '3',
+                    level: 'error',
+                    object_type: 'person'
+                }
+            ],
+            household: [
+                {
+                    key: 'errorTwo',
+                    cnt: '1',
+                    level: 'error',
+                    object_type: 'household'
+                }
+            ]
+        });
     });
 
     test('List interviews with audit filter', async () => {
@@ -804,10 +846,10 @@ describe('Queries with audits', () => {
     test('List interviews, validate audits', async () => {
 
         // Query by audit
-        const { interviews, totalCount } = await dbQueries.getList({ filters: { }, pageIndex: 0, pageSize: -1 });
+        const { interviews, totalCount } = await dbQueries.getList({ filters: {}, pageIndex: 0, pageSize: -1 });
         expect(totalCount).toEqual(5);
         expect(interviews.length).toEqual(5);
-        for (let interview of interviews) {
+        for (const interview of interviews) {
             if (interview.uuid === localUserInterviewAttributes.uuid) {
                 const audits = interview.audits;
                 expect(audits).toBeDefined();

packages/evolution-backend/src/models/interviews.db.queries.ts

@@ -13,6 +13,7 @@ import {
 } from 'evolution-common/lib/services/interviews/interview';
 import { _booleish, _removeBlankFields } from 'chaire-lib-common/lib/utils/LodashExtensions';
 import config from 'chaire-lib-common/lib/config/shared/project.config';
+import { AuditsByLevelAndObjectType } from 'evolution-common/lib/services/audits/types';
 
 const tableName = 'sv_interviews';
 const participantTable = 'sv_participants';
@@ -647,29 +648,40 @@ const getList = async <CustomSurvey, CustomHousehold, CustomHome, CustomPerson>(
  * list of errors with the error code and the total number of errors in
  * interviews
  */
-const getValidationErrors = async (params: {
+const getValidationAuditStats = async (params: {
     filters: { [key: string]: ValueFilterType };
-}): Promise<{ errors: { key: string; cnt: string }[] }> => {
+}): Promise<{ auditStats: AuditsByLevelAndObjectType }> => {
     try {
         const baseRawFilter =
             'i.is_active IS TRUE AND participant.is_valid IS TRUE AND participant.is_test IS NOT TRUE';
         const [rawFilter, bindings] = updateRawWhereClause(params.filters, baseRawFilter);
 
-        const validationErrorsQuery = knex
-            .select('error_code as key', knex.raw('count(error_code) cnt'))
+        const validationAuditStatsQuery = knex
+            .select('error_code as key', knex.raw('count(error_code) cnt'), 'level', 'object_type')
             .from(`${tableName} as i`)
             .innerJoin('sv_audits', 'id', 'interview_id')
             .leftJoin(`${participantTable} as participant`, 'i.participant_id', 'participant.id')
             .whereRaw(rawFilter, bindings)
-            .groupBy('error_code')
+            .groupBy('error_code', 'level', 'object_type')
             .orderBy('cnt', 'desc');
 
-        const errors = await validationErrorsQuery;
+        const audits = await validationAuditStatsQuery;
 
-        return { errors };
+        const auditStats: AuditsByLevelAndObjectType = {};
+        audits.forEach((audit) => {
+            if (!auditStats[audit.level]) {
+                auditStats[audit.level] = {};
+            }
+            if (!auditStats[audit.level][audit.object_type]) {
+                auditStats[audit.level][audit.object_type] = [];
+            }
+            auditStats[audit.level][audit.object_type].push({ cnt: audit.cnt, key: audit.key });
+        });
+
+        return { auditStats };
     } catch (error) {
         throw new TrError(
-            `Cannot get list of validation errors in table ${tableName} database (knex error: ${error})`,
+            `Cannot get list of validation audit stats in table ${tableName} database (knex error: ${error})`,
             'DBQCR0004',
             'DatabaseCannotListBecauseDatabaseError'
         );
@@ -760,6 +772,6 @@ export default {
     create,
     update,
     getList,
-    getValidationErrors,
+    getValidationAuditStats,
     getInterviewsStream
 };

packages/evolution-backend/src/services/auth/__tests__/participantAuthorization.test.ts

@@ -5,7 +5,7 @@
  * License text available at https://opensource.org/licenses/MIT
  */
 import { NextFunction, Request, Response } from 'express';
-import { v4 as uuidV4 } from 'uuid'
+import { v4 as uuidV4 } from 'uuid';
 import isAuthorized from '../participantAuthorization';
 import each from 'jest-each';
 import Interviews from '../../interviews/interviews';