diff --git a/packages/evolution-backend/src/models/__tests__/interviews.db.test.ts b/packages/evolution-backend/src/models/__tests__/interviews.db.test.ts index 31b62618..d4dc000b 100644 --- a/packages/evolution-backend/src/models/__tests__/interviews.db.test.ts +++ b/packages/evolution-backend/src/models/__tests__/interviews.db.test.ts @@ -86,7 +86,6 @@ const localUserInterviewAttributes = { }, validations: {}, logs: [], - audits: { errorOne: 3, errorThree: 1 } }; const facebookUserInterviewAttributes = { @@ -122,7 +121,6 @@ const googleUserInterviewAttributes = { }, validations: {}, logs: [], - audits: { errorOne: 3, errorTwo: 1 }, validated_data: { accessCode: '2222', home: { @@ -135,6 +133,7 @@ const googleUserInterviewAttributes = { beforeAll(async () => { jest.setTimeout(10000); + await truncate(knex, 'sv_audits'); await truncate(knex, 'sv_interviews'); await truncate(knex, 'sv_participants'); await create(knex, 'sv_participants', undefined, localUser as any); @@ -153,6 +152,7 @@ beforeAll(async () => { }); afterAll(async() => { + await truncate(knex, 'sv_audits'); await truncate(knex, 'sv_interviews'); await truncate(knex, 'users'); await truncate(knex, 'sv_participants'); @@ -579,13 +579,6 @@ describe('list interviews', () => { expect(filterResponsesBoolean.length).toEqual(1); expect((filterResponsesBoolean[0].responses as any).booleanField).toBeTruthy(); - // Query by audit - const { interviews: filterAudit, totalCount: countAudit } = - await dbQueries.getList({ filters: { 'audits': { value: 'errorThree' } }, pageIndex: 0, pageSize: -1 }); - expect(countAudit).toEqual(1); - expect(filterAudit.length).toEqual(1); - expect(filterAudit[0].uuid).toEqual(localUserInterviewAttributes.uuid); - }); test('Combine filter and paging', async () => { @@ -707,9 +700,40 @@ describe('list interviews', () => { }); -describe('get validation errors', () => { +describe('Queries with audits', () => { + + const errorOneCode = 'errorOne'; + const errorTwoCode = 'errorTwo'; + const errorThreeCode = 'errorThree'; - test('Get the complete list of errors', async () => { + beforeAll(async () => { + // Add 3 errors per of type one, and one of type three for one of the interviews + const firstInterview = await dbQueries.getInterviewByUuid(localUserInterviewAttributes.uuid); + if (firstInterview === undefined) { + throw 'error getting interview 1 for audits'; + } + await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id'); + await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id'); + await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id'); + await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorThreeCode, object_type: 'interview', object_uuid: firstInterview.uuid, version: 2 } as any, 'interview_id'); + + // Add 3 errors per of type one, and one of type two for another interview + const secondInterview = await dbQueries.getInterviewByUuid(googleUserInterviewAttributes.uuid); + if (secondInterview === undefined) { + throw 'error getting interview 2 for audits'; + } + await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id'); + await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id'); + await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id'); + await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorTwoCode, object_type: 'household', object_uuid: uuidV4(), version: 2 } as any, 'interview_id'); + + }); + + afterAll(async() => { + await truncate(knex, 'sv_audits'); + }); + + test('Get the complete list of validation errors', async () => { const { errors } = await dbQueries.getValidationErrors({ filters: {} }); expect(errors.length).toEqual(3); expect(errors).toEqual([ @@ -719,7 +743,7 @@ describe('get validation errors', () => { ]); }); - test('Use a validity filter', async () => { + test('Get validation errors with a validity filter', async () => { const { errors } = await dbQueries.getValidationErrors({ filters: { is_valid: { value: true } } }); expect(errors.length).toEqual(2); expect(errors).toEqual([ @@ -727,6 +751,59 @@ describe('get validation errors', () => { { key: 'errorTwo', cnt: '1' } ]); }); + + test('List interviews with audit filter', async () => { + + // Query by audit + const { interviews: filterAudit, totalCount: countAudit } = + await dbQueries.getList({ filters: { 'audits': { value: 'errorThree' } }, pageIndex: 0, pageSize: -1 }); + expect(countAudit).toEqual(1); + expect(filterAudit.length).toEqual(1); + expect(filterAudit[0].uuid).toEqual(localUserInterviewAttributes.uuid); + + }); + + test('List interviews with audit filter, no result', async () => { + + // Query by audit + const { interviews: filterAudit, totalCount: countAudit } = + await dbQueries.getList({ filters: { 'audits': { value: 'errorFour' } }, pageIndex: 0, pageSize: -1 }); + expect(countAudit).toEqual(0); + expect(filterAudit.length).toEqual(0); + + }); + + test('List interviews, validate audits', async () => { + + // Query by audit + const { interviews, totalCount } = await dbQueries.getList({ filters: { }, pageIndex: 0, pageSize: -1 }); + expect(totalCount).toEqual(5); + expect(interviews.length).toEqual(5); + for (let interview of interviews) { + if (interview.uuid === localUserInterviewAttributes.uuid) { + const audits = interview.audits; + expect(audits).toBeDefined(); + expect(Object.keys(audits as any).length).toEqual(2); + expect(audits).toMatchObject({ + [errorOneCode]: 3, + [errorThreeCode]: 1 + }); + } else if (interview.uuid === googleUserInterviewAttributes.uuid) { + const audits = interview.audits; + expect(audits).toBeDefined(); + expect(Object.keys(audits as any).length).toEqual(2); + expect(audits).toMatchObject({ + [errorOneCode]: 3, + [errorTwoCode]: 1 + }); + } else { + const audits = interview.audits; + expect(audits).toBeUndefined(); + } + } + + }); + }); describe('stream interviews query', () => { diff --git a/packages/evolution-backend/src/models/interviews.db.queries.ts b/packages/evolution-backend/src/models/interviews.db.queries.ts index b49f822e..82e3afea 100644 --- a/packages/evolution-backend/src/models/interviews.db.queries.ts +++ b/packages/evolution-backend/src/models/interviews.db.queries.ts @@ -416,8 +416,10 @@ const getRawWhereClause = ( return getBooleanFilter(`${tblAlias}.is_questionable`, filter); case 'uuid': return `${tblAlias}.${field} ${filter.op ? operatorSigns[filter.op] : operatorSigns.eq} '${filter.value}'`; - case 'audits': - if (typeof filter.value === 'string') { + case 'audits': { + if (typeof filter.value !== 'string') { + return undefined; + } const match = filter.value.match(dotSeparatedStringRegex); if (match === null) { throw new TrError( @@ -426,9 +428,10 @@ const getRawWhereClause = ( 'DatabaseInvalidWhereClauseUserEntry' ); } + // Add subquery to audits table + const auditSubQuery = knex('sv_audits').select('interview_id').distinct().where('error_code', filter.value); + return [`${tblAlias}.id in (${auditSubQuery.toSQL().sql})`, filter.value]; } - // Query whether the value exists - return `${tblAlias}.${field}->>'${filter.value}' is not null`; } const jsonObject = field.split('.'); // TODO only responses field order by is supported @@ -525,6 +528,16 @@ const getList = async ( const sortFields = params.sort || []; + const auditsCountQuery = knex('sv_audits') + .select('interview_id', 'error_code') + .count() + .groupBy('interview_id', 'error_code') + .orderBy('count') + .as('audits_cnt'); + const auditsQuery = knex(auditsCountQuery) + .select('interview_id', knex.raw('json_agg(json_build_object(error_code, count)) as audits')) + .groupBy('interview_id') + .as('audits'); const interviewsQuery = knex .select( 'i.id', @@ -533,12 +546,11 @@ const getList = async ( 'i.created_at', 'i.responses', 'i.validated_data', - 'i.audits', + 'audits.audits', 'i.is_valid', 'i.is_completed', 'i.is_validated', 'i.is_questionable', - 'i.audits', 'i.survey_id', 'participant.username', knex.raw('case when participant.facebook_id is null then false else true end facebook'), @@ -546,6 +558,7 @@ const getList = async ( ) .from(`${tableName} as i`) .leftJoin(`${participantTable} as participant`, 'i.participant_id', 'participant.id') + .leftJoin(auditsQuery, 'i.id', 'audits.interview_id') .whereRaw(rawFilter, bindings); // Add sort fields sortFields.forEach((field) => { @@ -558,7 +571,21 @@ const getList = async ( } const interviews = await interviewsQuery; - return { interviews: interviews.map((interview) => _removeBlankFields(interview)), totalCount }; + // TODO For backward compatibility, the type of the audits is an object with key => count. When we only use the new audits, this can be udpated + const auditsToObject = ({ audits, ...rest }) => { + const newAudits = + audits === undefined + ? undefined + : (audits as { [auditKey: string]: number }[]).reduce( + (accumulator, currentValue) => ({ ...accumulator, ...currentValue }), + {} + ); + return { + ...rest, + audits: newAudits + } as InterviewListAttributes; + }; + return { interviews: interviews.map((interview) => auditsToObject(_removeBlankFields(interview))), totalCount }; } catch (error) { throw new TrError( `Cannot get interview list in table ${tableName} database (knex error: ${error})`, @@ -587,12 +614,12 @@ const getValidationErrors = async (params: { const [rawFilter, bindings] = updateRawWhereClause(params.filters, baseRawFilter); const validationErrorsQuery = knex - .select('key', knex.raw('sum(value::numeric) cnt')) + .select('error_code as key', knex.raw('count(error_code) cnt')) .from(`${tableName} as i`) - .joinRaw('inner join lateral json_each_text(audits) on TRUE') + .innerJoin('sv_audits', 'id', 'interview_id') .leftJoin(`${participantTable} as participant`, 'i.participant_id', 'participant.id') .whereRaw(rawFilter, bindings) - .groupBy('key') + .groupBy('error_code') .orderBy('cnt', 'desc'); const errors = await validationErrorsQuery; diff --git a/packages/evolution-frontend/src/components/pageParts/validations/InterviewListComponent.tsx b/packages/evolution-frontend/src/components/pageParts/validations/InterviewListComponent.tsx index 1205c9f1..19786dd6 100644 --- a/packages/evolution-frontend/src/components/pageParts/validations/InterviewListComponent.tsx +++ b/packages/evolution-frontend/src/components/pageParts/validations/InterviewListComponent.tsx @@ -179,7 +179,7 @@ const InterviewListComponent: React.FunctionComponent props.t(`survey:validations:${error}`)) + .map((error: any) => props.t([`survey:validations:${error}`, `surveyAdmin:${error}`])) .join('\n')} /> ), diff --git a/packages/evolution-frontend/src/components/pageParts/validations/ValidationAuditFilter.tsx b/packages/evolution-frontend/src/components/pageParts/validations/ValidationAuditFilter.tsx index a286c431..5047c92b 100644 --- a/packages/evolution-frontend/src/components/pageParts/validations/ValidationAuditFilter.tsx +++ b/packages/evolution-frontend/src/components/pageParts/validations/ValidationAuditFilter.tsx @@ -90,7 +90,7 @@ export const ValidationAuditFilter = {options.map((key, i) => ( ))}