Skip to content

Commit

Permalink
audits: Use new audit table for audit filter
Browse files Browse the repository at this point in the history 
The audit filter uses a subquery to the sv_audits table instead of the
audits field of the interviews table.

The list of validation errors come from the sv_audits table as well.

Sequential tests are updated accordingly.
  • Loading branch information
@tahini
tahini committed Feb 5, 2024
1 parent ee684f7 commit 3e2c096
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 24 deletions.
101 changes: 89 additions & 12 deletions packages/evolution-backend/src/models/__tests__/interviews.db.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ const localUserInterviewAttributes = {
},
validations: {},
logs: [],
audits: { errorOne: 3, errorThree: 1 }
};

const facebookUserInterviewAttributes = {
Expand Down Expand Up @@ -122,7 +121,6 @@ const googleUserInterviewAttributes = {
},
validations: {},
logs: [],
audits: { errorOne: 3, errorTwo: 1 },
validated_data: {
accessCode: '2222',
home: {
Expand All @@ -135,6 +133,7 @@ const googleUserInterviewAttributes = {

beforeAll(async () => {
jest.setTimeout(10000);
await truncate(knex, 'sv_audits');
await truncate(knex, 'sv_interviews');
await truncate(knex, 'sv_participants');
await create(knex, 'sv_participants', undefined, localUser as any);
Expand All @@ -153,6 +152,7 @@ beforeAll(async () => {
});

afterAll(async() => {
await truncate(knex, 'sv_audits');
await truncate(knex, 'sv_interviews');
await truncate(knex, 'users');
await truncate(knex, 'sv_participants');
Expand Down Expand Up @@ -579,13 +579,6 @@ describe('list interviews', () => {
expect(filterResponsesBoolean.length).toEqual(1);
expect((filterResponsesBoolean[0].responses as any).booleanField).toBeTruthy();

// Query by audit
const { interviews: filterAudit, totalCount: countAudit } =
await dbQueries.getList({ filters: { 'audits': { value: 'errorThree' } }, pageIndex: 0, pageSize: -1 });
expect(countAudit).toEqual(1);
expect(filterAudit.length).toEqual(1);
expect(filterAudit[0].uuid).toEqual(localUserInterviewAttributes.uuid);

});

test('Combine filter and paging', async () => {
Expand Down Expand Up @@ -707,9 +700,40 @@ describe('list interviews', () => {

});

describe('get validation errors', () => {
describe('Queries with audits', () => {

const errorOneCode = 'errorOne';
const errorTwoCode = 'errorTwo';
const errorThreeCode = 'errorThree';

test('Get the complete list of errors', async () => {
beforeAll(async () => {
// Add 3 errors per of type one, and one of type three for one of the interviews
const firstInterview = await dbQueries.getInterviewByUuid(localUserInterviewAttributes.uuid);
if (firstInterview === undefined) {
throw 'error getting interview 1 for audits';
}
await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id');
await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id');
await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id');
await create(knex, 'sv_audits', undefined, { interview_id: firstInterview.id, error_code: errorThreeCode, object_type: 'interview', object_uuid: firstInterview.uuid, version: 2 } as any, 'interview_id');

// Add 3 errors per of type one, and one of type two for another interview
const secondInterview = await dbQueries.getInterviewByUuid(googleUserInterviewAttributes.uuid);
if (secondInterview === undefined) {
throw 'error getting interview 2 for audits';
}
await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id');
await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id');
await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorOneCode, object_type: 'person', object_uuid: uuidV4(), version: 2 } as any, 'interview_id');
await create(knex, 'sv_audits', undefined, { interview_id: secondInterview.id, error_code: errorTwoCode, object_type: 'household', object_uuid: uuidV4(), version: 2 } as any, 'interview_id');

});

afterAll(async() => {
await truncate(knex, 'sv_audits');
});

test('Get the complete list of validation errors', async () => {
const { errors } = await dbQueries.getValidationErrors({ filters: {} });
expect(errors.length).toEqual(3);
expect(errors).toEqual([
Expand All @@ -719,14 +743,67 @@ describe('get validation errors', () => {
]);
});

test('Use a validity filter', async () => {
test('Get validation errors with a validity filter', async () => {
const { errors } = await dbQueries.getValidationErrors({ filters: { is_valid: { value: true } } });
expect(errors.length).toEqual(2);
expect(errors).toEqual([
{ key: 'errorOne', cnt: '3' },
{ key: 'errorTwo', cnt: '1' }
]);
});

test('List interviews with audit filter', async () => {

// Query by audit
const { interviews: filterAudit, totalCount: countAudit } =
await dbQueries.getList({ filters: { 'audits': { value: 'errorThree' } }, pageIndex: 0, pageSize: -1 });
expect(countAudit).toEqual(1);
expect(filterAudit.length).toEqual(1);
expect(filterAudit[0].uuid).toEqual(localUserInterviewAttributes.uuid);

});

test('List interviews with audit filter, no result', async () => {

// Query by audit
const { interviews: filterAudit, totalCount: countAudit } =
await dbQueries.getList({ filters: { 'audits': { value: 'errorFour' } }, pageIndex: 0, pageSize: -1 });
expect(countAudit).toEqual(0);
expect(filterAudit.length).toEqual(0);

});

test('List interviews, validate audits', async () => {

// Query by audit
const { interviews, totalCount } = await dbQueries.getList({ filters: { }, pageIndex: 0, pageSize: -1 });
expect(totalCount).toEqual(5);
expect(interviews.length).toEqual(5);
for (let interview of interviews) {
if (interview.uuid === localUserInterviewAttributes.uuid) {
const audits = interview.audits;
expect(audits).toBeDefined();
expect(Object.keys(audits as any).length).toEqual(2);
expect(audits).toMatchObject({
[errorOneCode]: 3,
[errorThreeCode]: 1
});
} else if (interview.uuid === googleUserInterviewAttributes.uuid) {
const audits = interview.audits;
expect(audits).toBeDefined();
expect(Object.keys(audits as any).length).toEqual(2);
expect(audits).toMatchObject({
[errorOneCode]: 3,
[errorTwoCode]: 1
});
} else {
const audits = interview.audits;
expect(audits).toBeUndefined();
}
}

});

});

describe('stream interviews query', () => {
Expand Down
47 changes: 37 additions & 10 deletions packages/evolution-backend/src/models/interviews.db.queries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,8 +416,10 @@ const getRawWhereClause = (
return getBooleanFilter(`${tblAlias}.is_questionable`, filter);
case 'uuid':
return `${tblAlias}.${field} ${filter.op ? operatorSigns[filter.op] : operatorSigns.eq} '${filter.value}'`;
case 'audits':
if (typeof filter.value === 'string') {
case 'audits': {
if (typeof filter.value !== 'string') {
return undefined;
}
const match = filter.value.match(dotSeparatedStringRegex);
if (match === null) {
throw new TrError(
Expand All @@ -426,9 +428,10 @@ const getRawWhereClause = (
'DatabaseInvalidWhereClauseUserEntry'
);
}
// Add subquery to audits table
const auditSubQuery = knex('sv_audits').select('interview_id').distinct().where('error_code', filter.value);
return [`${tblAlias}.id in (${auditSubQuery.toSQL().sql})`, filter.value];
}
// Query whether the value exists
return `${tblAlias}.${field}->>'${filter.value}' is not null`;
}
const jsonObject = field.split('.');
// TODO only responses field order by is supported
Expand Down Expand Up @@ -525,6 +528,16 @@ const getList = async <CustomSurvey, CustomHousehold, CustomHome, CustomPerson>(

const sortFields = params.sort || [];

const auditsCountQuery = knex('sv_audits')
.select('interview_id', 'error_code')
.count()
.groupBy('interview_id', 'error_code')
.orderBy('count')
.as('audits_cnt');
const auditsQuery = knex(auditsCountQuery)
.select('interview_id', knex.raw('json_agg(json_build_object(error_code, count)) as audits'))
.groupBy('interview_id')
.as('audits');
const interviewsQuery = knex
.select(
'i.id',
Expand All @@ -533,19 +546,19 @@ const getList = async <CustomSurvey, CustomHousehold, CustomHome, CustomPerson>(
'i.created_at',
'i.responses',
'i.validated_data',
'i.audits',
'audits.audits',
'i.is_valid',
'i.is_completed',
'i.is_validated',
'i.is_questionable',
'i.audits',
'i.survey_id',
'participant.username',
knex.raw('case when participant.facebook_id is null then false else true end facebook'),
knex.raw('case when participant.google_id is null then false else true end google')
)
.from(`${tableName} as i`)
.leftJoin(`${participantTable} as participant`, 'i.participant_id', 'participant.id')
.leftJoin(auditsQuery, 'i.id', 'audits.interview_id')
.whereRaw(rawFilter, bindings);
// Add sort fields
sortFields.forEach((field) => {
Expand All @@ -558,7 +571,21 @@ const getList = async <CustomSurvey, CustomHousehold, CustomHome, CustomPerson>(
}
const interviews = await interviewsQuery;

return { interviews: interviews.map((interview) => _removeBlankFields(interview)), totalCount };
// TODO For backward compatibility, the type of the audits is an object with key => count. When we only use the new audits, this can be udpated
const auditsToObject = ({ audits, ...rest }) => {
const newAudits =
audits === undefined
? undefined
: (audits as { [auditKey: string]: number }[]).reduce(
(accumulator, currentValue) => ({ ...accumulator, ...currentValue }),
{}
);
return {
...rest,
audits: newAudits
} as InterviewListAttributes<CustomSurvey, CustomHousehold, CustomHome, CustomPerson>;
};
return { interviews: interviews.map((interview) => auditsToObject(_removeBlankFields(interview))), totalCount };
} catch (error) {
throw new TrError(
`Cannot get interview list in table ${tableName} database (knex error: ${error})`,
Expand Down Expand Up @@ -587,12 +614,12 @@ const getValidationErrors = async (params: {
const [rawFilter, bindings] = updateRawWhereClause(params.filters, baseRawFilter);

const validationErrorsQuery = knex
.select('key', knex.raw('sum(value::numeric) cnt'))
.select('error_code as key', knex.raw('count(error_code) cnt'))
.from(`${tableName} as i`)
.joinRaw('inner join lateral json_each_text(audits) on TRUE')
.innerJoin('sv_audits', 'id', 'interview_id')
.leftJoin(`${participantTable} as participant`, 'i.participant_id', 'participant.id')
.whereRaw(rawFilter, bindings)
.groupBy('key')
.groupBy('error_code')
.orderBy('cnt', 'desc');

const errors = await validationErrorsQuery;
Expand Down
2 changes: 1 addition & 1 deletion packages/evolution-frontend/src/components/pageParts/validations/InterviewListComponent.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ const InterviewListComponent: React.FunctionComponent<InterviewListComponentProp
icon={faExclamationTriangle}
className="faIconNoMargin _error _red"
title={Object.keys(value)
.map((error: any) => props.t(`survey:validations:${error}`))
.map((error: any) => props.t([`survey:validations:${error}`, `surveyAdmin:${error}`]))
.join('\n')}
/>
),
Expand Down
2 changes: 1 addition & 1 deletion packages/evolution-frontend/src/components/pageParts/validations/ValidationAuditFilter.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ export const ValidationAuditFilter = <CustomSurvey, CustomHousehold, CustomHome,
>
{options.map((key, i) => (
<option key={`validationError_${key}`} value={key}>
{key ? _truncate(t(`survey:validations:${key}`), { length: 70 }) : ''}
{key ? _truncate(t([`survey:validations:${key}`, `surveyAdmin:${key}`]), { length: 70 }) : ''}
</option>
))}
</select>
Expand Down

0 comments on commit 3e2c096

Please sign in to comment.