Skip to content

Commit

Permalink
interview filters: Allow an array of values
Browse files Browse the repository at this point in the history 
It is possible to receive an array of strings for filter values. In that
case, the result is and AND of those values in the resulting SQL query.
  • Loading branch information
@tahini @kaligrafy
tahini authored and kaligrafy committed Apr 3, 2024
1 parent 042821e commit 27c7bb9
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 75 deletions.
8 changes: 4 additions & 4 deletions packages/evolution-backend/src/api/survey.validation.routes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
* */
import express, { Request, Response } from 'express';
import moment from 'moment';
import Interviews from '../services/interviews/interviews';
import Interviews, { FilterType } from '../services/interviews/interviews';
import { updateInterview, copyResponsesToValidatedData } from '../services/interviews/interview';
import interviewUserIsAuthorized, { isUserAllowed } from '../services/auth/userAuthorization';
import projectConfig from '../config/projectConfig';
Expand Down Expand Up @@ -141,10 +141,10 @@ router.post('/validationList', async (req, res) => {
typeof pageIndex === 'number' ? pageSize : typeof pageSize === 'string' ? parseInt(pageSize) : -1;
const updatedAtNb = typeof updatedAt === 'string' ? parseInt(updatedAt) : 0;

const actualFilters: { [key: string]: string } = {};
const actualFilters: { [key: string]: FilterType } = {};
Object.keys(filters).forEach((key) => {
if (typeof filters[key] === 'string') {
actualFilters[key] = filters[key] as string;
if (typeof filters[key] === 'string' || Array.isArray(filters[key])) {
actualFilters[key] = filters[key];
} else if (typeof filters[key] === 'object' && filters[key].value !== undefined) {
actualFilters[key] = filters[key];
}
Expand Down
40 changes: 34 additions & 6 deletions packages/evolution-backend/src/models/__tests__/interviews.db.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,6 +579,19 @@ describe('list interviews', () => {
expect(filterResponsesBoolean.length).toEqual(1);
expect((filterResponsesBoolean[0].responses as any).booleanField).toBeTruthy();

// Query with array of values without results
const { interviews: filterAccessCodeArray, totalCount: countAccessCodeArray } =
await dbQueries.getList({ filters: { 'responses.accessCode': { value: [localUserInterviewAttributes.responses.accessCode.substring(0, 3), '222'], op: 'like' } }, pageIndex: 0, pageSize: -1 });
expect(countAccessCodeArray).toEqual(0);
expect(filterAccessCodeArray.length).toEqual(0);

// Query with array of values with results
const { interviews: filterAccessCodeArrayRes, totalCount: countAccessCodeArrayRes } =
await dbQueries.getList({ filters: { 'responses.accessCode': { value: [localUserInterviewAttributes.responses.accessCode.substring(0, 3), localUserInterviewAttributes.responses.accessCode.substring(0, 3)], op: 'like' } }, pageIndex: 0, pageSize: -1 });
expect(countAccessCodeArrayRes).toEqual(1);
expect(filterAccessCodeArrayRes.length).toEqual(1);
expect((filterAccessCodeArrayRes[0].responses as any).accessCode).toEqual(localUserInterviewAttributes.responses.accessCode);

});

test('Combine filter and paging', async () => {
Expand Down Expand Up @@ -707,7 +720,7 @@ describe('Queries with audits', () => {
const errorThreeCode = 'errorThree';

beforeAll(async () => {
// Add 3 errors per of type one, and one of type three for one of the interviews
// Add 3 errors per person of type one, and one of type three for one of the interviews
const firstInterview = await dbQueries.getInterviewByUuid(localUserInterviewAttributes.uuid);
if (firstInterview === undefined) {
throw 'error getting interview 1 for audits';
Expand Down Expand Up @@ -736,11 +749,9 @@ describe('Queries with audits', () => {
test('Get the complete list of validation errors', async () => {
const { errors } = await dbQueries.getValidationErrors({ filters: {} });
expect(errors.length).toEqual(3);
expect(errors).toEqual([
{ key: 'errorOne', cnt: '6' },
{ key: 'errorTwo', cnt: '1' },
{ key: 'errorThree', cnt: '1' }
]);
expect(errors.find(error => error.key === 'errorOne' && error.cnt === '6')).toBeDefined();
expect(errors.find(error => error.key === 'errorTwo' && error.cnt === '1')).toBeDefined();
expect(errors.find(error => error.key === 'errorThree' && error.cnt === '1')).toBeDefined();
});

test('Get validation errors with a validity filter', async () => {
Expand Down Expand Up @@ -773,6 +784,23 @@ describe('Queries with audits', () => {

});

test('List interviews with multiple audit filters', async () => {

// Query for errorThree and errorOne
const { interviews: filterAudit, totalCount: countAudit } =
await dbQueries.getList({ filters: { 'audits': { value: ['errorThree', 'errorOne'] } }, pageIndex: 0, pageSize: -1 });
expect(countAudit).toEqual(1);
expect(filterAudit.length).toEqual(1);
expect(filterAudit[0].uuid).toEqual(localUserInterviewAttributes.uuid);

// Query for errorThree and errorTwo, no result expected
const { interviews: filterAudit2, totalCount: countAudit2 } =
await dbQueries.getList({ filters: { 'audits': { value: ['errorThree', 'errorTwo'] } }, pageIndex: 0, pageSize: -1 });
expect(countAudit2).toEqual(0);
expect(filterAudit2.length).toEqual(0);

});

test('List interviews, validate audits', async () => {

// Query by audit
Expand Down
115 changes: 71 additions & 44 deletions packages/evolution-backend/src/models/interviews.db.queries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ export interface InterviewSearchAttributes {
surveyId?: number;
}

export type ValueFilterType = { value: string | string[] | boolean | number | null; op?: keyof OperatorSigns };

/** this will return the survey id or create a new survey in
* table sv_surveys if no existing survey shortname matches
* the project shortname
Expand Down Expand Up @@ -371,9 +373,9 @@ const addLikeBinding = (operator: keyof OperatorSigns | undefined, binding: stri
*/
const getRawWhereClause = (
field: string,
filter: { value: string | boolean | number | null; op?: keyof OperatorSigns },
filter: ValueFilterType,
tblAlias: string
): string | [string, string | boolean | number] | undefined => {
): (string | [string, string | boolean | number] | undefined)[] => {
// Make sure the field is a legitimate field to avoid sql injection. Field
// is either the name of a field, or a dot-separated path in a json object
// of the 'responses' field. We should not accept anything else.
Expand All @@ -389,58 +391,70 @@ const getRawWhereClause = (
'DatabaseInvalidWhereClauseUserEntry'
);
}
const getBooleanFilter = (
fieldName: string,
filter: { value: string | boolean | number | null; op?: keyof OperatorSigns }
) => {
const getBooleanFilter = (fieldName: string, filter: ValueFilterType) => {
const validityValue = _booleish(filter.value);
const notStr = filter.op === 'not' ? ' NOT ' : '';
if (validityValue !== null) {
return `${fieldName} IS ${notStr} ${validityValue ? 'TRUE' : 'FALSE'} `;
return [`${fieldName} IS ${notStr} ${validityValue ? 'TRUE' : 'FALSE'} `];
}
return `${fieldName} IS ${notStr} null`;
return [`${fieldName} IS ${notStr} null`];
};
switch (field) {
// For created_at and updated_at, we also accept a date range as an array two unix timestamps.
// Note that dates in the database are saved with time zones.
// In such a case, the filter operation parmeter is ignored (>= and <= are used).
case 'created_at':
if (Array.isArray(filter.value) && filter.value.length === 2) {
return `${tblAlias}.created_at >= to_timestamp(${filter.value[0]}) AND ${tblAlias}.created_at <= to_timestamp(${filter.value[1]}) `;
return [
`${tblAlias}.created_at >= to_timestamp(${filter.value[0]}) AND ${tblAlias}.created_at <= to_timestamp(${filter.value[1]}) `
];
} else {
return `${tblAlias}.created_at ${
filter.op ? operatorSigns[filter.op] : operatorSigns.eq
} to_timestamp(${filter.value}) `;
return [
`${tblAlias}.created_at ${filter.op ? operatorSigns[filter.op] : operatorSigns.eq} to_timestamp(${
filter.value
}) `
];
}
case 'updated_at':
if (Array.isArray(filter.value) && filter.value.length === 2) {
return `${tblAlias}.updated_at >= to_timestamp(${filter.value[0]}) AND ${tblAlias}.updated_at <= to_timestamp(${filter.value[1]}) `;
return [
`${tblAlias}.updated_at >= to_timestamp(${filter.value[0]}) AND ${tblAlias}.updated_at <= to_timestamp(${filter.value[1]}) `
];
} else {
return `${tblAlias}.updated_at ${
filter.op ? operatorSigns[filter.op] : operatorSigns.eq
} to_timestamp(${filter.value}) `;
return [
`${tblAlias}.updated_at ${filter.op ? operatorSigns[filter.op] : operatorSigns.eq} to_timestamp(${
filter.value
}) `
];
}
case 'is_valid':
return getBooleanFilter(`${tblAlias}.is_valid`, filter);
case 'is_questionable':
return getBooleanFilter(`${tblAlias}.is_questionable`, filter);
case 'uuid':
return `${tblAlias}.${field} ${filter.op ? operatorSigns[filter.op] : operatorSigns.eq} '${filter.value}'`;
return [
`${tblAlias}.${field} ${filter.op ? operatorSigns[filter.op] : operatorSigns.eq} '${filter.value}'`
];
case 'audits': {
if (typeof filter.value !== 'string') {
return undefined;
if (typeof filter.value !== 'string' && !Array.isArray(filter.value)) {
return [undefined];
}
const match = filter.value.match(dotSeparatedStringRegex);
if (match === null) {
const values = typeof filter.value === 'string' ? [filter.value] : filter.value;
const matches = values.map((value) => value.match(dotSeparatedStringRegex));
if (matches.find((m) => m === null) !== undefined) {
throw new TrError(
`Invalid value for where clause in ${tableName} database`,
'DBQCR0006',
'DatabaseInvalidWhereClauseUserEntry'
);
}
// Add subquery to audits table
const auditSubQuery = knex('sv_audits').select('interview_id').distinct().where('error_code', filter.value);
return [`${tblAlias}.id in (${auditSubQuery.toSQL().sql})`, filter.value];
return values.map((value) => [
`${tblAlias}.id in (${
knex('sv_audits').select('interview_id').distinct().where('error_code', value).toSQL().sql
})`,
value
]);
}
}
const jsonObject = field.split('.');
Expand All @@ -449,17 +463,28 @@ const getRawWhereClause = (
if (prefix !== undefined) {
const field = jsonObject.slice(1).join('.');
return filter.value === null
? `${prefix}->>'${field}' ${filter.op === 'not' ? ' IS NOT NULL' : ' IS NULL'}`
: [
`${prefix}->>'${field}' ${
filter.op !== undefined
? `${operatorSigns[filter.op] || operatorSigns.eq} ?`
: `${operatorSigns.eq} ?`
}`,
addLikeBinding(filter.op, filter.value)
];
? [`${prefix}->>'${field}' ${filter.op === 'not' ? ' IS NOT NULL' : ' IS NULL'}`]
: Array.isArray(filter.value)
? filter.value.map((value) => [
`${prefix}->>'${field}' ${
filter.op !== undefined
? `${operatorSigns[filter.op] || operatorSigns.eq} ?`
: `${operatorSigns.eq} ?`
}`,
addLikeBinding(filter.op, value)
])
: [
[
`${prefix}->>'${field}' ${
filter.op !== undefined
? `${operatorSigns[filter.op] || operatorSigns.eq} ?`
: `${operatorSigns.eq} ?`
}`,
addLikeBinding(filter.op, filter.value)
]
];
}
return undefined;
return [undefined];
};

/**
Expand All @@ -475,19 +500,21 @@ const getRawWhereClause = (
* and the second element is the array of bindings
*/
const updateRawWhereClause = (
filters: { [key: string]: { value: string | boolean | number | null; op?: keyof OperatorSigns } },
filters: { [key: string]: ValueFilterType },
baseFilter: string
): [string, (string | boolean | number)[]] => {
let rawFilter = baseFilter;
const bindings: (string | number | boolean)[] = [];
Object.keys(filters).forEach((key) => {
const whereClause = getRawWhereClause(key, filters[key], 'i');
if (typeof whereClause === 'string') {
rawFilter += ` AND ${whereClause}`;
} else if (Array.isArray(whereClause)) {
rawFilter += ` AND ${whereClause[0]}`;
bindings.push(whereClause[1]);
}
const whereClauses = getRawWhereClause(key, filters[key], 'i');
whereClauses.forEach((whereClause) => {
if (typeof whereClause === 'string') {
rawFilter += ` AND ${whereClause}`;
} else if (Array.isArray(whereClause)) {
rawFilter += ` AND ${whereClause[0]}`;
bindings.push(whereClause[1]);
}
});
});
return [rawFilter, bindings];
};
Expand All @@ -506,7 +533,7 @@ const updateRawWhereClause = (
* corresponding to the query
*/
const getList = async <CustomSurvey, CustomHousehold, CustomHome, CustomPerson>(params: {
filters: { [key: string]: { value: string | boolean | number | null; op?: keyof OperatorSigns } };
filters: { [key: string]: ValueFilterType };
pageIndex: number;
pageSize: number;
sort?: (string | { field: string; order: 'asc' | 'desc' })[];
Expand Down Expand Up @@ -621,7 +648,7 @@ const getList = async <CustomSurvey, CustomHousehold, CustomHome, CustomPerson>(
* interviews
*/
const getValidationErrors = async (params: {
filters: { [key: string]: { value: string | boolean | number | null; op?: keyof OperatorSigns } };
filters: { [key: string]: ValueFilterType };
}): Promise<{ errors: { key: string; cnt: string }[] }> => {
try {
const baseRawFilter =
Expand Down Expand Up @@ -669,7 +696,7 @@ const getValidationErrors = async (params: {
* @returns An interview stream
*/
const getInterviewsStream = function (params: {
filters: { [key: string]: { value: string | boolean | number | null; op?: keyof OperatorSigns } };
filters: { [key: string]: ValueFilterType };
select?: {
includeAudits?: boolean;
responses?: 'none' | 'participant' | 'validated' | 'both' | 'validatedIfAvailable';
Expand Down
36 changes: 36 additions & 0 deletions packages/evolution-backend/src/services/interviews/__tests__/interviews.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,42 @@ describe('Get all matching', () => {
});
});

test('Filters: various filters', async() => {
// string audit
await Interviews.getAllMatching({
filter: { audits: 'myAudit' }
});
expect(interviewsQueries.getList).toHaveBeenCalledTimes(1);
expect(interviewsQueries.getList).toHaveBeenCalledWith({
filters: { audits: { value: 'myAudit' } },
pageIndex: 0,
pageSize: -1
});

// array of string audit
await Interviews.getAllMatching({
filter: { audits: ['myAudit1', 'myAudit2'] }
});
expect(interviewsQueries.getList).toHaveBeenCalledTimes(2);
expect(interviewsQueries.getList).toHaveBeenCalledWith({
filters: { audits: { value: ['myAudit1', 'myAudit2'] } },
pageIndex: 0,
pageSize: -1
});

// object filter
await Interviews.getAllMatching({
filter: { audits: { value: 'myAudit', op: 'like' } }
});
expect(interviewsQueries.getList).toHaveBeenCalledTimes(3);
expect(interviewsQueries.getList).toHaveBeenCalledWith({
filters: { audits: { value: 'myAudit', op: 'like' } },
pageIndex: 0,
pageSize: -1
});

});

});

describe('Get Validation errors', () => {
Expand Down
Loading

0 comments on commit 27c7bb9

Please sign in to comment.