From 9690d382822f470d66065b480abbf2b586f1cc98 Mon Sep 17 00:00:00 2001 From: Elsa Date: Mon, 10 Feb 2025 17:31:17 -0500 Subject: [PATCH 1/3] organizeOutputBy=Patient query parameter --- README.md | 7 ++- src/services/export.service.js | 28 +++++------- test/services/export.service.test.js | 64 ++++++++++++++-------------- test/util/exportToNDJson.test.js | 4 -- 4 files changed, 47 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 5933b1b..687bda0 100644 --- a/README.md +++ b/README.md @@ -158,15 +158,20 @@ Endpoint: `POST [fhir base]/bulkstatus/[client id]/kickoff-import` The server supports the following query parameters: +From the [2.0.0 ci-build version of the Bulk Data Access IG](https://build.fhir.org/ig/HL7/bulk-data/export.html#query-parameters): + - `_type`: Filters the response to only include resources of the specified resource type(s) - If omitted, system-level requests will return all resources supported by the server within the scope of the client authorization - For Patient- and Group-level requests, the [Patient Compartment](https://www.hl7.org/fhir/compartmentdefinition-patient.html) is used as a point of reference for filtering the resource types that are returned. - `_outputFormat`: The server supports the following formats: `application/fhir+ndjson`, `application/ndjson+fhir`, `application/ndjson`, `ndjson` - `_typeFilter`: Filters the response to only include resources that meet the criteria of the specified comma-delimited FHIR REST queries. Returns an error for queries specified by the client that are unsupported by the server. Supports queries on the ValueSets (`type:in`, `code:in`, etc.) of a given resource type. - `patient`: Only applicable to POST requests for group-level and patient-level requests. When provided, the server SHALL NOT return resources in the patient compartment definition belonging to patients outside the list. Can support multiple patient references in a single request. -- `_bySubject`: Only applicable for group-level and patient-level requests. Creates export results, separating resources into files based on what subject they are associated with (rather than based on type). The only `_bySubject` value supported is `Patient`. This will result in an ndjson file for each patient in the returned data. If the `_type` parameter is used in conjunction with this parameter, `Patient` must be one of the types included in the passed value list. - `_elements`: Filters the content of the responses to omit unlisted, non-mandatory elements from the resources returned. These elements should be provided in the form `[resource type].[element name]` (e.g., `Patient.id`) which only filters the contents of those specified resources or in the form `[element name]` (e.g., `id`) which filters the contents of all of the returned resources. +From the [2.0.0 ci-build version of the argo24 branch of the Bulk Data Access IG](https://build.fhir.org/ig/HL7/bulk-data/branches/argo24/export.html#query-parameters): + +- `organizeOutputBy`: Applicable for all types of export requests. Creates export results, separating resources into files based on what resourceType they are to be organized by. The only `organizeOutputBy` value supported currently is `Patient`. This will result in an ndjson file for each patient in the returned data. If the `_type` parameter is used in conjunction with this parameter, `Patient` must be one of the types included in the passed value lists. + #### `_elements` Query Parameter The server supports the optional and experimental query parameter `_elements` as defined by the Bulk Data Access IG (here)[https://build.fhir.org/ig/HL7/bulk-data/export.html#query-parameters]. The `_elements` parameter is a string of comma-delimited HL7® FHIR® Elements used to filter the returned resources to only include listed elements and mandatory elements. Mandatory elements are defined as elements in the StructureDefinition of a resource type which have a minimum cardinality of 1. Because this server provides json-formatted data, `resourceType` is also an implied mandatory element for all Resources. diff --git a/src/services/export.service.js b/src/services/export.service.js index f95e3da..e54abb8 100644 --- a/src/services/export.service.js +++ b/src/services/export.service.js @@ -21,14 +21,6 @@ const bulkExport = async (request, reply) => { }) ); } - if (parameters._bySubject) { - reply.code(400).send( - createOperationOutcome('The "_bySubject" parameter cannot be used in a system-level export request.', { - issueCode: 400, - severity: 'error' - }) - ); - } if (validateExportParams(parameters, reply)) { request.log.info('Base >>> $export'); const clientEntry = await addPendingBulkExportRequest(); @@ -44,7 +36,7 @@ const bulkExport = async (request, reply) => { typeFilter: request.query._typeFilter, systemLevelExport: true, elements: elements, - byPatient: parameters._bySubject === 'Patient' + byPatient: parameters.organizeOutputBy === 'Patient' }; await exportQueue.createJob(job).save(); reply.code(202).header('Content-location', `${process.env.BULK_BASE_URL}/bulkstatus/${clientEntry}`).send(); @@ -74,7 +66,7 @@ const patientBulkExport = async (request, reply) => { await validatePatientReferences(parameters.patient, reply); } request.log.info('Patient >>> $export'); - const clientEntry = await addPendingBulkExportRequest(parameters._bySubject === 'Patient'); + const clientEntry = await addPendingBulkExportRequest(parameters.organizeOutputBy === 'Patient'); let types = request.query._type?.split(',') || parameters._type?.split(','); if (types) { @@ -93,7 +85,7 @@ const patientBulkExport = async (request, reply) => { patient: parameters.patient, systemLevelExport: false, elements: elements, - byPatient: parameters._bySubject === 'Patient' + byPatient: parameters.organizeOutputBy === 'Patient' }; await exportQueue.createJob(job).save(); reply.code(202).header('Content-location', `${process.env.BULK_BASE_URL}/bulkstatus/${clientEntry}`).send(); @@ -132,7 +124,7 @@ const groupBulkExport = async (request, reply) => { return splitRef[splitRef.length - 1]; }); - const clientEntry = await addPendingBulkExportRequest(parameters._bySubject === 'Patient'); + const clientEntry = await addPendingBulkExportRequest(parameters.organizeOutputBy === 'Patient'); let types = request.query._type?.split(',') || parameters._type?.split(','); if (types) { types = filterPatientResourceTypes(request, reply, types); @@ -151,7 +143,7 @@ const groupBulkExport = async (request, reply) => { systemLevelExport: false, patientIds: patientIds, elements: elements, - byPatient: parameters._bySubject === 'Patient' + byPatient: parameters.organizeOutputBy === 'Patient' }; await exportQueue.createJob(job).save(); reply.code(202).header('Content-location', `${process.env.BULK_BASE_URL}/bulkstatus/${clientEntry}`).send(); @@ -183,9 +175,9 @@ function validateExportParams(parameters, reply) { } } - if (parameters._bySubject && parameters._bySubject !== 'Patient') { + if (parameters.organizeOutputBy && parameters.organizeOutputBy !== 'Patient') { reply.code(400).send( - createOperationOutcome(`Server does not support the _bySubject parameter for values other than Patient.`, { + createOperationOutcome(`Server does not support the organizeOutputBy parameter for values other than Patient.`, { issueCode: 400, severity: 'error' }) @@ -215,12 +207,12 @@ function validateExportParams(parameters, reply) { ); return false; } - if (parameters._bySubject === 'Patient' && !requestTypes.includes('Patient')) { + if (parameters.organizeOutputBy === 'Patient' && !requestTypes.includes('Patient')) { reply .code(400) .send( createOperationOutcome( - `When _type is specified with _bySubject Patient, the Patient type must be included in the _type parameter.`, + `When _type is specified with organizeOutputBy Patient, the Patient type must be included in the _type parameter.`, { issueCode: 400, severity: 'error' } ) ); @@ -314,7 +306,7 @@ function validateExportParams(parameters, reply) { let unrecognizedParams = []; Object.keys(parameters).forEach(param => { - if (!['_outputFormat', '_type', '_typeFilter', 'patient', '_elements', '_bySubject'].includes(param)) { + if (!['_outputFormat', '_type', '_typeFilter', 'patient', '_elements', 'organizeOutputBy'].includes(param)) { unrecognizedParams.push(param); } }); diff --git a/test/services/export.service.test.js b/test/services/export.service.test.js index 2e2a567..9231709 100644 --- a/test/services/export.service.test.js +++ b/test/services/export.service.test.js @@ -293,28 +293,6 @@ describe('Check barebones bulk export logic (failure)', () => { }); }); - test('throws 400 error when "_bySubject" parameter used in system-level export', async () => { - await supertest(app.server) - .post('/$export') - .send({ - resourceType: 'Parameters', - parameter: [ - { - name: '_bySubject', - valueString: 'test' - } - ] - }) - .expect(400) - .then(response => { - expect(response.body.resourceType).toEqual('OperationOutcome'); - expect(response.body.issue[0].code).toEqual(400); - expect(response.body.issue[0].details.text).toEqual( - 'The "_bySubject" parameter cannot be used in a system-level export request.' - ); - }); - }); - test('throws 400 error when POST request body is not of resourceType "Parameters"', async () => { await supertest(app.server) .post('/$export') @@ -684,13 +662,33 @@ describe('Check group-level export logic (failure)', () => { }); }); -describe('Check by subject export logic', () => { +describe('Check organizeOutputBy=Patient export logic', () => { beforeEach(async () => { await bulkStatusSetup(); await app.ready(); }); - test('check 202 for Patient bySubject', async () => { + test('check 202 for system-level organizeOutputBy=Patient', async () => { + const createJobSpy = jest.spyOn(queue, 'createJob'); + await supertest(app.server) + .post('/$export') + .send({ + resourceType: 'Parameters', + parameter: [ + { + name: 'organizeOutputBy', + valueString: 'Patient' + } + ] + }) + .expect(202) + .then(response => { + expect(response.headers['content-location']).toBeDefined(); + expect(createJobSpy).toHaveBeenCalled(); + }); + }); + + test('check 202 for Patient organizeOutputBy=Patient', async () => { const createJobSpy = jest.spyOn(queue, 'createJob'); await supertest(app.server) .post('/Patient/$export') @@ -698,7 +696,7 @@ describe('Check by subject export logic', () => { resourceType: 'Parameters', parameter: [ { - name: '_bySubject', + name: 'organizeOutputBy', valueString: 'Patient' } ] @@ -710,7 +708,7 @@ describe('Check by subject export logic', () => { }); }); - test('check 202 for Group bySubject', async () => { + test('check 202 for Group organizeOutputBy=Patient', async () => { await createTestResource(testGroup, 'Group'); const createJobSpy = jest.spyOn(queue, 'createJob'); await supertest(app.server) @@ -719,7 +717,7 @@ describe('Check by subject export logic', () => { resourceType: 'Parameters', parameter: [ { - name: '_bySubject', + name: 'organizeOutputBy', valueString: 'Patient' } ] @@ -731,28 +729,28 @@ describe('Check by subject export logic', () => { }); }); - test('returns 400 for a _bySubject call that specifies a subject other than Patient', async () => { + test('returns 400 for a organizeOutputBy call that specifies a resource type other than Patient', async () => { await supertest(app.server) - .get('/Patient/$export?_bySubject=Other') + .get('/Patient/$export?organizeOutputBy=Other') .expect(400) .then(response => { expect(response.body.resourceType).toEqual('OperationOutcome'); expect(response.body.issue[0].code).toEqual(400); expect(response.body.issue[0].details.text).toEqual( - 'Server does not support the _bySubject parameter for values other than Patient.' + 'Server does not support the organizeOutputBy parameter for values other than Patient.' ); }); }); - test('returns 400 for a _bySubject call where _type does not include Patient', async () => { + test('returns 400 for a organizeOutputBy call where _type does not include Patient', async () => { await supertest(app.server) - .get('/Patient/$export?_bySubject=Patient&_type=Condition') + .get('/Patient/$export?organizeOutputBy=Patient&_type=Condition') .expect(400) .then(response => { expect(response.body.resourceType).toEqual('OperationOutcome'); expect(response.body.issue[0].code).toEqual(400); expect(response.body.issue[0].details.text).toEqual( - 'When _type is specified with _bySubject Patient, the Patient type must be included in the _type parameter.' + 'When _type is specified with organizeOutputBy Patient, the Patient type must be included in the _type parameter.' ); }); }); diff --git a/test/util/exportToNDJson.test.js b/test/util/exportToNDJson.test.js index cdd2191..f772419 100644 --- a/test/util/exportToNDJson.test.js +++ b/test/util/exportToNDJson.test.js @@ -101,10 +101,6 @@ describe('check export logic', () => { await exportToNDJson({ clientEntry: clientId, type: mockType, typeFilter: typeFilterWOValueSet }); expect(fs.existsSync(expectedFileNameWOValueSet)).toBe(false); }); - test('Expect folder created and export successful when _bySubject parameter is retrieved from request', async () => { - await exportToNDJson({ clientEntry: clientId, types: mockType, typeFilter: mockTypeFilter, byPatient: true }); - expect(fs.existsSync('./tmp/123456/testPatient.ndjson')).toBe(true); - }); }); describe('patientsQueryForType', () => { From aeefb0ad85d0b1a904133c095f886a92d957e544 Mon Sep 17 00:00:00 2001 From: Elsa Date: Wed, 12 Feb 2025 13:24:14 -0500 Subject: [PATCH 2/3] Bring back test relevant to organizeOutputBy --- test/util/exportToNDJson.test.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/util/exportToNDJson.test.js b/test/util/exportToNDJson.test.js index f772419..16a34d5 100644 --- a/test/util/exportToNDJson.test.js +++ b/test/util/exportToNDJson.test.js @@ -101,6 +101,10 @@ describe('check export logic', () => { await exportToNDJson({ clientEntry: clientId, type: mockType, typeFilter: typeFilterWOValueSet }); expect(fs.existsSync(expectedFileNameWOValueSet)).toBe(false); }); + test('Expect folder created and export successful when organizeOutputBy=Patient parameter is retrieved from request', async () => { + await exportToNDJson({ clientEntry: clientId, types: mockType, typeFilter: mockTypeFilter, byPatient: true }); + expect(fs.existsSync('./tmp/123456/testPatient.ndjson')).toBe(true); + }); }); describe('patientsQueryForType', () => { From dc242decfac3962cff9d4f51d9d792b37938d241 Mon Sep 17 00:00:00 2001 From: Elsa Date: Wed, 12 Feb 2025 16:11:50 -0500 Subject: [PATCH 3/3] Allow system level export for real this time --- README.md | 2 +- src/services/export.service.js | 13 +++++++++++-- src/util/exportToNDJson.js | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 687bda0..a0ac137 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ From the [2.0.0 ci-build version of the Bulk Data Access IG](https://build.fhir. From the [2.0.0 ci-build version of the argo24 branch of the Bulk Data Access IG](https://build.fhir.org/ig/HL7/bulk-data/branches/argo24/export.html#query-parameters): -- `organizeOutputBy`: Applicable for all types of export requests. Creates export results, separating resources into files based on what resourceType they are to be organized by. The only `organizeOutputBy` value supported currently is `Patient`. This will result in an ndjson file for each patient in the returned data. If the `_type` parameter is used in conjunction with this parameter, `Patient` must be one of the types included in the passed value lists. +- `organizeOutputBy`: Applicable for all types of export requests. Creates export results, separating resources into files based on what resourceType they are to be organized by. The only `organizeOutputBy` value supported currently is `Patient`. This will result in an ndjson file for each patient in the returned data. If the `_type` parameter is used in conjunction with this parameter, `Patient` must be one of the types included in the passed value lists. Note: in this server's implementation, resources that would otherwise be included in the export, but do not have references to resource type `Patient` are omitted from the export, following guidance from the IG's [Bulk Data Output File Organization](https://build.fhir.org/ig/HL7/bulk-data/branches/argo24/export.html#bulk-data-output-file-organization). #### `_elements` Query Parameter diff --git a/src/services/export.service.js b/src/services/export.service.js index e54abb8..bb838dd 100644 --- a/src/services/export.service.js +++ b/src/services/export.service.js @@ -23,9 +23,18 @@ const bulkExport = async (request, reply) => { } if (validateExportParams(parameters, reply)) { request.log.info('Base >>> $export'); - const clientEntry = await addPendingBulkExportRequest(); + const clientEntry = await addPendingBulkExportRequest(parameters.organizeOutputBy === 'Patient'); - const types = parameters._type?.split(','); + let types = request.query._type?.split(',') || parameters._type?.split(','); + // if parameters.organizeOutputBy=Patient, then we want to pre filter the types that could + // have patient references like we do for Patient level export + if (parameters.organizeOutputBy === 'Patient') { + if (types) { + types = filterPatientResourceTypes(request, reply, types); + } else { + types = patientResourceTypes; + } + } const elements = parameters._elements?.split(','); diff --git a/src/util/exportToNDJson.js b/src/util/exportToNDJson.js index 683acd4..87855aa 100644 --- a/src/util/exportToNDJson.js +++ b/src/util/exportToNDJson.js @@ -299,7 +299,7 @@ const getDocuments = async (collectionName, searchParameterQueries, valueSetQuer let vsQuery = {}; // Create patient id query (for Group export only) if (patientIds) { - if (patientIds.length == 0) { + if (patientIds.length === 0) { // if no patients in group, return no documents return { document: [], collectionName: collectionName }; }