From 338d293389875989373773cca7957c94bd727ef7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 19 Jun 2024 16:55:54 +1000 Subject: [PATCH 01/14] Fix type of gender column within ShareablePatientDemographics example --- input/fsh/examples.fsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input/fsh/examples.fsh b/input/fsh/examples.fsh index b3c7085..5323ea4 100644 --- a/input/fsh/examples.fsh +++ b/input/fsh/examples.fsh @@ -47,7 +47,7 @@ Usage: #example * column[+] * path = "gender" * name = "gender" - * type = "string" + * type = "code" * select[+] * forEach = "name.where(use = 'official').first()" * column[+] From 05d4d79b5c0ae99bc46691d4a1f82bcdc44440a3 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 19 Jun 2024 17:01:27 +1000 Subject: [PATCH 02/14] Fix grammar within example notes --- input/pagecontent/Binary-PatientAddresses-notes.md | 4 ++-- .../Binary-PatientAndContactAddressUnion-notes.md | 6 ++++-- input/pagecontent/Binary-PatientDemographics-notes.md | 4 ++-- input/pagecontent/Binary-UsCoreBloodPressures-notes.md | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/input/pagecontent/Binary-PatientAddresses-notes.md b/input/pagecontent/Binary-PatientAddresses-notes.md index c17cc2f..011c93a 100644 --- a/input/pagecontent/Binary-PatientAddresses-notes.md +++ b/input/pagecontent/Binary-PatientAddresses-notes.md @@ -1,4 +1,4 @@ -This will result in "patient_addresses" table as: +This will result in a "patient_addresses" table that looks like this: | patient_id | street | use | city | zip | |------------|--------------------------|----------|-------------|-------| @@ -7,4 +7,4 @@ This will result in "patient_addresses" table as: | 2 | 789 Brookside Ave\nApt 3 | home | Los Angeles | 90001 | | 3 | 987 Pinehurst Rd\nApt 4 | home | Chicago | 60601 | | 3 | 654 Evergreen Tce\nApt 5 | work | Houston | 77001 | -{:.table-data} \ No newline at end of file +{:.table-data} diff --git a/input/pagecontent/Binary-PatientAndContactAddressUnion-notes.md b/input/pagecontent/Binary-PatientAndContactAddressUnion-notes.md index 0214574..5e4076a 100644 --- a/input/pagecontent/Binary-PatientAndContactAddressUnion-notes.md +++ b/input/pagecontent/Binary-PatientAndContactAddressUnion-notes.md @@ -1,4 +1,5 @@ -This will result in "patient_and_contact_addresses" table as: +This will result in a "patient_and_contact_addresses" table that looks like +this: | resource_id | street | city | zip | is_patient | |-------------|--------------------------|-------------|-------|------------| @@ -7,4 +8,5 @@ This will result in "patient_and_contact_addresses" table as: | 1 | 456 Maplewood Dve\nApt 2 | New York | 10001 | false | | 2 | 789 Brookside Ave\nApt 3 | Los Angeles | 90001 | true | | 2 | 987 Pinehurst Rd\nApt 4 | Chicago | 60601 | false | -{:.table-data} \ No newline at end of file + +{:.table-data} diff --git a/input/pagecontent/Binary-PatientDemographics-notes.md b/input/pagecontent/Binary-PatientDemographics-notes.md index ceac422..d028b89 100644 --- a/input/pagecontent/Binary-PatientDemographics-notes.md +++ b/input/pagecontent/Binary-PatientDemographics-notes.md @@ -1,8 +1,8 @@ -This will result in a "patient_demographics" table as: +This will result in a "patient_demographics" table that looks like this: | id | gender | given_name | family_name | |----|--------|---------------|-------------| | 1 | female | Malvina Gerda | Vicario | | 2 | male | Yolotzin Adel | Bristow | | 3 | other | Jin Gomer | Aarens | -{:.table-data} \ No newline at end of file +{:.table-data} diff --git a/input/pagecontent/Binary-UsCoreBloodPressures-notes.md b/input/pagecontent/Binary-UsCoreBloodPressures-notes.md index ab0d999..57bf89b 100644 --- a/input/pagecontent/Binary-UsCoreBloodPressures-notes.md +++ b/input/pagecontent/Binary-UsCoreBloodPressures-notes.md @@ -1,4 +1,4 @@ -This will result in a "us_core_blood_pressures" table as: +This will result in a "us_core_blood_pressures" table that looks like this: | id | patient_id | effective_date_time | sbp_quantity_system | sbp_quantity_code | sbp_quantity_unit | sbp_quantity_value | dbp_quantity_system | dbp_quantity_code | dbp_quantity_unit | dbp_quantity_value | |----|------------|---------------------|---------------------------|-------------------|-------------------|--------------------|---------------------------|-------------------|-------------------|--------------------| From da444fcd82f12867c2c94572e2d11333a6d1c85a Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 19 Jun 2024 17:04:46 +1000 Subject: [PATCH 03/14] Update warning ignore message for owning committee --- input/ignoreWarnings.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/input/ignoreWarnings.txt b/input/ignoreWarnings.txt index c200429..3e855ff 100644 --- a/input/ignoreWarnings.txt +++ b/input/ignoreWarnings.txt @@ -7,5 +7,5 @@ # Publisher bug? Error in constraint 'sql-expressions' with expression '(path | forEach | forEachOrNull | union).count() = 1': Cannot assign field "sourceDefinition" because "res" is null -# Not a real HL7 project -When HL7 is publishing a resource, the owning committee must be stated using the http://hl7.org/fhir/StructureDefinition/structuredefinition-wg extension \ No newline at end of file +# Not an official HL7 project +When HL7 is publishing a resource, the owning committee must be stated using the http://hl7.org/fhir/StructureDefinition/structuredefinition-wg extension From 2d1bb00202e9e4bae1e69804167a0f70841de61e Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 18 Jun 2024 05:21:42 +1000 Subject: [PATCH 04/14] Add test-results.json to gitignore --- test_report/.gitignore | 1 + test_report/public/test-results.json | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 test_report/public/test-results.json diff --git a/test_report/.gitignore b/test_report/.gitignore index 4c94aa2..0e91b56 100644 --- a/test_report/.gitignore +++ b/test_report/.gitignore @@ -3,5 +3,6 @@ .DS_Store public/tests.json +public/test-results.json bun.lockb package-lock.json diff --git a/test_report/public/test-results.json b/test_report/public/test-results.json deleted file mode 100644 index 9a9a215..0000000 --- a/test_report/public/test-results.json +++ /dev/null @@ -1 +0,0 @@ -{"foreach.json":{"tests":[]},"fn_join.json":{"tests":[]},"where.json":{"tests":[]},"basic.json":{"tests":[]},"collection.json":{"tests":[{"result":{"passed":true}}]},"combinations.json":{"tests":[]},"fn_first.json":{"tests":[]},"fhirpath_numbers.json":{"tests":[]},"fn_empty.json":{"tests":[]},"fn_reference_keys.json":{"tests":[]},"fhirpath.json":{"tests":[]},"fn_boundary.json":{"tests":[]},"fn_extension.json":{"tests":[]},"logic.json":{"tests":[]},"constant_types.json":{"tests":[{"result":{"passed":true,"expected":[{"id":"d1","aidc":true},{"id":"d2","aidc":false},{"id":"d3","aidc":null}],"actual":[{"id":"d1","aidc":true},{"id":"d2","aidc":false},{"id":"d3","aidc":null}]}},{"result":{"passed":true,"expected":[{"id":"pt1","bool":true},{"id":"pt2","bool":false},{"id":"pt3","bool":null}],"actual":[{"id":"pt1","bool":true},{"id":"pt2","bool":false},{"id":"pt3","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"pt1","bool":true},{"id":"pt2","bool":false},{"id":"pt3","bool":null}],"actual":[{"id":"pt1","bool":true},{"id":"pt2","bool":false},{"id":"pt3","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"di1","bool":false},{"id":"di2","bool":true},{"id":"di3","bool":null}],"actual":[{"id":"di1","bool":false},{"id":"di2","bool":true},{"id":"di3","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"o1","bool":true},{"id":"o2","bool":false},{"id":"o3","bool":null},{"id":"o4","bool":null},{"id":"o5","bool":null}],"actual":[{"id":"o1","bool":true},{"id":"o2","bool":false},{"id":"o3","bool":null},{"id":"o4","bool":null},{"id":"o5","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"t1","bool":null},{"id":"t2","bool":null},{"id":"t3","bool":null},{"id":"t4","bool":null},{"id":"t5","bool":null},{"id":"t6","bool":null},{"id":"t7","bool":null},{"id":"t8","bool":true},{"id":"t9","bool":false}],"actual":[{"id":"t1","bool":null},{"id":"t2","bool":null},{"id":"t3","bool":null},{"id":"t4","bool":null},{"id":"t5","bool":null},{"id":"t6","bool":null},{"id":"t7","bool":null},{"id":"t8","bool":true},{"id":"t9","bool":false}]}},{"result":{"passed":true,"expected":[{"id":"o1","bool":true},{"id":"o2","bool":false},{"id":"o3","bool":null},{"id":"o4","bool":null},{"id":"o5","bool":null}],"actual":[{"id":"o1","bool":true},{"id":"o2","bool":false},{"id":"o3","bool":null},{"id":"o4","bool":null},{"id":"o5","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"t1","bool":null},{"id":"t2","bool":null},{"id":"t3","bool":null},{"id":"t4","bool":true},{"id":"t5","bool":false},{"id":"t6","bool":null},{"id":"t7","bool":null},{"id":"t8","bool":null},{"id":"t9","bool":null}],"actual":[{"id":"t1","bool":null},{"id":"t2","bool":null},{"id":"t3","bool":null},{"id":"t4","bool":true},{"id":"t5","bool":false},{"id":"t6","bool":null},{"id":"t7","bool":null},{"id":"t8","bool":null},{"id":"t9","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"cr1","bool":true},{"id":"cr2","bool":false},{"id":"cr3","bool":null}],"actual":[{"id":"cr1","bool":true},{"id":"cr2","bool":false},{"id":"cr3","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"o1","bool":null},{"id":"o2","bool":null},{"id":"o3","bool":null},{"id":"o4","bool":true},{"id":"o5","bool":false}],"actual":[{"id":"o1","bool":null},{"id":"o2","bool":null},{"id":"o3","bool":null},{"id":"o4","bool":true},{"id":"o5","bool":false}]}},{"result":{"passed":true,"expected":[{"id":"is1","bool":true},{"id":"is2","bool":false},{"id":"is3","bool":null}],"actual":[{"id":"is1","bool":true},{"id":"is2","bool":false},{"id":"is3","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"m1","bool":true},{"id":"m2","bool":false},{"id":"m3","bool":null}],"actual":[{"id":"m1","bool":true},{"id":"m2","bool":false},{"id":"m3","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"t1","bool":true},{"id":"t2","bool":false},{"id":"t3","bool":null},{"id":"t4","bool":null},{"id":"t5","bool":null},{"id":"t6","bool":null},{"id":"t7","bool":null},{"id":"t8","bool":null},{"id":"t9","bool":null}],"actual":[{"id":"t1","bool":true},{"id":"t2","bool":false},{"id":"t3","bool":null},{"id":"t4","bool":null},{"id":"t5","bool":null},{"id":"t6","bool":null},{"id":"t7","bool":null},{"id":"t8","bool":null},{"id":"t9","bool":null}]}},{"result":{"passed":true,"expected":[{"id":"t1","bool":null},{"id":"t2","bool":null},{"id":"t3","bool":null},{"id":"t4","bool":null},{"id":"t5","bool":null},{"id":"t6","bool":true},{"id":"t7","bool":false},{"id":"t8","bool":null},{"id":"t9","bool":null}],"actual":[{"id":"t1","bool":null},{"id":"t2","bool":null},{"id":"t3","bool":null},{"id":"t4","bool":null},{"id":"t5","bool":null},{"id":"t6","bool":true},{"id":"t7","bool":false},{"id":"t8","bool":null},{"id":"t9","bool":null}]}}]},"view_resource.json":{"tests":[{"result":{"passed":true}}]},"fn_oftype.json":{"tests":[]},"validate.json":{"tests":[{"result":{"passed":true}},{"result":{"passed":true}},{"result":{"passed":true}},{"result":{"passed":true}},{"result":{"passed":true}}]},"constant.json":{"tests":[{"result":{"passed":true}},{"result":{"passed":true}}]},"union.json":{"tests":[{"result":{"passed":true}},{"result":{"passed":true}}]}} \ No newline at end of file From 45e53de1ce3e996aa878b55edeb369694cd2a207 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 19 Jun 2024 17:12:02 +1000 Subject: [PATCH 05/14] Add result for ShareablePatientDemographics example --- .../Binary-ShareablePatientDemographics-notes.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 input/pagecontent/Binary-ShareablePatientDemographics-notes.md diff --git a/input/pagecontent/Binary-ShareablePatientDemographics-notes.md b/input/pagecontent/Binary-ShareablePatientDemographics-notes.md new file mode 100644 index 0000000..d028b89 --- /dev/null +++ b/input/pagecontent/Binary-ShareablePatientDemographics-notes.md @@ -0,0 +1,8 @@ +This will result in a "patient_demographics" table that looks like this: + +| id | gender | given_name | family_name | +|----|--------|---------------|-------------| +| 1 | female | Malvina Gerda | Vicario | +| 2 | male | Yolotzin Adel | Bristow | +| 3 | other | Jin Gomer | Aarens | +{:.table-data} From 1e03c151e1a42f263eb72f1abd0862849d8033c9 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 19 Jun 2024 17:23:19 +1000 Subject: [PATCH 06/14] Format index.md using Prettier --- input/pagecontent/index.md | 154 ++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 64 deletions(-) diff --git a/input/pagecontent/index.md b/input/pagecontent/index.md index 576ddfb..3c41349 100644 --- a/input/pagecontent/index.md +++ b/input/pagecontent/index.md @@ -1,17 +1,21 @@ -_This is an evolution of the original "SQL on FHIR" draft, which can -[still be found here](https://github.com/FHIR/sql-on-fhir-archived)._ +_This is an evolution of the original "SQL on FHIR" draft, which +can [still be found here](https://github.com/FHIR/sql-on-fhir-archived)._ -### Intro -The [FHIR®](https://hl7.org/fhir) standard is a great fit for RESTful and JSON-based -systems, helping make healthcare data liquidity real. This spec aims to take FHIR usage a step -futher, making FHIR work well with familiar and efficient SQL engines and surrounding ecosystems. +### Introduction -We do this by creating simple, tabular *views* of the underlying FHIR data that are tailored -to specific needs. Views are defined with [FHIRPath](https://hl7.org/fhirpath/) expressions in -a logical structure to specify things like column names and unnested items. +The [FHIR®](https://hl7.org/fhir) standard is a great fit for RESTful and +JSON-based systems, helping make healthcare data liquidity real. This spec aims +to take FHIR usage a step futher, making FHIR work well with familiar and +efficient SQL engines and surrounding ecosystems. -Let's start with a simple example, defining a "patient_demographics" view with the following -[ViewDefinition](StructureDefinition-ViewDefinition.html) structure: +We do this by creating simple, tabular *views* of the underlying FHIR data that +are tailored to specific needs. Views are defined +with [FHIRPath](https://hl7.org/fhirpath/) expressions in a logical structure to +specify things like column names and unnested items. + +Let's start with a simple example, defining a "patient_demographics" view with +the following [ViewDefinition](StructureDefinition-ViewDefinition.html) +structure: ```js { @@ -49,62 +53,74 @@ Let's start with a simple example, defining a "patient_demographics" view with t } ``` -This will result in a "patient_demographics" table that looks like this. The table can be persisted and queried -in your database of choice, using the view name as the table name: +This will result in a "patient_demographics" table that looks like this. The +table can be persisted and queried in your database of choice, using the view +name as the table name: | id | gender | given_name | family_name | |----|--------|---------------|-------------| | 1 | female | Malvina Gerda | Vicario | | 2 | male | Yolotzin Adel | Bristow | | 3 | other | Jin Gomer | Aarens | + {:.table-data} -Such tabular views can be created for any FHIR resource, with -[more examples here](artifacts.html#example-example-instances). See the -[View Definition page](StructureDefinition-ViewDefinition.html) for the full definition -of the above structure. +Such tabular views can be created for any FHIR resource, +with [more examples here](artifacts.html#example-example-instances). See +the [View Definition page](StructureDefinition-ViewDefinition.html) for the full +definition of the above structure. ### View Definition non-goals -View Definitions are intentionally constrained to a narrow set of functionality to make them easily -and broadly implementable, while deferring higher-level capabilities to database engines or processing -pipelines that solve those problems well. Therefore it's important to know what View Definitions -do *not* do, by design: + +View Definitions are intentionally constrained to a narrow set of functionality +to make them easily and broadly implementable, while deferring higher-level +capabilities to database engines or processing pipelines that solve those +problems well. Therefore it's important to know what View Definitions do *not* +do, by design: #### A single View Definition will not join different resources in any way -A single View Definition defines a tabular view of exactly one resource type, like a view of `Patient` -or a view of `Condition` resources. Any joins between resources are exclusively in downstream systems, -like between database tables computed by view definitions. This makes it possible for a wide set of -FHIR infrastructure to implement this spec, and lets database engines or processing pipelines join as -needed. + +A single View Definition defines a tabular view of exactly one resource type, +like a view of `Patient` or a view of `Condition` resources. Any joins between +resources are exclusively in downstream systems, like between database tables +computed by view definitions. This makes it possible for a wide set of FHIR +infrastructure to implement this spec, and lets database engines or processing +pipelines join as needed. #### View Definitions do not have sorting, aggregation, or limit capabilities -View Definitions define only the logical schema of views, and therefore defer sorting, aggergation or limit -operations to engines, along with cross-view joins. *View Runners* (described below) or future FHIR -server operations may accept limits or sort columns as part of their operations, so users at runtime can -specify what they need dynamically and independently of the definition of the view itself. + +View Definitions define only the logical schema of views, and therefore defer +sorting, aggergation or limit operations to engines, along with cross-view +joins. *View Runners* (described below) or future FHIR server operations may +accept limits or sort columns as part of their operations, so users at runtime +can specify what they need dynamically and independently of the definition of +the view itself. #### View Definitions are not aware of output formats -View Definitions themselves are independent of any tech stack and therefore unaware of the output format. -*View Runners* are the component that applies definitions to a particular stack, producing output like -a database table, Parquet file, CSV, or another format specific to the runner. +View Definitions themselves are independent of any tech stack and therefore +unaware of the output format. *View Runners* are the component that applies +definitions to a particular stack, producing output like a database table, +Parquet file, CSV, or another format specific to the runner. ### System Layers -The [View Definition](StructureDefinition-ViewDefinition.html) is the central element of this -spec, but in practice it is really one layer of an overall system. The layers are: +The [View Definition](StructureDefinition-ViewDefinition.html) is the central +element of this spec, but in practice it is really one layer of an overall +system. The layers are: - the *Data Layer* - the *View Layer* -- and the *Analytics Layer*. +- and the *Analytics Layer*. High-level diagram of layers **Figure 1: High-level diagram of layers** ### The Data Layer -The *Data Layer* is a set of lossless representations that collectively enable FHIR -to be used with a wide variety of different query technologies. It may + +The *Data Layer* is a set of lossless representations that collectively enable +FHIR to be used with a wide variety of different query technologies. It may optionally be persisted and annotated to make it or implementations of the view layer more efficient, but no specific Data Layer structure will be required by this specification. @@ -115,54 +131,64 @@ primarily applies when the underlying FHIR resources are stored in databases that the View layer will query. ### The View Layer -The *View Layer* defines portable, tabular views of FHIR data that can more easily -be consumed by a wide variety of analytic tools. The use of these tools is -described in *Analytics Layer* section. Our goal here is simply to get -the needed FHIR data in a form that matches user needs and common analytic -patterns. + +The *View Layer* defines portable, tabular views of FHIR data that can more +easily be consumed by a wide variety of analytic tools. The use of these tools +is described in *Analytics Layer* section. Our goal here is simply to get the +needed FHIR data in a form that matches user needs and common analytic patterns. The View Layer itself has two key components: * *View Definitions*, allowing users to define flattened views of FHIR data that -are portable between systems. -* *View Runners* are system-specific tools or libraries that apply view definitions to -the underlying data layer, optionally making use of annotations to optimize performance. + are portable between systems. +* *View Runners* are system-specific tools or libraries that apply view + definitions to + the underlying data layer, optionally making use of annotations to optimize + performance. -See the [View Definition documentation](StructureDefinition-ViewDefinition.html) for details and examples; -these are the central piece of this specification. +See the [View Definition documentation](StructureDefinition-ViewDefinition.html) +for details and examples; these are the central piece of this specification. -View Runners will be specific to the data -layer they use. Each data layer may have one or more corresponding view -runners, but a given View Definition can be run by many runners over many -data layers. +View Runners will be specific to the data layer they use. Each data layer may +have one or more corresponding view runners, but a given View Definition can be +run by many runners over many data layers. Example view runners may include: * A runner that creates a virtual, tabular view in an analytic database -* A runner that queries FHIR JSON directly and creates a table in a web application +* A runner that queries FHIR JSON directly and creates a table in a web + application * A runner that loads data directly into a notebook or other data analysis tool #### Generating Schemas -The output of many runners will have technology-specific schemas, such as database table -definitions or schema for structured files like Parquet. This will be runner- and technology- -specific, but runner implementaitons SHOULD offer a way to compute that schema from a ViewDefinition -when applicable. -For example, a runner that produces a table in a database system could return a "CREATE TABLE" or -"CREATE VIEW" statement based on the ViewDefinition, allowing the system to define tables prior to -populating them by evaluating the views over data. +The output of many runners will have technology-specific schemas, such as +database table definitions or schema for structured files like Parquet. This +will be runner- and technology- specific, but runner implementaitons SHOULD +offer a way to compute that schema from a ViewDefinition when applicable. -This would not apply to outputs that do not have common a schema specification, like CSV files. +For example, a runner that produces a table in a database system could return +a "CREATE TABLE" or "CREATE VIEW" statement based on the ViewDefinition, +allowing the system to define tables prior to populating them by evaluating the +views over data. + +This would not apply to outputs that do not have common a schema specification, +like CSV files. ### The Analytics Layer Finally, users must be able to easily leverage the above views with the analytic tools of their choice. This spec purposefully does not define what these are, -but common use cases may be SQL queries by consuming applications, dataframe-based -data science tools in Python or R, or integration with business intelligence tools. +but common use cases may be SQL queries by consuming applications, +dataframe-based data science tools in Python or R, or integration with business +intelligence tools. ### License -FHIR® is the registered trademark of HL7 and is used with the permission of HL7. Use of the FHIR trademark does not constitute endorsement of the contents of this repository by HL7, nor affirmation that this data is conformant to the various applicable standards + +FHIR® is the registered trademark of HL7 and is used with the permission of HL7. +Use of the FHIR trademark does not constitute endorsement of the contents of +this repository by HL7, nor affirmation that this data is conformant to the +various applicable standards. --- From 15e974f2826631eb7c82873e601b5e769275b0f5 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 19 Jun 2024 17:25:30 +1000 Subject: [PATCH 07/14] Fix typos on index.md --- input/pagecontent/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/input/pagecontent/index.md b/input/pagecontent/index.md index 3c41349..440888e 100644 --- a/input/pagecontent/index.md +++ b/input/pagecontent/index.md @@ -5,7 +5,7 @@ can [still be found here](https://github.com/FHIR/sql-on-fhir-archived)._ The [FHIR®](https://hl7.org/fhir) standard is a great fit for RESTful and JSON-based systems, helping make healthcare data liquidity real. This spec aims -to take FHIR usage a step futher, making FHIR work well with familiar and +to take FHIR usage a step further, making FHIR work well with familiar and efficient SQL engines and surrounding ecosystems. We do this by creating simple, tabular *views* of the underlying FHIR data that @@ -164,7 +164,7 @@ Example view runners may include: The output of many runners will have technology-specific schemas, such as database table definitions or schema for structured files like Parquet. This -will be runner- and technology- specific, but runner implementaitons SHOULD +will be runner- and technology- specific, but runner implementations SHOULD offer a way to compute that schema from a ViewDefinition when applicable. For example, a runner that produces a table in a database system could return From 1a38a0977fee702de6a2dd274d66f292be5b20e4 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 20 Jun 2024 14:40:17 +1000 Subject: [PATCH 08/14] Fix table formatting on index --- input/pagecontent/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/input/pagecontent/index.md b/input/pagecontent/index.md index 440888e..c835e7f 100644 --- a/input/pagecontent/index.md +++ b/input/pagecontent/index.md @@ -62,7 +62,6 @@ name as the table name: | 1 | female | Malvina Gerda | Vicario | | 2 | male | Yolotzin Adel | Bristow | | 3 | other | Jin Gomer | Aarens | - {:.table-data} Such tabular views can be created for any FHIR resource, From 86ea18ef3a78cae47fab48adce6674684db4b34a Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 20 Jun 2024 15:28:20 +1000 Subject: [PATCH 09/14] Combine index and purpose pages --- ...tructureDefinition-ViewDefinition-notes.md | 15 ++ input/pagecontent/index.md | 178 +++++++++--------- input/pagecontent/purpose.md | 96 ---------- sushi-config.yaml | 1 - 4 files changed, 104 insertions(+), 186 deletions(-) delete mode 100644 input/pagecontent/purpose.md diff --git a/input/pagecontent/StructureDefinition-ViewDefinition-notes.md b/input/pagecontent/StructureDefinition-ViewDefinition-notes.md index 1c75a10..74ba716 100644 --- a/input/pagecontent/StructureDefinition-ViewDefinition-notes.md +++ b/input/pagecontent/StructureDefinition-ViewDefinition-notes.md @@ -493,3 +493,18 @@ See [Processing Algorithm](implementer_guidance.html#processing-model) for a des process a FHIR resource as input for a `ViewDefinition`. Implementations do not need to follow this algorithm directly, but their outputs should be consistent with what this model produces. + +## Generating Schemas + +The output format produced by a View Runner will be technology-specific, such as +a SQL database query or a structured file like Parquet. View Runner +implementations SHOULD offer a way to compute the output schema from a +ViewDefinition when applicable. + +For example, a runner that produces a table in a database system could return +a "CREATE TABLE" or "CREATE VIEW" statement based on the ViewDefinition, +allowing the system to define tables prior to populating them by evaluating the +views over data. + +This would not apply to outputs that do not have a way of specifying their +schema, like CSV files. diff --git a/input/pagecontent/index.md b/input/pagecontent/index.md index c835e7f..06ce57b 100644 --- a/input/pagecontent/index.md +++ b/input/pagecontent/index.md @@ -1,55 +1,78 @@ _This is an evolution of the original "SQL on FHIR" draft, which can [still be found here](https://github.com/FHIR/sql-on-fhir-archived)._ -### Introduction +This specification proposes an approach to make large-scale analysis of FHIR +data accessible to a larger audience and portable between systems. The central +goal of this project is to make FHIR data work well with the best available +analytic tools, regardless of the technology stack. -The [FHIR®](https://hl7.org/fhir) standard is a great fit for RESTful and -JSON-based systems, helping make healthcare data liquidity real. This spec aims -to take FHIR usage a step further, making FHIR work well with familiar and -efficient SQL engines and surrounding ecosystems. +### Problem -We do this by creating simple, tabular *views* of the underlying FHIR data that -are tailored to specific needs. Views are defined -with [FHIRPath](https://hl7.org/fhirpath/) expressions in a logical structure to -specify things like column names and unnested items. +As the availability of FHIR data increases, there is a growing interest in +using it for analytic purposes. However, to use FHIR effectively analysts +require a thorough understanding of the specification, including its +conventions, semantics, and data types. + +FHIR is represented as a graph of resources, each of which includes nested data +elements. There are semantics defined for references between resources, data +types, terminology, extensions, and many other aspects of the specification. + +Most analytic and machine learning use cases require the preparation of FHIR +data using transformations and tabular projections from its original form. +The task of authoring these transformations and projections is not trivial +and there is currently no standard mechanism to support reuse. + +### Solution + +A standard format can be provided for defining tabular, use case-specific views +of FHIR data. Tools can be developed that use these views in queries capable of +being executed on a wide variety of different query engines. + +These views can be made available to users as an easier way to consume FHIR +data which is simpler to understand and easier to process with generic analytic +query tools. + +FHIR implementation guides could include definitions of simple, flattened views +that comprise essential data elements. The availability of these view +definitions will greatly reduce the need for analysts to perform repetitive and +redundant transformation tasks for common use cases. Let's start with a simple example, defining a "patient_demographics" view with the following [ViewDefinition](StructureDefinition-ViewDefinition.html) structure: -```js +```json { - "name": "patient_demographics", - "resource": "Patient", - "select": [ - { - "column": [ - { - "path": "getResourceKey()", - "name": "id" - }, + "name": "patient_demographics", + "resource": "Patient", + "select": [ { - "path": "gender", - "name": "gender" - } - ] - }, - { - // Create columns from the official name selected here. - "forEach": "name.where(use = 'official').first()", - "column": [ - { - "path": "given.join(' ')", - "name": "given_name", - "description": "A single given name field with all names joined together." + "column": [ + { + "path": "getResourceKey()", + "name": "id" + }, + { + "path": "gender", + "name": "gender" + } + ] }, { - "path": "family", - "name": "family_name" + "forEach": "name.where(use = 'official').first()", + "column": [ + { + "path": "given.join(' ')", + "name": "given_name", + "description": "A single given name field with all names joined together." + }, + { + "path": "family", + "name": "family_name" + } + ] } - ] - } - ] + ] } ``` @@ -69,7 +92,7 @@ with [more examples here](artifacts.html#example-example-instances). See the [View Definition page](StructureDefinition-ViewDefinition.html) for the full definition of the above structure. -### View Definition non-goals +### Non-goals View Definitions are intentionally constrained to a narrow set of functionality to make them easily and broadly implementable, while deferring higher-level @@ -105,48 +128,44 @@ Parquet file, CSV, or another format specific to the runner. ### System Layers The [View Definition](StructureDefinition-ViewDefinition.html) is the central -element of this spec, but in practice it is really one layer of an overall -system. The layers are: +element of this spec, but in practice it is only one layer within a larger +system. A broader view of the system includes three layers: -- the *Data Layer* -- the *View Layer* -- and the *Analytics Layer*. +- The *Data Layer*; +- The *View Layer*, and; +- The *Analytics Layer*. High-level diagram of layers **Figure 1: High-level diagram of layers** -### The Data Layer +#### Data Layer -The *Data Layer* is a set of lossless representations that collectively enable -FHIR to be used with a wide variety of different query technologies. It may -optionally be persisted and annotated to make it or implementations of the view -layer more efficient, but no specific Data Layer structure will be required by -this specification. +The Data Layer is a set of lossless representations that collectively enable +FHIR to be used with a wide variety of different query technologies. -Implementations are encouraged but not required to further annotate the FHIR -resources to help View layer implementations run efficient queries. This -primarily applies when the underlying FHIR resources are stored in databases -that the View layer will query. +The Data Layer may optionally be persisted and annotated to make implementations +of the View Layer more efficient, but no specific Data Layer structure will be +required by this specification. -### The View Layer +#### View Layer -The *View Layer* defines portable, tabular views of FHIR data that can more -easily be consumed by a wide variety of analytic tools. The use of these tools -is described in *Analytics Layer* section. Our goal here is simply to get the -needed FHIR data in a form that matches user needs and common analytic patterns. +The View Layer defines portable, tabular views of FHIR data that can be easily +consumed by a wide variety of analytic tools. The use of these tools is +described in the Analytics Layer section. Our goal here is to get the +required FHIR data into a form that matches user needs and common analytic +patterns. -The View Layer itself has two key components: +The View Layer has two key components: * *View Definitions*, allowing users to define flattened views of FHIR data that are portable between systems. -* *View Runners* are system-specific tools or libraries that apply view - definitions to - the underlying data layer, optionally making use of annotations to optimize +* *View Runners*, system-specific tools or libraries that apply view definitions + to the underlying data layer, optionally making use of annotations to optimize performance. -See the [View Definition documentation](StructureDefinition-ViewDefinition.html) -for details and examples; these are the central piece of this specification. +See [View Definition](StructureDefinition-ViewDefinition.html) for more details +and examples. View Runners will be specific to the data layer they use. Each data layer may have one or more corresponding view runners, but a given View Definition can be @@ -154,30 +173,15 @@ run by many runners over many data layers. Example view runners may include: -* A runner that creates a virtual, tabular view in an analytic database +* A runner that creates a virtual, tabular view in an analytic database. * A runner that queries FHIR JSON directly and creates a table in a web - application -* A runner that loads data directly into a notebook or other data analysis tool - -#### Generating Schemas - -The output of many runners will have technology-specific schemas, such as -database table definitions or schema for structured files like Parquet. This -will be runner- and technology- specific, but runner implementations SHOULD -offer a way to compute that schema from a ViewDefinition when applicable. + application. +* A runner that loads data directly into a notebook or other data analysis tool. -For example, a runner that produces a table in a database system could return -a "CREATE TABLE" or "CREATE VIEW" statement based on the ViewDefinition, -allowing the system to define tables prior to populating them by evaluating the -views over data. +#### The Analytics Layer -This would not apply to outputs that do not have common a schema specification, -like CSV files. - -### The Analytics Layer - -Finally, users must be able to easily leverage the above views with the analytic -tools of their choice. This spec purposefully does not define what these are, +Users must be able to easily leverage the above views with the analytic tools of +their choice. This specification purposefully does not define what these are, but common use cases may be SQL queries by consuming applications, dataframe-based data science tools in Python or R, or integration with business intelligence tools. @@ -188,7 +192,3 @@ FHIR® is the registered trademark of HL7 and is used with the permission of HL7 Use of the FHIR trademark does not constitute endorsement of the contents of this repository by HL7, nor affirmation that this data is conformant to the various applicable standards. - ---- - -**[Next: Purpose](purpose.html)** diff --git a/input/pagecontent/purpose.md b/input/pagecontent/purpose.md deleted file mode 100644 index 9561644..0000000 --- a/input/pagecontent/purpose.md +++ /dev/null @@ -1,96 +0,0 @@ -This specification proposes an approach to make large-scale analysis of FHIR data -accessible to a larger audience and portable between systems. The central goal -of this project is to make FHIR data work well with the best available analytic -tools, regardless of the technology stack. - - -### Problem - -As the availability of FHIR data increases, there is a growing interest in -using it for analytic purposes. However, to use FHIR effectively analysts -require a thorough understanding of the specification, including its -conventions, semantics, and data types. - -FHIR is represented as a graph of resources, each of which includes nested data -elements. There are semantics defined for references between resources, data -types, terminology, extensions, and many other aspects of the specification. - -Most analytic and machine learning use cases require the preparation of FHIR -data using transformations and tabular projections from its original form. -The task of authoring these transformations and projections is not trivial -and there is currently no standard mechanisms to support reuse. - -### Solution - -A standard format can be provided for defining tabular, use case-specific views -of FHIR data. Tools can be developed that use these views in queries capable of -being executed on a wide variety of different query engines. - -These views can be made available to users as an easier way to consume FHIR -data which is simpler to understand and easier to process with generic analytic -query tools. - -FHIR implementation guides could include definitions of simple, flattened views -that comprise essential data elements. The availability of these view -definitions will greatly reduce the need for analysts to perform repetitive and -redundant transformation tasks for common use cases. - -### Non-goals - -This is not a full analytic toolchain, but an attempt to adapt FHIR to such -toolchains. Therefore we scope this to create tabular views of resources, and -explicitly scope out higher-level analytic capabilities since many tools do -this well today. Examples of capabilities we scope out include: - -* Join operation between resources. This effort creates tabular views, but users -leverage the database engine or other tool of their choice to join them and -analyze at scale. -* Any form of data aggregation or statistical analysis. - -### Requirements - -The proposed system attempts to meet the following requirements: - -**A portable, unambiguous specification** -Any good standard is unambiguous and portable between technology stacks, institutions, -and deployments and this is no exception. - -**Leverage existing standards whenever possible** -Whenever practical we should avoid creating new standards and use existing approaches -to these problems. - -**Ability to select from repeated structures based on field values** -Flattened repeated structures in FHIR requires checking the content of those fields. -For example, creating a table of patient home addresses requires checking that -the address.use field is ‘home’. Similarly, a table with columns for systolic -and diastolic blood pressures needs to check the Observation.component.code fields -to select them properly. - -**Ability to filter based on code values or other criteria** -Many useful FHIR queries rely heavily on value sets to identify needed resources. -For instance, users may be interested in a table of statin meds for analysis, -requiring a value set of statin medication codes to allow such a flattened view -of statins. Therefore some form of value set-based filter should be used to create -the needed views. - -**Support running on a wide variety of databases and tools** -There are many excellent options for large-scale data analysis and new ones -continue to be created. Our efforts here should be generalizable across tools -as much as possible. We aim to cover many popular technologies and to avoid -features that are not widely implemented. - -**Support direct exports from data sources** -Some users have limited analytic needs and only need views over a small subset of -FHIR data that could be produced by a given system. Ideally a flattened FHIR definition -could be interpreted by a FHIR service so only the needed subset of data is -produced – whether directly in a tabular form or limited to the FHIR resources -needed for the views. - -**Common data transformation and loading should be supported** -It should be possible to run transformations on raw data or within a database using SQL, -that is, to use the ETL or ELT patterns. - - ---- - -**[Next: View Definitions](StructureDefinition-ViewDefinition.html)** diff --git a/sushi-config.yaml b/sushi-config.yaml index a61792e..d5a0017 100644 --- a/sushi-config.yaml +++ b/sushi-config.yaml @@ -71,7 +71,6 @@ parameters: # ╰────────────────────────────────────────────────────────────────────────────────────────────────╯ menu: Overview: index.html - Purpose: purpose.html View Definition: StructureDefinition-ViewDefinition.html Implementer Guidance: implementer_guidance.html Artifacts: artifacts.html From 17f95f00aa2f11a83d7cb03ffeebea7e1bda0a72 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 20 Jun 2024 16:04:14 +1000 Subject: [PATCH 10/14] Editorial updates to ViewDefinition intro --- input/fsh/models.fsh | 2 +- ...tructureDefinition-ViewDefinition-intro.md | 60 ++++++++++++------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/input/fsh/models.fsh b/input/fsh/models.fsh index d68051f..85327fa 100644 --- a/input/fsh/models.fsh +++ b/input/fsh/models.fsh @@ -21,7 +21,7 @@ Expression: "(forEach | forEachOrNull).count() <= 1" Logical: ViewDefinition Title: "View Definition" Description: """ -View definitions represent a tabular projection of a FHIR resource, where the columns and inclusion +A ViewDefinition represents a tabular projection of a FHIR resource, where the columns and inclusion criteria are defined by FHIRPath expressions. """ * url 0..1 uri "Canonical identifier for this view definition, represented as a URI (globally unique)" diff --git a/input/pagecontent/StructureDefinition-ViewDefinition-intro.md b/input/pagecontent/StructureDefinition-ViewDefinition-intro.md index 6c4e174..1032c26 100644 --- a/input/pagecontent/StructureDefinition-ViewDefinition-intro.md +++ b/input/pagecontent/StructureDefinition-ViewDefinition-intro.md @@ -1,36 +1,52 @@ -A *view definition* is the central piece of the [View Layer](index.html#system-layers) and represents a tabular projection of FHIR resources with the columns and filtering criteria defined by [FHIRPath](https://hl7.org/fhirpath/) expressions. The ViewDefinition logical model is described below. +It is the central piece of the [View Layer](index.html#system-layers) and +represents a tabular projection of FHIR resources with the columns and filtering +criteria defined by [FHIRPath](https://hl7.org/fhirpath/) expressions. The +logical model is described below. -## Key ViewDefinition Elements -The key elements of the ViewDefinition are: +## Key Elements ### Resource -Each ViewDefinition instance is tied to a single FHIR [ResourceType](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.resource), -such as *Patient* or *Observation*. It will then create zero to many rows for each resource -instance. [Examples](StructureDefinition-ViewDefinition-examples.html) include simple tabular -views of patients, unrolling patient addresses into an address table, specific views of needed -observations, and so on. + +Each ViewDefinition instance is tied to a single +FHIR [resource type](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.resource), +such as *Patient* or *Observation*. It will then create zero or more rows for +each resource +instance. [Examples](StructureDefinition-ViewDefinition-examples.html) include +simple tabular views of patients, unrolling patient addresses into an address +table, views of certain types of observations, and so on. ### Select -Each ViewDefinition instance has a [`select`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select) that specifies the content and names for the `column`s in the view. The content for each `column` is defined with [FHIRPath](https://hl7.org/fhirpath/) expressions that return specific data elements from the FHIR resources. + +Each ViewDefinition instance has +a [select](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select) +that specifies the content and names for the columns in the view. The content +for each column is defined with [FHIRPath](https://hl7.org/fhirpath/) +expressions that return specific data elements from the FHIR resources. ### Where -The ViewDefinition may include one or more [`where`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.where)s that may be used to further limit, or filter, the resources included in the view. -For instance, users may have different views for blood pressure observations or other observation types. -### Constant -The ViewDefinition may include one or more [`constant`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.constant)s, which are simply values that can be reused in FHIRPath expressions. +The ViewDefinition may include one or +more [where](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.where) +clauses that may be used to further limit, or filter, the resources included in +the view. For instance, users may have different views for blood pressure +observations or other observation types. +### Constants -### Descriptive Fields -The ViewDefinition may include several other fields with `name`, `status`, and other descriptive information about the view as seen below. +The ViewDefinition may include one or +more [constants](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.constant), +which are simply values that can be reused in FHIRPath expressions. -## Implementation +## View Runner -A system-specific *view runner* implementation can execute a *view definition* and -return the results as a table that can easily be used in the user's tech stack. See the -[system layers](index.html#system-layers) for details. +A *View Runner* implementation can execute a ViewDefinition and return the +results as a table that can be used for further processing using the user's +chosen tech stack. See [System Layers](index.html#system-layers) for details. ## Profiling -ViewDefinitions may be profiled to meet specific needs. For instance, the [ShareableViewDefinition](StructureDefinition-ShareableViewDefinition.html) profile -adds constraints for view definitions intended to be shared between systems. Implementers may create their -own ViewDefinition profiles for further specialized needs. + +ViewDefinitions may be profiled to meet specific needs. For instance, +the [ShareableViewDefinition](StructureDefinition-ShareableViewDefinition.html) +profile adds constraints for ViewDefinitions intended to be shared between +systems. Implementers may create their own ViewDefinition profiles for further +specialized needs. From df8082008627d76c7a9ff4c8f434cb48f0b1a6f7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 20 Jun 2024 17:05:25 +1000 Subject: [PATCH 11/14] Editorial updates to ViewDefinition notes --- ...tructureDefinition-ViewDefinition-intro.md | 2 +- ...tructureDefinition-ViewDefinition-notes.md | 470 +++++++++++------- 2 files changed, 304 insertions(+), 168 deletions(-) diff --git a/input/pagecontent/StructureDefinition-ViewDefinition-intro.md b/input/pagecontent/StructureDefinition-ViewDefinition-intro.md index 1032c26..e104975 100644 --- a/input/pagecontent/StructureDefinition-ViewDefinition-intro.md +++ b/input/pagecontent/StructureDefinition-ViewDefinition-intro.md @@ -9,7 +9,7 @@ logical model is described below. Each ViewDefinition instance is tied to a single FHIR [resource type](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.resource), -such as *Patient* or *Observation*. It will then create zero or more rows for +such as Patient or Observation. It will then create zero or more rows for each resource instance. [Examples](StructureDefinition-ViewDefinition-examples.html) include simple tabular views of patients, unrolling patient addresses into an address diff --git a/input/pagecontent/StructureDefinition-ViewDefinition-notes.md b/input/pagecontent/StructureDefinition-ViewDefinition-notes.md index 74ba716..4307987 100644 --- a/input/pagecontent/StructureDefinition-ViewDefinition-notes.md +++ b/input/pagecontent/StructureDefinition-ViewDefinition-notes.md @@ -1,101 +1,134 @@ ## Specifying a Select -A [`select`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select) specifies the content and names for the columns in the view. The content for each column is defined with a [FHIRPath](https://hl7.org/fhirpath/) expression that returns a specific data element from the FHIR resources. More complex views can be specified to create resource or reference keys, unnest a collection of items returned by a FHIRPath expression, nest or concatenate the results from multiple selects, and so on. +A [select](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select) +specifies the content and names for the columns in the view. The content for +each column is defined with a [FHIRPath](https://hl7.org/fhirpath/) expression +that returns a specific data element from the FHIR resources. More complex views +can be specified to create resource or reference keys, unnest a collection of +items returned by a FHIRPath expression, nest or concatenate the results from +multiple selects, and so on. ### Supported FHIRPath Functionality -The [FHIRPath](https://hl7.org/fhirpath/) expressions used in views are evaluated by a *view runner*. Only a -subset of FHIRPath features is required to be supported by all *view runners*, and -a set of additional features can be optionally supported. +The [FHIRPath](https://hl7.org/fhirpath/) expressions used in views are +evaluated by the View Runner. Only a subset of FHIRPath features is required to +be supported by all View Runners, and a set of additional features can be +optionally supported. -#### Required Core FHIRPath Expressions/Functions +#### Required FHIRPath Expressions/Functions -All *view runners* must implement these [FHIRPath](https://hl7.org/fhirpath/) capabilities: +All View Runners must implement these [FHIRPath](https://hl7.org/fhirpath/) +capabilities: * [Literals](https://hl7.org/fhirpath/#literals) for String, Integer and Decimal -* [where](https://hl7.org/fhirpath/#wherecriteria-expression-collection) function +* [where](https://hl7.org/fhirpath/#wherecriteria-expression-collection)function * [exists](https://hl7.org/fhirpath/#existscriteria-expression-boolean) function * [empty](https://hl7.org/fhirpath/#empty-boolean) function * [extension](https://hl7.org/fhir/R4/fhirpath.html#functions) function -* [join](https://build.fhir.org/ig/HL7/FHIRPath/#joinseparator-string-string) function* -* [ofType](https://hl7.org/fhirpath/#oftypetype-type-specifier-collection) function +* [join](https://build.fhir.org/ig/HL7/FHIRPath/#joinseparator-string-string) + function* +* [ofType](https://hl7.org/fhirpath/#oftypetype-type-specifier-collection) + function * [first](https://hl7.org/fhirpath/#first-collection) function -* [lowBoundary](https://build.fhir.org/fhirpath.html#functions) and [highBoundary](https://build.fhir.org/fhirpath.html#functions) functions (including on Period)* -* Boolean operators: [and](https://hl7.org/fhirpath/#and), [or](https://hl7.org/fhirpath/#or), [not](https://hl7.org/fhirpath/#not-boolean) -* Math operators: [addition (+)](https://hl7.org/fhirpath/#addition), [subtraction (-)](https://hl7.org/fhirpath/#subtraction), [multiplication (*)](https://hl7.org/fhirpath/#multiplication), [division (/)](https://hl7.org/fhirpath/#division) -* Comparison operators: [equals (=)](https://hl7.org/fhirpath/#equals), [not equals (!=)](https://hl7.org/fhirpath/#not-equals), [greater than (>)](https://hl7.org/fhirpath/#greater-than), [less or equal (<=)](https://hl7.org/fhirpath/#less-or-equal) +* [lowBoundary](https://build.fhir.org/ig/HL7/FHIRPath/#lowboundaryprecision-integer-decimal--date--datetime--time) + and [highBoundary](https://build.fhir.org/ig/HL7/FHIRPath/#highboundaryprecision-integer-decimal--date--datetime--time) + functions (including on [Period](https://hl7.org/fhir/datatypes.html#Period))* +* Boolean + operators: [and](https://hl7.org/fhirpath/#and), [or](https://hl7.org/fhirpath/#or), [not](https://hl7.org/fhirpath/#not-boolean) +* Math + operators: [addition (+)](https://hl7.org/fhirpath/#addition), [subtraction (-)](https://hl7.org/fhirpath/#subtraction), [multiplication (*)](https://hl7.org/fhirpath/#multiplication), [division (/)](https://hl7.org/fhirpath/#division) +* Comparison + operators: [equals (=)](https://hl7.org/fhirpath/#equals), [not equals (!=)](https://hl7.org/fhirpath/#not-equals), [greater than (>)](https://hl7.org/fhirpath/#greater-than), [less or equal (<=)](https://hl7.org/fhirpath/#less-or-equal) * [Indexer expressions](https://hl7.org/fhirpath/#index-integer-collection) -* Not yet part of the normative [FHIRPath](https://hl7.org/fhirpath/) release, currently in draft. +* Not yet part of the normative [FHIRPath](https://hl7.org/fhirpath/) +release, currently in draft. -#### Optional Core FHIRPath Functions +#### Optional FHIRPath Functions -*View runners* are encouraged to implement these functions as well to support a -broader set of use cases: +View Runners are encouraged to implement these functions to support a broader +set of use cases: -* [memberOf](https://hl7.org/fhir/R4/fhirpath.html#functions) function -* [toQuantity](https://hl7.org/fhirpath/#toquantityunit-string-quantity) function +* [memberOf](https://hl7.org/fhir/fhirpath.html#fn-memberOf) function +* [toQuantity](https://hl7.org/fhirpath/#toquantityunit-string-quantity)function #### Required Additional Functions -All *view runners* must implement these functions that extend the -FHIRPath specification. Despite not being in the [FHIRPath](https://hl7.org/fhirpath/) specification, they are necessary in the context of defining views: +All View Runners must implement these functions that extend the FHIRPath +specification. Despite not being in the [FHIRPath](https://hl7.org/fhirpath/) +specification, they are necessary in the context of defining views: * [getResourceKey](#getresourcekey--keytype) function * [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) function ##### getResourceKey() : *KeyType* -This is invoked at the root of a FHIR [Resource](https://build.fhir.org/resource.html) and returns -an opaque value to be used as the primary key for the row associated with the resource. In many cases -the value may just be the resource `id`, but exceptions are described below. This function is used in -tandem with [getReferenceKey()](#getreferencekeyresource-type-specifier--keytype), which returns +This is invoked at the root of a FHIR Resource and returns an opaque value to be +used as the primary key for the row associated with the resource. In many cases +the value may just be the resource `id`, but exceptions are described below. + +This function is used in tandem +with [getReferenceKey](#getreferencekeyresource-type-specifier--keytype), +which returns an equal value from references that point to this resource. -The returned *KeyType* is implementation dependent, but must be a FHIR primitive type that can be used -for efficient joins in the system's underlying data storage. Integers, strings, UUIDs, and other primitive -types are appropriate. +The returned *KeyType* is implementation dependent, but must be a FHIR primitive +type that can be used for efficient joins in the system's underlying data +storage. Integers, strings, UUIDs, and other primitive types may be appropriate. -See the [Joins with Resource and Reference Keys](#joins-with-resource-and-reference-keys) section below for details. +See +the [Joins with Resource and Reference Keys](#joins-with-resource-and-reference-keys) +section below for details. ##### getReferenceKey([resource: type specifier]) : *KeyType* -This is invoked on [Reference](https://hl7.org/fhir/references.html#Reference) elements and returns -an opaque value that represents the database key of the row being referenced. The value returned must -be equal to the [getResourceKey()](#getresourcekey--keytype) value returned on the resource itself. - -Users may pass an optional resource type (e.g., *Patient* or *Observation*) to indicate -the expected type that the reference should point to. The getReferenceKey() will return an empty collection -(effectively *null* since FHIRPath always returns collections) if the reference is not of the -expected type. For example, `Observation.subject.getReferenceKey(Patient)` would return a row key if the -subject is a *Patient*, or the empty collection (i.e., *{}*) if it is not. - -The returned *KeyType* is implementation dependent, but must be a FHIR primitive type that can be used -for efficient joins in the systems underlying data storage. Integers, strings, UUIDs, and other primitive -types are appropriate. - -The getReferenceKey() function has both required and optional functionality: - -* Implementations MUST support the relative literal form of reference (e.g., *Patient/123*), and MAY support -other types of references. If the implementation does not support the reference type, or is unable to -resolve the reference, it MUST return the empty collection (i.e., *{}*). -* Implementations MAY generate a list of unprocessable references through query responses, logging or -reporting. The details of how this information would be provided to the user is implementation specific. +This is invoked on [Reference](https://hl7.org/fhir/references.html#Reference) +elements and returns an opaque value that represents the database key of the row +being referenced. The value returned must be equal to +the [getResourceKey](#getresourcekey--keytype) value returned on the resource +itself. + +Users may pass an optional resource type (e.g., Patient or Observation) to +indicate the expected type that the reference should point to. The +getReferenceKey function will return an empty collection (effectively *null* +since FHIRPath always returns collections) if the reference is not of the +expected type. For example, `Observation.subject.getReferenceKey(Patient)` would +return a row key if the subject is a Patient, or the empty collection ( +i.e., `{}`) if it is not. + +The returned *KeyType* is implementation dependent, but must be a FHIR primitive +type that can be used for efficient joins in the systems underlying data +storage. Integers, strings, UUIDs, and other primitive types may be appropriate. + +The getReferenceKey function has both required and optional functionality: + +* Implementations MUST support the relative literal form of reference ( + e.g., `Patient/123`), and MAY support other types of references. If the + implementation does not support the reference type, or is unable to resolve + the reference, it MUST return the empty collection (i.e., `{}`). +* Implementations MAY generate a list of unprocessable references through query + responses, logging or reporting. The details of how this information would be + provided to the user is implementation specific. See the [Joins with Resource and Reference Keys](#joins-with-resource-and-reference-keys) section below for details. ### Joins with Resource and Reference Keys + While ViewDefinitions do not directly implement joins across resources, the views produced should be easily joined by the database or analytic tools of the -user's choice. This can be done by including primary and foreign keys as part of the tabular -view output, which can be done with the [getResourceKey()](#getresourcekey--keytype) and -[getReferenceKey()](#getreferencekeyresource-type-specifier--keytype) functions. - -Users may call [getResourceKey()](#getresourcekey--keytype) to obtain a resources primary key, -and call [getReferenceKey()](#getreferencekeyresource-type-specifier--keytype) to get +user's choice. This can be done by including primary and foreign keys as part of +the tabular +view output, which can be done with +the [getResourceKey](#getresourcekey--keytype) and +[getReferenceKey](#getreferencekeyresource-type-specifier--keytype) functions. + +Users may call [getResourceKey](#getresourcekey--keytype) to obtain a +resources primary key, +and call [getReferenceKey](#getreferencekeyresource-type-specifier--keytype)to +get the corresponding foreign key from a reference pointing at that resource/row. -For example, a minimal view of *Patient* resources could look like this: +For example, a minimal view of Patient resources could look like this: ```js { @@ -115,7 +148,8 @@ For example, a minimal view of *Patient* resources could look like this: } ``` -A view of *Observation* resources would then have its own row key and a foreign key to easily join to the view of *Patient* resources, like this: +A view of Observation resources would then have its own row key and a foreign +key to easily join to the view of Patient resources, like this: ```js { @@ -134,26 +168,25 @@ A view of *Observation* resources would then have its own row key and a foreign "name": "patient_id" } ] - }], - "where": [ - // An expression that selects observations that have a patient subject. - ] + }] } ``` -Users of the views could then join `simple_obs.patient_id` to `active_patients.id` using common -join semantics. +Users of the views could then join `simple_obs.patient_id` +to `active_patients.id` using common join semantics. ### Suggested Implementations for getResourceKey() and getReferenceKey() -While [getResourceKey()](#getresourcekey--keytype) and -[getReferenceKey()](#getreferencekeyresource-type-specifier--keytype) must return matching -values for the same row, *how* they do so is left to the implementation. This is by design, -allowing ViewDefinitions to be run across a wide set of systems that have different data invariants -or pre-processing capabilities. + +While [getResourceKey](#getresourcekey--keytype) +and [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) must +return matching values for the same row, *how* they do so is left to the +implementation. This is by design, allowing ViewDefinitions to be run across a +wide set of systems that have different data invariants or pre-processing +capabilities. Here are some implementation options to meet different needs: - +
@@ -163,7 +196,7 @@ Here are some implementation options to meet different needs: - + @@ -171,39 +204,61 @@ Here are some implementation options to meet different needs: - +
Approach
Return the Resource IDIf the system can guarantee that each resource has a simple id and the corresponding references have simple, relative ids that point to it (e.g., Patient/123), getResourceKey() and getReferenceKey() implementations may simply return those values. This is the simplest case and will apply to many (but not all) systems.If the system can guarantee that each resource has a simple id and the corresponding references have simple, relative ids that point to it (e.g., Patient/123), getResourceKey and getReferenceKey implementations may simply return those values. This is the simplest case and will apply to many (but not all) systems.
Return a "Primary" Identifier
Pre-Process to Create or Resolve KeysThe most difficult case is systems where the resource id is a not a reliable row key, and resources have multiple identifiers with no clear way to select one for the key.

In this case, implementations will likely have to fall back to some form of preprocessing to determine appropriate keys. This may be accomplished by:

  • Pre-processing all data to have clear resource ids or "primary" identifiers and using one of the options above.
  • Building some form of cross-link table dynamically within the implementation itself based on the underlying data. For instance, if an implementation's getResourceKey() uses a specific identifier system, getReferenceKey() could use a pre-built cross-link table to find the appropriate identifier-based key to return.
The most difficult case is systems where the resource id is a not a reliable row key, and resources have multiple identifiers with no clear way to select one for the key.

In this case, implementations will likely have to fall back to some form of preprocessing to determine appropriate keys. This may be accomplished by:

  • Pre-processing all data to have clear resource ids or "primary" identifiers and using one of the options above.
  • Building some form of cross-link table dynamically within the implementation itself based on the underlying data. For instance, if an implementation's getResourceKey uses a specific identifier system, getReferenceKey could use a pre-built cross-link table to find the appropriate identifier-based key to return.
-There are many variations and alternatives to the above. This spec simply asserts that implementations must -be able to produce a row key for each resource and a matching key for references pointing at that resource, -and intentionally leaves the specific mechanism to the implementation. +There are many variations and alternatives to the above. This specification +simply asserts that implementations must be able to produce a row key for each +resource and a matching key for references pointing at that resource, and +intentionally leaves the specific mechanism to the implementation. ### Contained Resources -This specification requires implementers to extract contained resources, -needed for *view definitions*, into independent resources that can then be accessed via -[getReferenceKey()](#getreferencekeyresource-type-specifier--keytype) like any other resource. Implementations SHOULD -normalize these resources appropriately whenever possible, such as eliminating duplicate resources contained in many parent resources. Note that this may change in a later version of this specification, allowing users to explicitly create separate views for contained resources that could be distinct from top-level resource views. -Contained resources have different semantics than other resources since they don't have an independent identity, -and the same logical record may be duplicated across many containing resources. This makes SQL best practices difficult since the data is denormalized and ambiguous. For instance, `Patient.generalPractitioner` may be a contained resource that may or may not be the same practitioner seen in other *Patient* resources. Therefore, the approach in this specification requires systems to pre-process the data into normalized, independent resources if needed. - -For the same reason, the output from running a ViewDefinition will not include contained resources. For instance, a view of -*Practitioner* resources will include top-level *Practitioner* resources but not contained *Practitioner* resources from inside the *Patient* resources. +This specification requires implementers to extract contained resources that +need to be included in views into independent resources that can then be +accessed via [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) +like any +other resource. Implementations SHOULD normalize these resources appropriately +whenever possible, such as eliminating duplicate resources contained in many +parent resources. Note that this may change in a later version of this +specification, allowing users to explicitly create separate views for contained +resources that could be distinct from top-level resource views. + +Contained resources have different semantics than other resources since they +don't have an independent identity, and the same logical record may be +duplicated across many containing resources. This makes SQL best practices +difficult since the data is denormalized and ambiguous. For +instance, `Patient.generalPractitioner` may be a contained resource that may or +may not be the same practitioner seen in other Patient resources. Therefore, the +approach in this specification requires systems to pre-process the data into +normalized, independent resources if needed. + +For the same reason, the output from running a ViewDefinition will not include +contained resources. For instance, a view of Practitioner resources will include +top-level Practitioner resources but not contained Practitioner resources from +inside the Patient resources. ### Unnesting semantics -It is often desirable to unroll the collection returned from a [FHIRPath](https://hl7.org/fhirpath/) expression into a separate row for each item in the collection. This is done with [`forEach`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) -or [`forEachOrNull`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull). +It is often desirable to unroll the collection returned from +a [FHIRPath](https://hl7.org/fhirpath/) expression into a separate row for each +item in the collection. This is done +with [forEach](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) +or [forEachOrNull](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull). -For instance, each Patient resource -can have multiple addresses, which users can expand into a separate *patient_addresses* table with -one row per address. Each row would still have a *patient_id* field to know which patient that address row is -associated with. You can see this in the -[PatientAddresses example](Binary-PatientAddresses.html), which unrolls addresses as described above. +For instance, each Patient resource can have multiple addresses, which users can +expand into a separate `patient_addresses` table with one row per address. Each +row would still have a `patient_id` field to know which patient that address row +is associated with. You can see this in +the [PatientAddresses example](Binary-PatientAddresses.html), which unrolls +addresses as described above. -[`forEach`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) and [`forEachOrNull`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) apply both to the columns within a `select` and any nested `select`s it contains. Therefore, the following `select`s will produce the same results: +[forEach](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) +and [forEachOrNull](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) +apply both to the columns within a `select` and any nested `select`s it +contains. Therefore, the following `select`s will produce the same results: ```js "select": [{ @@ -219,11 +274,25 @@ associated with. You can see this in the }] ``` -While a [`forEach`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) is similar to an *INNER JOIN* supported by many SQL engines, a [`forEachOrNull`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) is analogous to -a *LEFT OUTER JOIN*. [`forEach`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) will produce a row only if the [FHIRPath](https://hl7.org/fhirpath/) expression returns one or more items in the collection. On the other hand, [`forEachOrNull`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) will produce a row even if the [FHIRPath](https://hl7.org/fhirpath/) expression returns an empty collection. With [`forEachOrNull`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull), all columns will be included but, if nothing is returned by the [`forEachOrNull`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) expression, the row produced will have *null* values. - -To illustrate this, the following expression in a Patient view uses [`forEach`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach), and will therefore return a row for each *Patient* resource -only if the *Patient* resource has at least one `Patient.address`, similar to an *INNER JOIN* in SQL: +While a [forEach](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) +is similar to an `INNER JOIN` supported by many SQL engines, +a [forEachOrNull](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) +is analogous to +a `LEFT OUTER JOIN`. [forEach](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach) +will produce a row only if the [FHIRPath](https://hl7.org/fhirpath/) expression +returns one or more items in the collection. On the other +hand, [forEachOrNull](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) +will produce a row even if the [FHIRPath](https://hl7.org/fhirpath/) expression +returns an empty collection. +With [forEachOrNull](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull), +all columns will be included but, if nothing is returned by +the [forEachOrNull](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) +expression, the row produced will have null values. + +To illustrate this, the following expression in a Patient view +uses [forEach](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEach), +and will therefore return a row for each Patient resource only if the Patient +resource has at least one `Patient.address`, similar to an `INNER JOIN` in SQL: ```js "select": [ @@ -237,9 +306,12 @@ only if the *Patient* resource has at least one `Patient.address`, similar to an ] ``` -In contrast, this view with [`forEachOrNull`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) will produce the "id" column for *every* *Patient* resource. If the *Patient* resource has no -`Patient.address`, there will be a single row for the *Patient* resource and the "zip" column will be *null*. For *Patient* resources with one or more -`Patient.address`, the result will be identical to the expression above. +In contrast, this view +with [forEachOrNull](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.forEachOrNull) +will produce the `id` column for *every* Patient resource. If the Patient +resource has no `Patient.address`, there will be a single row for the Patient +resource and the `zip` column will contain null. For Patient resources with one +or more `Patient.address`, the result will be identical to the expression above. ```js "select": [ @@ -254,20 +326,30 @@ In contrast, this view with [`forEachOrNull`](StructureDefinition-ViewDefinition ``` ### Multiple Select Expressions -A ViewDefinition may have multiple `select`s, which can be organized as siblings or in parent/child relationships. This is typically done -as different `select`s may use different `forEach` or `forEachOrNull` expressions to unroll different parts of the resources. -The multiple rows produced using `forEach` or `forEachOrNull` from a `select` are joined to others with the following rules: +A ViewDefinition may have multiple `select`s, which can be organized as siblings +or in parent/child relationships. This is typically done as different `select`s +may use different `forEach` or `forEachOrNull`expressions to unroll different +parts of the resources. -* Parent/child `select`s will repeat values from the parent `select` for each item in the child `select`. -* Sibling `select`s are effectively cross joined, where each row in each `select` is duplicated for every row in sibling `select`s. +The multiple rows produced using `forEach` or `forEachOrNull` from a `select`are +joined to others with the following rules: -The [example view definitions](StructureDefinition-ViewDefinition-examples.html) illustrate this behavior. +* Parent/child `select`s will repeat values from the parent `select` for each + item in the child `select`. +* Sibling `select`s are effectively cross joined, where each row in + each `select` is duplicated for every row in sibling `select`s. + +The [examples](StructureDefinition-ViewDefinition-examples.html) illustrate this +behavior. ### Unions -A `select` can have an optional `unionAll`, which contains a list of `select`s that are used to create a union. `unionAll` effectively concatenates the results of the nested `select`s that it contains, but without a guarantee that row ordering will be preserved. Each `select` contained in the `unionAll` must produce the same columns including their specified names and FHIR types. -For instance, to create a table of all `Patient.address` and `Patient.contact.address`, we could use `unionAll`: +A `select` can have an optional `unionAll`, which contains a list of `select`s +that are used to create a union. `unionAll` effectively concatenates the results +of the nested `select`s that it contains, but without a guarantee that row +ordering will be preserved. Each `select` contained in the `unionAll` must +produce the same columns including their specified names and FHIR types. ```js "select": { @@ -290,21 +372,38 @@ For instance, to create a table of all `Patient.address` and `Patient.contact.ad } ``` -The above example uses `forEach` to select different data elements from the resources to be included in the union. For other use cases, it is possible to define the columns directly -in the `select`. See the [PatientAndContactAddressUnion example](Binary-PatientAndContactAddressUnion.html) for a complete version of the above. - -The columns produced from the `unionAll` list are effectively added to the parent `select`, following any other columns from its parent for column ordering. See the [column ordering](#column-ordering) section below for details. +The above example uses `forEach` to select different data elements from the +resources to be included in the union. For other use cases, it is possible to +define the columns directly in the `select`. See +the [PatientAndContactAddressUnion example](Binary-PatientAndContactAddressUnion.html) +for a complete version of the above. -The `select`s in a `unionAll` MUST have matching columns. Specifically, each nested selection structure MUST produce the same number of columns with the same names and order, and the values for the columns MUST be of the same types as determined by the [column types](#column-types) part of this specification. +The columns produced from the `unionAll` list are effectively added to the +parent `select`, following any other columns from its parent for column +ordering. See the [column ordering](#column-ordering) section below for details. -`unionAll` behaves similarly to the *UNION ALL* in SQL and will not filter out duplicate rows. Note that each `select` can contain only one `unionAll` list since these items must be combined in a single, logical -*UNION ALL*. Nested `select`s can be used when multiple `unionAll`s are needed within a single view. +The `select`s in a `unionAll` MUST have matching columns. Specifically, each +nested selection structure MUST produce the same number of columns with the same +names and order, and the values for the columns MUST be of the same types as +determined by the [column types](#column-types) part of this specification. +`unionAll` behaves similarly to the `UNION ALL` in SQL and will not filter out +duplicate rows. Note that each `select` can contain only one `unionAll` list +since these items must be combined in a single, logical `UNION ALL`. +Nested `select`s can be used when multiple `unionAll`s are needed within a +single view. ### Composing Multiple Selects and Unions -`unionAll` produces rows that can be used just like a `select` expression. These rows can be used by containing `select`s or `unionAll`s without needing any special knowledge of how they were produced. This means that `unionAll` and `select` operations can be nested with intuitive behavior, similar to how functions can be nested in many programming languages. -For instance, the two expressions below will return the same rows despite the first being a single `unionAll` and the second being composed of a nested `select` that contains additional `unionAll`s. +`unionAll` produces rows that can be used just like a `select` expression. These +rows can be used by containing `select`s or `unionAll`s without needing any +special knowledge of how they were produced. This means that `unionAll` +and `select` operations can be nested with intuitive behavior, similar to how +functions can be nested in many programming languages. + +For instance, the two expressions below will return the same rows despite the +first being a single `unionAll` and the second being composed of a +nested `select` that contains additional `unionAll`s. ```js "select": { @@ -316,7 +415,7 @@ For instance, the two expressions below will return the same rows despite the fi { "forEach": "b", "column": [] // snip - } + }, { "forEach": "c", "column": [] // snip @@ -351,17 +450,25 @@ And, the equivalent with a nested structure: ] } ``` -Note the former example is preferred due to its simplicity and the latter is included purely for illustrative purposes. + +Note the former example is preferred due to its simplicity and the latter is +included purely for illustrative purposes. ### Column Ordering -*View runners* that support column ordering in their output format MUST order the columns of the result according to the rules defined in this section. +View Runners that support column ordering in their output format MUST order +the columns of the result according to the rules defined in this section. -`select`s that have nested `select`s will place the columns of the parent `select` before the columns of the nested `select`, and the columns from a `unionAll` list are placed last. +`select`s that have nested `select`s will place the columns of the +parent `select` before the columns of the nested `select`, and the columns from +a `unionAll` list are placed last. -To change the column ordering, it is possible to place the columns or the `unionAll` in a nested `select`, which can be ordered relative to other nested `select`s as desired. +To change the column ordering, it is possible to place the columns or +the `unionAll` in a nested `select`, which can be ordered relative to other +nested `select`s as desired. -For example, the `column`s in this ViewDefinition will appear in alphabetical order: +For example, the `column`s in this ViewDefinition will appear in alphabetical +order: ```js { @@ -403,29 +510,48 @@ For example, the `column`s in this ViewDefinition will appear in alphabetical or ``` ### Column Types -All values in a given column must be of a single data type. The -data type can be explicitly specified with the -[`collection`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.collection) and -[`type`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.type) for a column. In most cases, the data type for a column can be determined by the `path` expression, allowing users to interactively build and evaluate a ViewDefinition without needing to look up and explicitly specify the data type. - -If the column is a primitive type (typical of tabular output), its type is inferred under the following conditions: - -1. If the [`collection`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.collection) is not -set to *true*, the returned data type must be a single value. -2. If the `path` is a series of *parent.child.subPath* navigation steps from a known data type, either from the root resource -or a child of an ofType() function, then the data type for each column is determined by the structure definition it comes from. -3. If the terminal expression is one of the [supported FHIRPath functions](#supported-fhirpath-functionality) with a defined return type, then the column will be of that data type. For instance, if the `path` ends in exists() or lowBoundary(), the data type for the column would be boolean or an instant type, respectively. -4. A path that ends in ofType() will be of the type given to that function. - -Note that type inference is an optional feature and some implementations may -not support it. Therefore, a ViewDefinition that is intended to be shared between -different implementations should have the [`type`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.type) - for each column set explicitly, even for primitives. It is reasonable for an implementation to -treat any non-specified types as strings. Moreover, non-primitive data types will not be supported by all implementations. Therefore, it is important to always explicitly set the [`type`](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.type) so each column can have its data type easily determined. - -Importantly, the above determines the FHIR type produced for the column. How that type is physically manifested depends -on the implementation. Implementations may map these to native database types or have each column simply produce a string, -as would be done in a CSV-based implementation. See the [database type hints](#type-hinting-with-tags) section below if finer + +All values in a given column must be of a single data type. The data type can be +explicitly specified with +the [collection](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.collection) +and [type](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.type) +for a column. In most cases, the data type for a column can be determined by +the `path` expression, allowing users to interactively build and evaluate a +ViewDefinition without needing to look up and explicitly specify the data type. + +If the column is a primitive type (typical of tabular output), its type is +inferred under the following conditions: + +1. If + the [collection](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.collection) + is not set to `true`, the returned data type must be a single value. +2. If the `path` is a series of `parent.child.subPath` navigation steps from a + known data type, either from the root resource or a child of an `ofType` + function, then the data type for each column is determined by the structure + definition it comes from. +3. If the terminal expression is one of + the [supported FHIRPath functions](#supported-fhirpath-functionality) with a + defined return type, then the column will be of that data type. For instance, + if the `path` ends in `exists()` or `lowBoundary()`, the data type for the + column would be boolean or an instant type, respectively. +4. A path that ends in `ofType()` will be of the type given to that function. + +Note that type inference is an optional feature and some implementations may not +support it. Therefore, a ViewDefinition that is intended to be shared between +different implementations should have +the [type](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.type) +for each column set explicitly, even for primitives. It is reasonable for an +implementation to treat any non-specified types as strings. Moreover, +non-primitive data types will not be supported by all implementations. +Therefore, it is important to always explicitly set +the [type](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select.column.type) +so each column can have its data type easily determined. + +Importantly, the above determines the FHIR type produced for the column. How +that type is physically manifested depends on the implementation. +Implementations may map these to native database types or have each column +simply produce a string, as would be done in a CSV-based implementation. See +the [database type hints](#type-hinting-with-tags) section below if finer database-specific type control is needed. ### Type Hinting with Tags @@ -440,25 +566,29 @@ common and can simplify analysis in some systems. ```json { - "resourceType": "ViewDefinition", - "name": "patient_birth_date", - "resource": "Patient", - "description": "A view of simple patient birth dates", - "select": [{ - "column": [ - { "path": "id" }, - { - "name": "birth_date", - "path": "birthDate", - "tags": [ - { - "name": "ansi/type", - "value": "DATE" - } - ] - } + "resourceType": "ViewDefinition", + "name": "patient_birth_date", + "resource": "Patient", + "description": "A view of simple patient birth dates", + "select": [ + { + "column": [ + { + "path": "id" + }, + { + "name": "birth_date", + "path": "birthDate", + "tags": [ + { + "name": "ansi/type", + "value": "DATE" + } + ] + } + ] + } ] - }] } ``` @@ -468,8 +598,13 @@ Behavior is undefined and left to the runner if the expression returns a value that is incompatible with the underlying database type. ## Using Constants -A ViewDefinition may include one or more constants, which are simply values that can be reused in [FHIRPath](https://hl7.org/fhirpath/) expressions. This can improve readability and reduce redundancy. Constants can be -used in `path` expressions by simply using *%[name]*. Effectively, these placeholders are replaced by the value of the constant before the [FHIRPath](https://hl7.org/fhirpath/) expression is evaluated. + +A ViewDefinition may include one or more constants, which are simply values that +can be reused in [FHIRPath](https://hl7.org/fhirpath/) expressions. This can +improve readability and reduce redundancy. Constants can be used in `path` +expressions by simply using `%[name]`. Effectively, these placeholders are +replaced by the value of the constant before +the [FHIRPath](https://hl7.org/fhirpath/) expression is evaluated. This is an example of a constant used in the `where` constraint of a view: @@ -489,7 +624,8 @@ This is an example of a constant used in the `where` constraint of a view: ## Processing Algorithm (Model) -See [Processing Algorithm](implementer_guidance.html#processing-model) for a description of how to +See [Processing Algorithm](implementer_guidance.html#processing-model) for a +description of how to process a FHIR resource as input for a `ViewDefinition`. Implementations do not need to follow this algorithm directly, but their outputs should be consistent with what this model produces. From d89bb48393b2c017c3cf1867df1e4664aa9ee143 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 20 Jun 2024 17:20:01 +1000 Subject: [PATCH 12/14] Combine view definition and implementer guidance pages --- ...tructureDefinition-ViewDefinition-notes.md | 531 +++++++++++------- input/pagecontent/implementer_guidance.md | 147 ----- sushi-config.yaml | 1 - 3 files changed, 343 insertions(+), 336 deletions(-) delete mode 100644 input/pagecontent/implementer_guidance.md diff --git a/input/pagecontent/StructureDefinition-ViewDefinition-notes.md b/input/pagecontent/StructureDefinition-ViewDefinition-notes.md index 4307987..590da93 100644 --- a/input/pagecontent/StructureDefinition-ViewDefinition-notes.md +++ b/input/pagecontent/StructureDefinition-ViewDefinition-notes.md @@ -1,21 +1,11 @@ -## Specifying a Select - -A [select](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select) -specifies the content and names for the columns in the view. The content for -each column is defined with a [FHIRPath](https://hl7.org/fhirpath/) expression -that returns a specific data element from the FHIR resources. More complex views -can be specified to create resource or reference keys, unnest a collection of -items returned by a FHIRPath expression, nest or concatenate the results from -multiple selects, and so on. - -### Supported FHIRPath Functionality +## Supported FHIRPath Functionality The [FHIRPath](https://hl7.org/fhirpath/) expressions used in views are evaluated by the View Runner. Only a subset of FHIRPath features is required to be supported by all View Runners, and a set of additional features can be optionally supported. -#### Required FHIRPath Expressions/Functions +### Required FHIRPath Expressions/Functions All View Runners must implement these [FHIRPath](https://hl7.org/fhirpath/) capabilities: @@ -44,15 +34,15 @@ capabilities: * Not yet part of the normative [FHIRPath](https://hl7.org/fhirpath/) release, currently in draft. -#### Optional FHIRPath Functions +### Optional FHIRPath Functions View Runners are encouraged to implement these functions to support a broader set of use cases: * [memberOf](https://hl7.org/fhir/fhirpath.html#fn-memberOf) function -* [toQuantity](https://hl7.org/fhirpath/#toquantityunit-string-quantity)function +* [toQuantity](https://hl7.org/fhirpath/#toquantityunit-string-quantity) function -#### Required Additional Functions +### Required Additional Functions All View Runners must implement these functions that extend the FHIRPath specification. Despite not being in the [FHIRPath](https://hl7.org/fhirpath/) @@ -61,7 +51,7 @@ specification, they are necessary in the context of defining views: * [getResourceKey](#getresourcekey--keytype) function * [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) function -##### getResourceKey() : *KeyType* +#### getResourceKey() : *KeyType* This is invoked at the root of a FHIR Resource and returns an opaque value to be used as the primary key for the row associated with the resource. In many cases @@ -80,7 +70,7 @@ See the [Joins with Resource and Reference Keys](#joins-with-resource-and-reference-keys) section below for details. -##### getReferenceKey([resource: type specifier]) : *KeyType* +#### getReferenceKey([resource: type specifier]) : *KeyType* This is invoked on [Reference](https://hl7.org/fhir/references.html#Reference) elements and returns an opaque value that represents the database key of the row @@ -112,135 +102,17 @@ The getReferenceKey function has both required and optional functionality: See the [Joins with Resource and Reference Keys](#joins-with-resource-and-reference-keys) section below for details. -### Joins with Resource and Reference Keys - -While ViewDefinitions do not directly implement joins across resources, the -views produced should be easily joined by the database or analytic tools of the -user's choice. This can be done by including primary and foreign keys as part of -the tabular -view output, which can be done with -the [getResourceKey](#getresourcekey--keytype) and -[getReferenceKey](#getreferencekeyresource-type-specifier--keytype) functions. - -Users may call [getResourceKey](#getresourcekey--keytype) to obtain a -resources primary key, -and call [getReferenceKey](#getreferencekeyresource-type-specifier--keytype)to -get -the corresponding foreign key from a reference pointing at that resource/row. - -For example, a minimal view of Patient resources could look like this: - -```js -{ - "name": "active_patients", - "resource": "Patient", - "select": [{ - "column": [ - { - "path": "getResourceKey()", - "name": "id" - }, - { - "path": "active" - } - ] - }] -} -``` - -A view of Observation resources would then have its own row key and a foreign -key to easily join to the view of Patient resources, like this: - -```js -{ - "name": "simple_obs", - "resource": "Observation", - "select": [{ - "column": [ - { - "path": "getResourceKey()", - "name": "id" - }, - { - // The `Patient` parameter is optional, but ensures the returned value - // will either be a patient row key or null. - "path": "subject.getReferenceKey(Patient)", - "name": "patient_id" - } - ] - }] -} -``` - -Users of the views could then join `simple_obs.patient_id` -to `active_patients.id` using common join semantics. - -### Suggested Implementations for getResourceKey() and getReferenceKey() - -While [getResourceKey](#getresourcekey--keytype) -and [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) must -return matching values for the same row, *how* they do so is left to the -implementation. This is by design, allowing ViewDefinitions to be run across a -wide set of systems that have different data invariants or pre-processing -capabilities. - -Here are some implementation options to meet different needs: - - - - - - - - - - - - - - - - - - - - - - -
ApproachDetails
Return the Resource IDIf the system can guarantee that each resource has a simple id and the corresponding references have simple, relative ids that point to it (e.g., Patient/123), getResourceKey and getReferenceKey implementations may simply return those values. This is the simplest case and will apply to many (but not all) systems.
Return a "Primary" IdentifierSince the resource id is by definition a system-specific identifier, it may change as FHIR data is exported and loaded between systems, and therefore not be a reliable target for references. For instance, a bulk export from one source system could be loaded into a target system that applies its own ids to the resources, requiring that joins be done on the resource's identifier rather than its id.

In this case, implementations will need to determine row keys based on the resource identifier and corresponding identifiers in the references.

The simplest variation of this is when there is only one identifier for each resource. In other cases, the implementation may be able to select a "primary" identifier, based on the identifier.system namespace, identifier.use code, or other property. For instance, if the primary Identifier.system is example_primary_system, implementations can select the desired identifier to use as a row key by checking for that.

In either case, the resource identifier and corresponding reference identifier can then be converted to a row key, perhaps by building a string or computing a cryptographic hash of the identifiers themselves. The best approach is left to the implementation.
Pre-Process to Create or Resolve KeysThe most difficult case is systems where the resource id is a not a reliable row key, and resources have multiple identifiers with no clear way to select one for the key.

In this case, implementations will likely have to fall back to some form of preprocessing to determine appropriate keys. This may be accomplished by:

  • Pre-processing all data to have clear resource ids or "primary" identifiers and using one of the options above.
  • Building some form of cross-link table dynamically within the implementation itself based on the underlying data. For instance, if an implementation's getResourceKey uses a specific identifier system, getReferenceKey could use a pre-built cross-link table to find the appropriate identifier-based key to return.
- -There are many variations and alternatives to the above. This specification -simply asserts that implementations must be able to produce a row key for each -resource and a matching key for references pointing at that resource, and -intentionally leaves the specific mechanism to the implementation. - -### Contained Resources - -This specification requires implementers to extract contained resources that -need to be included in views into independent resources that can then be -accessed via [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) -like any -other resource. Implementations SHOULD normalize these resources appropriately -whenever possible, such as eliminating duplicate resources contained in many -parent resources. Note that this may change in a later version of this -specification, allowing users to explicitly create separate views for contained -resources that could be distinct from top-level resource views. - -Contained resources have different semantics than other resources since they -don't have an independent identity, and the same logical record may be -duplicated across many containing resources. This makes SQL best practices -difficult since the data is denormalized and ambiguous. For -instance, `Patient.generalPractitioner` may be a contained resource that may or -may not be the same practitioner seen in other Patient resources. Therefore, the -approach in this specification requires systems to pre-process the data into -normalized, independent resources if needed. +## Specifying a Select -For the same reason, the output from running a ViewDefinition will not include -contained resources. For instance, a view of Practitioner resources will include -top-level Practitioner resources but not contained Practitioner resources from -inside the Patient resources. +A [select](StructureDefinition-ViewDefinition-definitions.html#diff_ViewDefinition.select) +specifies the content and names for the columns in the view. The content for +each column is defined with a [FHIRPath](https://hl7.org/fhirpath/) expression +that returns a specific data element from the FHIR resources. More complex views +can be specified to create resource or reference keys, unnest a collection of +items returned by a FHIRPath expression, nest or concatenate the results from +multiple selects, and so on. -### Unnesting semantics +## Unnesting Semantics It is often desirable to unroll the collection returned from a [FHIRPath](https://hl7.org/fhirpath/) expression into a separate row for each @@ -325,7 +197,7 @@ or more `Patient.address`, the result will be identical to the expression above. ] ``` -### Multiple Select Expressions +## Multiple Select Expressions A ViewDefinition may have multiple `select`s, which can be organized as siblings or in parent/child relationships. This is typically done as different `select`s @@ -343,7 +215,7 @@ joined to others with the following rules: The [examples](StructureDefinition-ViewDefinition-examples.html) illustrate this behavior. -### Unions +## Unions A `select` can have an optional `unionAll`, which contains a list of `select`s that are used to create a union. `unionAll` effectively concatenates the results @@ -393,7 +265,7 @@ since these items must be combined in a single, logical `UNION ALL`. Nested `select`s can be used when multiple `unionAll`s are needed within a single view. -### Composing Multiple Selects and Unions +## Composing Multiple Selects and Unions `unionAll` produces rows that can be used just like a `select` expression. These rows can be used by containing `select`s or `unionAll`s without needing any @@ -454,7 +326,7 @@ And, the equivalent with a nested structure: Note the former example is preferred due to its simplicity and the latter is included purely for illustrative purposes. -### Column Ordering +## Column Ordering View Runners that support column ordering in their output format MUST order the columns of the result according to the rules defined in this section. @@ -509,7 +381,7 @@ order: } ``` -### Column Types +## Column Types All values in a given column must be of a single data type. The data type can be explicitly specified with @@ -554,7 +426,175 @@ simply produce a string, as would be done in a CSV-based implementation. See the [database type hints](#type-hinting-with-tags) section below if finer database-specific type control is needed. -### Type Hinting with Tags +## Using Constants + +A ViewDefinition may include one or more constants, which are simply values that +can be reused in [FHIRPath](https://hl7.org/fhirpath/) expressions. This can +improve readability and reduce redundancy. Constants can be used in `path` +expressions by simply using `%[name]`. Effectively, these placeholders are +replaced by the value of the constant before +the [FHIRPath](https://hl7.org/fhirpath/) expression is evaluated. + +This is an example of a constant used in the `where` constraint of a view: + +```js +{ + // + "constant": [{ + "name": "bp_code", + "valueCode": "8480-6" + }], + // + "where": [{ + "path": "code.coding.exists(system='http://loinc.org' and code=%bp_code)" + }], +} +``` + +## Joins with Resource and Reference Keys + +While ViewDefinitions do not directly implement joins across resources, the +views produced should be easily joined by the database or analytic tools of the +user's choice. This can be done by including primary and foreign keys as part of +the tabular +view output, which can be done with +the [getResourceKey](#getresourcekey--keytype) and +[getReferenceKey](#getreferencekeyresource-type-specifier--keytype) functions. + +Users may call [getResourceKey](#getresourcekey--keytype) to obtain a +resources primary key, +and call [getReferenceKey](#getreferencekeyresource-type-specifier--keytype)to +get +the corresponding foreign key from a reference pointing at that resource/row. + +For example, a minimal view of Patient resources could look like this: + +```js +{ + "name": "active_patients", + "resource": "Patient", + "select": [{ + "column": [ + { + "path": "getResourceKey()", + "name": "id" + }, + { + "path": "active" + } + ] + }] +} +``` + +A view of Observation resources would then have its own row key and a foreign +key to easily join to the view of Patient resources, like this: + +```js +{ + "name": "simple_obs", + "resource": "Observation", + "select": [{ + "column": [ + { + "path": "getResourceKey()", + "name": "id" + }, + { + // The `Patient` parameter is optional, but ensures the returned value + // will either be a patient row key or null. + "path": "subject.getReferenceKey(Patient)", + "name": "patient_id" + } + ] + }] +} +``` + +Users of the views could then join `simple_obs.patient_id` +to `active_patients.id` using common join semantics. + +### Suggested Implementations for getResourceKey() and getReferenceKey() + +While [getResourceKey](#getresourcekey--keytype) +and [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) must +return matching values for the same row, *how* they do so is left to the +implementation. This is by design, allowing ViewDefinitions to be run across a +wide set of systems that have different data invariants or pre-processing +capabilities. + +Here are some implementation options to meet different needs: + + + + + + + + + + + + + + + + + + + + + + +
ApproachDetails
Return the Resource IDIf the system can guarantee that each resource has a simple id and the corresponding references have simple, relative ids that point to it (e.g., Patient/123), getResourceKey and getReferenceKey implementations may simply return those values. This is the simplest case and will apply to many (but not all) systems.
Return a "Primary" IdentifierSince the resource id is by definition a system-specific identifier, it may change as FHIR data is exported and loaded between systems, and therefore not be a reliable target for references. For instance, a bulk export from one source system could be loaded into a target system that applies its own ids to the resources, requiring that joins be done on the resource's identifier rather than its id.

In this case, implementations will need to determine row keys based on the resource identifier and corresponding identifiers in the references.

The simplest variation of this is when there is only one identifier for each resource. In other cases, the implementation may be able to select a "primary" identifier, based on the identifier.system namespace, identifier.use code, or other property. For instance, if the primary Identifier.system is example_primary_system, implementations can select the desired identifier to use as a row key by checking for that.

In either case, the resource identifier and corresponding reference identifier can then be converted to a row key, perhaps by building a string or computing a cryptographic hash of the identifiers themselves. The best approach is left to the implementation.
Pre-Process to Create or Resolve KeysThe most difficult case is systems where the resource id is a not a reliable row key, and resources have multiple identifiers with no clear way to select one for the key.

In this case, implementations will likely have to fall back to some form of preprocessing to determine appropriate keys. This may be accomplished by:

  • Pre-processing all data to have clear resource ids or "primary" identifiers and using one of the options above.
  • Building some form of cross-link table dynamically within the implementation itself based on the underlying data. For instance, if an implementation's getResourceKey uses a specific identifier system, getReferenceKey could use a pre-built cross-link table to find the appropriate identifier-based key to return.
+ +There are many variations and alternatives to the above. This specification +simply asserts that implementations must be able to produce a row key for each +resource and a matching key for references pointing at that resource, and +intentionally leaves the specific mechanism to the implementation. + +## Contained Resources + +This specification requires implementers to extract contained resources that +need to be included in views into independent resources that can then be +accessed via [getReferenceKey](#getreferencekeyresource-type-specifier--keytype) +like any +other resource. Implementations SHOULD normalize these resources appropriately +whenever possible, such as eliminating duplicate resources contained in many +parent resources. Note that this may change in a later version of this +specification, allowing users to explicitly create separate views for contained +resources that could be distinct from top-level resource views. + +Contained resources have different semantics than other resources since they +don't have an independent identity, and the same logical record may be +duplicated across many containing resources. This makes SQL best practices +difficult since the data is denormalized and ambiguous. For +instance, `Patient.generalPractitioner` may be a contained resource that may or +may not be the same practitioner seen in other Patient resources. Therefore, the +approach in this specification requires systems to pre-process the data into +normalized, independent resources if needed. + +For the same reason, the output from running a ViewDefinition will not include +contained resources. For instance, a view of Practitioner resources will include +top-level Practitioner resources but not contained Practitioner resources from +inside the Patient resources. + +## Generating Schemas + +The output format produced by a View Runner will be technology-specific, such as +a SQL database query or a structured file like Parquet. View Runner +implementations SHOULD offer a way to compute the output schema from a +ViewDefinition when applicable. + +For example, a runner that produces a table in a database system could return +a "CREATE TABLE" or "CREATE VIEW" statement based on the ViewDefinition, +allowing the system to define tables prior to populating them by evaluating the +views over data. + +This would not apply to outputs that do not have a way of specifying their +schema, like CSV files. + +## Type Hinting with Tags Since these analytic views are often used as SQL tables, it can be useful to provide database type information to ensure the desired tables or views are @@ -597,50 +637,165 @@ Another use case may be for users to select database-specific numeric types. Behavior is undefined and left to the runner if the expression returns a value that is incompatible with the underlying database type. -## Using Constants +## Processing Algorithm -A ViewDefinition may include one or more constants, which are simply values that -can be reused in [FHIRPath](https://hl7.org/fhirpath/) expressions. This can -improve readability and reduce redundancy. Constants can be used in `path` -expressions by simply using `%[name]`. Effectively, these placeholders are -replaced by the value of the constant before -the [FHIRPath](https://hl7.org/fhirpath/) expression is evaluated. +The following description provides an algorithm for how to process a FHIR +resource as input for a ViewDefinition. Implementations do not need to follow +this algorithm directly, but their outputs should be consistent with what this +model produces. -This is an example of a constant used in the `where` constraint of a view: +### Validate Columns (entry point) -```js -{ - // - "constant": [{ - "name": "bp_code", - "valueCode": "8480-6" - }], - // - "where": [{ - "path": "code.coding.exists(system='http://loinc.org' and code=%bp_code)" - }], -} -``` +**Purpose**: This step ensures that a ViewDefinition's columns are valid, by setting up a recursive call. -## Processing Algorithm (Model) +**Inputs** +* `V`: a `ViewDefinition` to validate -See [Processing Algorithm](implementer_guidance.html#processing-model) for a -description of how to -process a FHIR resource as input for a `ViewDefinition`. Implementations do not -need to follow this algorithm directly, but their outputs should be consistent -with what this model produces. +1. Call `ValidateColumns(V, C)` according to the recursive step below. -## Generating Schemas +### `ValidateColumns(S, C)` (recursive step) -The output format produced by a View Runner will be technology-specific, such as -a SQL database query or a structured file like Parquet. View Runner -implementations SHOULD offer a way to compute the output schema from a -ViewDefinition when applicable. +**Purpose:** This step ensures that column names are unique across `S` and disjoint from `C` -For example, a runner that produces a table in a database system could return -a "CREATE TABLE" or "CREATE VIEW" statement based on the ViewDefinition, -allowing the system to define tables prior to populating them by evaluating the -views over data. +**Inputs** +* `S`: a single Selection Structure +* `C`: a list of Columns that exist prior to this call -This would not apply to outputs that do not have a way of specifying their -schema, like CSV files. +**Outputs** +* `Ret`: a list of Columns + +**Errors** +* Column Already Defined +* Union Branches Inconsistent + +0. Initialize `Ret` to equal `C` + +1. For each Column `col` in `S.column[]` + * If a Column with name `col.name` already exists in `Ret`, throw "Column Already Defined" + * Otherwise, append `col` to `Ret` + +2. For each Selection Structure `sel` in `S.select[]` + * For each Column `c` in `Validate(sel, Ret)` + * If a Column with name `c.name` already exists in `Ret`, throw "Column Already Defined" + * Otherwise, append `c` to the end of `Ret` + +3. If `S.unionAll[]` is present + 1. Define `u0` as `Validate(S.unionAll[0], Ret)` + 2. For each Selection Structure `sel` in `S.unionAll[]` + * Define `u` as `ValidateColumns(sel, Ret)` + * If the list of names from `u0` is different from the list of names from `u`, throw "Union Branches Inconsistent" + * Otherwise, continue + 3. For each Column `col` in `u0` + * Append `col` to `Ret` + +4. Return `Ret` + +### Process a Resource (entry point) + +**Purpose:** This step emits all rows produced by a ViewDefinition on an input +Resource, by setting up a recursive call. + +**Inputs** + +* `V`: a `ViewDefinition` +* `R`: a FHIR Resource to process with `V` + +**Emits:** one output row at a time + +1. Ensure resource type is correct + * If `R.resourceType` is different from `V.resource`, return immediately + without emitting any rows + * Otherwise, continue +2. If `V.where` is defined, ensure constraints are met + * Evaluate `fhirpath(V.where.path, R)` to determine whether `R` is a + candidate for `V` + * If `R` is not a candidate for `V`, return immediately without emitting + any rows + * Otherwise, continue +3. Emit all rows from `Process(S, V)` + +### `Process(S, N)` (recursive step) + +**Purpose:** This step emits all rows for a given Selection Structure and Node. +We first generate sets of "partial rows" (i.e., sets of incomplete column +bindings from the various clauses of `V`) and combine them to emit complete +rows. For example, if there are two sets of partial rows: + +* `[{"a": 1},{"a": 2}]` with bindings for the variable `a` +* `[{"b": 3},{"b": 4}]` with bindings for the variable `b` + +Then the Cartesian product of these sets consists of four complete rows: + +```json +[ + {"a": 1, "b": 3}, + {"a": 1, "b": 4}, + {"a": 2, "b": 3}, + {"a": 2, "b": 4} +] +``` + +**Inputs** +* `S`: a Selection Structure +* `N`: a Node (element) from a FHIR resource + +**Errors** +* Multiple values found but not expected for column + +**Emits:** One output row at a time + +1. Define a list of Nodes `foci` as + * If `S.forEach` is defined: `fhirpath(S.forEach, N)` + * Else if `S.forEachOrNull` is defined: `fhirpath(S.forEachOrNull, N)` + * Otherwise: `[N]` (a list with just the input node) + +2. For each element `f` of `foci` + 1. Initialize an empty list `parts` (each element of `parts` will be a list + of partial rows) + 2. Process Columns: + * For each Column `col` of `S.column`, define `val` + as `fhirpath(col.path, f)` + 1. Define `b` as a row whose column named `col.name` takes the value + * If `val` was the empty set: `null` + * Else if `val` has a single element `e`: `e` + * Else if `col.collection` is true: `val` + * Else: throw "Multiple values found but not expected for + column" + 2. Append `[b]` to `parts` + + * (Note: append a list so the final element of `parts` is now a list + containing the single row `b`). + 3. Process Selects: + * For each selection structure `sel` of `S.select` + 1. Define `rows` as the collection of all rows emitted + by `Process(sel, f)` + 2. Append `rows` to `parts` + + * (Note: do not append the elements but the whole list, so the final + element of `parts` is now the list `rows`) + 4. Process UnionAlls: + 1. Initialize `urows` as an empty list of rows + 2. For each selection structure `u` of `S.unionAll` + * For each row `r` in `Process(u, f)` + * Append `r` to `urows` + 3. Append `urows` to `parts` + + * (Note: do not append the elements but the whole list, so the final + element of `parts` is now the list `urows`) + 5. For every list of partial rows `prows` in the Cartesian product + of `parts` + 1. Initialize a blank row `r` + 2. For each partial row `p` in `prows` + * Add `p`'s column bindings to the row `r` + 3. Emit the row `r` + * (Note: the Cartesian product is always between a Selection + Structure and its direct children, not deeper descendants. Because + the process is recursive, rows generated by, for example, + a `.select[0].select[0].select[0]` will eventually bubble up to + the top level, but the bubbling happens one level at a time.) +3. If `foci` is an empty list and `S.forEachOrNull` is defined (Note: when this + condition is met, no rows have been emitted so far) + 1. Initialize a blank row `r` + 2. For each Column `c` in `ValidateColumns(V, [])` + * Bind the column `c.name` to `null` in the row `r` + 3. Emit the row `r` diff --git a/input/pagecontent/implementer_guidance.md b/input/pagecontent/implementer_guidance.md deleted file mode 100644 index 8e5601c..0000000 --- a/input/pagecontent/implementer_guidance.md +++ /dev/null @@ -1,147 +0,0 @@ -This spec defines the ViewDefinition model and logical system layers, but intentionally -leaves the details of those layers to implementations. However, implementations themselves -are encouraged to share common approaches when helpful. - -### Implementation Patterns - -TODO: add links to implementation patterns (like Spark, JSON, etc) as these efforts materialize. - -### Processing Model - -The following description provides an algorithm for how to process a FHIR -resource as input for a `ViewDefinition`. Implementations do not need to follow -this algorithm directly, but their outputs should be consistent with what this -model produces. - -#### Validate Columns (entry point) - -**Purpose**: This step ensures that a ViewDefinition's columns are valid, by setting up a recursive call. - -**Inputs** -* `V`: a `ViewDefinition` to validate - -1. Call `ValidateColumns(V, C)` according to the recursive step below. - -### `ValidateColumns(S, C)` (recursive step) - -**Purpose:** This step ensures that column names are unique across `S` and disjoint from `C` - -**Inputs** -* `S`: a single Selection Structure -* `C`: a list of Columns that exist prior to this call - -**Outputs** -* `Ret`: a list of Columns - -**Errors** -* Column Already Defined -* Union Branches Inconsistent - -0. Initialize `Ret` to equal `C` - -1. For each Column `col` in `S.column[]` - * If a Column with name `col.name` already exists in `Ret`, throw "Column Already Defined" - * Otherwise, append `col` to `Ret` - -2. For each Selection Structure `sel` in `S.select[]` - * For each Column `c` in `Validate(sel, Ret)` - * If a Column with name `c.name` already exists in `Ret`, throw "Column Already Defined" - * Otherwise, append `c` to the end of `Ret` - -3. If `S.unionAll[]` is present - 1. Define `u0` as `Validate(S.unionAll[0], Ret)` - 2. For each Selection Structure `sel` in `S.unionAll[]` - * Define `u` as `ValidateColumns(sel, Ret)` - * If the list of names from `u0` is different from the list of names from `u`, throw "Union Branches Inconsistent" - * Otherwise, continue - 3. For each Column `col` in `u0` - * Append `col` to `Ret` - -4. Return `Ret` - -### Process a Resource (entry point) - -**Purpose:** This step emits all rows produced by a ViewDefinition on an input Resource, by setting up a recursive call. - -**Inputs** -* `V`: a `ViewDefinition` -* `R`: a FHIR Resource to process with `V` - -**Emits:** one output row at a time - -1. Ensure resource type is correct - * If `R.resourceType` is different from `V.resource`, return immediately without emitting any rows - * Otherwise, continue -2. If `V.where` is defined, ensure constraints are met - * Evaluate `fhirpath(V.where.path, R)` to determine whether `R` is a candidate for `V` - * If `R` is not a candidate for `V`, return immediately without emitting any rows - * Otherwise, continue -3. Emit all rows from `Process(S, V)` - -### `Process(S, N)` (recursive step) - -**Purpose:** This step emits all rows for a given Selection Structure and Node. We first generate sets of "partial rows" (i.e., sets of incomplete column bindings from the various clauses of `V`) and combine them to emit complete rows. For example, if there are two sets of partial rows: - -* `[{"a": 1},{"a": 2}]` with bindings for the variable `a` -* `[{"b": 3},{"b": 4}]` with bindings for the variable `b` - -Then the Cartesian product of these sets consists of four complete rows: - -```js -[ - {"a": 1, "b": 3}, - {"a": 1, "b": 4}, - {"a": 2, "b": 3}, - {"a": 2, "b": 4} -] -``` - -**Inputs** -* `S`: a Selection Structure -* `N`: a Node (element) from a FHIR resource - -**Errors** -* Multiple values found but not expected for column - -**Emits:** One output row at a time - -1. Define a list of Nodes `foci` as - * If `S.forEach` is defined: `fhirpath(S.forEach, N)` - * Else if `S.forEachOrNull` is defined: `fhirpath(S.forEachOrNull, N)` - * Otherwise: `[N]` (a list with just the input node) - -2. For each element `f` of `foci` - 1. Initialize an empty list `parts` (each element of `parts` will be a list of partial rows) - 2. Process Columns: - * For each Column `col` of `S.column`, define `val` as `fhirpath(col.path, f)` - 1. Define `b` as a row whose column named `col.name` takes the value - * If `val` was the empty set: `null` - * Else if `val` has a single element `e`: `e` - * Else if `col.collection` is true: `val` - * Else: throw "Multiple values found but not expected for column" - 2. Append `[b]` to `parts` - * (Note: append a list so the final element of `parts` is now a list containing the single row `b`). - 3. Process Selects: - * For each selection structure `sel` of `S.select` - 1. Define `rows` as the collection of all rows emitted by `Process(sel, f)` - 2. Append `rows` to `parts` - * (Note: do not append the elements but the whole list, so the final element of `parts` is now the list `rows`) - 4. Process UnionAlls: - 1. Initialize `urows` as an empty list of rows - 2. For each selection structure `u` of `S.unionAll` - * For each row `r` in `Process(u, f)` - * Append `r` to `urows` - 3. Append `urows` to `parts` - * (Note: do not append the elements but the whole list, so the final element of `parts` is now the list `urows`) - 5. For every list of partial rows `prows` in the Cartesian product of `parts` - 1. Initialize a blank row `r` - 2. For each partial row `p` in `prows` - * Add `p`'s column bindings to the row `r` - 3. Emit the row `r` - * (Note: the Cartesian product is always between a Selection Structure and its direct children, not deeper descendants. Because the process is recursive, rows generated by, for example, a `.select[0].select[0].select[0]` will eventually bubble up to the top level, but the bubbling happens one level at a time.) -3. If `foci` is an empty list and `S.forEachOrNull` is defined - * (Note: when this condition is met, no rows have been emitted so far) - 1. Initialize a blank row `r` - 2. For each Column `c` in `ValidateColumns(V, [])` - * Bind the column `c.name` to `null` in the row `r` - 3. Emit the row `r` diff --git a/sushi-config.yaml b/sushi-config.yaml index d5a0017..a9a93d5 100644 --- a/sushi-config.yaml +++ b/sushi-config.yaml @@ -72,7 +72,6 @@ parameters: menu: Overview: index.html View Definition: StructureDefinition-ViewDefinition.html - Implementer Guidance: implementer_guidance.html Artifacts: artifacts.html Contributing: contributing.html From f9a3449d9a1db4b4d4440dd28c388aebd13a372c Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 20 Jun 2024 17:31:37 +1000 Subject: [PATCH 13/14] Update contributing page --- input/pagecontent/contributing.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/input/pagecontent/contributing.md b/input/pagecontent/contributing.md index 88eea18..a4ed493 100644 --- a/input/pagecontent/contributing.md +++ b/input/pagecontent/contributing.md @@ -1,18 +1,27 @@ -Contributors and early users are welcome! Here are some places to start: +Contributors are welcome! Here are some places to start: -* Contribute to [github discussion](https://github.com/FHIR/sql-on-fhir-v2/discussions) +* Join the discussion on the [FHIR chat](https://chat.fhir.org/#narrow/stream/179219-analytics-on-FHIR) * Join us on weekly meetings, Tuesday 3pm US Eastern time, Zoom links posted in [FHIR chat](https://chat.fhir.org/#narrow/stream/179219-analytics-on-FHIR) -* Ask any questions in [FHIR chat](https://chat.fhir.org/#narrow/stream/179219-analytics-on-FHIR) +* Contribute issues or pull requests on [GitHub](https://github.com/FHIR/sql-on-fhir-v2) ### Credits + * Nikolai Ryzhikov @niquola (Health Samurai) +* Ryan Brush @rbrush (Google) +* John Grimes @johngrimes (CSIRO) +* Josh Mandel @jmandel (Microsoft) * Dan Gottlieb @gotdan (Central Square Solutions) +* Arjun Sanyal @arjun (NCQA) +* Igor Kislitsyn @Yngwarr (Health Samurai) +* Maxim Putinstev @mput (Health Samurai) +* Bashir Sadjad @bashir2 (Google) +* Alex Walley @awalley-ncqa (NCQA) * Vadim Peretokin @vadi2 (Philips) * Marat Surmashev @aitem (Health Samurai) -* Ryan Brush @rbrush (Google) * Brian Kaney @bkaney (Vermonster) -* Josh Mandel @jmandel (Microsoft) -* John Grimes @johngrimes (CSIRO) +* Carl Anderson @barabo (Mayo Clinic) +* Joel Montavon @joelmontavon (Pharmacy Quality Alliance) +* Daniel Kapitan @dkapitan (PharmAccess Foundation) * FHIR Community - https://chat.fhir.org/ Work is sponsored and supported by: From cdd99e25176df65947cf47e5b93b6100f32b46eb Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 20 Jun 2024 17:35:07 +1000 Subject: [PATCH 14/14] Update version to 2.0.0-pre --- sushi-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sushi-config.yaml b/sushi-config.yaml index a9a93d5..4d745b1 100644 --- a/sushi-config.yaml +++ b/sushi-config.yaml @@ -9,7 +9,7 @@ name: SQLonFHIR title: SQL on FHIR # description: Example Implementation Guide for getting started with SUSHI status: draft # draft | active | retired | unknown -version: 0.0.1-pre +version: 2.0.0-pre fhirVersion: 5.0.0 # https://www.hl7.org/fhir/valueset-FHIR-version.html copyrightYear: 2023+ releaseLabel: ci-build # ci-build | draft | qa-preview | ballot | trial-use | release | update | normative+trial-use @@ -58,7 +58,7 @@ publisher: parameters: excludettl: true path-test: tests - + # validation: [allow-any-extensions, no-broken-links] # # ╭────────────────────────────────────────────menu.xml────────────────────────────────────────────╮