Skip to content

Commit

Permalink
references #114 Separate analysis SQL from ETL code.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mgurley
mgurley committed Oct 17, 2019
1 parent 95717d2 commit 1cb9e15
Show file tree
Hide file tree
Showing 4 changed files with 440 additions and 0 deletions.
125 changes: 125 additions & 0 deletions analysis/SQL for Use Cases PostgreSQL.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
-- Time between diagnosis and treatment, record level
SET search_path TO omop, public;
select
ed.person_id as person_id,
c.concept_name as cancer_type,
SELECT case split_part('1@2@', '@', 3) m.value_source_value
when '@' then 'Unknown' -- No records
when '0' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '00' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '05' then 'Non-Metastatic' -- No clinical or radiographic evidence of distant mets|- Tumor cells found in circulating blood, bone marrow or other distant lymph node tissue less than or equal to 0.2 mm
when '1' then 'Metastatic'
when '2' then 'Metastatic'
when '10' then 'Metastatic'
when '20' then 'Metastatic'
when '30' then 'Metastatic'
when '40' then 'Metastatic'
when '50' then 'Metastatic'
when '60' then 'Metastatic'
when '70' then 'Metastatic'
when '8' then 'Unknown' -- Unknown
when '9' then 'Unknown' -- Unknown
when '88' then 'Unknown' -- Death certificate only
when '99' then 'Unknown' -- Unknwon
else 'Unknown' end
FROM measurement m
--WHERE ed.episode_id = m.modifier_of_event_id
AND m.modifier_of_field_concept_id = 1000000003 -- 'epsiode.episode_id'
AND m.measurement_concept_id
-- NAACCR items for metastases. Does not include site-specific items.
IN (
35918335 -- EOD Mets
,35918581 -- Mets at DX-Bone
,35918692 -- Mets at DX-Brain
,35918491 -- Mets at DX-Distant LN
,35918290 -- Mets at DX-Liver
,35918559 -- Mets at DX-Lung
,35918527 -- Mets at DX-Other
)
as is_metastatic,
DATE_PART('day', et.episode_start_datetime - ed.episode_start_datetime) as time_daignosis_treatment_days
from episode ed
left join
(select min(episode_start_datetime) as episode_start_datetime, episode_parent_id
from episode
where episode_concept_id in (32531, 32532)
and episode_parent_id is not null
group by episode_parent_id
) as et
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.episode_id = et.episode_parent_id
join concept_relationship cr1
on ed.episode_object_concept_id = cr1.concept_id_2
and cr1.relationship_id = 'Maps to'
left join concept_relationship cr2
on cr1.concept_id_1 = cr2.concept_id_1
and cr2.relationship_id = 'ICDO to Schema'
join concept c
on cr2.concept_id_2 = c.concept_id


-- Survival from diagnosis, record level
SET search_path TO omop, public;

select
ed.episode_id,
ed.person_id,
c.concept_name as cancer_type,
case coalesce(reverse(substring(reverse(m.value_source_value) from 1
for position('@' in reverse(m.value_source_value)) - 1)), '@')
when '@' then 'Unknown' -- No records
when '0' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '00' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '05' then 'Non-Metastatic' -- No clinical or radiographic evidence of distant mets|- Tumor cells found in circulating blood, bone marrow or other distant lymph node tissue less than or equal to 0.2 mm
when '1' then 'Metastatic'
when '2' then 'Metastatic'
when '10' then 'Metastatic'
when '20' then 'Metastatic'
when '30' then 'Metastatic'
when '40' then 'Metastatic'
when '50' then 'Metastatic'
when '60' then 'Metastatic'
when '70' then 'Metastatic'
when '8' then 'Unknown' -- Unknown
when '9' then 'Unknown' -- Unknown
when '88' then 'Unknown' -- Death certificate only
when '99' then 'Unknown' -- Unknwon
else 'Unknown' end
as is_metastatic,
case dt.death_datetime is null when true then 1 else 0 end as vital_status,
round(DATE_PART('day', coalesce(dt.death_datetime, op.last_followup_date) - ed.episode_start_datetime)/30)
as survival_from_diagnosis_months
from episode ed
left join person p
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.person_id = p.person_id
LEFT JOIN death dt ON p.person_id = dt.person_id
left join
(select max(observation_period_end_date) as last_followup_date, person_id
from observation_period
group by person_id
) as op
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.person_id = op.person_id
join concept_relationship cr1
on ed.episode_object_concept_id = cr1.concept_id_2
and cr1.relationship_id = 'Maps to'
left join concept_relationship cr2
on cr1.concept_id_1 = cr2.concept_id_1
and cr2.relationship_id = 'ICDO to Schema'
join concept c
on cr2.concept_id_2 = c.concept_id
left join measurement m
on ed.episode_id = m.modifier_of_event_id
and m.modifier_of_field_concept_id = 1000000003 -- 'epsiode.episode_id'
and m.measurement_concept_id
-- NAACCR items for metastases. Does not include site-specific items.
in (
35918335 -- EOD Mets
,35918581 -- Mets at DX-Bone
,35918692 -- Mets at DX-Brain
,35918491 -- Mets at DX-Distant LN
,35918290 -- Mets at DX-Liver
,35918559 -- Mets at DX-Lung
,35918527 -- Mets at DX-Other
)
132 changes: 132 additions & 0 deletions analysis/SQL for Use Cases average.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
-- Average time between diagnosis and treatment
select e.cancer_type, e.is_metastatic,
avg(e.time_diagnosis_treatment_days) as avg_time_diagnosis_treatment_days
from
(select
c.concept_name as cancer_type,
case coalesce(reverse(substring(reverse(m.value_source_value) from 1
for position('@' in reverse(m.value_source_value)) - 1)), '@')
when '@' then 'Unknown' -- No records
when '0' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '00' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '05' then 'Non-Metastatic' -- No clinical or radiographic evidence of distant mets|- Tumor cells found in circulating blood, bone marrow or other distant lymph node tissue less than or equal to 0.2 mm
when '1' then 'Metastatic'
when '2' then 'Metastatic'
when '10' then 'Metastatic'
when '20' then 'Metastatic'
when '30' then 'Metastatic'
when '40' then 'Metastatic'
when '50' then 'Metastatic'
when '60' then 'Metastatic'
when '70' then 'Metastatic'
when '8' then 'Unknown' -- Unknown
when '9' then 'Unknown' -- Unknown
when '88' then 'Unknown' -- Death certificate only
when '99' then 'Unknown' -- Unknwon
else 'Unknown' end
as is_metastatic,
DATE_PART('day', et.episode_start_datetime - ed.episode_start_datetime) as time_diagnosis_treatment_days
from episode ed
left join
(select min(episode_start_datetime) as episode_start_datetime, episode_parent_id
from episode
where episode_concept_id in (32531, 32532)
and episode_parent_id is not null
group by episode_parent_id
) as et
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.episode_id = et.episode_parent_id
join concept_relationship cr1
on ed.episode_object_concept_id = cr1.concept_id_2
and cr1.relationship_id = 'Maps to'
left join concept_relationship cr2
on cr1.concept_id_1 = cr2.concept_id_1
and cr2.relationship_id = 'ICDO to Schema'
join concept c
on cr2.concept_id_2 = c.concept_id
left join measurement m
on ed.episode_id = m.modifier_of_event_id
and m.modifier_of_field_concept_id = 1000000003 -- 'epsiode.episode_id'
and m.measurement_concept_id
-- NAACCR items for metastases. Does not include site-specific items.
in (
35918335 -- EOD Mets
,35918581 -- Mets at DX-Bone
,35918692 -- Mets at DX-Brain
,35918491 -- Mets at DX-Distant LN
,35918290 -- Mets at DX-Liver
,35918559 -- Mets at DX-Lung
,35918527 -- Mets at DX-Other
)
) as e
where e.time_diagnosis_treatment_days is not null
group by e.cancer_type, e.is_metastatic

-- Average survival from diagnosis(only for deceased patients)
select e.cancer_type, e.is_metastatic,
avg(e.survival_from_diagnosis_months) as avg_survival_from_diagnosis_months
from
(
select
ed.person_id,
c.concept_name as cancer_type,
case coalesce(reverse(substring(reverse(m.value_source_value) from 1
for position('@' in reverse(m.value_source_value)) - 1)), '@')
when '@' then 'Unknown' -- No records
when '0' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '00' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '05' then 'Non-Metastatic' -- No clinical or radiographic evidence of distant mets|- Tumor cells found in circulating blood, bone marrow or other distant lymph node tissue less than or equal to 0.2 mm
when '1' then 'Metastatic'
when '2' then 'Metastatic'
when '10' then 'Metastatic'
when '20' then 'Metastatic'
when '30' then 'Metastatic'
when '40' then 'Metastatic'
when '50' then 'Metastatic'
when '60' then 'Metastatic'
when '70' then 'Metastatic'
when '8' then 'Unknown' -- Unknown
when '9' then 'Unknown' -- Unknown
when '88' then 'Unknown' -- Death certificate only
when '99' then 'Unknown' -- Unknwon
else 'Unknown' end
as is_metastatic,
case p.death_datetime is null when true then 1 else 0 end as vital_status,
round(DATE_PART('day', coalesce(p.death_datetime, op.last_followup_date) - ed.episode_start_datetime)/30)
as survival_from_diagnosis_months
from episode ed
left join person p
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.person_id = p.person_id
left join
(select max(observation_period_end_date) as last_followup_date, person_id
from observation_period
group by person_id
) as op
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.person_id = op.person_id
join concept_relationship cr1
on ed.episode_object_concept_id = cr1.concept_id_2
and cr1.relationship_id = 'Maps to'
left join concept_relationship cr2
on cr1.concept_id_1 = cr2.concept_id_1
and cr2.relationship_id = 'ICDO to Schema'
join concept c
on cr2.concept_id_2 = c.concept_id
left join measurement m
on ed.episode_id = m.modifier_of_event_id
and m.modifier_of_field_concept_id = 1000000003 -- 'epsiode.episode_id'
and m.measurement_concept_id
-- NAACCR items for metastases. Does not include site-specific items.
in (
35918335 -- EOD Mets
,35918581 -- Mets at DX-Bone
,35918692 -- Mets at DX-Brain
,35918491 -- Mets at DX-Distant LN
,35918290 -- Mets at DX-Liver
,35918559 -- Mets at DX-Lung
,35918527 -- Mets at DX-Other
)
) as e
where e.vital_status = 0
group by e.cancer_type, e.is_metastatic
120 changes: 120 additions & 0 deletions analysis/SQL for Use Cases.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
-- Time between diagnosis and treatment, record level
select
ed.person_id as person_id,
c.concept_name as cancer_type,
case coalesce(reverse(substring(reverse(m.value_source_value) from 1
for position('@' in reverse(m.value_source_value)) - 1)), '@')
when '@' then 'Unknown' -- No records
when '0' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '00' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '05' then 'Non-Metastatic' -- No clinical or radiographic evidence of distant mets|- Tumor cells found in circulating blood, bone marrow or other distant lymph node tissue less than or equal to 0.2 mm
when '1' then 'Metastatic'
when '2' then 'Metastatic'
when '10' then 'Metastatic'
when '20' then 'Metastatic'
when '30' then 'Metastatic'
when '40' then 'Metastatic'
when '50' then 'Metastatic'
when '60' then 'Metastatic'
when '70' then 'Metastatic'
when '8' then 'Unknown' -- Unknown
when '9' then 'Unknown' -- Unknown
when '88' then 'Unknown' -- Death certificate only
when '99' then 'Unknown' -- Unknwon
else 'Unknown' end
as is_metastatic,
DATE_PART('day', et.episode_start_datetime - ed.episode_start_datetime) as time_daignosis_treatment_days
from episode ed
left join
(select min(episode_start_datetime) as episode_start_datetime, episode_parent_id
from episode
where episode_concept_id in (32531, 32532)
and episode_parent_id is not null
group by episode_parent_id
) as et
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.episode_id = et.episode_parent_id
join concept_relationship cr1
on ed.episode_object_concept_id = cr1.concept_id_2
and cr1.relationship_id = 'Maps to'
left join concept_relationship cr2
on cr1.concept_id_1 = cr2.concept_id_1
and cr2.relationship_id = 'ICDO to Schema'
join concept c
on cr2.concept_id_2 = c.concept_id
left join measurement m
on ed.episode_id = m.modifier_of_event_id
and m.modifier_of_field_concept_id = 1000000003 -- 'epsiode.episode_id'
and m.measurement_concept_id
-- NAACCR items for metastases. Does not include site-specific items.
in (
35918335 -- EOD Mets
,35918581 -- Mets at DX-Bone
,35918692 -- Mets at DX-Brain
,35918491 -- Mets at DX-Distant LN
,35918290 -- Mets at DX-Liver
,35918559 -- Mets at DX-Lung
,35918527 -- Mets at DX-Other
)

-- Survival from diagnosis, record level
select
ed.person_id,
c.concept_name as cancer_type,
case coalesce(reverse(substring(reverse(m.value_source_value) from 1
for position('@' in reverse(m.value_source_value)) - 1)), '@')
when '@' then 'Unknown' -- No records
when '0' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '00' then 'Non-Metastatic' -- No distant metastasis|Unknown if distant metastasis
when '05' then 'Non-Metastatic' -- No clinical or radiographic evidence of distant mets|- Tumor cells found in circulating blood, bone marrow or other distant lymph node tissue less than or equal to 0.2 mm
when '1' then 'Metastatic'
when '2' then 'Metastatic'
when '10' then 'Metastatic'
when '20' then 'Metastatic'
when '30' then 'Metastatic'
when '40' then 'Metastatic'
when '50' then 'Metastatic'
when '60' then 'Metastatic'
when '70' then 'Metastatic'
when '8' then 'Unknown' -- Unknown
when '9' then 'Unknown' -- Unknown
when '88' then 'Unknown' -- Death certificate only
when '99' then 'Unknown' -- Unknwon
else 'Unknown' end
as is_metastatic,
case p.death_datetime is null when true then 1 else 0 end as vital_status,
round(DATE_PART('day', coalesce(p.death_datetime, op.last_followup_date) - ed.episode_start_datetime)/30)
as survival_from_diagnosis_months
from episode ed
left join person p
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.person_id = p.person_id
left join
(select max(observation_period_end_date) as last_followup_date, person_id
from observation_period
group by person_id
) as op
on ed.episode_concept_id = 32528 -- first disease occurrence
and ed.person_id = op.person_id
join concept_relationship cr1
on ed.episode_object_concept_id = cr1.concept_id_2
and cr1.relationship_id = 'Maps to'
left join concept_relationship cr2
on cr1.concept_id_1 = cr2.concept_id_1
and cr2.relationship_id = 'ICDO to Schema'
join concept c
on cr2.concept_id_2 = c.concept_id
left join measurement m
on ed.episode_id = m.modifier_of_event_id
and m.modifier_of_field_concept_id = 1000000003 -- 'epsiode.episode_id'
and m.measurement_concept_id
-- NAACCR items for metastases. Does not include site-specific items.
in (
35918335 -- EOD Mets
,35918581 -- Mets at DX-Bone
,35918692 -- Mets at DX-Brain
,35918491 -- Mets at DX-Distant LN
,35918290 -- Mets at DX-Liver
,35918559 -- Mets at DX-Lung
,35918527 -- Mets at DX-Other
)
Loading

0 comments on commit 1cb9e15

Please sign in to comment.