From 1f6bbdc362d3a2efe625d864040fac157e4af0cb Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 1 Jul 2021 17:41:50 +0300 Subject: [PATCH 01/45] Create load_input.sql this script populates the following tables: drug_concept_stage, internal_relationship_stage, and relationship_to_concept with the use of concept_manual, concept_relationship_manual and class_drugs_scraper. --- ATC/load_input.sql | 2469 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2469 insertions(+) create mode 100644 ATC/load_input.sql diff --git a/ATC/load_input.sql b/ATC/load_input.sql new file mode 100644 index 000000000..f61b69c25 --- /dev/null +++ b/ATC/load_input.sql @@ -0,0 +1,2469 @@ +/********************* +**** INPUT TABLES **** +**********************/ +-- for ATC we don't need ds_stage AND pc_stage +DROP TABLE IF EXISTS drug_concept_stage CASCADE; +DROP TABLE IF EXISTS internal_relationship_stage; +DROP TABLE IF EXISTS relationship_to_concept CASCADE;; + +CREATE TABLE drug_concept_stage +( concept_name VARCHAR(255), + vocabulary_id VARCHAR(20), + concept_class_id VARCHAR(20), + standard_concept VARCHAR(1), + concept_code VARCHAR(50), + possible_excipient VARCHAR(1), + domain_id VARCHAR(20), + valid_start_date DATE, + valid_end_date DATE, + invalid_reason VARCHAR(1), + source_concept_class_id VARCHAR(20)); + +CREATE TABLE internal_relationship_stage +( concept_code_1 VARCHAR(50), + concept_code_2 VARCHAR(50)); + +CREATE TABLE relationship_to_concept +( concept_code_1 VARCHAR(50), + vocabulary_id_1 VARCHAR(20), + concept_id_2 INT, + precedence INT, + conversion_factor FLOAT); + +--create indexes AND constraints +DROP INDEX if exists irs_concept_code_1; +DROP INDEX if exists irs_concept_code_2; +DROP INDEX if exists dcs_concept_code; +DROP INDEX if exists ds_drug_concept_code; +DROP INDEX if exists ds_ingredient_concept_code; +DROP INDEX if exists dcs_unique_concept_code; +DROP INDEX if exists irs_unique_concept_code; + +CREATE INDEX irs_concept_code_1 + ON internal_relationship_stage (concept_code_1); +CREATE INDEX irs_concept_code_2 + ON internal_relationship_stage (concept_code_2); +CREATE INDEX dcs_concept_code + ON drug_concept_stage (concept_code); +CREATE UNIQUE INDEX dcs_unique_concept_code + ON drug_concept_stage (concept_code); +CREATE INDEX irs_unique_concept_code + ON internal_relationship_stage (concept_code_1, concept_code_2); +/************************************* +***** internal_relationship_stage **** +**************************************/ +-- increase the LENGTH for concept_code_1 AND concept_code_2 fields to infinity +ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_1 TYPE VARCHAR; +ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_2 TYPE VARCHAR; +----------------------- +-- oral formulations -- +------------------------ +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_oral; +CREATE TABLE dev_oral +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217214 -- Oral Product (Dose Form Group) +AND relationship_id = 'RxNorm inverse is a' +AND d.concept_name !~* 'sublingual'); -- will be processed separately + +-- add links between Oral ATC Drug Classes AND RxN/RxE Dose Forms into the internal_relationship_stage +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT +a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_oral b +WHERE a.concept_name ~* 'oral|systemic|chewing gum' -- respective ATC dose forms (preliminarily converted from adm_r and added to the names in concept_manual) +AND a.invalid_reason IS NULL; -- indicates alive ATC code + +-- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_oral b +WHERE a.concept_name ~* 'oral|systemic|chewing gum' +AND a.invalid_reason IS NULL +) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code + FROM t1 a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) -- remove all unnecessary information after the semicolon + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_oral b +WHERE a.concept_name ~* 'oral|systemic|chewing gum' +AND a.invalid_reason IS NULL +) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code + FROM t1 a + JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = upper(TRIM(REGEXP_REPLACE(a.concept_name, ';.*$', ''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +----------------------------- +-- sublingual formulations -- +----------------------------- +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_sub; +CREATE TABLE dev_sub +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217214 +AND relationship_id = 'RxNorm inverse is a' +AND d.concept_name ~* 'sublingual'); -- should be separated FROM oral forms in the ATC vocabulary. + +-- add links between Sublingual ATC Drug Classes and Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_sub b +WHERE a.concept_name ~* 'sublingual' +AND a.invalid_reason IS NULL; -- 507 + +-- add links between Sublingual ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_sub b +WHERE a.concept_name ~* 'sublingual' +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 +FROM t1 a +JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Sublingual ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_sub b +WHERE a.concept_name ~* 'sublingual' +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +----------------------------- +-- parenteral formulations -- +----------------------------- +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_parenteral; +CREATE TABLE dev_parenteral +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217210 -- Injectable Product +AND relationship_id = 'RxNorm inverse is a'); -- returns all children of Injectable Product + +-- add links between Injectable ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_parenteral b +WHERE a.concept_name ~* 'parenteral|systemic' -- respective ATC routes +AND a.invalid_reason IS NULL +; + +-- add links between Parenteral ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_parenteral b +WHERE a.concept_name ~* 'parenteral|systemic' +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept c + ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Parenteral ATC Drug Classes AND Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_parenteral b +WHERE a.concept_name ~* 'parenteral|systemic' +AND a.invalid_reason IS NULL + ) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +------------------------ +-- nasal formulations -- +------------------------ +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_nasal; +CREATE TABLE dev_nasal +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217213 -- Nasal Product +AND relationship_id = 'RxNorm inverse is a'); -- returns all children of Nasal Product + +-- add those which are out of ancestry (Nasal Pin) +INSERT INTO dev_nasal +SELECT * FROM concept WHERE concept_id = 43563498; + +-- add links between Nasal ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_nasal b +WHERE a.concept_name ~* 'nasal' +AND a.invalid_reason IS NULL; + +-- add links between Nasal ATC Drug Classes AND Standars Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_nasal b +WHERE a.concept_name ~* 'nasal' +AND a.invalid_reason IS NULL +) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Nasal ATC Drug Classes AND Standars Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_nasal b +WHERE a.concept_name ~* 'nasal' +AND a.invalid_reason IS NULL +) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +-------------------------- +-- topical formulations -- +-------------------------- +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE IF EXISTS dev_topic; +CREATE TABLE dev_topic +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 IN ( + 36217206,36244040,36244034,36217219,-- Topical Product|Soap Product|Shampoo Product|Drug Implant Product + 36217223,36217212,36217224) -- Paste Product|Mucosal Product|Prefilled Applicator Product +AND relationship_id = 'RxNorm inverse is a'); -- returns all children of Topical Product + +-- add those which are out of Dose Form ancestry +INSERT INTO dev_topic +SELECT * FROM concept WHERE concept_id IN (43126087); -- Medicated Nail Polish + +-- add links between 1) genuine Topical ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_topic b +WHERE a.concept_name ~* 'topical' +AND a.invalid_reason IS NULL; -- exclude transdermal systems + +-- add links between 2) Transdermal or Implantable ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_topic b +WHERE a.concept_name ~* 'topical' +AND a.invalid_reason IS NULL; + +-- add links between Topical ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_topic b +WHERE a.concept_name ~* 'topical' +AND a.invalid_reason IS NULL +) -- respective ATC routes +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Ingredient name AS a code +FROM t1 a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Topical ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_topic b +WHERE a.concept_name ~* 'topical' +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Transdermal or Implantable ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_topic b +WHERE a.concept_name ~* 'transdermal|implant' -- respective ATC routes +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2-- Standard Ingredient name AS a code +FROM t1 a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Transdermal or Implantable ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_topic b +WHERE a.concept_name ~* 'transdermal|implant' -- respective ATC routes +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +----------------------------- +-- mouthwash formulations -- +----------------------------- +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_mouth; +CREATE TABLE dev_mouth +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36244022 -- Mouthwash Product (Dose Form Group) +AND relationship_id = 'RxNorm inverse is a' +AND d.concept_name ~* 'mouthwash'); + +-- add links between Local Oral ATC Drug Classes AND Rx Mouthwash formulations +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_mouth b +WHERE a.concept_name ~* 'local oral' -- respective ATC route +AND a.invalid_reason IS NULL; -- 21 + +-- add links between Local Oral ATC Drug Classes AND Ingredients using the concept table (up to date no need to use tne concept_synonym but you may check) +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_mouth b +WHERE a.concept_name ~* 'local oral' -- respective ATC routes +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 +FROM t1 a +JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +------------------------- +-- rectal formulations -- +------------------------- +-- create a temporary table WITH all related Dose Forms +DROP TABLE IF EXISTS dev_rectal; +CREATE TABLE dev_rectal +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217211 -- Rectal Product +AND relationship_id = 'RxNorm inverse is a'); + +-- add links between Rectal ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_rectal b +WHERE a.concept_name ~* 'rectal' -- respective ATC route +AND a.invalid_reason IS NULL; -- 1150 + +-- add links between Rectal ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_rectal b +WHERE a.concept_name ~* 'rectal' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Rectal ATC Drug Classes AND Standard Ingredients using the concept_synonym synonym +INSERT INTO internal_relationship_stage +( concept_code_1, + concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_rectal b +WHERE a.concept_name ~* 'rectal' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +-------------------------- +-- vaginal formulations -- +-------------------------- +-- create a temporary table WITH all related Dose Forms +DROP TABLE IF EXISTS dev_vaginal; +CREATE TABLE dev_vaginal +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + AND d.invalid_reason IS NULL +WHERE concept_id_1 = 36217209 +AND relationship_id = 'RxNorm inverse is a'); -- Vaginal Product + +-- add links between Vaginal ATC Drug Classes and Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_vaginal b +WHERE a.concept_name ~* 'vaginal' -- respective ATC routes +AND a.invalid_reason IS NULL; + +-- add links between Vaginal ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_vaginal b +WHERE a.concept_name ~* 'vaginal' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Vaginal ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_vaginal b +WHERE a.concept_name ~* 'vaginal' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +--------------------------- +-- urethral formulations -- +--------------------------- +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE IF EXISTS dev_urethral; +CREATE TABLE dev_urethral +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217225 -- Urethral Product +AND relationship_id = 'RxNorm inverse is a'); + +-- add links between Urethral ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_urethral b +WHERE a.concept_name ~* 'urethral' -- respective ATC route +AND a.invalid_reason IS NULL; + +-- add links between Urethral ATC Drug Classes AND Standard Ingredients using the concept table (up to date no need to use tne concept_synonym but you may check) +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) + WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_urethral b +WHERE a.concept_name ~* 'urethral' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code +c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +------------------------------ +--- ophtalmic formulations --- +------------------------------ +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_opht; +CREATE TABLE dev_opht +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217218 -- Ophthalmic Product (Dose Form Group) +AND relationship_id = 'RxNorm inverse is a' +AND d.concept_name ~* 'ophthalmic'); + +-- add links between Ophthalmic ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_opht b +WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route +AND a.invalid_reason IS NULL; + +-- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_opht b +WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 +FROM t1 a +JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_opht b +WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +--------------------------- +-- inhalant formulations -- +--------------------------- +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_inhal; +CREATE TABLE dev_inhal +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 IN (36217207, 36244037) -- Inhalant Product| Oral Spray Product +AND relationship_id = 'RxNorm inverse is a'); + +-- add links between Inhalant ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_inhal b +WHERE a.concept_name ~* 'inhalant' -- respective ATC route +AND a.invalid_reason IS NULL); + +-- add links between Inhalant ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_inhal b +WHERE a.concept_name ~* 'inhalant' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 +FROM t1 a + JOIN concept c ON UPPER (c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Inhalant ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_inhal b +WHERE a.concept_name ~* 'inhalant' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 +FROM t1 a + JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +-------------------------------- +-- Ingredients W/O Dose Forms -- +-------------------------------- +-- add links between ATC Drug Classes, which do not have Dose Forms, AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code AS concept_code_1, -- for such Drug Classes use ATC code only + c.concept_name AS concept_code_2 +FROM concept_manual a + JOIN concept c ON TRIM(UPPER (REGEXP_REPLACE (c.concept_name,'\s+|\W+','','g'))) = TRIM( UPPER (REGEXP_REPLACE (a.concept_name,'\s+|\W+| \(.*\)|, combinations.*|;.*$','','g'))) + AND a.concept_class_id = 'ATC 5th' + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (a.concept_code, c.concept_name) not in (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); + +-- add links between ATC Drug Classes, which do not have Dose Forms, AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code AS concept_code_1, -- for such Drug Classes use ATC code only + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM concept_manual a + JOIN concept_synonym b ON TRIM(UPPER (REGEXP_REPLACE (b.concept_synonym_name,'\s+|\W+','','g'))) = TRIM(UPPER (REGEXP_REPLACE (a.concept_name,'\s+|\W+|, combinations.*|;.*$','','g'))) + JOIN concept c + ON c.concept_id = b.concept_id + AND a.concept_class_id = 'ATC 5th' + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (a.concept_code, c.concept_name) not IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); + +/***************************** +**** combined ATC Classes **** +******************************/ +-- obtain 1st ATC Combo Ingredient using the concept table and full name match +drop table if exists dev_combo; +create unlogged table dev_combo AS ( +WITH t1 AS +( + SELECT * + FROM class_drugs_scraper + WHERE LENGTH(class_code) = 7 + AND (class_name ~* '\yand\y' + OR class_name LIKE '%,%' AND class_name LIKE '%combination%') + AND change_type IN ('A','') +) +SELECT DISTINCT class_code, + class_name, + adm_r, + SPLIT_PART(class_name,' and ',1) AS class, + c.concept_id, + c.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship +FROM t1 a + JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (class_name,' and ',1))) +WHERE c.standard_concept = 'S' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id ~ 'Rx' +); + +-- obtain 1st ATC Combo Ingredient using the concept table and full name match +INSERT INTO dev_combo +WITH t1 AS +( + SELECT * + FROM class_drugs_scraper + WHERE LENGTH(class_code) = 7 + AND (class_name ~* '\yand\y' + OR class_name LIKE '%,%' AND class_name LIKE '%combination%') + AND change_type IN ('A','') +) +SELECT DISTINCT class_code, + class_name, + adm_r, + SPLIT_PART(class_name,' and ',1) AS class, + c.concept_id, + c.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship +FROM t1 a + JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (class_name,' and ',1))),'\w+') +WHERE c.standard_concept = 'S' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id ~ 'Rx' +AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); + +-- obtain 1st ATC Combo Ingredient using the concept_synonym table and full name match +INSERT INTO dev_combo +WITH t1 AS +( + SELECT * + FROM class_drugs_scraper + WHERE LENGTH(class_code) = 7 + AND (class_name ~* '\yand\y' + OR class_name LIKE '%,%' AND class_name LIKE '%combination%') + AND change_type IN ('A','') +) +SELECT DISTINCT class_code, + class_name, + adm_r, + SPLIT_PART(class_name,' and ',1) AS class, + d.concept_id, + d.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship +FROM t1 a +JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE(TRIM(lower(SPLIT_PART (class_name,' and ', 1))), SUBSTRING(TRIM(lower(SPLIT_PART (class_name,' and ', 1))), '\w+')) +JOIN concept d ON d.concept_id = cs.concept_id +WHERE d.standard_concept = 'S' +AND d.concept_class_id = 'Ingredient' +AND d.vocabulary_id ~ 'Rx' +AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); + +-- obtain 2nd ATC Combo Ingredient using the concept table and full name match +INSERT INTO dev_combo +WITH t1 AS +( + SELECT * + FROM class_drugs_scraper + WHERE LENGTH(class_code) = 7 + AND (class_name ~* '\yand\y' + OR class_name LIKE '%,%' AND class_name LIKE '%combination%') + AND change_type IN ('A','') +) +SELECT DISTINCT class_code, + class_name, + adm_r, + SPLIT_PART(class_name,' and ',2) AS class, + c.concept_id, + c.concept_name, + 2 AS rnk -- stands for the Secondary lateral relationship +FROM t1 a + JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (a.class_name,' and ',2))) +WHERE c.standard_concept = 'S' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id ~ 'Rx'; + +-- obtain 2nd ATC Combo Ingredient using the concept table and partial name match +INSERT INTO dev_combo +WITH t1 AS +( + SELECT * + FROM class_drugs_scraper + WHERE LENGTH(class_code) = 7 + AND (class_name ~* '\yand\y' + OR class_name LIKE '%,%' AND class_name LIKE '%combination%') + AND change_type IN ('A','') +) +SELECT DISTINCT class_code, + class_name, + adm_r, + SPLIT_PART(class_name,' and ', 2) AS class, + c.concept_id, + c.concept_name, + 2 AS rnk +FROM t1 a + JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (class_name,' and ', 2))),'\w+') +WHERE c.standard_concept = 'S' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id ~ 'Rx' +AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) +AND c.concept_id NOT IN (19049024, 19136048); + +-- obtain 2nd ATC Combo Ingredient using the concept_synonym table and partial name match +INSERT INTO dev_combo +WITH t1 AS +( + SELECT * + FROM class_drugs_scraper + WHERE LENGTH(class_code) = 7 + AND (class_name ~* '\yand\y' + OR class_name LIKE '%,%' AND class_name LIKE '%combination%') + AND change_type IN ('A','') +) +SELECT DISTINCT class_code, + class_name, + adm_r, + SPLIT_PART(class_name,' and ',2) AS class, + d.concept_id, + d.concept_name, + 2 AS rnk +FROM t1 a +JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE ( TRIM(lower(SPLIT_PART (class_name,' and ', 2))), SUBSTRING ( TRIM(lower(SPLIT_PART (class_name,' and ', 2))), '\w+')) +JOIN concept d ON d.concept_id = cs.concept_id +WHERE d.standard_concept = 'S' +AND d.concept_class_id = 'Ingredient' +AND d.vocabulary_id ~ '^Rx' +AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); + +-- add manual mappings for ATC Combos using concept_relationship_manual +INSERT INTO dev_combo +WITH t1 AS +( + SELECT * + FROM class_drugs_scraper + WHERE LENGTH(class_code) = 7 + AND (class_name ~* '\yand\y' + OR class_name ~* 'combination|quinupristin\/dalfopristin') + AND change_type IN ('A','') +) +SELECT DISTINCT +class_code, + class_name, + adm_r, + '' AS class, -- leave it empty + c.concept_id, + c.concept_name, + CASE WHEN relationship_id = 'ATC - RxNorm pr lat' THEN 1 + WHEN relationship_id = 'ATC - RxNorm sec lat' THEN 2 + WHEN relationship_id = 'ATC - RxNorm pr up' THEN 3 + ELSE 4 -- stands for 'ATC - RxNorm sec up' + END AS rnk +FROM t1 a +JOIN concept_relationship_manual r ON r.concept_code_1= a.class_code +JOIN concept c ON c.concept_code = r.concept_code_2 AND c.vocabulary_id ~ 'Rx' +AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' +WHERE (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) +AND r.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up'); -- 1022 + +-- add Acetylsalicylic acid +INSERT INTO dev_combo +SELECT class_code, + class_name, + adm_r, + 'acetylsalicylic acid', + 1112807, + 'aspirin', + CASE WHEN class_name ~ '^acetylsalicylic' THEN 1 ELSE 2 END +FROM dev_combo +WHERE (class_code, concept_id) NOT IN ( select class_code, 1112807 FROM dev_combo WHERE class_name ~* 'acetylsalicylic') +AND class_name ~* 'acetylsalicylic'; + +-- ethinylestradiol +INSERT INTO dev_combo +SELECT class_code, + class_name, + adm_r, + 'ethinylestradiol', + 1549786, + 'ethinyl estradiol', + CASE WHEN class_name ~* '^ethinylestradiol' THEN 1 ELSE 2 END +FROM dev_combo +WHERE (class_code, concept_id) NOT IN ( select class_code, 1549786 FROM dev_combo WHERE class_name ~* 'ethinylestradiol') +AND class_name ~* 'ethinylestradiol'; + +-- estrogen +INSERT INTO dev_combo +SELECT class_code, + class_name, + adm_r, + 'ethinylestradiol', + 19049228, + 'estrogens', + CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo +WHERE (class_code, concept_id) NOT IN ( select class_code, 19049228 FROM dev_combo WHERE class_name ~* 'estrogen') +AND class_name ~ 'estrogen' + UNION ALL +SELECT class_code, + class_name, + adm_r, + 'estrogens, conjugated (USP)', + 1549080, + 'estrogens, conjugated (USP)', + CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo +WHERE (class_code, concept_id) NOT IN ( select class_code, 1549080 FROM dev_combo WHERE class_name ~* 'estrogen') +AND class_name ~ 'estrogen' + UNION ALL +SELECT class_code, + class_name, + adm_r, + 'estrogens, esterified (USP)', + 1551673, + 'estrogens, esterified (USP)', + CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo +WHERE (class_code, concept_id) NOT IN ( select class_code, 1551673 FROM dev_combo WHERE class_name ~* 'estrogen') +AND class_name ~ 'estrogen' + UNION ALL +SELECT class_code, + class_name, + adm_r, + 'synthetic conjugated estrogens, A', + 1596779, + 'synthetic conjugated estrogens, A', + CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo +WHERE (class_code, concept_id) NOT IN ( select class_code, 1596779 FROM dev_combo WHERE class_name ~* 'estrogen') +AND class_name ~* 'estrogen' + UNION ALL +SELECT class_code, + class_name, + adm_r, + 'synthetic conjugated estrogens, B' AS class, + 1586808, + 'synthetic conjugated estrogens, B', + CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo +WHERE (class_code, concept_id) NOT IN ( select class_code, 1586808 FROM dev_combo WHERE class_name ~* 'estrogen') +AND class_name ~* 'estrogen'; + +-- remove erroneous automap +DELETE FROM dev_combo WHERE class_code = 'M05BX53' AND concept_id = 19000815; -- strontium + +-- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.class_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code +a.concept_id +FROM dev_combo a, + dev_oral b, + concept_manual c +WHERE c.concept_name ~ 'oral|systemic|chewing gum' +AND c.concept_code = a.class_code +) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code + FROM t1 a + JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Oral ATC Drug Classes AND Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.class_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.class_name, +a.concept_id +FROM dev_combo a, + dev_parenteral b, + concept_manual c +WHERE c.concept_name ~ 'parenteral|systemic' +AND c.concept_code = a.class_code +) -- Oral formulations + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code + FROM t1 a + JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Vaginal ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.class_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.class_name, +a.concept_id +FROM dev_combo a, + dev_vaginal b, + concept_manual c +WHERE c.concept_name ~* 'vaginal' +AND c.concept_code = a.class_code +) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code + FROM t1 a + JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between ATC Drug Classes W/O Dose Forms AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT +class_code AS concept_code_1, -- ATC code + Dose Form name AS a code +class_name, -- ATC name to be used AS a key for JOIN +concept_id +FROM dev_combo +WHERE adm_r is null +) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code + FROM t1 a + JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); + +/****************************** +******* manual mapping ******** +*******************************/ +-- add manually mapped ATC Drug Classes to Standard Ingredients using concept_relationship_manual +INSERT INTO internal_relationship_stage +( concept_code_1, concept_code_2) +WITH t1 AS (select distinct a.concept_code_1, a.relationship_id, c.concept_name + FROM concept_relationship_manual a + JOIN class_drugs_scraper b + ON b.class_code = a.concept_code_1 + JOIN concept c + ON c.concept_code = a.concept_code_2 + AND c.vocabulary_id = a.vocabulary_id_2 + AND c.standard_concept = 'S' AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient') +SELECT DISTINCT concept_code_1, + concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code, +FROM t1 a + JOIN class_drugs_scraper b ON b.class_code = a.concept_code_1 AND b,change_type IN ('A', '') + AND a.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up') + AND (concept_code_1, concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1),concept_code_2 + FROM internal_relationship_stage); + +/********************************** +******* drug_concept_stage ******** +***********************************/ +TRUNCATE drug_concept_stage; +-- change length of concept_code field +ALTER TABLE drug_concept_stage ALTER COLUMN concept_code TYPE VARCHAR; + +-- add all ATC Drug Classes using the internal_relationship_stage table +INSERT INTO drug_concept_stage +( + concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date +) +SELECT DISTINCT concept_code_1, -- ATC code + name + 'ATC', + 'Drug Product', + NULL, + concept_code_1, + NULL, + 'Drug', + TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage; + +-- add ATC Drug Attributes in the form of Rx Dose Form names using the internal_relationship_stage table +INSERT INTO drug_concept_stage +( + concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date +) +SELECT DISTINCT concept_code_2 AS concept_name, -- ATC pseudo-attribute IN the form of OMOP Dose Form name + 'ATC' AS vocabulary_id, + 'Dose Form' AS concept_class_id, + NULL AS standard_concept, + concept_code_2 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, + TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage, + concept +WHERE concept_code_2 = concept_name +AND concept_class_id = 'Dose Form' +AND vocabulary_id ~ 'RxNorm' +AND invalid_reason IS NULL; + +-- add ATC Drug Attributes IN the form of Standard Ingredient names using the internal_relationship_stage table +INSERT INTO drug_concept_stage +( concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date) +SELECT DISTINCT concept_code_2, -- ATC pseudo-attribute IN the form of OMOP Ingredient name + 'ATC', + 'Ingredient', + NULL, + concept_code_2, + NULL, + 'Drug', + TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage, + concept +WHERE upper (concept_code_2) = upper(concept_name) +AND concept_class_id = 'Ingredient' +AND vocabulary_id ~ 'RxNorm' +AND standard_concept = 'S' +AND invalid_reason IS NULL +AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); + +-- obtain additional ingredients for those ATC codes which are still unmapped using fuzzy match +INSERT INTO internal_relationship_stage +WITH t1 AS +( + -- define concepts to map + SELECT DISTINCT class_code, + class_name + FROM class_drugs_scraper + WHERE ( + -- totally lost + class_code NOT IN (SELECT SPLIT_PART(concept_code,' ',1) FROM drug_concept_stage) + AND LENGTH(class_code) = 7 + AND class_code NOT IN (SELECT concept_code_1 + FROM concept_relationship_manual + WHERE relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up')) + AND class_code NOT IN ('B03AD04', 'V09GX01', 'V09XX03') -- ferric oxide polymaltose complexes | thallium (201Tl) chloride | selenium (75Se) norcholesterol + AND class_name !~* '^indium|^iodine|^yttrium|^RIFAMPICIN|coagulation factor' + AND change_type IN ('', 'A')) + OR ( + -- absent IN the internal_relationship_stage + class_code IN (SELECT SPLIT_PART(concept_code,' ',1) FROM drug_concept_stage) +AND class_code NOT IN (SELECT SPLIT_PART(concept_code_1,' ',1) + FROM internal_relationship_stage a + JOIN concept c + ON c.concept_name = a.concept_code_2 + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient') +AND LENGTH(class_code) = 7) +OR + -- with absent Ingredient IN drug_relationship_stage +(class_code IN (SELECT SPLIT_PART(concept_code_1,' ',1) +FROM internal_relationship_stage +GROUP BY concept_code_1 +HAVING COUNT(1) = 1) AND class_code NOT IN ( + SELECT SPLIT_PART(concept_code_1,' ',1) + FROM internal_relationship_stage a + JOIN concept c + ON LOWER (c.concept_name) = LOWER (a.concept_code_2) + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S')) +), +-- fuzzy macth 1 using name similarity +t2 AS +(SELECT a.*, + c.* +FROM t1 a + JOIN concept_synonym b ON lower (b.concept_synonym_name) LIKE lower (concat ('%',class_name,'%')) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S'), +-- fuzzy match WITH levenshtein +t3 AS +(SELECT * +FROM t1 a + JOIN concept c + ON devv5.levenshtein (lower (class_name),lower (concept_name)) = 1 + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND class_code NOT IN (SELECT class_code FROM t2) +), +-- match with non-standard and crosswalk to Standard +t4 AS (SELECT a.*,d.* +FROM t1 a + JOIN concept c ON lower(REGEXP_REPLACE (c.concept_name, '\s+|\W+', '', 'g')) = lower(TRIM(REGEXP_REPLACE(a.class_name,';.*$|, combinations?|IN combinations?', '', 'g'))) + AND c.domain_id = 'Drug' + JOIN concept_relationship r ON r.concept_id_1 = c.concept_id + JOIN concept d ON d.concept_id = r.concept_id_2 AND d.standard_concept = 'S' AND d.concept_class_id = 'Ingredient'), +t5 AS +(SELECT * +FROM t1 a + JOIN concept c + ON lower (c.concept_name) = lower (SUBSTRING (class_name,'\w+')) + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND a.class_code NOT IN (SELECT concept_code_1 FROM concept_relationship_manual) + AND concept_id NOT IN (19018544, 19071128, 1195334, 19124906, 19049024, 40799093, 19136048) --calcium|copper|choline|magnesium|potassium|Serum|sodium + AND class_code NOT IN (SELECT class_code FROM t2) +AND class_code NOT IN (SELECT class_code FROM t3) +), +t6 AS ( +SELECT class_code,class_name,concept_id,concept_name FROM t2 +UNION ALL +SELECT class_code,class_name,concept_id,concept_name FROM t3 +UNION ALL +SELECT class_code,class_name,concept_id,concept_name FROM t4 +UNION ALL +SELECT class_code,class_name,concept_id,concept_name FROM t5) +SELECT DISTINCT class_code, --class_name, + concept_name +FROM t6 +WHERE (class_code,concept_name) NOT IN (SELECT concept_code_1, + concept_code_2 + FROM internal_relationship_stage) + AND concept_id <> 43013482; -- butyl ester of methyl vinyl ether-maleic anhydride copolymer (125 kD) + +/********************************** +*** FUTHER WORK WITH ATC COMBOS *** +***********************************/ +-- assemble mappings for ATC Classes indicating Ingredient Groups using the the concept_ancestor AND/OR concept tables along WITH word pattern matching +-- take descendants of Acid preparations +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'acid preparations' AS class, + concept_id, + c.concept_name, + CASE WHEN class_name ~* '^acid' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, + concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id = 21600704-- ATC code of Acid preparations + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' +WHERE class_name ~* 'acid preparations'; + +-- Sulfonamides +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'sulfonamides' AS class, + concept_id, + c.concept_name, + CASE WHEN class_name ~* '^sulfonamides|^combinations of sulfonamides' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, + concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id = 21603038-- ATC code of sulfonamides + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + JOIN concept_relationship b ON b.concept_id_1 = ancestor_concept_id + AND b.invalid_reason is null AND b.relationship_id = 'ATC - RxNorm pr lat' +WHERE class_name ~* 'sulfonamides' AND class_name !~* '^short-acting sulfonamides|^intermediate-acting sulfonamides|^long-acting sulfonamides' + AND LENGTH (class_code) = 7; + +-- take descendants of Amino acids +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'amino acids', + concept_id, + concept_name, + CASE WHEN class_name ~* '^amino acids' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21601215, 21601034) -- 21601215 B05XB Amino acids| 21601034 B02AA Amino acids + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + WHERE class_name ~* 'amino\s*acid' + AND class_code <> 'B03AD01'; -- ferrous amino acid complex + +-- take descendants of Analgesics +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'analgesics', + concept_id, + concept_name, + CASE WHEN class_name ~* '^analgesics' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21604253) -- 21604253 N02 ANALGESICS ATC 2nd + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + AND concept_id NOT IN (939506, 950435, 964407) -- sodium bicarbonate|citric acid|salicylic acid + WHERE class_name ~* 'anae?lgesics?' AND class_name !~* '\yexcl' + AND LENGTH(class_code) = 7; + +-- take ingredients indicating Animals +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'animals', + concept_id, + concept_name, + CASE WHEN class_name ~* '^animals' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept +WHERE class_name ~* 'Animals' +AND LENGTH(class_code) = 7 +AND (concept_id IN (19091701,19056189,40170543,40170448,40170341,40170416,40175840,40175865,40170916,40175984,40161698,40170420, +19095690,40170741,40170848,40161809,40161813,45892235,40171114,45892234,37496548,40170660,40172147,40175843,40175898,40175933,40171110, +40175911,40171275,40172704,40171317,40175983,40171135,35201802,40238446,40175899,40227400,40175938,19061053,19112547,43013524,40170475, +40170818,40161805,40167658,1340875,42903998,963757,40171594,37496553,40172160,35201545,40175931,35201783,789889,35201778,40175951,35201548, +40161124,42709317,40161676,40161750,40170521,40161754,40170973,40170979,40170876,40175917) +OR ( +concept_name ~* 'rabbit|\ycow\y|\ydog\y|\ycat\y|goose|\yhog\y|\ygland\y|hamster|\yduck|oyster|\yhorse\y|\ylamb|pancreas|brain|kidney|\ybone\y|heart|spleen|lungs|^Pacific|\yfish|\yegg\y|\ypork|shrimp|\yveal|\ytuna|chicken' +AND concept_name ~* 'extract' AND vocabulary_id LIKE 'RxNorm%' +AND standard_concept = 'S' AND concept_class_id = 'Ingredient' +AND concept_id NOT IN (46276144,40170814,40226703,43560374,40227355,42903998,40227484,19086386)) +); + +-- take descendants of Antiinfectives +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'antiinfectives', + concept_id, + concept_name, + CASE WHEN class_name ~* '^anti-?infectives' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21605189, 21603552, 21605145, 21601168, 21605188, 21605146) -- Antiinfectives| ANTIINFECTIVES| ANTIINFECTIVES | Antiinfectives | Antiinfectives + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + AND concept_id NOT IN (19044522)-- zinc sulfate + WHERE class_name ~* 'anti-?infectives?' --AND class_name ~* '\yexcl' + AND LENGTH(class_code) = 7; + +-- take ingredients indicating Cadmium compounds +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'cadmium compounds', + concept_id, + concept_name, + CASE WHEN class_name ~* '^cadmium compounds' THEN 3 ELSE 4 END ::INT AS rnk -- groups don't have primary lateral ings +FROM class_drugs_scraper, concept +WHERE lower(concept_name) LIKE '%cadmium %' +AND concept_class_id = 'Ingredient' +AND vocabulary_id LIKE 'RxNorm%' +AND concept_id <> 45775350 +AND class_name ~* 'cadmium compounds?' --AND class_name ~* '\yexcl' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Calcium (different salts) +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'calcium (different salts IN combination)', + concept_id, + concept_name, + CASE WHEN class_name ~* '^calcium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept +WHERE concept_name ~* '\ycalcium\y' +AND concept_class_id = 'Ingredient' +AND vocabulary_id LIKE 'RxNorm%' +AND concept_id NOT IN (42903945,43533002,1337191,19007595,43532262,19051475) -- calcium ion|calcium hydride|calcium hydroxide|calcium oxide|calcium peroxide|anhydrous calcium iodide +AND class_name ~* 'calcium' AND class_name ~* '\ysalt' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Calcium compounds +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'calcium compounds', + concept_id, + concept_name, + CASE WHEN class_name ~* '^calcium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept +WHERE concept_name ~* '\ycalcium\y' +AND concept_class_id = 'Ingredient' +AND vocabulary_id LIKE 'RxNorm%' +AND concept_id NOT IN (19014944,42903945) +AND class_name ~* 'calcium' AND class_name ~* '\ycompound' +AND LENGTH (class_code) = 7; + +-- take descendants of Laxatives +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'contact laxatives', + concept_id, + concept_name, + CASE WHEN class_name ~* '^contact laxatives' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21600537) + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' +WHERE class_name ~* 'contact' AND class_name ~* 'laxatives?' +AND LENGTH(class_code) = 7; + +-- take descendants of Corticosteroids +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'corticosteroids', + concept_id, + concept_name, + CASE WHEN class_name ~* '^corticosteroids' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21605042, 21605164, 21605200, 21605165, 21605199, 21601607, 975125) + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' +WHERE class_name ~* 'corticosteroids?' +AND LENGTH(class_code) = 7; + +-- take descendants of Cough suppressants +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'cough suppressants', + concept_id, + concept_name, + CASE WHEN class_name ~* '^cough suppressants|^other cough suppressants' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21603440, 21603366, 21603409, 21603395, 21603436) + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + AND concept_id NOT IN (943191,1139042,1189220,1781321,19008366,19039512,19041843,19050346,19058933,19071861,19088167,19095266,42904041) + WHERE class_name ~* 'cough' AND class_name ~* 'suppressants?' +AND LENGTH(class_code) = 7; + +-- take descendants of Diuretics +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'diuretics', + concept_id, + concept_name, + CASE WHEN class_name ~* '^diuretics' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id = 21601461 + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + WHERE class_name ~* 'diuretics?' +AND LENGTH (class_code) = 7; + +-- take descendants of Magnesium (different salts IN combination) +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'magnesium (different salts IN combination)', + concept_id, + concept_name, + CASE WHEN class_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21600892) + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + WHERE class_name ~* 'magnesium' AND class_name ~* 'salt' +AND LENGTH(class_code) = 7; + +-- take ingredients indicating Magnesium (different salts IN combination) +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'magnesium (different salts IN combination)', + concept_id, + concept_name, + CASE WHEN class_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept c + WHERE class_name ~* 'magnesium' AND class_name ~* 'salt' + AND concept_name ~ 'magnesium' AND standard_concept = 'S' AND concept_class_id = 'Ingredient' +AND LENGTH(class_code)=7 +AND (class_code, concept_id) NOT IN (select class_code, concept_id FROM dev_combo) +AND concept_id NOT IN (43532017, 37498676); -- magnesium cation | magnesium Mg-28 + +-- take ingredients indicating Multivitamins +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'multivitamins', + concept_id, + concept_name, + CASE WHEN class_name ~* '^multivitamins' THEN 1 ELSE 2 END ::INT AS rnk +FROM class_drugs_scraper, + concept +WHERE concept_id = 36878782 + AND class_name ~* 'multivitamins?' + AND LENGTH (class_code) = 7 ; + +-- take descendants of Opium alkaloids WITH morphine +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'opium alkaloids WITH morphine', + concept_id, + concept_name, + CASE WHEN class_name ~* '^opium alkaloids WITH morphine' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21604255) -- Natural opium alkaloids + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + AND concept_id NOT IN (19112635) + WHERE class_name ~* 'opium alkaloids WITH morphine' + AND LENGTH (class_code) = 7; + +-- take descendants of Opium derivatives +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'opium derivatives', + concept_id, + concept_name, + CASE WHEN class_name ~* '^opium derivatives' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id = 21603396 + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + AND concept_id NOT IN (19021930, 1201620) + WHERE class_name ~* 'opium derivatives' + AND LENGTH (class_code) = 7; + +-- take descendants of Organic nitrates +INSERT INTO dev_combo +SELECT DISTINCT +class_code, class_name, adm_r, +'organic nitrates', + concept_id, + concept_name, + CASE WHEN class_name ~* '^organic nitrates' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21600316) + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + WHERE class_name ~* 'organic nitrates' + AND LENGTH (class_code) = 7; + +-- take descendants of Psycholeptics +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'psycholeptics', + concept_id, + concept_name, + CASE WHEN class_name ~* '^psycholeptics' THEN 3 WHEN CLASS_NAME ~ 'excl\. psycholeptics' THEN 0 ELSE 4 END ::INT AS rnk -- 0 stands for excluded drugs +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id = 21604489 + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + AND concept_id NOT IN (742594) + WHERE class_name ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' + AND LENGTH (class_code) = 7; + +-- take descendants of Selenium compounds +INSERT INTO dev_combo +SELECT DISTINCT +class_code, class_name, adm_r, +'selenium compounds', + concept_id, + concept_name, + CASE WHEN class_name ~* '^selenium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21600908) + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + WHERE class_name ~* 'selenium compounds' + AND LENGTH (class_code) = 7; + +-- take descendants of Silver compounds +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'silver compounds', + concept_id, + concept_name, + CASE WHEN class_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id IN (21602248) + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' + WHERE class_name ~* 'silver compounds' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Silver +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'silver compounds', + concept_id, + concept_name, + CASE WHEN class_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND concept_name ~* 'silver\y' + AND ('silver compounds', concept_id) NOT IN (select class, concept_id FROM dev_combo) +AND class_name ~* 'silver compounds' + AND LENGTH (class_code) = 7; + +-- take descendants of Sulfonylureas +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'sulfonylureas', + concept_id, + concept_name, + CASE WHEN class_name ~* '^sulfonylureas?' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id + AND ancestor_concept_id = 21600749 + AND vocabulary_id LIKE 'RxNorm%' + AND concept_class_id = 'Ingredient' +WHERE class_name ~* 'sulfonylureas?' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Snake venom antiserum +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'snake venom antiserum', + concept_id, + concept_name, + CASE WHEN class_name ~* '^snake venom antiserum' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'antiserum' AND concept_name ~* 'snake' +AND class_name ~* 'snake venom antiserum' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Aluminium preparations +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'aluminium preparations', + concept_id, + concept_name, + CASE WHEN class_name ~* '^aluminium preparations' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'aluminium|aluminum' + AND class_name ~* 'aluminium preparations' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Aluminium compounds +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'aluminium compounds', + concept_id, + concept_name, + CASE WHEN class_name ~* '^aluminium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'aluminium|aluminum' + AND class_name ~* 'aluminium compounds' + AND LENGTH(class_code) = 7; + +-- take ingredients indicating Lactic acid producing organisms +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'lactic acid producing organisms', + concept_id, + concept_name, + CASE WHEN class_name ~* '^lactic acid producing organisms' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'lactobacil' + AND class_name ~* 'lactic acid producing organisms' + AND LENGTH(class_code) = 7; + +-- take ingredients indicating Lactobacillus +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'lactobacillus', + concept_id, + concept_name, + CASE WHEN class_name ~* '^lactobacillus' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'lactobacil' + AND class_name ~* 'lactobacillus' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Magnesium compounds +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'magnesium compounds', + concept_id, + concept_name, + CASE WHEN class_name ~* '^magnesium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'magnesium' + AND class_name ~* 'magnesium compounds' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Grass pollen +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'grass pollen', + concept_id, + concept_name, + CASE WHEN class_name ~* '^grass pollen' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'grass' AND concept_name ~* 'pollen' + AND class_name ~* 'grass pollen' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Oil +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'oil', + concept_id, + concept_name, + 3 -- hardcoded + FROM class_drugs_scraper, + concept c +WHERE vocabulary_id IN ('RxNorm','RxNorm Extension') +AND concept_class_id = 'Ingredient' +AND standard_concept = 'S' +AND concept_name ~* '\yoil\y|\yoleum\y' +AND class_name ~* '^oil$' +AND LENGTH (class_code) = 7; + +-- take ingredients indicating Flowers +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'flowers', + concept_id, + concept_name, + CASE WHEN class_name ~* '^flowers' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* '\yflower\y' AND concept_name ~* 'extract' + AND class_name ~* '^flowers' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Fumaric acid derivatives +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'fumaric acid derivatives', + concept_id, + concept_name, + CASE WHEN class_name ~* '^fumaric acid derivatives' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'fumarate\y' + AND class_name ~* 'fumaric acid derivatives' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Glycerol +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'glycerol', + concept_id, + concept_name, + CASE WHEN class_name ~* '^glycerol' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'glycerol\y' + AND class_name ~* '^glycerol$' + AND LENGTH (class_code) = 7; + +-- take descendants of Proton pump inhibitors +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'proton pump inhibitors', + concept_id, + concept_name, + CASE WHEN class_name ~* '^proton pump inhibitors' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' + AND ancestor_concept_id IN (21600095) + WHERE class_name ~* 'proton pump inhibitors?' + AND LENGTH (class_code) = 7; + +-- take descendants of Thiazides +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'thiazides', + concept_id, + concept_name, + CASE WHEN class_name ~* '^thiazides' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper, concept_ancestor s + JOIN concept c + ON descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' + AND ancestor_concept_id IN (21601463) + WHERE class_name ~* 'thiazides' + AND LENGTH (class_code) = 7; + +-- take ingredients indicating Electrolytes + INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'electrolytes', + concept_id, + concept_name, + 3 -- hardcoded rank for electrolytes (no 4) + FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* ('^magnesium sulfate|^ammonium chloride|^sodium chloride|^sodium acetate|^magnesium chloride^|potassium lactate|^sodium glycerophosphate|^magnesium phosphate|^potassium chloride|^calcium chloride' + || '^sodium bicarbonate|^hydrochloric acid|^potassium acetate|^zinc chloride|^sodium phosphate|^potassium bicarbonate|^succinic acid|^sodium lactate|^sodium gluconate|^sodium fumarate') + AND class_name ~* 'electrolytes' + AND LENGTH(class_code) = 7; + +-- bismuth preparations +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'bismuth preparations', + concept_id, + concept_name, + 3 -- hardcoded rank for bismuth preparations (no 4) + FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* ('\ybismuth') + AND class_name ~* 'bismuth preparations' + AND LENGTH (class_code) = 7; + +-- artificial tears +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'artificial tears', + concept_id, + concept_name, + 3 -- hardcoded rank for bismuth preparations (no 4) + FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') +-- AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'carboxymethylcellulose$|polyvinyl alcohol$|hydroxypropyl methylcellulose$|^hypromellose$|hydroxypropyl cellulose$|^hyaluronic acid|^hyaluronate' + AND concept_class_id = 'Ingredient' +-- AND concept_name ~* 'Ophthalmic Solution' + AND class_name ~* 'artificial tears' + AND LENGTH (class_code) = 7; + +-- potassium-sparing agents +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'potassium-sparing agents', + concept_id, + concept_name, + CASE WHEN class_name ~* '^potassium-sparing agents' THEN 3 ELSE 4 END ::INT AS rnk +FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* '\yAmiloride|Triamterene|Spironolactone|Eplerenone|Finerenone|Canrenone|Canrenoic acid' + AND class_name ~* 'potassium-sparing agents' + AND LENGTH (class_code) = 7; + +-- excl\.trimethoprim +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'excl. trimethoprim', + concept_id, + concept_name, + 0 -- hardcoded rank + FROM class_drugs_scraper,concept c +WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND concept_class_id = 'Ingredient' + AND standard_concept = 'S' + AND concept_name ~* 'trimethoprim' + AND class_name ~* 'excl. trimethoprim' + AND LENGTH (class_code) = 7; + +-- fuzzy match (should be checked before INSERT) +INSERT INTO dev_combo +SELECT DISTINCT a.class_code, + a.class_name, + a.adm_r, + a.class, + c.concept_id, + c.concept_name, + CASE + WHEN class_code IN ('A12CC30', 'G01AX14', 'V03AE04') THEN 3 + WHEN lower(SUBSTRING(SPLIT_PART(a.class_name,' and ',1),'^...')) = lower(SUBSTRING(c.concept_name,'^...')) THEN 1 + WHEN lower(SUBSTRING(SPLIT_PART(a.class_name,' and ',2),'^...')) = lower(SUBSTRING(c.concept_name,'^...')) THEN 2 + WHEN class = 'arginine' THEN 1 + WHEN class = 'lysine' THEN 2 + WHEN class = 'trastuzumab' THEN 2 + WHEN class_name ~ 'AND iron' THEN 2 + WHEN class_name ~ '^iron' THEN 1 + WHEN class_name ~ 'AND potassium' THEN 2 + ELSE 3 END +FROM dev_combo a, + devv5.concept_synonym b, + concept c +WHERE lower(b.concept_synonym_name) LIKE concat('%',SPLIT_PART(a.class,' (',1),'%') +AND b.concept_id = c.concept_id +AND c.concept_class_id = 'Ingredient' +AND c.standard_concept = 'S' +AND invalid_reason IS NULL +AND c.concept_id NOT IN (40171179, 719174, 19006043,19022417, 43525936, 43013670, 43532256) -- white-tailed deer hair extract |5-hydroxylysine|carbocysteine-lysine +AND (a.class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) +AND class <> ''; + +-- fix Vitamin D AND analogues IN combination +UPDATE dev_combo + SET rnk = 3 +WHERE rnk = 1 +AND class_code = 'A11CC20'; + +-- fix erroneous rnk of 1 for Ingredient groups +UPDATE dev_combo + SET rnk = 3 +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND rnk = 1; + +-- remove erroneous Ingredient match +DELETE +FROM dev_combo +WHERE class_code = 'V03AE04' +AND concept_name IN ('calcium','magnesium','magnesite'); + +DELETE +FROM dev_combo +WHERE class_code = 'A06AG11' +AND class_name = 'sodium lauryl sulfoacetate, incl. combinations' +AND concept_name = 'sodium' +AND rnk = 1; + +DELETE +FROM dev_combo +WHERE class_code = 'A01AA51' +AND class_name = 'sodium fluoride, combinations' +AND concept_name = 'sodium' +AND rnk = 1; + +DELETE +FROM dev_combo +WHERE class_code = 'A06AB58' +AND class_name = 'sodium picosulfate, combinations' +AND concept_name = 'sodium' +AND rnk = 1; + +DELETE +FROM dev_combo +WHERE class_code = 'B05XA06' +AND class_name = 'potassium phosphate, incl. combinations WITH other potassium salts' +AND concept_name = 'potassium' +AND rnk = 1; + +DELETE +FROM dev_combo +WHERE class_code = 'A12BA51' +AND class_name = 'potassium chloride, combinations' +AND concept_name = 'potassium' +AND rnk = 1; + +DELETE +FROM dev_combo +WHERE class_code = 'C01DA58' +AND class_name = 'isosorbide dinitrate, combinations' +AND concept_name = 'isosorbide' +AND rnk = 1; + +-- fix erroneous rnk of 3 for J07AG53 +UPDATE dev_combo + SET rnk = 1 +WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND rnk = 3; -- 5 + +-- remove ATC classes other than 5th +DELETE +FROM dev_combo +WHERE LENGTH(class_code) < 7; + +-- 1300751 105669 polysaccharide iron complex +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + class_name, + 1300751, + 'polysaccharide iron complex', + 1 +FROM dev_combo +WHERE class_code = 'B03AD01'; + +DELETE +FROM dev_combo +WHERE class_code = 'B03AD01' +AND rnk = 4; + +-- add missing Ingredient +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'codeine', + 1201620, + 'codeine', + 1 +FROM dev_combo +WHERE class_code = 'N02AA59'; + +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'codeine', + 1189596, + 'dihydrocodeine', + 1 +FROM dev_combo +WHERE class_code = 'N02AA59'; + +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'paracetamol', + 1125315, + 'acetaminophen', + 1 +FROM dev_combo +WHERE class_code = 'N02BE51'; + +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + adm_r, + 'acetylsalicylic acid', + 1112807, + 'aspirin', + 1 +FROM dev_combo +WHERE class_code = 'N02BA51'; + +-- add links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of OMOP Ingredient names using dev_combo table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT class_code, -- ATC + c.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code +FROM dev_combo a + JOIN concept c ON c.concept_id = a.concept_id AND c.standard_concept = 'S' +AND (a.class_code, c.concept_name) NOT IN ( +SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 +FROM internal_relationship_stage) +WHERE LENGTH (class_code) = 7 +AND rnk <> 0; + +-- add more links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of OMOP Ingredient names using dev_combo table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT class_code, -- ATC + c.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code +FROM dev_combo a + JOIN concept c ON lower(c.concept_name) = lower(a.concept_name) + AND c.standard_concept = 'S' +AND (a.class_code, c.concept_name) NOT IN ( +SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 +FROM internal_relationship_stage) +WHERE LENGTH (class_code) = 7 +AND rnk <> 0; + +-- add ATC Groupers to DCS AS Drug Products +INSERT INTO drug_concept_stage +( concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date +) +SELECT DISTINCT b.class_name, -- ATC code+name + 'ATC', + 'Drug Product', + NULL, + concept_code_1, + NULL, + 'Drug', + TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage a +JOIN class_drugs_scraper b ON b.class_code = SPLIT_PART(a.concept_code_1, ' ', 1) +WHERE concept_code_1 NOT IN (select concept_code FROM drug_concept_stage); + +-- add ATC Drug Attributes IN the form of Standard Ingredient names using internal_relationship_stage +INSERT INTO drug_concept_stage +( concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date) +SELECT DISTINCT concept_code_2 AS concept_name, -- ATC pseudo-attribute IN the form of OMOP Dose Form name + 'ATC' AS vocabulary_id, + c.concept_class_id AS concept_class_id, + NULL AS standard_concept, -- check all standard_concept values + concept_code_2 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, + TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage, + concept c WHERE lower(c.concept_name) = lower(concept_code_2) +AND concept_class_id = 'Ingredient' +AND vocabulary_id IN ('RxNorm', 'RxNorm Extension') +AND invalid_reason IS NULL +AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); + +/*************************************** +******* relationship_to_concept ******** +****************************************/ +-- add mappings of ATC Drug Attributes to OMOP Equivalents +TRUNCATE relationship_to_concept; +ALTER TABLE relationship_to_concept ALTER COLUMN concept_code_1 TYPE VARCHAR; + +-- add links between ATC Drug Attributes AND their OMOP equivalents +INSERT INTO relationship_to_concept +( concept_code_1, vocabulary_id_1, concept_id_2) +SELECT DISTINCT concept_code_2 AS concept_code_1, -- ATC attribute IN the form of OMOP Dose Form OR Ingredient name + 'ATC' AS vocabulary_id_1, + c.concept_id AS concept_id_2 -- OMOP concept_id +FROM internal_relationship_stage + JOIN concept c + ON lower(concept_code_2) = lower(c.concept_name) + AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') + AND c.invalid_reason IS NULL; + +-- run load_interim.sql From 2f2bcb72a012c6b7d899daf27121701e2335a9a3 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 1 Jul 2021 17:53:19 +0300 Subject: [PATCH 02/45] Update load_input.sql refactoring --- ATC/load_input.sql | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index f61b69c25..3c9ce8e28 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -1,11 +1,26 @@ -/********************* -**** INPUT TABLES **** -**********************/ --- for ATC we don't need ds_stage AND pc_stage +/************************************************************************** +* Copyright 2016 Observational Health Data Sciences AND Informatics (OHDSI) +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may NOT use this file except IN compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to IN writing, software +* distributed under the License is distributed ON an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* Authors: Anna Ostropolets, Polina Talapova +* Date: Jul 2021 +**************************************************************************/ DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; -DROP TABLE IF EXISTS relationship_to_concept CASCADE;; +DROP TABLE IF EXISTS relationship_to_concept CASCADE; +-- for ATC we don't need ds_stage AND pc_stage CREATE TABLE drug_concept_stage ( concept_name VARCHAR(255), vocabulary_id VARCHAR(20), @@ -887,8 +902,8 @@ FROM concept_manual a **** combined ATC Classes **** ******************************/ -- obtain 1st ATC Combo Ingredient using the concept table and full name match -drop table if exists dev_combo; -create unlogged table dev_combo AS ( +DROP TABLE if exists dev_combo; +CREATE TABLE dev_combo AS ( WITH t1 AS ( SELECT * @@ -1258,7 +1273,7 @@ WITH t1 AS (select distinct a.concept_code_1, a.relationship_id, c.concept_name SELECT DISTINCT concept_code_1, concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code, FROM t1 a - JOIN class_drugs_scraper b ON b.class_code = a.concept_code_1 AND b,change_type IN ('A', '') + JOIN class_drugs_scraper b ON b.class_code = a.concept_code_1 AND b.change_type IN ('A', '') AND a.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up') AND (concept_code_1, concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1),concept_code_2 FROM internal_relationship_stage); @@ -1266,7 +1281,6 @@ FROM t1 a /********************************** ******* drug_concept_stage ******** ***********************************/ -TRUNCATE drug_concept_stage; -- change length of concept_code field ALTER TABLE drug_concept_stage ALTER COLUMN concept_code TYPE VARCHAR; @@ -2451,7 +2465,6 @@ AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); ******* relationship_to_concept ******** ****************************************/ -- add mappings of ATC Drug Attributes to OMOP Equivalents -TRUNCATE relationship_to_concept; ALTER TABLE relationship_to_concept ALTER COLUMN concept_code_1 TYPE VARCHAR; -- add links between ATC Drug Attributes AND their OMOP equivalents From f891fff75792adc2866aaea3e13e00a59bf2da10 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 1 Jul 2021 18:34:00 +0300 Subject: [PATCH 03/45] Update load_input.sql fix --- ATC/load_input.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 3c9ce8e28..dc1ef120e 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -2380,6 +2380,14 @@ SELECT DISTINCT class_code, FROM dev_combo WHERE class_code = 'N02BA51'; +--remove vitamin D as a false Ingredient +DELETE +FROM dev_combo +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE concept_id = 19009405) +AND class_name ~* 'colecalc' +AND class_name !~* 'vitamin D' +AND concept_id = 19009405; + -- add links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of OMOP Ingredient names using dev_combo table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) From ee69588c2408ccd752d8bec7976276540c25cc46 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 1 Jul 2021 19:57:59 +0300 Subject: [PATCH 04/45] Update load_input.sql fix --- ATC/load_input.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index dc1ef120e..3d0348f49 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -1662,14 +1662,14 @@ SELECT DISTINCT class_code, 'corticosteroids', concept_id, concept_name, - CASE WHEN class_name ~* '^corticosteroids' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN class_name ~* '^corticosteroid|^combinations of corticosteroids?' THEN 3 ELSE 4 END ::INT AS rnk FROM class_drugs_scraper, concept_ancestor JOIN concept c ON descendant_concept_id = c.concept_id AND ancestor_concept_id IN (21605042, 21605164, 21605200, 21605165, 21605199, 21601607, 975125) AND vocabulary_id LIKE 'RxNorm%' AND concept_class_id = 'Ingredient' -WHERE class_name ~* 'corticosteroids?' +WHERE class_name ~* 'corticosteroid' AND LENGTH(class_code) = 7; -- take descendants of Cough suppressants From 25c0d759f78b1c00852e5c2c6ae3aeaf7d2e76f9 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Wed, 7 Jul 2021 18:13:07 +0300 Subject: [PATCH 05/45] Create load_interim.sql --- ATC/load_interim.sql | 981 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 981 insertions(+) create mode 100644 ATC/load_interim.sql diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql new file mode 100644 index 000000000..ef49d86db --- /dev/null +++ b/ATC/load_interim.sql @@ -0,0 +1,981 @@ +/******************** +***** REFERENCE ***** +*********************/ +-- create temporary table containing links bwetween ATC Сlass codes and combinations of (ATC_code + Dose Form) from drug_concept_stage +DROP TABLE if exists reference; +CREATE TABLE reference +AS +SELECT DISTINCT class_code, + concept_code +FROM drug_concept_stage + LEFT JOIN class_drugs_scraper ON SPLIT_PART (concept_name,' ',1) = class_code; + +-- create a table with aggregated RxE ingredients: rx_combo +DROP TABLE IF EXISTS rx_combo; +CREATE TABLE rx_combo AS +SELECT drug_concept_id, + string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo +FROM devv5.drug_strength + JOIN concept ON concept_id = drug_concept_id AND + concept_class_id IN ('Clinical Drug Form', 'Ingredient') -- 'Clinical Drug Comp' doesn't exist in ATCs +GROUP BY drug_concept_id ; + +--ambiguous_class_ingredient +DROP TABLE IF EXISTS ambiguous_class_ingredient_tst; +CREATE UNLOGGED TABLE ambiguous_class_ingredient_tst +AS +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_name AS concept_code_2, + rnk +FROM dev_combo a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.class_code + JOIN drug_concept_stage c + ON c.concept_code = i.concept_code_2 + AND c.concept_class_id = 'Ingredient' + AND lower (a.concept_name) = lower (c.concept_name); + +-- add rnk 0 - excluded Ingredients +INSERT INTO ambiguous_class_ingredient_tst +SELECT DISTINCT class_code, + class_name, + concept_name AS concept_code_2, + rnk +FROM dev_combo +WHERE rnk = 0; -- 3726 + +-- separate pure ATC combinations (Primary lateral + Secondary lateral) +DROP TABLE IF EXISTS pure_combos; +CREATE TABLE pure_combos +AS +SELECT DISTINCT class_code, + concept_code_2, + class_name, + rnk +FROM ambiguous_class_ingredient_tst +WHERE class_code IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 1) -- Primary lateral +AND class_code IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 2) -- Secondary lateral +AND class_code NOT IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 3) -- exclude Priamry upward +AND class_code NOT IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 4); -- exclude Secondary upward + -- 1629 + +-- separate Primary upward Ingredients (ATC Groupers mentioned at the beginning of ATC Class name) -- rnk 3 only +DROP TABLE IF EXISTS ing_pr_up; +CREATE TABLE ing_pr_up AS +SELECT DISTINCT class_code, + concept_code_2, + class_name, + rnk +FROM ambiguous_class_ingredient_tst +WHERE class_code IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 3) -- include Priamry upward +AND class_code NOT IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 1) -- exclude Primary lateral +AND class_code NOT IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 2) -- exclude Secondary lateral +AND class_code NOT IN (SELECT class_code + FROM ambiguous_class_ingredient_tst + WHERE rnk = 4);-- exclude Secondary upward + +-- add the same Ingredients marked as rnk=1 to create permutations, assume that there will not be more than 3 ingredients in combination +INSERT INTO ing_pr_up +SELECT class_code, concept_code_2,class_name, 1 + FROM ing_pr_up; -- 3186 + +-- combine pure ATC combos with ATC groupers +INSERT INTO ambiguous_class_ingredient_tst +SELECT DISTINCT class_code, class_name, concept_code_2, rnk +FROM ing_pr_up +UNION +SELECT DISTINCT class_code, class_name, concept_code_2, rnk +FROM pure_combos +; -- 7920 + +-- create index to make the script faster +CREATE INDEX ambiguous_class_ingredient_test ON ambiguous_class_ingredient_tst (class_code, concept_code_2, rnk); + +-- create a table with aggregated ATC ingredients: full_combo (for 3 ingredients only) +-- separate direct ingredients +DROP TABLE IF EXISTS ing_pr_lat; +CREATE UNLOGGED TABLE ing_pr_lat +AS +SELECT DISTINCT a.*, -- distinct is required here + concept_id_2, + 1 AS precedence -- will be changed a bit later for some ATC codes + FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE rnk = 1; -- 4119 + +-- separate Secondary lateral ingredients (rnk = 2) +DROP TABLE IF EXISTS ing_sec_lat; +CREATE UNLOGGED TABLE ing_sec_lat +AS +SELECT DISTINCT a.*, + concept_id_2, + 1 as precedence -- distinct is required here (can be filled in a futher release) + FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE rnk = 2; -- 1213 + +-- separate Primary upward ingredients (rnk = 3) +DROP TABLE IF EXISTS ing_pr_up; +CREATE UNLOGGED TABLE ing_pr_up +AS +SELECT DISTINCT a.*, + concept_id_2, + 1 as precedence +FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE rnk = 3; -- 4086 + +-- separate Secondary upward ingredients (rnk = 4) +DROP TABLE IF EXISTS ing_sec_up; +CREATE UNLOGGED TABLE ing_sec_up +AS +SELECT DISTINCT a.*, -- distinct is required here + concept_id_2, + precedence + FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE rnk = 4; -- Secondary upward + +-- obtain the full list of all possible combinations of ingredients for a one ATC-combo (48m 46s) +DROP TABLE if exists full_combo; +CREATE UNLOGGED TABLE full_combo +AS +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id_2||COALESCE('-' || b.concept_id_2,'') ||COALESCE('-' || c.concept_id_2,'') ||COALESCE('-' || d.concept_id_2,'') AS i_combo +FROM ing_pr_lat a + JOIN ing_sec_lat b USING (class_code)-- rank 2 + LEFT JOIN ing_pr_up c USING (class_code)-- rank 3 + LEFT JOIN ing_sec_up d USING (class_code)-- rank 4 +ORDER BY class_code; + +-- create table with Ingredient permutations +DROP TABLE IF EXISTS permutations; +CREATE UNLOGGED TABLE permutations +AS +SELECT distinct a.class_code, a.class_name, a.concept_id_2||COALESCE('-' || b.concept_id_2, '')||COALESCE('-' || c.concept_id_2, '') AS i_combo + FROM ing_pr_lat a -- from Primary lateral +LEFT JOIN ing_sec_lat b ON b.class_code = a.class_code -- to the 1st Secondary lateral +LEFT JOIN ing_sec_lat c ON c.class_code = a.class_code -- and the 2nd Secondary lateral +WHERE b.concept_id_2<>c.concept_id_2 AND b.concept_id_2<>a.concept_id_2;--34262 + +-- add newly created permutations to the full_combo table in oreder to enrich the set of aggregated ATC ingredients +INSERT INTO full_combo +SELECT * FROM permutations; -- 34262 + +-- Separate 1 Pr lateral in combination +DROP TABLE if exists ing_pr_lat_combo; + +CREATE TABLE ing_pr_lat_combo +AS +(SELECT * +FROM dev_combo a +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); + +-- create temporary table with all possible i_combos for Primary laterla in combinations (with unspecified drugs) +DROP TABLE if exists ing_pr_lat_combo_to_drug; + +CREATE TABLE ing_pr_lat_combo_to_drug +AS +(WITH t1 +AS +(SELECT drug_concept_id, + REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo +FROM rx_combo +WHERE i_combo LIKE '%-%') + SELECT DISTINCT a.class_code, + a.class_name, + d.i_combo +FROM ing_pr_lat_combo a + JOIN t1 b ON b.i_combo = a.concept_id + JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id); + +-- 2 Pr lat in combinations +DROP TABLE if exists ing_pr_lat_combo_excl; + +CREATE TABLE ing_pr_lat_combo_excl +AS +(SELECT * +FROM dev_combo a +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); + +DROP TABLE if exists ing_pr_lat_combo_excl_to_drug; +CREATE TABLE ing_pr_lat_combo_excl_to_drug +AS +(WITH t1 +AS +(SELECT drug_concept_id, + REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo +FROM rx_combo +WHERE i_combo LIKE '%-%') SELECT DISTINCT a.class_code,a.class_name,d.i_combo FROM ing_pr_lat_combo_excl a JOIN t1 b ON b.i_combo = a.concept_id JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code AND a1.rnk <> a.rnk AND a1.rnk = 0 JOIN t1 f ON f.i_combo = a1.concept_id +-- excluded +JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id JOIN rx_combo d1 ON d1.drug_concept_id = f.drug_concept_id +-- excluded +AND d1.drug_concept_id <> d.drug_concept_id);; + +-- 3 Pr up+Sec up +DROP TABLE if exists ing_pr_sec_up_combo; +CREATE TABLE ing_pr_sec_up_combo +AS +(SELECT * +FROM dev_combo a +WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) +); + +DROP TABLE if exists ing_pr_sec_up_combo_to_drug; + +CREATE TABLE ing_pr_sec_up_combo_to_drug +AS +(WITH t1 +AS +(SELECT drug_concept_id, + REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo +FROM rx_combo +WHERE i_combo LIKE '%-%') +SELECT DISTINCT a.class_code, + a.class_name, + c.i_combo +FROM ing_pr_sec_up_combo a + JOIN t1 b + ON b.i_combo = a.concept_id + AND a.rnk = 3 + JOIN ing_pr_sec_up_combo a1 + ON a1.class_code = a.class_code + AND a1.concept_id <> a.concept_id + JOIN t1 j + ON j.i_combo = a1.concept_id + AND a1.rnk = 4 + JOIN rx_combo c + ON c.drug_concept_id = b.drug_concept_id + AND c.drug_concept_id = j.drug_concept_id); + +-- 4 Pr up+Sec up with excluded ingreds +DROP TABLE if exists ing_pr_sec_up_combo_excl; + +CREATE TABLE ing_pr_sec_up_combo_excl +AS +(SELECT * +FROM dev_combo a +WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); + +-- currently no match, but query works +DROP TABLE if exists ing_pr_sec_up_combo_excl_to_drug; + +CREATE TABLE ing_pr_sec_up_combo_excl_to_drug +AS +(WITH t1 +AS +(SELECT drug_concept_id, + REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo +FROM rx_combo +WHERE i_combo LIKE '%-%') +SELECT DISTINCT a.class_code, + a.class_name, + c.i_combo +FROM ing_pr_sec_up_combo_excl a + JOIN t1 b + ON b.i_combo = a.concept_id + AND a.rnk = 3 + JOIN ing_pr_sec_up_combo_excl a1 + ON a1.class_code = a.class_code + AND a1.concept_id <> a.concept_id + JOIN t1 j + ON j.i_combo = a1.concept_id + AND a1.rnk = 4 + JOIN ing_pr_sec_up_combo_excl a2 + ON a2.class_code = a.class_code + AND a2.rnk = 0 + JOIN rx_combo c + ON c.drug_concept_id = b.drug_concept_id + AND c.drug_concept_id = j.drug_concept_id + JOIN rx_combo c1 + ON c1.drug_concept_id = j.drug_concept_id + AND c1.drug_concept_id = c.drug_concept_id + AND c.i_combo !~ a2.concept_id::VARCHAR); + +INSERT INTO full_combo +SELECT * +FROM ing_pr_lat_combo_to_drug +UNION +SELECT * +FROM ing_pr_lat_combo_excl_to_drug +UNION +SELECT * +FROM ing_pr_sec_up_combo_to_drug +UNION +SELECT * +FROM ing_pr_sec_up_combo_excl_to_drug; -- 10086 + +-- create a table full_combo_reodered to order aggregated ATC ingredients by an Ingredient +DROP TABLE IF EXISTS full_combo_reodered; +CREATE UNLOGGED TABLE full_combo_reodered AS +SELECT DISTINCT fc.class_code, + fc.class_name, + l.i_combo +FROM full_combo fc +CROSS JOIN LATERAL(SELECT STRING_AGG(s0.ing, '-' ORDER BY s0.ing::INT) AS i_combo FROM ( + SELECT UNNEST(STRING_TO_ARRAY(fc.i_combo, '-')) AS ing + ) AS s0) l; + +-- create index to make the script faster -- 30m 7s (ask Timur to check whether these indices are useful) - 55m 38s +CREATE INDEX i_full_combo_reodered ON full_combo_reodered (class_code, i_combo); + +-- create full_combo_with_form table containig aggregated ATC ingredients + Dose Form -- 6.03s +DROP TABLE full_combo_with_form; +CREATE UNLOGGED TABLE full_combo_with_form +AS +SELECT DISTINCT a.class_code, + a.class_name, + a.i_combo, + r.concept_id_2::int +FROM full_combo_reodered a + JOIN internal_relationship_stage i ON class_code = substring (concept_code_1, '\w+') -- cut ATC code before space character + JOIN drug_concept_stage b + ON b.concept_code = i.concept_code_2 + AND b.concept_class_id = 'Dose Form' + JOIN relationship_to_concept r ON r.concept_code_1 = i.concept_code_2; + +-- add ATC combos without forms from the 'reference' table -- 27 740 577 rows affected +INSERT INTO full_combo_with_form +(class_code, i_combo) +SELECT DISTINCT f.class_code, + i_combo + FROM full_combo_reodered f + JOIN reference r ON r.class_code = f.class_code +WHERE r.concept_code = r.class_code +; -- 17 (7640) + +CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo,concept_id_2); -- 10m 8s +DROP TABLE IF EXISTS combo_not_limited_to_higher_ATC; +CREATE UNLOGGED TABLE combo_not_limited_to_higher_ATC +AS +WITH t1 AS +( + SELECT drug_concept_id, regexp_split_to_table (i_combo, '-')::INT AS i_combo FROM rx_combo +WHERE i_combo LIKE '%-%' -- at least two ingredients +AND drug_concept_id NOT IN (SELECT drug_concept_id FROM ing_excl) -- filter out drugs containig excluded ingredients if any +) +SELECT DISTINCT b.class_code, + d.class_name, + a.drug_concept_id +FROM t1 a-- Pr up + JOIN ing_pr_sec_up b -- Primary and Secondary upward with Ingredients + ON b.concept_id_2 = a.i_combo + JOIN dev_combo d on d.class_code = b.class_code + and d.rnk = 3 + JOIN t1 k -- Sec up + on k.drug_concept_id = a.drug_concept_id + JOIN ing_pr_sec_up c on c.concept_id_2 = k.i_combo + JOIN dev_combo d2 on d2.class_code = c.class_code + and a.i_combo <> k.i_combo + and d2.rnk = 4; + +/******************************* +******** CLASS TO DRUG ********* +********************************/ +-- create table with one-step links between ATC combos of 5th class and Rx/RxN Drug Products using the full macth of an Ingredient and Dose Form +DROP TABLE IF EXISTS class_to_drug; +CREATE TABLE class_to_drug AS +-- separate all Rx/RxN combo drugs of 'Clinical Drug Form' concept class +WITH t1 AS +( + SELECT c.concept_id, -- Standard Drug Product + c.concept_name, + c.concept_class_id, + c.vocabulary_id, + r.concept_id_2, -- Dose Form + a.i_combo -- combination of Standard Ingredient IDs as a key for join +FROM rx_combo a + JOIN concept c ON c.concept_id = a.drug_concept_id + JOIN concept_relationship r ON r.concept_id_1 = c.concept_id +WHERE c.concept_class_id = 'Clinical Drug Form' +AND c.vocabulary_id LIKE 'RxNorm%' +AND c.invalid_reason IS NULL +AND r.relationship_id = 'RxNorm has dose form' +AND r.invalid_reason IS NULL + ) + SELECT DISTINCT f.class_code, -- ATC + f.class_name, + r.concept_id, -- Standard Drug Product + r.concept_name, + r.concept_class_id, + 1 AS ORDER -- specific number for cases with the same order of ids in i_combo? + FROM full_combo_with_form f + JOIN t1 r + ON r.i_combo = f.i_combo -- combination of Standard Ingredient IDs + AND r.concept_id_2 = f.concept_id_2 ; + +-- add additional links using the table of 'combo_not_limited_to_higher_ATC' (this step gives errors!!!) +INSERT INTO class_to_drug + WITH t1 AS ( + -- separate Rx/RxN combo Drug Forms + SELECT a.concept_id, -- Rx + a.concept_name, + a.concept_class_id, + a.vocabulary_id, + c.concept_id_2, -- Rx Dose Form + r.class_code, -- ATC + r.class_name +FROM combo_not_limited_to_higher_ATC r + JOIN concept a ON r.drug_concept_id = a.concept_id + JOIN concept_relationship c ON c.concept_id_1 = a.concept_id + WHERE a.concept_class_id = 'Clinical Drug Form' + AND a.vocabulary_id LIKE 'RxNorm%' + AND a.invalid_reason IS NULL + AND relationship_id = 'RxNorm has dose form' + AND c.invalid_reason IS NULL + ) +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.concept_class_id, + 6 -- combo_not_limited_to_higher_ATC +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.class_code + JOIN relationship_to_concept rtc ON i.concept_code_2 = rtc.concept_code_1 +WHERE a.concept_id_2 = rtc.concept_id_2 +AND (a.class_code, a.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug) -- prevent duplicates +; -- 1416 + +-- contraceptive packs +INSERT INTO class_to_drug +SELECT class_code, + class_name, + c.concept_id, + c.concept_name, + c.concept_class_id -- was added 17.06 + "order" +FROM class_to_drug ctd + JOIN concept_ancestor ON ctd.concept_id = ancestor_concept_id + JOIN concept c ON descendant_concept_id = c.concept_id +WHERE class_code ~ 'G03FB|G03AB' +AND c.concept_class_id IN ('Clinical Pack') +AND (ctd.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug); -- 440 + +-- insert mono-ATC codes +DROP TABLE IF EXISTS mono_ing; +CREATE UNLOGGED TABLE mono_ing +AS +SELECT DISTINCT a.class_code, + a.class_name, + rtc.concept_id_2 AS ing_id +FROM class_drugs_scraper a + JOIN internal_relationship_stage i ON class_code = SUBSTRING (concept_code_1,'\w+') + JOIN drug_concept_stage d + ON lower (d.concept_code) = lower (i.concept_code_2) + AND concept_class_id = 'Ingredient' + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 +WHERE LENGTH(class_code) = 7 +AND class_code NOT IN (SELECT class_code FROM dev_combo) +AND class_code NOT IN (SELECT class_code FROM class_drugs_scraper where class_code ~ '^J07|^A10A' AND length(class_code)=7) +AND class_name !~* '\yand\y' +and class_code not in (select class_code from class_to_drug) +; -- 5462 + +-- add Forms +DROP TABLE if exists mono_ing_with_form; + +CREATE TABLE mono_ing_with_form +AS +(SELECT class_code, + class_name, + ing_id, + concept_id_2 AS form_id +FROM mono_ing + JOIN internal_relationship_stage i ON class_code = SUBSTRING (i.concept_code_1,'\w+') + JOIN drug_concept_stage d + ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Dose Form' + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2);; + +INSERT INTO class_to_drug +with t1 as (SELECT distinct drug_concept_id, i_combo::INT AS i_combo FROM rx_combo +WHERE i_combo NOT LIKE '%-%') +SELECT DISTINCT k.class_code, k.class_name, a.concept_id, a.concept_name, a.concept_class_id,2 +FROM mono_ing_with_form k -- ATC +JOIN t1 r -- Rx +ON r.i_combo = k.ing_id +JOIN concept a ON r.drug_concept_id = a.concept_id +JOIN concept_relationship c ON c.concept_id_1 = a.concept_id AND k.form_id = c.concept_id_2 + WHERE a.concept_class_id = 'Clinical Drug Form' + AND a.vocabulary_id LIKE 'RxNorm%' + AND a.invalid_reason IS NULL + AND relationship_id = 'RxNorm has dose form' + AND c.invalid_reason IS NULL + AND (k.class_code, a.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug); -- 14119 + +-- additional links +INSERT INTO class_to_drug +SELECT DISTINCT m.class_code, + m.class_name, + c.concept_id, + c.concept_name, + c.concept_class_id, + 3 +FROM mono_ing m + JOIN internal_relationship_stage i ON class_code = SUBSTRING (concept_code_1,'\w+') + JOIN concept c ON m.ing_id = c.concept_id +WHERE i.concept_code_1 not in (select concept_code_1 from internal_relationship_stage where concept_code_1 LIKE '% %') -- exclude ATC classes with Forms (they contain space inbetween) + AND (m.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug) +;-- 1168 + +-- add manual links using concept_relationship_manual -- TO DO: relatioship_id for J07BM J07BM from Subsumes to ATC - RxNorn pr lat +-- check in old class_to_drug whether there are additional ingredients +INSERT INTO class_to_drug +SELECT DISTINCT a.class_code, +f.concept_name as class_name, +c.concept_id, +c.concept_name, +c.concept_class_id, +--relationship_id, +1 +from class_drugs_scraper a +join concept_relationship_manual b on b.concept_code_1 = a.class_code +join concept_manual f on f.concept_code = a.class_code +AND b.relationship_Id in ('ATC - RxNorm') --'ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up') +JOIN concept c on c.concept_code = b.concept_code_2 and c.vocabulary_id = b.vocabulary_id_2 +and c.vocabulary_id in ('RxNorm', 'RxNorm Extension') + AND (a.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug) + and f.invalid_reason is null; -- 4920 + +DELETE +FROM class_to_drug +WHERE (class_code = 'A02BA07' AND concept_class_id = 'Clinical Drug Form') -- Branded Drug 'Tritec' +OR class_code = 'G03GA08' -- choriogonadotropin alfa (no Standard Precise Ingredient) +OR class_code='N05AF02' -- clopentixol +OR class_code IN ('D07AB02','D07BB04') -- hydrocortisone butyrate + combo that so far doesn't exist +OR class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral +OR (class_code = 'B02BD14' AND concept_name LIKE '%Tretten%') -- 2 --catridecacog +OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog +; + +INSERT INTO class_to_drug +SELECT 'B02BD11','catridecacog', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE (vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' + AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') + OR concept_id = 35603348 -- the whole hierarchy (35603348 1668002 factor XIII Injection [Tretten] Branded Drug Form) -- 2 +; +INSERT INTO class_to_drug +SELECT 'B02BD14','susoctocog alfa', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE (vocabulary_id LIKE 'RxNorm%' + AND concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') + OR concept_id IN (35603348, 44109089) -- the whole hierarchy +; -- 3 + +INSERT INTO class_to_drug +SELECT 'A02BA07','ranitidine bismuth citrate', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Tritec%' + AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' +; -- 1 + +INSERT INTO class_to_drug +SELECT 'G03GA08','choriogonadotropin alfa', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~ 'choriogonadotropin alfa' + AND standard_concept = 'S' AND concept_class_id ~ 'Clinical Drug Comp' +; -- 1 + + +INSERT INTO class_to_drug +SELECT 'N05AF02','clopenthixol', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Sordinol|Ciatyl' + AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' +; -- 4 + +INSERT INTO class_to_drug +SELECT 'D07AB02','hydrocortisone butyrate', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Hydrocortisone butyrate' AND concept_class_id = 'Clinical Drug' + AND standard_concept = 'S' +;-- 6 + +-- remove pentaerithrityl tetranitrate; oral +DELETE +FROM class_to_drug +WHERE class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral +; -- 7 + +INSERT INTO class_to_drug +SELECT 'C01DA05','pentaerithrityl tetranitrate', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ILIKE '%Pentaerythritol Tetranitrate%' and + standard_concept = 'S' AND concept_class_id = 'Clinical Drug Comp' -- forms do not exist +; -- 9 + +-- clean up erroneous amount of ingredients +DELETE +FROM class_to_drug +WHERE class_name LIKE '%,%and%' + AND class_name NOT LIKE '%,%,%and%' + AND NOT class_name ~* 'comb|other|whole root|selective' + AND concept_name NOT LIKE '% / % / %'; -- 135 + +-- process Packs, which should be added to the ATC hiearchy in parallel (to be shown in Athena as well) +DROP TABLE IF EXISTS pack_a; + create table pack_a AS ( + SELECT DISTINCT concept_id_1, + drug_concept_id, + string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo, + count(drug_concept_id) over (partition by concept_id_1) as cnt + FROM drug_strength + JOIN concept_relationship r + on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL + JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' + GROUP BY drug_concept_id,concept_id_1); + +DROP TABLE IF EXISTS pack_b; + create table pack_b as + (SELECT DISTINCT class_code, class_name, r.concept_code_1, r.concept_id_2 + FROM dev_combo + JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code + JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' + JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 + WHERE class_name LIKE '% and %'); + +DROP TABLE IF EXISTS pack_c; + create table pack_c AS (SELECT DISTINCT class_code, r.concept_code_1, r.concept_id_2 + FROM dev_combo + JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code + JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' + JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 + WHERE class_name LIKE '% and %'); + +DROP TABLE IF EXISTS pack_all; +CREATE TABLE pack_all as + SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name + FROM pack_a a + JOIN pack_a aa on aa.concept_id_1 = a.concept_id_1 + JOIN concept cc on concept_id = a.concept_id_1 + JOIN pack_b b on cast(b.concept_id_2 AS varchar) = aa.i_combo + JOIN pack_c c on cast(c.concept_id_2 AS varchar) = a.i_combo + WHERE a.drug_concept_id != aa.drug_concept_id + AND b.class_code = c.class_code + AND b.concept_code_1 != c.concept_code_1 + AND a.cnt = 2; + +-- errnoues pack match +DELETE +FROM pack_all + WHERE (class_code='A02BD11' AND concept_id=42731634) + OR (class_code='R03AL08' AND concept_id=43045404); + +DROP TABLE IF EXISTS pack_temp; +CREATE TABLE pack_temp +as +with a AS (SELECT concept_id_1, + string_agg(ingredient_concept_id::varchar, '-' ORDER BY ingredient_concept_id) AS i_combo + FROM drug_strength + JOIN concept_relationship r on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL + JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' + GROUP BY concept_id_1), + b AS ( +SELECT DISTINCT class_code, class_name, r.concept_code_1, r.concept_id_2 +FROM dev_combo +JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code +JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' +JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 +WHERE class_name LIKE '%, combinations') +SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name +FROM a +JOIN b on i_combo ~ cast(b.concept_id_2 AS varchar) +and i_combo!=cast(b.concept_id_2 AS varchar) +JOIN concept cc on concept_id_1 = concept_id +; + +INSERT INTO pack_all +with a AS ( + SELECT p.concept_id, p.concept_name, p.concept_class_id, class_code,class_name, r.concept_id_2 + FROM pack_temp p + JOIN internal_relationship_stage ON substring(concept_code_1, '\w+') = class_code + JOIN drug_concept_stage d ON concept_code_2 = concept_code AND d.concept_class_id = 'Dose Form' + JOIN relationship_to_concept r ON r.concept_code_1 = concept_code_2), + b AS (SELECT p.*, r2.concept_id_2 + FROM pack_temp p + JOIN devv5.concept_ancestor r on concept_id = descendant_concept_id + JOIN concept_relationship r2 + ON r2.concept_id_1 = ancestor_concept_id AND r2.invalid_reason IS NULL + AND relationship_id = 'RxNorm has dose form') +SELECT a.concept_id, a.concept_name, a.concept_class_id, a.class_code, a.class_name +FROM a +JOIN b USING (concept_id, concept_id_2) +WHERE (concept_id, a.class_code) NOT IN (SELECT concept_id, class_code FROM pack_all) +UNION +SELECT p.* +FROM pack_temp p +JOIN reference r on concept_code = p.class_code +WHERE (concept_id, p.class_code) NOT IN (SELECT concept_id, class_code FROM pack_all) +; -- 3945 + +INSERT INTO class_to_drug +( + class_code, + class_name, + concept_id, + concept_name, + concept_class_id, + "order" +) +SELECT DISTINCT class_code, + class_name, + concept_id, + concept_name, + concept_class_id, + 8 +FROM pack_all; -- 4882 + +-- 4.11 fix packs +INSERT INTO class_to_drug +SELECT DISTINCT class_code, + class_name, + c.concept_id, + c.concept_name, + c.concept_class_id, + 8 +FROM class_to_drug f + JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) + JOIN devv5.concept c + ON c.concept_id = descendant_concept_id + AND c.concept_class_id LIKE '%Pack%' +WHERE f.class_code ~ 'G03FB|G03AB' -- packs +; -- 1809 + +-- get rid of all other forms except Packs +-- 4.12 Progestogens and estrogens +DELETE +FROM class_to_drug +WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens +AND concept_class_id IN ('Clinical Drug Form','Ingredient'); -- 68 + +-- 4.14 Solution for the first run: for inambiguous ATC classes (those that classify an ingredient through only one class) +-- we relate this ATC class to the entire group of drugs that have this ingredient. +DROP TABLE IF EXISTS interim; +CREATE TABLE interim +AS +WITH a AS +( + SELECT DISTINCT class_code, + class_name, + c.concept_id, + c.concept_name + FROM class_to_drug crd + JOIN devv5.concept_ancestor ON crd.concept_id = descendant_concept_id + JOIN concept c + ON c.concept_id = ancestor_concept_id + AND c.concept_class_id = 'Ingredient' +) +SELECT * +FROM (SELECT COUNT(*) OVER (PARTITION BY TRIM(REGEXP_REPLACE(class_name,'\(.*\)',''))) AS cnt, + class_code, + class_name, + a.concept_id, + a.concept_name -- regexp for (vit C) + FROM a) a +WHERE cnt = 1 +AND class_code NOT IN (SELECT class_code FROM class_drugs_scraper +WHERE class_code ~ '^J07|^A10A' +AND length(class_code)=7) +--and class_code NOT IN ('G02BB02') +; + +DELETE +FROM interim +WHERE class_name IN ( + SELECT class_name + FROM (SELECT DISTINCT class_code, class_name FROM class_drugs_scraper) a + GROUP BY class_name + HAVING count(1) > 1); -- 58 + +DELETE +FROM interim +WHERE concept_id IN ( + SELECT a.concept_id + FROM interim a + JOIN interim b ON a.concept_id = b.concept_id + WHERE a.class_code != b.class_code); -- 105 + +DROP TABLE IF EXISTS interim_2; +CREATE TABLE interim_2 AS +WITH a AS ( +SELECT DISTINCT class_code, class_name, c.concept_id,c.concept_name +FROM class_to_drug crd +JOIN devv5.concept_ancestor on crd.concept_id = descendant_concept_id +JOIN concept c on c.concept_id = ancestor_concept_id AND c.concept_class_id = 'Ingredient' + ), +b AS (SELECT concept_id FROM a +GROUP BY concept_id having count(1)=1), +c AS (SELECT a.class_code, a.class_name, a.concept_id FROM a JOIN b USING(concept_id) + ) +SELECT c.* +FROM c +JOIN interim USING(class_code) +WHERE class_code NOT IN (SELECT class_code FROM class_to_drug_manual) +; + +DELETE +FROM interim_2 +WHERE class_name IN ( + SELECT class_name + FROM (SELECT DISTINCT class_code, class_name FROM class_drugs_scraper) a + GROUP BY class_name + HAVING count(1) > 1); --0 + +-- -- inambiguous only for forms +DELETE FROM class_to_drug +WHERE class_code IN (SELECT class_code FROM interim) +; -- 6988 (9196) + +-- add missing forms for the monocomponent ATC Drug Classes +INSERT INTO class_to_drug -- ingredients are partially inambiguous +(class_code, class_name, concept_id, concept_name, concept_class_id, "order") +SELECT class_code, class_name, c.concept_id, c.concept_name, c.concept_class_id, 2 +FROM interim i +JOIN devv5.concept_ancestor ca on i.concept_id = ancestor_concept_id +JOIN concept c on descendant_concept_id = c.concept_id +WHERE c.concept_class_id = 'Clinical Drug Form' AND c.concept_name NOT LIKE '% / %' +and (class_code, c.concept_id) not in (select class_code, concept_id from class_to_drug) +; -- 8653 + +-- remove absolutely inambiguous Ingredient +DELETE FROM class_to_drug +WHERE class_code IN(SELECT class_code FROM interim_2) +; -- 4019 +-- add their maps +INSERT INTO class_to_drug -- ingredients are absolutely inambiguous +(class_code, class_name, concept_id, concept_name, concept_class_id, "order") +SELECT DISTINCT class_code, class_name, concept_id, c.concept_name, c.concept_class_id, 3 +FROM interim_2 i +JOIN concept c USING (concept_id) +where (class_code, c.concept_id) not in (select class_code, concept_id from class_to_drug) +; -- 1415 + +-- also adding those that don't have forms, but are unique +with ing AS ( +SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt +FROM ( +SELECT DISTINCT substring(i.concept_code_1,'\w+'), r.concept_code_1, concept_id_2 +FROM relationship_to_concept r +JOIN internal_relationship_stage i on i.concept_code_2 = r.concept_code_1 +JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' + ) a GROUP BY concept_id_2,concept_code_1 ), +drug AS ( +SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 +FROM internal_relationship_stage i +WHERE not exists (SELECT 1 FROM internal_relationship_stage i2 + WHERE i.concept_code_2=i2.concept_code_2 + AND substring(i.concept_code_1,'\w+')!=substring(i2.concept_code_1,'\w+')) +), + drug_name AS ( +SELECT DISTINCT class_code,class_name, concept_id_2 +FROM ing +JOIN drug ON concept_code_2=concept_code_1 +JOIN class_drugs_scraper ON code = class_code +WHERE cnt=1 AND class_name = concept_code_1 +and code NOT IN (SELECT class_code FROM class_to_drug) + ), +all_drug AS ( +SELECT class_code,class_name, concept_id,concept_name, concept_class_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt +FROM drug_name +JOIN concept on concept_id_2 = concept_id +) +INSERT INTO class_to_drug +SELECT class_code, class_name, concept_id, concept_name,concept_class_id,3 +FROM all_drug +WHERE cnt=1 +and (class_code, concept_id) not in (select class_code, concept_id from class_to_drug) +; -- 25 + +UPDATE class_to_drug +SET "order" = 5 +WHERE "order" in (98,99); -- 649 + +UPDATE class_to_drug +SET "order" = 4 +WHERE class_name ~ ' and ' +AND NOT class_name ~ 'adrenergics|analgesics|antispasmodics|antibacterials|antiflatulents|imidazoles|minerals|polyfructosans|triazoles|natural phospholipids|lactic acid producing organisms|antiflatulents|beta-lactamase inhibitor|sulfonylureas|antibiotics|antiinfectives|antiseptics|artificial tears|contact laxatives|corticosteroids|ordinary salt combinations|imidazoles/triazoles|cough suppressants|diuretics|drugs for obstructive airway diseases|expectorants|belladonna alkaloids|mucolytics|mydriatics|non-opioid analgesics|organic nitrates|potassium-sparing agents|proton pump inhibitors|psycholeptics|thiazides|snake venom antiserum|fat emulsions|amino acids|acid preparations|sulfur compounds|stramoni preparations|protamines|penicillins|comt inhibitor|cannabinoids|decarboxylase inhibitor|edetates|barbiturates|excl|combinations of|derivate|with' +AND "order" in (98,99); -- 89 + +-- calcium compounds +UPDATE class_to_drug +SET "order" = 7 +WHERE class_name ~ 'compounds' +AND "order" = 2; -- 2 + +-- working with duplicates to remove a and b vs a/b, comb +DELETE FROM class_to_drug +WHERE (class_code,concept_id) IN( +SELECT b.class_code, b.concept_id FROM class_to_drug a +JOIN class_to_drug b on a.concept_id = b.concept_id +WHERE a."order" = 4 AND b."order" in (6,7)) +; -- 0 + +-- 'Progestogens and estrogens, sequential preparations' should be Clinical Packs only +-- check manual links for this!!! -- fix concept_class_ids +DELETE +FROM class_to_drug +WHERE class_code ~ 'G03FB|G03AB' +AND concept_class_id NOT IN ('Clinical Pack') +; -- 1371 + +update class_to_drug +set "order" = 99 +where "order" is null; -- 440 + +-- add missing alive OLD links(to solve) +insert into class_to_drug +WITH t1 AS +( + SELECT atc_code, + concept_id, + concept_code, + concept_name + FROM dev_atc.atc_all_relationships + WHERE vocabulary_id ~ 'Rx' and relationship_id = 'ATC - RxNorm' + and concept_class_id = 'Clinical Drug Form' +EXCEPT + SELECT atc_code,concept_id,concept_code,concept_name FROM dev_atc.atc_all_new_relationships_2 +WHERE vocabulary_id ~ 'Rx' and relationship_id = 'ATC - RxNorm' + and concept_class_id = 'Clinical Drug Form' +) +SELECT distinct atc_code, d.concept_name as class_name, c.concept_id, c.concept_name, c.concept_class_id, 23 +FROM t1 a + JOIN concept c + ON c.concept_id = a.concept_id + JOIN concept_manual d on d.concept_code = a.atc_code + AND c.standard_concept = 'S' + AND d.invalid_reason is null + where (atc_code, c.concept_id) not in (select class_code, concept_id from class_to_drug); -- 878 (855) From 38b63f44085f9024dc912892d56398f82595cca4 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Wed, 7 Jul 2021 18:14:38 +0300 Subject: [PATCH 06/45] Update load_stage.sql --- ATC/load_stage.sql | 2039 +++++++------------------------------------- 1 file changed, 330 insertions(+), 1709 deletions(-) diff --git a/ATC/load_stage.sql b/ATC/load_stage.sql index 1a44e0745..c89b37216 100644 --- a/ATC/load_stage.sql +++ b/ATC/load_stage.sql @@ -13,26 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* Authors: Anna Ostropolets, Timur Vakhitov -* Date: Jan 2020 +* Authors: Anna Ostropolets, Polina Talapova, Timur Vakhitov +* Date: Jul 2021 **************************************************************************/ - -/* -Prerequisites: -1. Get the latest file FROM source, name it class_drugs_scraper. The file should contain class code, class name, additional information. -For example, for ATC the file will be created in the following manner: -SELECT id, atc_code AS class_code, atc_name AS class_name, ddd, u, adm_r, note, description, ddd_description -FROM atc_drugs_scraper; -2. Prepare input tables (drug_concept_stage, internal_relationship_stage, relationship_to_concept) -according to the general rules for drug vocabularies -3. Prepare the following tables: -- reference (represents the original code and concatenated code AND its drug forms INRxNorm format) -- class_to_drug_manual (stores manual mappings, i.g. Insulins) -- ambiguous_class_ingredient (class, code, class_name, ingredients of ATC AS ing, flag [ing for the main ingredient,with for additional ingredients,excl for those that should be excluded]). -For groups of ingredients (e.g. antibiotics) list all the possible variations -*/ - --- 1. Update latest_UPDATE field to new date +-- Update latest_UPDATE field to new date DO $_$ BEGIN PERFORM VOCABULARY_PACK.SetLatestUpdate( @@ -43,1021 +27,32 @@ BEGIN ); END $_$; - - --- 2. Truncate all working tables AND remove indices +-- Truncate all working tables AND remove indices TRUNCATE TABLE concept_stage; TRUNCATE TABLE concept_relationship_stage; TRUNCATE TABLE concept_synonym_stage; TRUNCATE TABLE pack_content_stage; TRUNCATE TABLE drug_strength_stage; --- 3. Preliminary work --- 3.0 create table with manual work -DROP TABLE IF EXISTS class_to_drug_manual_tbd; -CREATE TABLE class_to_drug_manual_tbd AS - SELECT * FROM class_drugs_scraper -WHERE class_code ~ '^J07|^A10A' -AND length(class_code)=7; - --- 3.1 create table with combo drugs to be used later --- 3.1.1 First change names for D07X to represent that they are combos -UPDATE class_drugs_scraper -SET class_name = class_name||', combinations' - WHERE class_code ~ 'D07X' AND length(class_code)=7 AND class_name NOT LIKE 'combinations of%';--D07XB30 - -DROP TABLE IF EXISTS class_1_comb; -CREATE TABLE class_1_comb AS -SELECT * -FROM class_drugs_scraper -WHERE class_name ~ 'comb| and |excl|derivate|other|with' - AND length(class_code) = 7 - -- AND NOT class_name ~ 'decarboxylase inhibitor' XXX - AND class_code NOT IN (SELECT class_code FROM class_to_drug_manual_tbd) -- manual XXX -; - --- add to github script -DELETE -FROM class_1_comb -WHERE class_code in ('S01XA20','G02BB01','G02BA02','G02BA03','G02BB02') -;--artificial tears and other indifferent preparations, actually not combo; G02% IUD and vaginal rings - --- create unified table with combinations, combos a+b -DROP TABLE IF EXISTS ambiguous_class_ingredient_tst; -CREATE TABLE ambiguous_class_ingredient_tst AS - -- a+b -SELECT class_code,class_name, concept_code_2, CASE WHEN rnk = 1 THEN 'ing' ELSE 'with' END AS flag, rnk -FROM ( - SELECT class_code, - concept_code_2, - class_name, - coalesce(rnk, CASE - WHEN rnk IS NULL - THEN rank() OVER (PARTITION BY concept_code_1, rnk ORDER BY concept_code_2) + 1 END) AS rnk - FROM ( - SELECT DISTINCT class_code, - i.concept_code_2, - class_name, - CASE - WHEN concept_code_2 = - regexp_replace(regexp_replace(class_name, ' and.*', ''), '\, .*', '') THEN 1 - ELSE NULL END AS rnk, - concept_code_1 - FROM class_1_comb - LEFT JOIN reference USING (class_code) - JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d - ON lower(d.concept_code) = lower(concept_code_2) AND concept_class_id = 'Ingredient' - WHERE class_name ~ ' and ' - AND NOT class_name ~ 'excl|combinations of|derivate|other|with') a - ) a -UNION --- complex combinations -SELECT class_code, class_name, ing, flag, rnk -FROM ambiguous_class_ingredient -; - --- one strange parsing -UPDATE ambiguous_class_ingredient_tst -SET flag = 'ing', rnk = 1 -WHERE class_code='A11GB01' AND concept_code_2='ascorbic acid (vit c)'; - -UPDATE ambiguous_class_ingredient_tst -SET rnk = 2 -WHERE class_code='A11GB01' AND concept_code_2='calcium'; - - -DELETE -FROM ambiguous_class_ingredient_tst -WHERE (class_code, concept_code_2, flag) IN (SELECT a.class_code,a.concept_code_2,a.flag - FROM ambiguous_class_ingredient_tst a - JOIN ambiguous_class_ingredient_tst b - ON a.class_code = b.class_code AND a.concept_code_2 = b.concept_code_2 - WHERE a.flag = 'with' AND b.flag = 'ing'); - --- insert pure combinations -DROP TABLE IF EXISTS pure_combos; -CREATE TABLE pure_combos - as - SELECT * FROM ( - SELECT class_code,i.concept_code_2, class_name, 'ing' AS flag, 1 AS rnk - FROM class_1_comb - LEFT JOIN reference USING (class_code) - JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d ON lower(d.concept_code) = lower(concept_code_2) AND concept_class_id = 'Ingredient' - WHERE class_name IN ('combinations','various combinations') -UNION - SELECT class_code, i.concept_code_2, class_name, 'with', 2 - FROM class_1_comb a - JOIN concept c ON regexp_replace(c.concept_code, '..$', '') = regexp_replace(a.class_code, '..$', '') and - c.concept_class_id = 'ATC 5th' -- getting siblings for combinations - JOIN internal_relationship_stage i ON c.concept_code = substring(i.concept_code_1,'\w+') - JOIN drug_concept_stage d ON lower(d.concept_code) = lower(i.concept_code_2) AND d.concept_class_id = 'Ingredient' - WHERE a.class_name IN ('combinations','various combinations') -) a -; - -UPDATE pure_combos v -SET rnk = 1, flag = 'ing' -FROM ( -SELECT class_code, min(concept_code_2) OVER (PARTITION BY class_code ORDER BY concept_code_2) AS min_code -FROM pure_combos -WHERE class_code NOT IN (SELECT class_code FROM pure_combos WHERE rnk=1) - ) a -WHERE (v.class_code =a.class_code AND v.concept_code_2 = a.min_code) -; - -DROP TABLE IF EXISTS combo_of; -CREATE TABLE combo_of - as - SELECT class_code, i.concept_code_2, class_name, 'with' AS flag, 2 as rnk - FROM class_1_comb a - JOIN concept c ON regexp_replace(c.concept_code, '..$', '') = regexp_replace(a.class_code, '..$', '') and - c.concept_class_id = 'ATC 5th' -- getting siblings for combinations - JOIN internal_relationship_stage i ON c.concept_code = substring(i.concept_code_1,'\w+') - JOIN drug_concept_stage d ON lower(d.concept_code) = lower(i.concept_code_2) AND d.concept_class_id = 'Ingredient' - WHERE class_name ~ '(combinations of|in combination)' - AND NOT class_name ~ ' and |derivate|other drugs|^other |with' AND class_name NOT IN ('combinations','various combinations') -UNION - SELECT class_code, i.concept_code_2, class_name, 'with' AS flag, 2 as rnk - FROM class_1_comb a - JOIN concept c ON regexp_replace(c.concept_code, '.$', '') = regexp_replace(a.class_code, '...$', '') AND concept_name like '%plain%' - JOIN concept cc ON c.concept_code = regexp_replace(cc.concept_code, '..$', '') AND cc.concept_class_id = 'ATC 5th' - JOIN internal_relationship_stage i ON cc.concept_code = substring(i.concept_code_1,'\w+') - JOIN drug_concept_stage d ON lower(d.concept_code) = lower(i.concept_code_2) AND d.concept_class_id = 'Ingredient' - WHERE class_name ~ '(combinations of|in combination)' - AND NOT class_name ~ ' and |derivate|other drugs|^other |with' AND class_name NOT IN ('combinations','various combinations') -; - --- insert into combo_of same ingredients to create permutations, assume that there won't be > 3 ingredients in combination -INSERT INTO combo_of -SELECT class_code, concept_code_2,class_name, 'ing', 1 - FROM combo_of -; - -INSERT INTO ambiguous_class_ingredient_tst -SELECT DISTINCT class_code, class_name, concept_code_2, flag, rnk -FROM combo_of -UNION -SELECT DISTINCT class_code, class_name, concept_code_2, flag, rnk -FROM pure_combos -; - - -CREATE INDEX ambiguous_class_ingredient_test ON ambiguous_class_ingredient_tst (class_code, concept_code_2, flag); - --- 3.2 create a table with aggregated RxE ingredients -DROP TABLE IF EXISTS rx_combo; -CREATE TABLE rx_combo AS -SELECT drug_concept_id, - string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo -FROM devv5.drug_strength - JOIN concept ON concept_id = drug_concept_id AND - concept_class_id IN ('Clinical Drug Form', 'Ingredient') -- 'Clinical Drug Comp' doesn't exist -GROUP BY drug_concept_id -; - -DROP TABLE IF EXISTS full_combo; -CREATE TABLE full_combo -AS -WITH hold AS ( -SELECT a.*, concept_id_2, precedence -FROM ambiguous_class_ingredient_tst a -JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE flag = 'ing'), - ing AS ( -SELECT a.*, concept_id_2, precedence -FROM ambiguous_class_ingredient_tst a -JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE flag = 'with' AND rnk = 2 - ), - ing2 AS ( -SELECT a.*, concept_id_2, precedence -FROM ambiguous_class_ingredient_tst a -JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE flag = 'with' AND rnk = 3 - ), - ing3 AS ( -SELECT a.*, concept_id_2, precedence -FROM ambiguous_class_ingredient_tst a -JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE flag = 'with' AND rnk = 4 - ) -SELECT hold.class_code, hold.class_name, hold.concept_id_2||COALESCE('-' || ing.concept_id_2, '')||COALESCE('-' || ing2.concept_id_2, '')||COALESCE('-' || ing3.concept_id_2, '') AS i_combo - FROM hold -JOIN ing USING (class_code) -LEFT JOIN ing2 USING (class_code) -LEFT JOIN ing3 USING (class_code) -ORDER BY class_code -; - --- adding another layer of permuted concepts. E.g. combination of corticosteroids now can have 3 ingredients -DROP TABLE IF EXISTS permutations; -CREATE TABLE permutations -AS -WITH hold AS - (SELECT a.*, concept_id_2, precedence -FROM ambiguous_class_ingredient_tst a -JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE flag = 'ing'), - ing AS (SELECT a.*, concept_id_2, precedence -FROM ambiguous_class_ingredient_tst a -JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE flag = 'with' AND rnk = 2 - AND NOT EXISTS (SELECT 1 FROM ambiguous_class_ingredient_tst b WHERE b.class_code = a.class_code AND rnk=3)) -SELECT hold.class_code, hold.class_name, hold.concept_id_2||COALESCE('-' || i.concept_id_2, '')||COALESCE('-' || i2.concept_id_2, '') AS i_combo - FROM hold -JOIN ing i ON i.class_code = hold.class_code -JOIN ing i2 ON i2.class_code = hold.class_code -WHERE i.concept_id_2!=i2.concept_id_2 AND i.concept_id_2!=hold.concept_id_2 -; - -INSERT INTO full_combo -SELECT DISTINCT * -FROM permutations; - -CREATE INDEX i_full_combo ON full_combo (class_code, i_combo); - -DROP TABLE IF EXISTS full_combo_reodered; -CREATE TABLE full_combo_reodered - AS -WITH a AS (SELECT class_code,class_name, i_combo, rank() OVER (PARTITION BY class_code ORDER BY i_combo) AS rnk - FROM (SELECT DISTINCT * FROM full_combo) a), - b AS (SELECT class_code,class_name, rnk, cast(unnest(string_to_array(i_combo, '-')) AS int) AS ing - FROM a) -SELECT class_code, class_name, string_agg(ing::varchar, '-' ORDER BY ing) AS i_combo -FROM b -GROUP BY class_code, class_name, rnk; -- to array - -CREATE INDEX i_full_combo_reodered ON full_combo_reodered (class_code, i_combo); - -DROP TABLE IF EXISTS full_combo_with_form; -CREATE TABLE full_combo_with_form -AS -SELECT class_code, class_name, i_combo, concept_id_2 -FROM full_combo_reodered -JOIN internal_relationship_stage irs on class_code = substring (irs.concept_code_1,'\w+') -JOIN drug_concept_stage dc on dc.concept_code = irs.concept_code_2 AND dc.concept_class_id = 'Dose Form' -JOIN relationship_to_concept rtc on rtc.concept_code_1 = irs.concept_code_2 - -UNION ALL - -SELECT f.class_code, class_name, i_combo, null -- take from reference those that do not have forms -FROM full_combo_reodered f -JOIN reference r on r.class_code = f.class_code -WHERE r.concept_code = r.class_code -; - -CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo,concept_id_2); - --- Assembling final table --- Order: -1. Manual -2. Mono: Ingredient A; form -3. Mono: Ingredient A -4. Combo: Ingredient A + Ingredient B -5. Combo: Ingredient A + group B -6. Combo: Ingredient A, combination; form -7. Combo: Ingredient A, combination -8. Any packs - -Group A + Group B -other things like Calcium compounds - -DROP TABLE IF EXISTS class_to_drug; --747 -CREATE TABLE class_to_drug -as - with rxnorm AS ( - SELECT a.concept_id, a.concept_name,a.concept_class_id,a.vocabulary_id, c.concept_id_2,r.i_combo - FROM rx_combo r - JOIN concept a ON r.drug_concept_id = a.concept_id - JOIN concept_relationship c ON c.concept_id_1 = a.concept_id - WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id LIKE 'RxNorm%' - AND a.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason IS NULL - ) - SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id, 98 as order - FROM full_combo_with_form f - JOIN rxnorm r on r.i_combo = f.i_combo AND r.concept_id_2 = f.concept_id_2 -; - --- adding everything we can for combinations -DROP TABLE IF EXISTS combo_not_limited_to_higherATC; -CREATE TABLE combo_not_limited_to_higherATC -AS - WITH combo AS ( --250 - SELECT c.class_code, c.class_name, concept_id_2 - FROM class_1_comb c - JOIN internal_relationship_stage i ON c.class_code = substring(i.concept_code_1, '\w+') - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 - WHERE c.class_name ~ '(, combinations)|(in combination with other drugs)' - AND NOT c.class_name ~ 'with|and thiazides|and other diuretics' -- gives errors in C07BB52 AND C07CB53 if removed - ) - SELECT DISTINCT class_code, class_name, drug_concept_id - FROM rx_combo - JOIN combo on i_combo ~ cast(concept_id_2 AS VARCHAR) AND i_combo LIKE '%-%' -- at least two ingredients - WHERE not exists(SELECT 1 - FROM ambiguous_class_ingredient_tst a2 - JOIN relationship_to_concept rtc2 ON rtc2.concept_code_1 = a2.concept_code_2 - WHERE a2.class_code = combo.class_code - AND a2.flag = 'excl' - AND i_combo ~ cast(rtc2.concept_id_2 AS VARCHAR)) -; - --- inserting those few ATCs that are like A + group B, combinations -INSERT INTO combo_not_limited_to_higherATC -SELECT DISTINCT a.class_code, a.class_name, drug_concept_id -FROM ambiguous_class_ingredient_tst a - JOIN ambiguous_class_ingredient_tst b ON a.class_code = b.class_code AND a.flag = 'ing' AND b.flag = 'with' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = a.concept_code_2 - JOIN relationship_to_concept rtc2 ON rtc2.concept_code_1 = b.concept_code_2 - JOIN rx_combo on i_combo ~ cast(rtc.concept_id_2 AS VARCHAR) AND i_combo ~ cast(rtc2.concept_id_2 AS VARCHAR) AND - i_combo ~ '.*-.*-' -where a.class_name ~ '( and .*, combinations$)|(combinations with other drugs)' -; - - WITH rxnorm AS ( - SELECT a.concept_id, a.concept_name,a.concept_class_id,a.vocabulary_id, c.concept_id_2,r.class_code, r.class_name - FROM combo_not_limited_to_higherATC r - JOIN concept a ON r.drug_concept_id = a.concept_id - JOIN concept_relationship c ON c.concept_id_1 = a.concept_id - WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id LIKE 'RxNorm%' - AND a.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason IS NULL - ) -INSERT INTO class_to_drug -SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id, 6 -FROM rxnorm -JOIN internal_relationship_stage i ON substring(i.concept_code_1, '\w+') = class_code -JOIN relationship_to_concept rtc ON i.concept_code_2 = rtc.concept_code_1 -WHERE rxnorm.concept_id_2 = rtc.concept_id_2 -; - -INSERT INTO class_to_drug -SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id, 7 -FROM combo_not_limited_to_higherATC -JOIN concept ON concept_id = drug_concept_id -JOIN reference r USING (class_code) -WHERE r.concept_code=r.class_code -; - ---insert all forms for those ATC that do not specify forms --821 -WITH rxnorm AS ( - SELECT a.concept_id, a.concept_name,a.concept_class_id,a.vocabulary_id, c.concept_id_2,r.i_combo - FROM rx_combo r - JOIN concept a ON r.drug_concept_id = a.concept_id - JOIN concept_relationship c ON c.concept_id_1 = a.concept_id - WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id LIKE 'RxNorm%' - AND a.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason IS NULL -) -INSERT -INTO class_to_drug -SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id,99 -FROM full_combo_with_form f - JOIN rxnorm r ON r.i_combo = f.i_combo -WHERE f.concept_id_2 IS NULL -; - ---Combinations of propranolol AND hydralazine or dihydralazine are classified INC07FX01. [combinations with other drugs] XXX --- 3.6.4 start removing incorrectly assigned combo based ON WHO rank --- XXX https://www.whocc.no/atc_ddd_index/?code=N02BA51&showdescription=yes --- zero rank (no official rank is present) -SELECT * --XXX -FROM class_to_drug -WHERE class_code ~ 'M03BA73|M03BA72|N02AC74|M03BB72|N02BB52|M03BB73|M09AA72|N02AB72|N02BB72|N02BA77' - AND concept_name ~* - 'Salicylamide|Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Methadone|etamizole|Ergotamine'--acetylsalicylic -; ---starts the official rank -DELETE --90 -FROM class_to_drug -WHERE class_code ~ 'N02BB74|N02BB54' - AND concept_name ~* 'Phenazone|Salicylamide|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' -; -DELETE --52 -FROM class_to_drug -WHERE class_code ~ 'N02BA75|N02BA55' - AND concept_name ~* 'Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' -; -DELETE --6 -FROM class_to_drug -WHERE class_code ~ 'N02BB71|N02BB51' - AND concept_name ~* 'Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' -; -DELETE --50 -FROM class_to_drug -WHERE class_code ~ 'N02BA71|N02BA51' - AND concept_name ~* 'Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' -; -DELETE -- 1 -FROM class_to_drug -WHERE class_code ~ 'N02BE71|N02BE51' - AND concept_name ~* 'Dipyrocetyl|Bucetin|Phenacetin' -; -DELETE --108 -FROM class_to_drug -WHERE class_code ~ '^N02' - AND concept_name ~ 'Codeine' - AND NOT class_name ~ 'codeine'; - --- PPI and aspirin -- XXX changes this logic -DELETE -FROM class_to_drug -WHERE class_code ~ 'N02BA51' - AND concept_name ~* 'meprazole|Pantoprazole|Rabeprazol'; - --- contraceptive packs -INSERT INTO class_to_drug -SELECT class_code, class_name, c.concept_id,c.concept_name,"order" -FROM class_to_drug ctd -JOIN devv5.concept_ancestor ON ctd.concept_id = ancestor_concept_id -JOIN concept c ON descendant_concept_id = c.concept_id -WHERE class_code ~ 'G03FB|G03AB' - AND c.concept_class_id IN ('Clinical Pack'); - -DELETE FROM class_to_drug -WHERE class_code ~ 'G03FB|G03AB' - AND concept_class_id NOT IN ('Clinical Pack'); - --- insert mono-ATC codes -DROP TABLE IF EXISTS mono_ing; -CREATE TABLE mono_ing as -SELECT DISTINCT class_code, class_name, concept_id_2 AS ing_id - FROM class_drugs_scraper -JOIN internal_relationship_stage i on class_code = substring(concept_code_1,'\w+') -JOIN drug_concept_stage d on lower(d.concept_code) = lower(i.concept_code_2) AND concept_class_id = 'Ingredient' -JOIN relationship_to_concept rtc on rtc.concept_code_1 = i.concept_code_2 - WHERE length(class_code)=7 -; - -DELETE FROM mono_ing -WHERE class_code IN (SELECT class_code FROM class_1_comb) -OR class_code IN (SELECT class_code FROM class_to_drug_manual_tbd) -OR (class_name LIKE '% and %' AND class_code!='S01XA20') -; - -INSERT INTO class_to_drug --587434 -with form AS ( - SELECT class_code, class_name, ing_id, concept_id_2 AS form_id - FROM mono_ing - JOIN internal_relationship_stage i on class_code = substring(i.concept_code_1, '\w+') - JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Dose Form' - JOIN relationship_to_concept rtc on rtc.concept_code_1 = i.concept_code_2 -) -SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id,2 -FROM form -JOIN rx_combo r on i_combo = cast(ing_id AS varchar) --XXX ingr?? -JOIN concept a ON r.drug_concept_id = a.concept_id -JOIN concept_relationship c ON c.concept_id_1 = a.concept_id AND form_id = c.concept_id_2 - WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id LIKE 'RxNorm%' - AND a.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason IS NULL -; - -INSERT INTO class_to_drug --785 -SELECT DISTINCT class_code, class_name, c.concept_id, c.concept_name, c.concept_class_id, 3 -FROM mono_ing m -JOIN internal_relationship_stage i on class_code = substring(concept_code_1,'\w+') -JOIN concept c on m.ing_id = c.concept_id - WHERE i.concept_code_1 NOT LIKE '% %' -; - -INSERT INTO class_to_drug -SELECT class_code, class_name, concept_id, concept_name, concept_class_id, 1 -FROM class_to_drug_manual; - --- 3.8.1 manually excluded drugs based on Precise Ingredients -DELETE -FROM class_to_drug -WHERE class_code IN ('B02BD14','B02BD11') -and concept_class_id = 'Ingredient'; - -INSERT INTO class_to_drug -SELECT 'B02BD11','catridecacog', concept_id, concept_name, concept_class_id,1 -FROM concept -WHERE (vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' - AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') - OR concept_id = 35603348 -- the whole hierarchy -; - -INSERT INTO class_to_drug -SELECT 'B02BD14','susoctocog alfa', concept_id, concept_name, concept_class_id,1 -FROM concept -WHERE (vocabulary_id LIKE 'RxNorm%' - AND concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') - OR concept_id IN (35603348, 44109089) -- the whole hierarchy -; - -DELETE -FROM class_to_drug -WHERE class_code = 'A02BA07' - AND concept_class_id = 'Clinical Drug Form'; --Tritec -INSERT INTO class_to_drug -SELECT 'A02BA07','ranitidine bismuth citrate', concept_id, concept_name, concept_class_id,1 -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Tritec%' - AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' -; - -DELETE -FROM class_to_drug -- choriogonadotropin alfa -WHERE class_code = 'G03GA08'; -INSERT INTO class_to_drug -SELECT 'G03GA08','choriogonadotropin alfa', concept_id, concept_name, concept_class_id,1 -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%choriogonadotropin alfa%' - AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug Form' -; - -DELETE -FROM class_to_drug -WHERE class_code='N05AF02'; -- clopentixol -INSERT INTO class_to_drug -SELECT 'N05AF02','clopenthixol', concept_id, concept_name, concept_class_id,1 -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Sordinol|Ciatyl' - AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' -; - -DELETE -FROM class_to_drug -WHERE class_code IN('D07AB02','D07BB04'); -- hydrocortisone butyrate + combo that so far doesn't exist -INSERT INTO class_to_drug -SELECT 'D07AB02','hydrocortisone butyrate', concept_id, concept_name, concept_class_id,1 -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Hydrocortisone butyrate' AND concept_class_id = 'Clinical Drug' - AND standard_concept = 'S' -; - -DELETE -FROM class_to_drug -WHERE class_code = 'C01DA05'; - -INSERT INTO class_to_drug -SELECT 'C01DA05','pentaerithrityl tetranitrate', concept_id, concept_name, concept_class_id,1 -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Pentaerythritol Tetranitrate%' and - standard_concept = 'S' AND concept_class_id = 'Clinical Drug Comp' -- forms do not exist -; - -DELETE -FROM class_to_drug -WHERE class_code = 'B02BD14' - AND concept_name LIKE '%Tretten%' -; --catridecacog - -DELETE -FROM class_to_drug -- XXX -WHERE class_name = 'amino acids'; - -DELETE -FROM class_to_drug -WHERE class_name LIKE '%,%and%' - AND class_name NOT LIKE '%,%,%and%' - AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT LIKE '% / % / %'; - ---combinations FROM ATC 4th, need to be fixed afterwards -DELETE -FROM class_to_drug -WHERE class_name NOT LIKE '% / %' -AND class_code ~ 'S01CB|G03EK|G03CC|D10AA' -- D10AA smth for acne, unclear; S01CB with whatever you want, G% is strange hormones -; - -DROP TABLE IF EXISTS pack_all; -CREATE TABLE pack_all -AS - WITH a AS ( - SELECT concept_id_1, - drug_concept_id, - string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo, - count(drug_concept_id) over (partition by concept_id_1) as cnt - FROM drug_strength - JOIN concept_relationship r - on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL - JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' - GROUP BY drug_concept_id,concept_id_1), - b as - (SELECT class_code, class_name, r.concept_code_1, r.concept_id_2 - FROM class_1_comb - JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 - WHERE class_name LIKE '% and %'), - c AS (SELECT class_code, r.concept_code_1, r.concept_id_2 - FROM class_1_comb - JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 - WHERE class_name LIKE '% and %') - SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name - FROM a - JOIN a aa on aa.concept_id_1 = a.concept_id_1 - JOIN concept cc on concept_id = a.concept_id_1 - JOIN b on cast(b.concept_id_2 AS varchar) = aa.i_combo - JOIN c on cast(c.concept_id_2 AS varchar) = a.i_combo - WHERE a.drug_concept_id != aa.drug_concept_id - AND b.class_code = c.class_code - AND b.concept_code_1 != c.concept_code_1 - AND a.cnt = 2 -; - --- insert 3-component drugs -INSERT INTO pack_all - WITH a AS ( - SELECT concept_id_1, - drug_concept_id, - string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo, - count(drug_concept_id) over (partition by concept_id_1) as cnt - FROM drug_strength - JOIN concept_relationship r - on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL - JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' - GROUP BY drug_concept_id,concept_id_1), - b as - (SELECT class_code, class_name, r.concept_code_1, r.concept_id_2 - FROM class_1_comb - JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 - WHERE class_name LIKE '% and %'), - c AS (SELECT class_code, r.concept_code_1, r.concept_id_2 - FROM class_1_comb - JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 - WHERE class_name LIKE '% and %'), - d AS (SELECT class_code, r.concept_code_1, r.concept_id_2 - FROM class_1_comb - JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 - WHERE class_name LIKE '% and %') - SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name - FROM a - JOIN a aa on aa.concept_id_1 = a.concept_id_1 - JOIN a aaa on aaa.concept_id_1 = a.concept_id_1 - JOIN concept cc on concept_id = a.concept_id_1 - JOIN b on cast(b.concept_id_2 AS varchar) = aa.i_combo - JOIN c on cast(c.concept_id_2 AS varchar) = a.i_combo - JOIN d on cast(d.concept_id_2 AS varchar) = aaa.i_combo - WHERE a.drug_concept_id != aa.drug_concept_id - AND b.class_code = c.class_code - AND d.class_code = c.class_code - AND b.concept_code_1 != c.concept_code_1 - AND d.concept_code_1 != c.concept_code_1 AND d.concept_code_1 != b.concept_code_1 - AND a.cnt = 3 -; --- XXX 4-component drugs -DELETE -FROM pack_all - WHERE (class_code='A02BD11' AND concept_id=42731634) - OR (class_code='R03AL08' AND concept_id=43045404); - - -DROP TABLE IF EXISTS pack_temp; -CREATE TABLE pack_temp -as -with a AS (SELECT concept_id_1, - string_agg(ingredient_concept_id::varchar, '-' ORDER BY ingredient_concept_id) AS i_combo - FROM drug_strength - JOIN concept_relationship r on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL - JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' - GROUP BY concept_id_1), - b AS ( -SELECT DISTINCT class_code, class_name, r.concept_code_1, r.concept_id_2 -FROM class_1_comb -JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code -JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' -JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 -WHERE class_name LIKE '%, combinations') -SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name -FROM a -JOIN b on i_combo ~ cast(b.concept_id_2 AS varchar) -and i_combo!=cast(b.concept_id_2 AS varchar) -JOIN concept cc on concept_id_1 = concept_id -; - - -INSERT INTO pack_all -with a AS ( - SELECT p.concept_id, p.concept_name, p.concept_class_id, class_code,class_name, r.concept_id_2 - FROM pack_temp p - JOIN internal_relationship_stage ON substring(concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d ON concept_code_2 = concept_code AND d.concept_class_id = 'Dose Form' - JOIN relationship_to_concept r ON r.concept_code_1 = concept_code_2), - b AS (SELECT p.*, r2.concept_id_2 - FROM pack_temp p - JOIN devv5.concept_ancestor r on concept_id = descendant_concept_id - JOIN concept_relationship r2 - ON r2.concept_id_1 = ancestor_concept_id AND r2.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form') -SELECT a.concept_id, a.concept_name, a.concept_class_id, a.class_code, a.class_name -FROM a -JOIN b USING (concept_id, concept_id_2) -WHERE (concept_id, a.class_code) NOT IN (SELECT concept_id, class_code FROM pack_all) -UNION -SELECT p.* -FROM pack_temp p -JOIN reference r on concept_code = p.class_code -WHERE (concept_id, p.class_code) NOT IN (SELECT concept_id, class_code FROM pack_all) -; - -INSERT INTO class_to_drug -(class_code, class_name, concept_id, concept_name, concept_class_id, "order") -SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id, 8 -FROM pack_all; - --- 4.11 fix packs -INSERT INTO class_to_drug -SELECT DISTINCT class_code, class_name,c.concept_id, c.concept_name,c.concept_class_id, 8 -FROM class_to_drug f - JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = cast(f.concept_id AS INT) - JOIN devv5.concept c ON c.concept_id = descendant_concept_id AND c.concept_class_id LIKE '%Pack%' -WHERE f.class_code ~ 'G03FB|G03AB'; -- packs - --- 4.12 -DELETE -FROM class_to_drug -WHERE class_code ~ 'G03FB|G03AB' - AND concept_class_id IN ('Clinical Drug Form', 'Ingredient'); - --- XXX Maybe introduce later -/* -SELECT * -FROM class_to_drug -WHERE class_name LIKE '%and estrogen%' -- if there are regular estiol/estradiol/EE - AND concept_id IN (SELECT concept_id - FROM class_to_drug GROUP BY concept_id HAVING COUNT(1) > 1); - */ - --- 4.14 Solution for the first run: for inambiguous ATC classes (those that classify an ingredient through only one class) --- we relate this ATC class to the entire group of drugs that have this ingredient. -DROP TABLE IF EXISTS interim; -CREATE TABLE interim -AS -WITH a AS (SELECT DISTINCT class_code, class_name, c.concept_id,c.concept_name -FROM class_to_drug crd -JOIN devv5.concept_ancestor on crd.concept_id = descendant_concept_id -JOIN concept c on c.concept_id = ancestor_concept_id AND c.concept_class_id = 'Ingredient') -SELECT * -FROM (SELECT count (*) OVER (PARTITION BY trim(regexp_replace(class_name, '\(.*\)','')) ) AS cnt, class_code, class_name, a.concept_id,a.concept_name -- regexp for (vit C) -FROM a ) a -WHERE cnt=1 -AND class_code NOT IN (SELECT class_code FROM class_to_drug_manual) ---and class_code NOT IN ('G02BB02') -; - -DELETE -FROM interim -WHERE class_name IN ( - SELECT class_name - FROM (SELECT DISTINCT class_code, class_name FROM class_drugs_scraper) a - GROUP BY class_name - HAVING count(1) > 1); - -DELETE -FROM interim -WHERE concept_id IN ( - SELECT a.concept_id - FROM interim a - JOIN interim b ON a.concept_id = b.concept_id - WHERE a.class_code != b.class_code); - - -DROP TABLE IF EXISTS interim_2; -CREATE TABLE interim_2 AS -WITH a AS ( -SELECT DISTINCT class_code, class_name, c.concept_id,c.concept_name -FROM class_to_drug crd -JOIN devv5.concept_ancestor on crd.concept_id = descendant_concept_id -JOIN concept c on c.concept_id = ancestor_concept_id AND c.concept_class_id = 'Ingredient' - ), -b AS (SELECT concept_id FROM a -GROUP BY concept_id having count(1)=1), -c AS (SELECT a.class_code, a.class_name, a.concept_id FROM a JOIN b USING(concept_id) - ) -SELECT c.* -FROM c -JOIN interim USING(class_code) -WHERE class_code NOT IN (SELECT class_code FROM class_to_drug_manual) -; - -DELETE -FROM interim_2 -WHERE class_name IN ( - SELECT class_name - FROM (SELECT DISTINCT class_code, class_name FROM class_drugs_scraper) a - GROUP BY class_name - HAVING count(1) > 1); - -DELETE FROM class_to_drug -- inambiguous only for forms -WHERE class_code IN (SELECT class_code FROM interim) -; - -INSERT INTO class_to_drug -- ingredients are partially inambiguous -(class_code, class_name, concept_id, concept_name, concept_class_id, "order") -SELECT class_code, class_name, c.concept_id, c.concept_name, c.concept_class_id, 2 -FROM interim i -JOIN devv5.concept_ancestor ca on i.concept_id = ancestor_concept_id -JOIN concept c on descendant_concept_id = c.concept_id -WHERE c.concept_class_id = 'Clinical Drug Form' AND c.concept_name NOT LIKE '% / %' -; - -DELETE FROM class_to_drug -WHERE class_code IN(SELECT class_code FROM interim_2) -; -INSERT INTO class_to_drug -- ingredients are absolutely inambiguous -(class_code, class_name, concept_id, concept_name, concept_class_id, "order") -SELECT class_code, class_name, concept_id, c.concept_name, c.concept_class_id, 3 -FROM interim_2 i -JOIN concept c USING (concept_id) -; - --- also adding those that don't have forms, but are unique -with ing AS ( -SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt -FROM ( -SELECT DISTINCT substring(i.concept_code_1,'\w+'), r.concept_code_1, concept_id_2 -FROM relationship_to_concept r -JOIN internal_relationship_stage i on i.concept_code_2 = r.concept_code_1 -JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' - ) a GROUP BY concept_id_2,concept_code_1 ), -drug AS ( -SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 -FROM internal_relationship_stage i -WHERE not exists (SELECT 1 FROM internal_relationship_stage i2 - WHERE i.concept_code_2=i2.concept_code_2 - AND substring(i.concept_code_1,'\w+')!=substring(i2.concept_code_1,'\w+')) -), - drug_name AS ( -SELECT DISTINCT class_code,class_name, concept_id_2 -FROM ing -JOIN drug ON concept_code_2=concept_code_1 -JOIN class_drugs_scraper ON code = class_code -WHERE cnt=1 AND class_name = concept_code_1 -and code NOT IN (SELECT class_code FROM class_to_drug) - ), -all_drug AS ( -SELECT class_code,class_name, concept_id,concept_name, concept_class_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt -FROM drug_name -JOIN concept on concept_id_2 = concept_id -) -INSERT INTO class_to_drug -SELECT class_code, class_name, concept_id, concept_name,concept_class_id,3 -FROM all_drug -WHERE cnt=1 -; - -UPDATE class_to_drug -SET "order" = 4 -WHERE class_name ~ ' and ' -AND NOT class_name ~ 'adrenergics|analgesics|antispasmodics|antibacterials|antiflatulents|imidazoles|minerals|polyfructosans|triazoles|natural phospholipids|lactic acid producing organisms|antiflatulents|beta-lactamase inhibitor|sulfonylureas|antibiotics|antiinfectives|antiseptics|artificial tears|contact laxatives|corticosteroids|ordinary salt combinations|imidazoles/triazoles|cough suppressants|diuretics|drugs for obstructive airway diseases|expectorants|belladonna alkaloids|mucolytics|mydriatics|non-opioid analgesics|organic nitrates|potassium-sparing agents|proton pump inhibitors|psycholeptics|thiazides|snake venom antiserum|fat emulsions|amino acids|acid preparations|sulfur compounds|stramoni preparations|protamines|penicillins|comt inhibitor|cannabinoids|decarboxylase inhibitor|edetates|barbiturates|excl|combinations of|derivate|with' -AND "order" in (98,99); - -UPDATE class_to_drug -SET "order" = 5 -WHERE "order" in (98,99); - --- calcium compounds -UPDATE class_to_drug -SET "order" = 7 -WHERE class_name ~ 'compounds' -AND "order" = 2; - - --- working with duplicates to remove a and b vs a/b, comb -DELETE FROM class_to_drug -WHERE (class_code,concept_id) IN( -SELECT b.class_code, b.concept_id FROM class_to_drug a -JOIN class_to_drug b on a.concept_id = b.concept_id -WHERE a."order" = 4 AND b."order" in (6,7)) -; - ---5. Get tables for see what is missing ---5.1 Check new class codes that should be worked out -SELECT * -FROM class_drugs_scraper -WHERE class_code NOT IN - (SELECT concept_code FROM concept WHERE vocabulary_id = 'ATC'); - ---5.2 Take a look at new standard ingredients that did not exist before the last ATC release -SELECT * -FROM concept -WHERE vocabulary_id IN('RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND valid_start_date > (SELECT latest_update - FROM devv5.vocabulary_conversion - WHERE vocabulary_id_v5 = 'ATC'); - ---5.3 Take a look at the ATC codes that aren't covered in the release -SELECT * -FROM class_drugs_scraper -WHERE class_code NOT IN (SELECT class_code FROM class_to_drug) -and length (class_code) =7 -; - --- 6. Add ATC --- create temporary table atc_tmp_table -DROP TABLE IF EXISTS atc_tmp_table; -CREATE UNLOGGED TABLE atc_tmp_table AS -SELECT rxcui, - code, - concept_name, - 'ATC' AS vocabulary_id, - 'C' AS standard_concept, - concept_code, - concept_class_id -FROM ( - SELECT DISTINCT rxcui, - code, - SUBSTR(str, 1, 255) AS concept_name, - code AS concept_code, - CASE - WHEN LENGTH(code) = 1 - THEN 'ATC 1st' - WHEN LENGTH(code) = 3 - THEN 'ATC 2nd' - WHEN LENGTH(code) = 4 - THEN 'ATC 3rd' - WHEN LENGTH(code) = 5 - THEN 'ATC 4th' - WHEN LENGTH(code) = 7 - THEN 'ATC 5th' - END AS concept_class_id - FROM sources.rxnconso - WHERE sab = 'ATC' - AND tty IN ( - 'PT', - 'IN' - ) - AND code != 'NOCODE' - ) AS s1; - - -INSERT INTO atc_tmp_table -SELECT NULL, - class_code, - class_name, - 'ATC', - 'C', - class_code, - CASE - WHEN LENGTH(class_code) = 1 - THEN 'ATC 1st' - WHEN LENGTH(class_code) = 3 - THEN 'ATC 2nd' - WHEN LENGTH(class_code) = 4 - THEN 'ATC 3rd' - WHEN LENGTH(class_code) = 5 - THEN 'ATC 4th' - WHEN LENGTH(class_code) = 7 - THEN 'ATC 5th' - END AS concept_class_id -FROM class_drugs_scraper -WHERE class_code NOT IN (SELECT code FROM atc_tmp_table); - -CREATE INDEX idx_atc_code ON atc_tmp_table (code); -CREATE INDEX idx_atc_ccode ON atc_tmp_table (concept_code); -ANALYZE atc_tmp_table; +-- Add ATC codes using concept_manual (processed inside function below) +DO $_$ +BEGIN + PERFORM VOCABULARY_PACK.ProcessManualConcepts(); +END $_$; +--Add manual relationships +DO $_$ +BEGIN + PERFORM VOCABULARY_PACK.ProcessManualRelationships(); +END $_$; --- 7. Add atc_tmp_table to concept_stage -INSERT INTO concept_stage ( - concept_name, - domain_id, - vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT concept_name, - 'Drug' AS domain_id, - dv.vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - v.latest_UPDATE AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM atc_tmp_table dv, - vocabulary v -WHERE v.vocabulary_id = dv.vocabulary_id; +--5. Manual synonyms +DO $_$ +BEGIN + PERFORM VOCABULARY_PACK.ProcessManualSynonyms(); +END $_$; --- 8. Create all sorts of relationships to self, RxNorm AND SNOMED +-- Add 1) crosslinks 'SNOMED - ATC eq' between SNOMED drugs and higher ATC classes (not 5th) and 2) internal 'Is a' relationships using mrconso INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1068,71 +63,7 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT DISTINCT d.concept_code AS concept_code_1, - e.concept_code AS concept_code_2, - 'VA Class to ATC eq' AS relationship_id, - 'VA Class' AS vocabulary_id_1, - 'ATC' AS vocabulary_id_2, - v.latest_UPDATE AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM atc_tmp_table d -JOIN sources.rxnconso r ON r.rxcui = d.rxcui - AND r.code != 'NOCODE' -JOIN atc_tmp_table e ON r.rxcui = e.rxcui - AND r.code = e.concept_code -JOIN vocabulary v ON v.vocabulary_id = d.vocabulary_id -WHERE d.concept_class_id LIKE 'VA Class' - AND e.concept_class_id LIKE 'ATC%' - -UNION ALL - --- Cross-link between drug class Chemical Structure AND ATC -SELECT DISTINCT d.concept_code AS concept_code_1, - e.concept_code AS concept_code_2, - 'NDFRT to ATC eq' AS relationship_id, - 'NDFRT' AS vocabulary_id_1, - 'ATC' AS vocabulary_id_2, - v.latest_UPDATE AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM atc_tmp_table d -JOIN sources.rxnconso r ON r.rxcui = d.rxcui - AND r.code != 'NOCODE' -JOIN atc_tmp_table e ON r.rxcui = e.rxcui - AND r.code = e.concept_code -JOIN vocabulary v ON v.vocabulary_id = d.vocabulary_id -WHERE d.concept_class_id = 'Chemical Structure' - AND e.concept_class_id IN ( - 'ATC 1st', - 'ATC 2nd', - 'ATC 3rd', - 'ATC 4th' - ) - -UNION ALL - --- Cross-link between drug class ATC AND Therapeutic Class -SELECT DISTINCT d.concept_code AS concept_code_1, - e.concept_code AS concept_code_2, - 'NDFRT to ATC eq' AS relationship_id, - 'NDFRT' AS vocabulary_id_1, - 'ATC' AS vocabulary_id_2, - v.latest_UPDATE AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM atc_tmp_table d -JOIN sources.rxnconso r ON r.rxcui = d.rxcui - AND r.code != 'NOCODE' -JOIN atc_tmp_table e ON r.rxcui = e.rxcui - AND r.code = e.concept_code -JOIN vocabulary v ON v.vocabulary_id = d.vocabulary_id -WHERE d.concept_class_id LIKE 'Therapeutic Class' - AND e.concept_class_id LIKE 'ATC%' - -UNION ALL - --- Cross-link between drug class SNOMED AND ATC classes (not ATC 5th) +-- crosslinks between SNOMED Drug Class AND ATC Classes (not ATC 5th) SELECT DISTINCT d.concept_code AS concept_code_1, e.concept_code AS concept_code_2, 'SNOMED - ATC eq' AS relationship_id, @@ -1148,13 +79,12 @@ JOIN sources.rxnconso r ON r.code = d.concept_code JOIN sources.rxnconso r2 ON r.rxcui = r2.rxcui AND r2.sab = 'ATC' AND r2.code != 'NOCODE' -JOIN atc_tmp_table e ON r2.code = e.concept_code +JOIN concept_manual e ON r2.code = e.concept_code AND e.concept_class_id != 'ATC 5th' -- Ingredients only to RxNorm + AND e.vocabulary_id = 'ACT' WHERE d.vocabulary_id = 'SNOMED' AND d.invalid_reason IS NULL - UNION ALL - -- Hierarchy inside ATC SELECT uppr.concept_code AS concept_code_1, lowr.concept_code AS concept_code_2, @@ -1167,299 +97,159 @@ SELECT uppr.concept_code AS concept_code_1, FROM concept_stage uppr, concept_stage lowr, vocabulary v -WHERE ( - ( - LENGTH(uppr.concept_code) IN ( - 4, - 5 - ) - AND lowr.concept_code = SUBSTR(uppr.concept_code, 1, LENGTH(uppr.concept_code) - 1) - ) - OR ( - LENGTH(uppr.concept_code) IN ( - 3, - 7 - ) - AND lowr.concept_code = SUBSTR(uppr.concept_code, 1, LENGTH(uppr.concept_code) - 2) - ) - ) +WHERE uppr.invalid_reason is null AND lowr.invalid_reason is null -- to exclude deprecated or updated from the hierarchy +AND ( +(LENGTH(uppr.concept_code) IN (4,5) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 1)) +OR (LENGTH(uppr.concept_code) IN (3,7) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 2)) +) AND uppr.vocabulary_id = 'ATC' AND lowr.vocabulary_id = 'ATC' - AND v.vocabulary_id = 'ATC'; - --- 9. Add new relationships -INSERT INTO concept_relationship_stage ( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT DISTINCT cs.class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM class_to_drug cs --manual source table -JOIN concept c ON c.concept_id = cs.concept_id -WHERE NOT EXISTS ( - SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.class_code - AND crs.vocabulary_id_1 = 'ATC' - AND crs.concept_code_2 = c.concept_code - AND crs.vocabulary_id_2 = c.vocabulary_id - AND crs.relationship_id = 'ATC - RxNorm' - ) -AND c.concept_class_id != 'Ingredient' -- 6 for unambiguous AND 1 for manual -; + AND v.vocabulary_id = 'ATC'; --6493 - --- Primary unambiguous -INSERT INTO concept_relationship_stage (concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason) -SELECT DISTINCT substring(irs.concept_code_1, '\w+') AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason +-- Add new 'ATC - RxNorm' links between ATC classes and RxN/RxE using class_to_drug table +INSERT INTO concept_relationship_stage +( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason +) +SELECT DISTINCT cs.class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','yyyymmdd') AS valid_end_date, + NULL AS invalid_reason +FROM class_to_drug cs +--manual source table + JOIN concept c ON c.concept_id = cs.concept_id +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.class_code + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = c.concept_code + AND crs.vocabulary_id_2 = c.vocabulary_id + AND crs.relationship_id = 'ATC - RxNorm') +AND c.concept_class_id != 'Ingredient' +AND (cs.class_code,c.concept_code) NOT IN (SELECT concept_code_1, + concept_code_2 + FROM concept_relationship_stage); + +-- Add 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between an ATC class and RxN/RxE drug using input tables and ambiguous_class_ingredient_tst +INSERT INTO concept_relationship_stage +( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason +) +with t1 as ( +SELECT DISTINCT SUBSTRING(irs.concept_code_1,'\w+') AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason FROM internal_relationship_stage irs - JOIN relationship_to_concept rtc on irs.concept_code_2 = rtc.concept_code_1 - JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE rtc.concept_code_1 - NOT IN ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', - 'polyfructosans', 'triazoles', 'natural phospholipids', - 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', - 'antibiotics', 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', - 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', - 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', - 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', - 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', - 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', - 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', - 'barbiturates') - AND not exists(SELECT 1 - FROM ambiguous_class_ingredient_tst t - WHERE t.concept_code_2 = rtc.concept_code_1 - AND t.class_code = substring(irs.concept_code_1, '\w+') - AND flag = 'with') -; - -INSERT INTO concept_relationship_stage (concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason) -SELECT DISTINCT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'ing' - JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE not exists(SELECT 1 - FROM ambiguous_class_ingredient_tst a2 - WHERE a.class_code = a2.class_code - AND a.rnk = a2.rnk - AND a.concept_code_2 != a2.concept_code_2) - AND rtc.concept_code_1 - NOT IN ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', - 'polyfructosans', 'triazoles', 'natural phospholipids', - 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', - 'antibiotics', 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', - 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', - 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', - 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', - 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', - 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', - 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', - 'barbiturates') - AND (class_code, concept_code,'ATC - RxNorm pr lat') not in - (SELECT concept_code_1,concept_code_2, relationship_id FROM concept_relationship_stage) - AND class_name != 'combinations' -; - ---manual table -INSERT INTO concept_relationship_stage ( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT class_code AS concept_code_1, - concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM class_to_drug_manual_ing -WHERE flag = 0 -UNION -SELECT class_code AS concept_code_1, - concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM class_to_drug_manual_ing -WHERE flag = 1 - AND class_name != 'combinations' - AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) - AND class_code NOT IN ('J06AA03');--snake venom antiserum - - --- Secondary unambiguous + JOIN relationship_to_concept rtc ON irs.concept_code_2 = rtc.concept_code_1 + JOIN concept c + ON concept_id_2 = c.concept_id + AND c.concept_class_id = 'Ingredient' +WHERE + NOT EXISTS (SELECT 1 +FROM dev_combo t +WHERE lower(t.concept_name) = lower(rtc.concept_code_1) +AND t.class_code = SUBSTRING(irs.concept_code_1,'\w+') +AND t.rnk in (2,3,4) +)) +select * from t1 +where (concept_code_1, concept_code_2) NOT IN (SELECT concept_code_1, + concept_code_2 + FROM concept_relationship_stage); + +-- Add more 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between an ATC class and RxN/RxE drug using rtc and ambiguous_class_ingredient_tst INSERT INTO concept_relationship_stage - (concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT DISTINCT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'with' - JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE not exists(SELECT 1 - FROM ambiguous_class_ingredient_tst a2 - WHERE a.class_code = a2.class_code - AND a.rnk = a2.rnk - AND a.concept_code_2 != a2.concept_code_2) - AND rtc.concept_code_1 - NOT IN ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', - 'polyfructosans', 'triazoles', 'natural phospholipids', - 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', - 'antibiotics', 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', - 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', - 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', - 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', - 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', - 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', - 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', - 'barbiturates') - AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) - AND class_name != 'combinations' - AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage); +( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason +) +SELECT DISTINCT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM dev_combo a + JOIN relationship_to_concept rtc + ON lower(a.concept_name) = lower(rtc.concept_code_1) + AND rnk = 1 + JOIN concept c + ON concept_id_2 = c.concept_id + AND c.concept_class_id = 'Ingredient' +WHERE + (class_code,concept_code,'ATC - RxNorm pr lat') NOT IN (SELECT concept_code_1, + concept_code_2, + relationship_id +FROM concept_relationship_stage) AND class_name != 'combinations' +; -- 1 ---manual table -INSERT INTO concept_relationship_stage - (concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT DISTINCT class_code AS concept_code_1, - concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM class_to_drug_manual_ing -WHERE flag > 1 - AND class_name != 'combinations' - AND (class_code, concept_code) NOT IN ( - SELECT concept_code_1,concept_code_2 - FROM concept_relationship_stage); +--select * from concept_relationship_stage where concept_code_1 = 'N02AA59'; --- Primary ambiguous +-- add 'ATC - RxNorm sec lat' relationships indicating Secondary unambiguous links between an ATC class and RxN/RxE drug using rtc and ambiguous_class_ingredient_tst INSERT INTO concept_relationship_stage - (concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'ing' - JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) - AND rtc.concept_code_1 - IN('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', - 'polyfructosans', 'triazoles','natural phospholipids', - 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', 'antibiotics', - 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', - 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', - 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', - 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', - 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', - 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', - 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', 'barbiturates') -UNION -SELECT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'ing' - JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE exists(SELECT 1 - FROM ambiguous_class_ingredient_tst a2 - WHERE a.class_code = a2.class_code - AND a.rnk = a2.rnk - AND a.concept_code_2 != a2.concept_code_2) -; - +( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason +) +SELECT DISTINCT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM dev_combo a + JOIN relationship_to_concept rtc + ON lower(a.concept_name) = lower(rtc.concept_code_1) + AND rnk = 2 + JOIN concept c + ON concept_id_2 = c.concept_id + AND c.concept_class_id = 'Ingredient' +WHERE + (class_code,concept_code) NOT IN (SELECT concept_code_1, + concept_code_2 +FROM concept_relationship_stage) +AND (class_code, + concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage); -- 381 (171 (9817 (2755)_ ) + +-- add 'ATC - RxNorm pr up' relationships meaning Primary ambiguous links between an ATC class and RxN/RxE drug using rtc and ambiguous_class_ingredient_tst INSERT INTO concept_relationship_stage (concept_code_1, concept_code_2, @@ -1470,33 +260,26 @@ INSERT INTO concept_relationship_stage valid_end_date, invalid_reason ) -SELECT DISTINCT substring(irs.concept_code_1, '\w+') AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM internal_relationship_stage irs - JOIN relationship_to_concept rtc ON irs.concept_code_2 = rtc.concept_code_1 - JOIN concept c ON concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE rtc.concept_code_1 - IN('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', - 'polyfructosans', 'triazoles','natural phospholipids', - 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', 'antibiotics', - 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', - 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', - 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', - 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', - 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', - 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', - 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', 'barbiturates') - AND (substring(irs.concept_code_1, '\w+'), concept_code) not in - (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) - AND substring(irs.concept_code_1, '\w+') NOT IN (SELECT class_code FROM ambiguous_class_ingredient_tst) -; - +SELECT DISTINCT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM dev_combo a + JOIN relationship_to_concept rtc + ON lower (a.concept_name) = lower (rtc.concept_code_1) + AND rnk = 3 + JOIN concept c + ON concept_id_2 = c.concept_id + AND c.concept_class_id = 'Ingredient' +WHERE (class_code,concept_code) NOT IN (SELECT concept_code_1, + concept_code_2 + FROM concept_relationship_stage); -- 3382 + +-- add 'ATC - RxNorm sec up' relationships indicating Secondary ambiguous links between ATC classes and RxN/RxE drugs using rtc and ambiguous_class_ingredient_tst INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1507,137 +290,72 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT class_code AS concept_code_1, - concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM class_to_drug_manual_ing -WHERE flag = 1 - AND class_name != 'combinations' - AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) - AND class_code IN ('J06AA03');--snake venom antiserum - --- combinations, unambiguous, all secondary -INSERT INTO concept_relationship_stage ( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM (SELECT class_code,c.concept_code,c.vocabulary_id - FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 - JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' - WHERE class_name = 'combinations' - UNION - SELECT class_code,concept_code, vocabulary_id - FROM class_to_drug_manual_ing - WHERE class_name = 'combinations' - ) c -WHERE (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage); - - --- Secondary ambiguous -INSERT INTO concept_relationship_stage ( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT DISTINCT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc ON a.concept_code_2 = rtc.concept_code_1 - JOIN concept c ON concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE (exists(SELECT 1 - FROM ambiguous_class_ingredient_tst a2 - WHERE a.class_code = a2.class_code - AND a.rnk = a2.rnk - AND a.concept_code_2 != a2.concept_code_2) - OR rtc.concept_code_1 - IN - ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', - 'polyfructosans', 'triazoles', 'natural phospholipids', - 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', 'antibiotics', - 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', - 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', - 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', - 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', - 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', - 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', - 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', 'barbiturates') - ) - AND flag = 'with' - AND class_name != 'combinations' - and (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) -; - --- add 'x,combinations' -WITH main_ing AS ( - SELECT class_code, concept_id_2 - FROM class_to_drug c - JOIN internal_relationship_stage i ON c.class_code = substring(i.concept_code_1, '\w+') - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 - WHERE class_name ~ '(, combinations)|(in combination with other drugs)' - AND NOT class_name ~ 'with|and thiazides|and other diuretics' -- gives errors INC07BB52 AND C07CB53 if removed +SELECT DISTINCT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM dev_combo a + JOIN relationship_to_concept rtc ON lower(a.concept_name) = lower(rtc.concept_code_1) + JOIN concept c + ON concept_id_2 = c.concept_id + AND c.concept_class_id = 'Ingredient' +WHERE +a.rnk = 4 +AND (class_code,concept_code) NOT IN (SELECT concept_code_1, + concept_code_2 + FROM concept_relationship_stage); -- 7881 + +-- add 'ATC - RxNorm sec up' relationships for 'x, combinations' +INSERT INTO concept_relationship_stage +( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason ) -INSERT INTO concept_relationship_stage ( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT DISTINCT class_code AS concept_code_1, - cc.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - cc.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason +WITH main_ing AS +( + SELECT class_code,class_name, + concept_id_2 + FROM class_to_drug c + JOIN internal_relationship_stage i ON c.class_code = SUBSTRING (i.concept_code_1,'\w+') + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 + WHERE (class_name ~ '(, combinations)' + AND NOT class_name ~ '\ywith|and thiazides|and other diuretics') + OR class_name ~ '(in combination with other drugs)' -- gives errors INC07BB52 AND C07CB53 if removed +) +SELECT DISTINCT class_code AS concept_code_1, + cc.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + cc.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason FROM class_to_drug c - JOIN devv5.concept_ancestor ON descendant_concept_id = c.concept_id - JOIN concept cc ON cc.concept_id = ancestor_concept_id AND cc.standard_concept = 'S' - AND cc.concept_class_id = 'Ingredient' AND cc.vocabulary_id LIKE 'Rx%' -WHERE class_name ~ '(, combinations)|(in combination with other drugs)' - AND NOT class_name ~ 'with|and thiazides|and other diuretics' - AND (c.class_code, cc.concept_id) NOT IN (SELECT class_code, concept_id_2 FROM main_ing) - AND (class_code, cc.concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) -; - - - --- XXX 8.1 temporary deprecate old rels + JOIN devv5.concept_ancestor ON descendant_concept_id = c.concept_id + JOIN concept cc + ON cc.concept_id = ancestor_concept_id + AND cc.standard_concept = 'S' + AND cc.concept_class_id = 'Ingredient' + AND cc.vocabulary_id LIKE 'Rx%' +WHERE (c.class_code,cc.concept_id) NOT IN (SELECT class_code, concept_id_2 FROM main_ing) +AND (class_code,cc.concept_code) NOT IN (SELECT concept_code_1, + concept_code_2 + FROM concept_relationship_stage) +AND ((class_name ~ '(, combinations)' AND NOT class_name ~ '\ywith|and thiazides|and other diuretics') +OR class_name ~ '(in combination with other drugs)') +;-- 336 + +-- deprecate links between ATC classes and dead RxN/RxE (should we kill links from updated ATC to RxN/RxE?) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1648,55 +366,30 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT c.concept_code AS concept_code_1, - cc.concept_code AS concept_code_2, - c.vocabulary_id AS vocabulary_id_1, - cc.vocabulary_id AS vocabulary_id_2, - relationship_id AS relationship_id, +SELECT c.concept_code AS concept_code_1, + cc.concept_code AS concept_code_2, + c.vocabulary_id AS vocabulary_id_1, + cc.vocabulary_id AS vocabulary_id_2, + relationship_id AS relationship_id, cr.valid_start_date AS valid_start_date, - current_date AS valid_end_date, - 'D' AS invalid_reason + CURRENT_DATE AS valid_end_date, + 'D' AS invalid_reason FROM concept_relationship cr - JOIN concept c on concept_id_1 = c.concept_id - JOIN concept cc on concept_id_2 = cc.concept_id + JOIN concept c ON concept_id_1 = c.concept_id + JOIN concept cc ON concept_id_2 = cc.concept_id +AND cc.standard_concept is null -- non-standard WHERE c.vocabulary_id = 'ATC' - AND cc.vocabulary_id LIKE 'RxNorm%' - AND relationship_id IN ('ATC - RxNorm', 'ATC - RxNorm name') -- 'Maps to', 'Drug class of drug', - AND cr.invalid_reason IS NULL - and (c.concept_code, relationship_id, cc.concept_code) NOT IN - (SELECT concept_code_1,relationship_id, concept_code_2 FROM concept_relationship_stage) -; - -INSERT INTO concept_relationship_stage ( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT c.concept_code, - cc.concept_code, - c.vocabulary_id, - cc.vocabulary_id, +AND cc.vocabulary_id LIKE 'RxNorm%' +AND relationship_id ~ 'ATC - RxNorm' -- to cover all types of ATC links +AND cr.invalid_reason IS NULL +AND (c.concept_code,relationship_id,cc.concept_code) NOT IN ( +SELECT concept_code_1, relationship_id, - cr.valid_start_date, - cr.valid_end_date, - cr.invalid_reason -FROM concept_relationship cr - JOIN concept c on c.concept_id = concept_id_1 - JOIN concept cc on cc.concept_id = concept_id_2 -WHERE cr.invalid_reason IS NULL - AND cr.relationship_id = 'Maps to' - AND c.vocabulary_id = 'ATC' - AND cc.vocabulary_id IN('RxNorm', 'RxNorm Extension', 'CVX') - AND (c.concept_code, cc.concept_code) IN(SELECT concept_code_1,concept_code_2 - FROM concept_relationship_stage - WHERE relationship_id = 'ATC - RxNorm pr lat'); + concept_code_2 +FROM concept_relationship_stage) +; -- 4994 (245) => 239 => 254 --- primary lateral always go inside, >25 - calcium, 23-24 J07% +-- add 'Maps to' for 'ATC - RxNorm pr lat' for monocomponent ATC classes which do not have doubling Standard ingredients (1-to-many mapping is permissive only for real combo ATC classes) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1707,7 +400,7 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT concept_code_1, +SELECT DISTINCT concept_code_1, concept_code_2, vocabulary_id_1, vocabulary_id_2, @@ -1718,14 +411,15 @@ SELECT concept_code_1, FROM ( SELECT *, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt FROM concept_relationship_stage -WHERE relationship_id IN('ATC - RxNorm pr lat') -- 'Maps to', 'Drug class of drug', +WHERE relationship_id IN ('ATC - RxNorm pr lat') and invalid_reason IS NULL -and (concept_code_1, vocabulary_id_1, 'Maps to', concept_code_2,vocabulary_id_2) - NOT IN (SELECT concept_code_1, vocabulary_id_1, relationship_id, concept_code_2,vocabulary_id_2 FROM concept_relationship_stage) + -- exclude Maps to CVX ) a -WHERE cnt<25 -; +WHERE cnt = 1-- can be just one , old version - cnt<25 +and (concept_code_1, 'Maps to') + NOT IN (SELECT concept_code_1, relationship_id FROM concept_relationship_stage); -- 4608 +-- more than one INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1736,26 +430,26 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT concept_code_1, +with t1 as ( +SELECT distinct a.*, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt +FROM concept_relationship_stage a +JOIN atc_one_to_many_excl b on b.atc_code = a.concept_code_1 +JOIN concept c on c.concept_code = a.concept_code_2 and c.vocabulary_id = a.vocabulary_id_2 and c.standard_concept = 'S' +and b.concept_code <> a.concept_code_2 +WHERE a.relationship_id IN ('ATC - RxNorm pr lat') +and a.invalid_reason IS NULL) + select DISTINCT concept_code_1, concept_code_2, vocabulary_id_1, vocabulary_id_2, 'Maps to', valid_start_date, valid_end_date, - invalid_reason -FROM ( -SELECT *, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt -FROM concept_relationship_stage -WHERE relationship_id IN('ATC - RxNorm sec lat') -- 'Maps to', 'Drug class of drug', -and invalid_reason IS NULL -and (concept_code_1, vocabulary_id_1, 'Maps to', concept_code_2,vocabulary_id_2) - NOT IN (SELECT concept_code_1, vocabulary_id_1, relationship_id, concept_code_2,vocabulary_id_2 FROM concept_relationship_stage) - ) a -WHERE cnt<10 -; + invalid_reason from t1 where concept_code_1 in (select concept_code_1 from t1 group by concept_code_1 having count (1)=1) + and (concept_code_1, 'Maps to') NOT IN (SELECT concept_code_1, relationship_id FROM concept_relationship_stage) + ; -- 66 --- add deprecated Maps to +-- add 'Maps to' to Standard Ingredients for polycomponent ATC classes having 'ATC - RxNorm sec lat' INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1766,26 +460,33 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT c.concept_code, - cc.concept_code, - c.vocabulary_id, - cc.vocabulary_id, - relationship_id, - cr.valid_start_date, - current_date, - 'D' -FROM concept_relationship cr - JOIN concept c on c.concept_id = concept_id_1 - JOIN concept cc on cc.concept_id = concept_id_2 -WHERE cr.invalid_reason IS NULL - AND cr.relationship_id = 'Maps to' - AND c.vocabulary_id = 'ATC' AND c.invalid_reason IS NULL - AND cc.vocabulary_id IN('RxNorm', 'RxNorm Extension', 'CVX') - AND (c.concept_code, cr.relationship_id, cc.concept_code) NOT IN (SELECT concept_code_1,relationship_id, concept_code_2 - FROM concept_relationship_stage) - ; + with t1 as ( +SELECT *, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt +FROM concept_relationship_stage +WHERE relationship_id IN('ATC - RxNorm sec lat') -- 'Maps to', 'Drug class of drug', +and invalid_reason IS NULL +) +select distinct a.concept_code_1, +--b.class_name, + a.concept_code_2, + -- c.concept_name, + a.vocabulary_id_1, + a.vocabulary_id_2, + 'Maps to', + a.valid_start_date, + a.valid_end_date, + a.invalid_reason from t1 a +join class_drugs_scraper b on a.concept_code_1 = b.class_code +join concept c on c.concept_code = a.concept_code_2 and c.vocabulary_id = a.vocabulary_id_2 +where class_name !~ 'thiazides|agents|combinations' +and cnt<10 +and (b.class_code, c.concept_id) not in (select atc_code, concept_id from atc_one_to_many_excl) +and concept_code_2 not in ('OMOP995053','142141','4100', '117466') +AND (concept_code_1||concept_code_2) not in ('G03FA10'||'4083') +and (concept_code_1, vocabulary_id_1, 'Maps to', concept_code_2,vocabulary_id_2) + NOT IN (SELECT concept_code_1, vocabulary_id_1, relationship_id, concept_code_2,vocabulary_id_2 FROM concept_relationship_stage);-- 448 --- 16. Add synonyms to concept_synonym stage for each of the rxcui/code combinations in atc_tmp_table +-- Add synonyms to concept_synonym stage for each of the rxcui/code combinations in atc_tmp_table (can we concept_synonym manual?) INSERT INTO concept_synonym_stage ( synonym_concept_code, synonym_name, @@ -1796,117 +497,37 @@ SELECT DISTINCT dv.concept_code AS synonym_concept_code, SUBSTR(r.str, 1, 1000) AS synonym_name, dv.vocabulary_id AS synonym_vocabulary_id, 4180186 AS language_concept_id -FROM atc_tmp_table dv -JOIN sources.rxnconso r ON dv.code = r.code - AND dv.rxcui = r.rxcui +FROM concept_manual dv +JOIN sources.rxnconso r ON dv.concept_code = r.code AND r.code != 'NOCODE' - AND r.lat = 'ENG'; - - --- 16.2 update concept names for combos -UPDATE concept_stage c -SET concept_name = name -FROM (SELECT c2.concept_name||' - '||c.concept_name AS name, c.concept_code -FROM concept_stage c -JOIN concept_relationship_stage cr ON cr.concept_code_1 = c.concept_code -JOIN concept_stage c2 ON c2.concept_code = cr.concept_code_2 AND relationship_id = 'Is a' -WHERE c.concept_name IN ('various', 'combinations','others')) a -WHERE (c.concept_code = a.concept_code); - -UPDATE concept_stage c -SET concept_name = concept_name || ', combinations' -WHERE concept_code IN ( - SELECT DISTINCT concept_code - FROM concept_stage - JOIN concept_relationship_stage cr ON concept_code = concept_code_1 - WHERE NOT concept_name ~ ' and |comb|/| with |complex' - AND cr.invalid_reason IS NULL - AND relationship_id ~ 'sec') -; - --- 16.3 add forms to names -DROP TABLE IF EXISTS new_name_form; -CREATE TABLE new_name_form -AS -WITH name AS ( - SELECT class_code, - class_name, - CASE - WHEN adm_r = 'O' THEN 'oral' - WHEN adm_r = 'Chewing gum' THEN 'chewing gum' - WHEN adm_r LIKE '%Inhal%' THEN 'inhalant' - WHEN adm_r = 'Instill.sol.' THEN 'instillation solution' - WHEN adm_r = 'N' THEN 'nasal' - WHEN adm_r = 'P' THEN 'parenteral' - WHEN adm_r = 'R' THEN 'rectal' - WHEN adm_r = 'SL' THEN 'sublingual' - WHEN adm_r LIKE 'TD%' THEN 'transdermal' - WHEN adm_r = 'V' THEN 'vaginal' - WHEN adm_r IN('implant', 'intravesical', 'ointment', 'oral aerosol', 'urethral') THEN adm_r - WHEN adm_r = 's.c. implant' THEN 'implant' - ELSE null END AS form - FROM class_drugs_scraper - WHERE adm_r IS NOT NULL - UNION - SELECT class_code, - class_name, - CASE - WHEN class_code LIKE 'A01AB%' THEN 'local oral' - WHEN concept_code LIKE '%Oral%' THEN 'oral' - WHEN concept_code LIKE '%Rectal%' THEN 'rectal' - WHEN concept_code LIKE '%Vaginal%' THEN 'vaginal' - WHEN concept_code LIKE '%Injectable%' THEN 'parenteral' - WHEN concept_code LIKE '%Inhal%' THEN 'inhalant' - WHEN concept_code LIKE '%Nasal%' THEN 'nasal' - WHEN concept_code LIKE '%Ophthalmic%' THEN 'ophthalmic' - WHEN concept_code LIKE '%Topical%' THEN 'topical' - WHEN concept_code LIKE '%Ophthalmic%' THEN 'ophthalmic' - ELSE null END AS form - FROM reference - JOIN class_drugs_scraper USING (class_code) - WHERE class_code != concept_code - AND adm_r IS NULL -), -forms AS ( -SELECT class_code,class_name, - regexp_replace(string_agg(form, ', ' ORDER BY class_code, form),'oral, parenteral','systemic') AS form_list -FROM name -WHERE form IS NOT NULL -GROUP BY class_code,class_name) -SELECT class_code, class_name||'; '||form_list AS new_name - FROM forms; - -UPDATE concept_stage -SET concept_name = new_name -FROM (SELECT class_code, new_name FROM new_name_form) a -WHERE (concept_code = class_code); - + AND r.lat = 'ENG' + AND r.sab = 'ATC' + AND r.tty IN ( + 'PT', + 'IN' + ) + ; -- 7105 (6440) -- to compare with old script using except --- 18. Working with replacement mappings +-- perform mapping replacement using function below (only to Standard concepts?) DO $_$ BEGIN PERFORM VOCABULARY_PACK.CheckReplacementMappings(); END $_$; --- 20. Add mapping FROM deprecated to fresh concepts +-- Add mapping FROM deprecated to fresh concepts DO $_$ BEGIN PERFORM VOCABULARY_PACK.AddFreshMAPSTO(); END $_$; - --- 19. Deprecate 'Maps to' mappings to deprecated AND upgraded concepts +-- Deprecate 'Maps to' mappings to deprecated AND upgraded concepts (the step of such deprecation can be deleted) but what should we do with ancestor? DO $_$ BEGIN PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO(); END $_$; - --- 21. DELETE ambiguous 'Maps to' mappings +-- DELETE ambiguous 'Maps to' mappings DO $_$ BEGIN PERFORM VOCABULARY_PACK.DELETEAmbiguousMAPSTO(); END $_$; - -DELETE FROM concept_relationship_stage -WHERE vocabulary_id_1='SNOMED' AND vocabulary_id_2 = 'RxNorm'; From 7d1efa8125841c46874b0de28347964cbfdd6e8e Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 3 Aug 2021 10:01:15 +0300 Subject: [PATCH 07/45] Update load_input.sql minor changes, code refactoring --- ATC/load_input.sql | 1777 +++++++++++++++++++++++++------------------- 1 file changed, 997 insertions(+), 780 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 3d0348f49..170d7ef7b 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -16,11 +16,11 @@ * Authors: Anna Ostropolets, Polina Talapova * Date: Jul 2021 **************************************************************************/ +-- in ATC we do not use ds_stage AND pc_stage DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; -DROP TABLE IF EXISTS relationship_to_concept CASCADE; +DROP TABLE IF EXISTS relationship_to_concept CASCADE;; --- for ATC we don't need ds_stage AND pc_stage CREATE TABLE drug_concept_stage ( concept_name VARCHAR(255), vocabulary_id VARCHAR(20), @@ -97,7 +97,8 @@ a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form FROM concept_manual a, dev_oral b WHERE a.concept_name ~* 'oral|systemic|chewing gum' -- respective ATC dose forms (preliminarily converted from adm_r and added to the names in concept_manual) -AND a.invalid_reason IS NULL; -- indicates alive ATC code +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- indicates alive ATC code -- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -168,7 +169,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_sub b WHERE a.concept_name ~* 'sublingual' -AND a.invalid_reason IS NULL; -- 507 +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- 507 -- add links between Sublingual ATC Drug Classes AND Standard Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -239,7 +241,7 @@ FROM concept_manual a, dev_parenteral b WHERE a.concept_name ~* 'parenteral|systemic' -- respective ATC routes AND a.invalid_reason IS NULL -; +AND a.concept_class_id = 'ATC 5th'; -- add links between Parenteral ATC Drug Classes AND Standard Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -314,7 +316,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_nasal b WHERE a.concept_name ~* 'nasal' -AND a.invalid_reason IS NULL; +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- add links between Nasal ATC Drug Classes AND Standars Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -390,7 +393,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_topic b WHERE a.concept_name ~* 'topical' -AND a.invalid_reason IS NULL; -- exclude transdermal systems +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- exclude transdermal systems -- add links between 2) Transdermal or Implantable ATC Drug Classes AND Rx Dose Forms INSERT INTO internal_relationship_stage @@ -515,7 +519,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_mouth b WHERE a.concept_name ~* 'local oral' -- respective ATC route -AND a.invalid_reason IS NULL; -- 21 +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- 21 -- add links between Local Oral ATC Drug Classes AND Ingredients using the concept table (up to date no need to use tne concept_synonym but you may check) INSERT INTO internal_relationship_stage @@ -562,7 +567,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_rectal b WHERE a.concept_name ~* 'rectal' -- respective ATC route -AND a.invalid_reason IS NULL; -- 1150 +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- 1150 -- add links between Rectal ATC Drug Classes AND Standard Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -634,7 +640,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_vaginal b WHERE a.concept_name ~* 'vaginal' -- respective ATC routes -AND a.invalid_reason IS NULL; +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- add links between Vaginal ATC Drug Classes AND Standard Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -704,7 +711,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_urethral b WHERE a.concept_name ~* 'urethral' -- respective ATC route -AND a.invalid_reason IS NULL; +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- add links between Urethral ATC Drug Classes AND Standard Ingredients using the concept table (up to date no need to use tne concept_synonym but you may check) INSERT INTO internal_relationship_stage @@ -752,7 +760,8 @@ SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC FROM concept_manual a, dev_opht b WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route -AND a.invalid_reason IS NULL; +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -787,6 +796,78 @@ FROM concept_manual a, WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route AND a.invalid_reason IS NULL ) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM t1 a + JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + JOIN concept c + ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL + WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + ------------------------- +--- otic formulations --- +-------------------------- +-- create a temporary table WITH all related RxN/RxE Dose Forms +DROP TABLE if exists dev_otic; +CREATE TABLE dev_otic +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = 36217217 -- Otic Product (Dose Form Group) +AND relationship_id = 'RxNorm inverse is a' +); + +-- add links between Ophthalmic ATC Drug Classes AND Rx Dose Forms +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name -- OMOP Dose Form name treated AS a code +FROM concept_manual a, + dev_otic b +WHERE a.concept_name ~* '\yotic' -- respective ATC route +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; + +-- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_otic b +WHERE a.concept_name ~* '\yotic' -- respective ATC route +AND a.invalid_reason IS NULL +) +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 +FROM t1 a +JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept_synonym table +INSERT INTO internal_relationship_stage +(concept_code_1, concept_code_2) +WITH t1 +AS +(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_name -- ATC name to be used AS a key for JOIN +FROM concept_manual a, + dev_otic b +WHERE a.concept_name ~* '\yotic' -- respective ATC route +AND a.invalid_reason IS NULL +) SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code FROM t1 a @@ -817,12 +898,13 @@ AND relationship_id = 'RxNorm inverse is a'); -- add links between Inhalant ATC Drug Classes AND Rx Dose Forms INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code +SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code b.concept_name -- OMOP Dose Form name treated AS a code FROM concept_manual a, dev_inhal b WHERE a.concept_name ~* 'inhalant' -- respective ATC route -AND a.invalid_reason IS NULL); +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; -- add links between Inhalant ATC Drug Classes AND Standard Ingredients using the concept table INSERT INTO internal_relationship_stage @@ -896,180 +978,156 @@ FROM concept_manual a AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.invalid_reason IS NULL - WHERE (a.concept_code, c.concept_name) not IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); - + WHERE (a.concept_code, c.concept_name) not IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); /***************************** **** combined ATC Classes **** ******************************/ --- obtain 1st ATC Combo Ingredient using the concept table and full name match +-- separate all ATC Combo Classes +DROP TABLE if exists combo_pull; +CREATE TABLE combo_pull +AS +(WITH t1 +AS +(SELECT DISTINCT concept_code AS class_code, + concept_name AS class_name, + SPLIT_PART(concept_name,';','1') AS nm +FROM concept_manual +WHERE SPLIT_PART(concept_name,';','1') ~ ' and |combinat|preparations|acids|animals|antiinfectives|compounds|lytes\y|flowers|^glycerol|grass pollen|bacillus|^oil|alkaloids|\/|antiserum|organisms|antiseptics' +AND invalid_reason IS NULL +AND concept_class_id = 'ATC 5th' +AND concept_name !~* 'varicella/zoster|tositumomab/iodine') SELECT*FROM t1); + +-- create a table for ATC Combo Class mappings, start with the 1st ATC Combo Ingredient using the concept table and full name match DROP TABLE if exists dev_combo; -CREATE TABLE dev_combo AS ( -WITH t1 AS -( - SELECT * - FROM class_drugs_scraper - WHERE LENGTH(class_code) = 7 - AND (class_name ~* '\yand\y' - OR class_name LIKE '%,%' AND class_name LIKE '%combination%') - AND change_type IN ('A','') -) -SELECT DISTINCT class_code, - class_name, - adm_r, - SPLIT_PART(class_name,' and ',1) AS class, - c.concept_id, - c.concept_name, - 1 AS rnk -- stands for the Primary lateral relationship -FROM t1 a - JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (class_name,' and ',1))) +CREATE TABLE dev_combo +AS +(SELECT DISTINCT class_code, + class_name, + SPLIT_PART(nm,' and ',1) AS class, + c.concept_id, + c.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship + FROM combo_pull a + JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (nm,' and ',1))) WHERE c.standard_concept = 'S' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx' -); +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id ~ 'Rx'); --- obtain 1st ATC Combo Ingredient using the concept table and full name match +-- obtain 1st ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo -WITH t1 AS -( - SELECT * - FROM class_drugs_scraper - WHERE LENGTH(class_code) = 7 - AND (class_name ~* '\yand\y' - OR class_name LIKE '%,%' AND class_name LIKE '%combination%') - AND change_type IN ('A','') -) SELECT DISTINCT class_code, class_name, - adm_r, - SPLIT_PART(class_name,' and ',1) AS class, + SPLIT_PART(nm,' and ',1) AS class, c.concept_id, c.concept_name, 1 AS rnk -- stands for the Primary lateral relationship -FROM t1 a - JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (class_name,' and ',1))),'\w+') +FROM combo_pull a + JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 1))),'\w*\s*-?\s*\w+') WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); --- obtain 1st ATC Combo Ingredient using the concept_synonym table and full name match +-- obtain 1st ATC Combo Ingredient using the concept_synonym table and name match INSERT INTO dev_combo -WITH t1 AS -( - SELECT * - FROM class_drugs_scraper - WHERE LENGTH(class_code) = 7 - AND (class_name ~* '\yand\y' - OR class_name LIKE '%,%' AND class_name LIKE '%combination%') - AND change_type IN ('A','') -) SELECT DISTINCT class_code, class_name, - adm_r, - SPLIT_PART(class_name,' and ',1) AS class, + SPLIT_PART(nm,' and ', 1) AS class, d.concept_id, d.concept_name, 1 AS rnk -- stands for the Primary lateral relationship -FROM t1 a -JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE(TRIM(lower(SPLIT_PART (class_name,' and ', 1))), SUBSTRING(TRIM(lower(SPLIT_PART (class_name,' and ', 1))), '\w+')) +FROM combo_pull a +JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE(TRIM(lower(SPLIT_PART (nm,' and ', 1))), SUBSTRING(TRIM(lower(SPLIT_PART (nm,' and ', 1))), '\w*\s*-?\s*\w+')) JOIN concept d ON d.concept_id = cs.concept_id WHERE d.standard_concept = 'S' AND d.concept_class_id = 'Ingredient' AND d.vocabulary_id ~ 'Rx' -AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); +AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); -- 15 + +-- obtain 1st ATC Combo Ingredient using the concept table and partial name match +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + SPLIT_PART(nm,' and ',1) AS class, + c.concept_id, + c.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship +FROM combo_pull a + JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 1))),'^\w+') +WHERE c.standard_concept = 'S' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id ~ 'Rx' +AND (class_code) NOT IN (select class_code FROM dev_combo); -- 3 -- obtain 2nd ATC Combo Ingredient using the concept table and full name match INSERT INTO dev_combo -WITH t1 AS -( - SELECT * - FROM class_drugs_scraper - WHERE LENGTH(class_code) = 7 - AND (class_name ~* '\yand\y' - OR class_name LIKE '%,%' AND class_name LIKE '%combination%') - AND change_type IN ('A','') -) SELECT DISTINCT class_code, class_name, - adm_r, - SPLIT_PART(class_name,' and ',2) AS class, + SPLIT_PART(nm,' and ', 2) AS class, c.concept_id, c.concept_name, 2 AS rnk -- stands for the Secondary lateral relationship -FROM t1 a - JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (a.class_name,' and ',2))) +FROM combo_pull a + JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (a.nm,' and ', 2))) WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx'; +AND c.vocabulary_id ~ 'Rx'; -- 249 -- obtain 2nd ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo -WITH t1 AS -( - SELECT * - FROM class_drugs_scraper - WHERE LENGTH(class_code) = 7 - AND (class_name ~* '\yand\y' - OR class_name LIKE '%,%' AND class_name LIKE '%combination%') - AND change_type IN ('A','') -) SELECT DISTINCT class_code, class_name, - adm_r, - SPLIT_PART(class_name,' and ', 2) AS class, + SPLIT_PART(nm,' and ', 2) AS class, c.concept_id, c.concept_name, 2 AS rnk -FROM t1 a - JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (class_name,' and ', 2))),'\w+') +FROM combo_pull a + JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 2))),'\w*\s*-?\s*\w+') WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND c.concept_id NOT IN (19049024, 19136048); +AND c.concept_id NOT IN (19049024, 19136048); -- 4 -- obtain 2nd ATC Combo Ingredient using the concept_synonym table and partial name match INSERT INTO dev_combo -WITH t1 AS -( - SELECT * - FROM class_drugs_scraper - WHERE LENGTH(class_code) = 7 - AND (class_name ~* '\yand\y' - OR class_name LIKE '%,%' AND class_name LIKE '%combination%') - AND change_type IN ('A','') -) SELECT DISTINCT class_code, class_name, - adm_r, - SPLIT_PART(class_name,' and ',2) AS class, + SPLIT_PART(nm,' and ',2) AS class, d.concept_id, d.concept_name, 2 AS rnk -FROM t1 a -JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE ( TRIM(lower(SPLIT_PART (class_name,' and ', 2))), SUBSTRING ( TRIM(lower(SPLIT_PART (class_name,' and ', 2))), '\w+')) +FROM combo_pull a +JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE ( TRIM(lower(SPLIT_PART (nm,' and ', 2))), SUBSTRING (TRIM(lower(SPLIT_PART (nm,' and ', 2))), '\w*\s*-?\s*\w+')) JOIN concept d ON d.concept_id = cs.concept_id WHERE d.standard_concept = 'S' AND d.concept_class_id = 'Ingredient' AND d.vocabulary_id ~ '^Rx' -AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); +AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); -- 1 + +-- last hope for the 2nd one +INSERT INTO dev_combo +SELECT DISTINCT class_code, + class_name, + SPLIT_PART(nm,' and ', 2) AS class, + c.concept_id, + c.concept_name, + 2 AS rnk +FROM combo_pull a + JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 2))),'^\w+') +WHERE c.standard_concept = 'S' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id ~ 'Rx' +AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) +AND c.concept_id NOT IN (19049024, 19136048) +and class_code in ('R03AK10', 'R03AL01', 'R03AL03', 'R03AL04', 'R03AL05', 'R03AL06', 'R03AL07', 'R03AL10') ; -- 4 -- add manual mappings for ATC Combos using concept_relationship_manual INSERT INTO dev_combo -WITH t1 AS -( - SELECT * - FROM class_drugs_scraper - WHERE LENGTH(class_code) = 7 - AND (class_name ~* '\yand\y' - OR class_name ~* 'combination|quinupristin\/dalfopristin') - AND change_type IN ('A','') -) SELECT DISTINCT class_code, class_name, - adm_r, - '' AS class, -- leave it empty + c.concept_name AS class, -- leave it empty c.concept_id, c.concept_name, CASE WHEN relationship_id = 'ATC - RxNorm pr lat' THEN 1 @@ -1077,44 +1135,41 @@ class_code, WHEN relationship_id = 'ATC - RxNorm pr up' THEN 3 ELSE 4 -- stands for 'ATC - RxNorm sec up' END AS rnk -FROM t1 a +FROM combo_pull a JOIN concept_relationship_manual r ON r.concept_code_1= a.class_code JOIN concept c ON c.concept_code = r.concept_code_2 AND c.vocabulary_id ~ 'Rx' AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' WHERE (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND r.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up'); -- 1022 +AND r.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up'); -- 17075 -- add Acetylsalicylic acid INSERT INTO dev_combo SELECT class_code, class_name, - adm_r, 'acetylsalicylic acid', 1112807, 'aspirin', CASE WHEN class_name ~ '^acetylsalicylic' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1112807 FROM dev_combo WHERE class_name ~* 'acetylsalicylic') -AND class_name ~* 'acetylsalicylic'; +AND class_name ~* 'acetylsalicylic'; -- 276 -- ethinylestradiol INSERT INTO dev_combo SELECT class_code, class_name, - adm_r, 'ethinylestradiol', 1549786, 'ethinyl estradiol', CASE WHEN class_name ~* '^ethinylestradiol' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1549786 FROM dev_combo WHERE class_name ~* 'ethinylestradiol') -AND class_name ~* 'ethinylestradiol'; +AND class_name ~* 'ethinylestradiol'; -- 44 -- estrogen INSERT INTO dev_combo SELECT class_code, class_name, - adm_r, 'ethinylestradiol', 19049228, 'estrogens', @@ -1125,7 +1180,6 @@ AND class_name ~ 'estrogen' UNION ALL SELECT class_code, class_name, - adm_r, 'estrogens, conjugated (USP)', 1549080, 'estrogens, conjugated (USP)', @@ -1136,7 +1190,6 @@ AND class_name ~ 'estrogen' UNION ALL SELECT class_code, class_name, - adm_r, 'estrogens, esterified (USP)', 1551673, 'estrogens, esterified (USP)', @@ -1147,7 +1200,6 @@ AND class_name ~ 'estrogen' UNION ALL SELECT class_code, class_name, - adm_r, 'synthetic conjugated estrogens, A', 1596779, 'synthetic conjugated estrogens, A', @@ -1158,14 +1210,13 @@ AND class_name ~* 'estrogen' UNION ALL SELECT class_code, class_name, - adm_r, 'synthetic conjugated estrogens, B' AS class, 1586808, 'synthetic conjugated estrogens, B', CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1586808 FROM dev_combo WHERE class_name ~* 'estrogen') -AND class_name ~* 'estrogen'; +AND class_name ~* 'estrogen'; -- 1235 -- remove erroneous automap DELETE FROM dev_combo WHERE class_code = 'M05BX53' AND concept_id = 19000815; -- strontium @@ -1180,7 +1231,7 @@ a.concept_id FROM dev_combo a, dev_oral b, concept_manual c -WHERE c.concept_name ~ 'oral|systemic|chewing gum' +WHERE c.concept_name ~* 'oral|systemic|chewing gum' AND c.concept_code = a.class_code ) SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code @@ -1245,7 +1296,7 @@ class_code AS concept_code_1, -- ATC code + Dose Form name AS a code class_name, -- ATC name to be used AS a key for JOIN concept_id FROM dev_combo -WHERE adm_r is null +WHERE class_name !~';' ) SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code @@ -1253,7 +1304,7 @@ WHERE adm_r is null JOIN concept c ON c.concept_id = a.concept_id AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); -- 1198 /****************************** ******* manual mapping ******** @@ -1263,24 +1314,25 @@ INSERT INTO internal_relationship_stage ( concept_code_1, concept_code_2) WITH t1 AS (select distinct a.concept_code_1, a.relationship_id, c.concept_name FROM concept_relationship_manual a - JOIN class_drugs_scraper b - ON b.class_code = a.concept_code_1 + JOIN concept_manual b + ON b.concept_code = a.concept_code_1 JOIN concept c ON c.concept_code = a.concept_code_2 AND c.vocabulary_id = a.vocabulary_id_2 AND c.standard_concept = 'S' AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient') SELECT DISTINCT concept_code_1, - concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code, + a.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code, FROM t1 a - JOIN class_drugs_scraper b ON b.class_code = a.concept_code_1 AND b.change_type IN ('A', '') + JOIN concept_manual b ON b.concept_code = a.concept_code_1 AND b.invalid_reason is null and b.concept_class_id = 'ATC 5th' AND a.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up') - AND (concept_code_1, concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1),concept_code_2 - FROM internal_relationship_stage); + AND (concept_code_1, a.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1),concept_code_2 + FROM internal_relationship_stage); -- 7208 /********************************** ******* drug_concept_stage ******** ***********************************/ +TRUNCATE drug_concept_stage; -- change length of concept_code field ALTER TABLE drug_concept_stage ALTER COLUMN concept_code TYPE VARCHAR; @@ -1461,786 +1513,744 @@ WHERE (class_code,concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage) AND concept_id <> 43013482; -- butyl ester of methyl vinyl ether-maleic anhydride copolymer (125 kD) - -/********************************** -*** FUTHER WORK WITH ATC COMBOS *** -***********************************/ +/***************************************** +*** FUTHER WORK WITH ATC COMBO CLASSES *** +******************************************/ -- assemble mappings for ATC Classes indicating Ingredient Groups using the the concept_ancestor AND/OR concept tables along WITH word pattern matching -- take descendants of Acid preparations INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'acid preparations' AS class, - concept_id, + c.concept_id, c.concept_name, - CASE WHEN class_name ~* '^acid' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, - concept_ancestor + CASE WHEN a.concept_name ~* '^acid' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, -- ATC + concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id = 21600704-- ATC code of Acid preparations + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id = 21600704-- ATC code of Acid preparations AND vocabulary_id LIKE 'RxNorm%' AND concept_class_id = 'Ingredient' -WHERE class_name ~* 'acid preparations'; +WHERE a.concept_name ~* 'acid preparations' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 8 -- Sulfonamides INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'sulfonamides' AS class, concept_id, c.concept_name, - CASE WHEN class_name ~* '^sulfonamides|^combinations of sulfonamides' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, - concept_ancestor + CASE WHEN a.concept_name ~* '^sulfonamides|^combinations of sulfonamides' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, -- ATC + concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id = 21603038-- ATC code of sulfonamides - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id = 21603038-- ATC code of sulfonamides + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' JOIN concept_relationship b ON b.concept_id_1 = ancestor_concept_id AND b.invalid_reason is null AND b.relationship_id = 'ATC - RxNorm pr lat' -WHERE class_name ~* 'sulfonamides' AND class_name !~* '^short-acting sulfonamides|^intermediate-acting sulfonamides|^long-acting sulfonamides' - AND LENGTH (class_code) = 7; +WHERE a.concept_name ~* 'sulfonamides' AND a.concept_name !~* '^short-acting sulfonamides|^intermediate-acting sulfonamides|^long-acting sulfonamides' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Amino acids INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'amino acids', - concept_id, - concept_name, - CASE WHEN class_name ~* '^amino acids' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^amino acids' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, -- ATC + concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21601215, 21601034) -- 21601215 B05XB Amino acids| 21601034 B02AA Amino acids - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - WHERE class_name ~* 'amino\s*acid' - AND class_code <> 'B03AD01'; -- ferrous amino acid complex + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21601215, 21601034) -- 21601215 B05XB Amino acids| 21601034 B02AA Amino acids + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + WHERE a.concept_name ~* 'amino\s*acid' + AND a.concept_code <> 'B03AD01' + AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- ferrous amino acid complex -- take descendants of Analgesics INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'analgesics', - concept_id, - concept_name, - CASE WHEN class_name ~* '^analgesics' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^analgesics' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21604253) -- 21604253 N02 ANALGESICS ATC 2nd - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - AND concept_id NOT IN (939506, 950435, 964407) -- sodium bicarbonate|citric acid|salicylic acid - WHERE class_name ~* 'anae?lgesics?' AND class_name !~* '\yexcl' - AND LENGTH(class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21604253) -- 21604253 N02 ANALGESICS ATC 2nd + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN (939506, 950435, 964407) -- sodium bicarbonate|citric acid|salicylic acid + WHERE a.concept_name ~* 'analgesics?' AND a.concept_name !~* '\yexcl' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Animals INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'animals', - concept_id, - concept_name, - CASE WHEN class_name ~* '^animals' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept -WHERE class_name ~* 'Animals' -AND LENGTH(class_code) = 7 -AND (concept_id IN (19091701,19056189,40170543,40170448,40170341,40170416,40175840,40175865,40170916,40175984,40161698,40170420, + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^animals' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, + concept c +WHERE a.concept_name ~* 'Animals' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' +AND (c.concept_id IN (19091701,19056189,40170543,40170448,40170341,40170416,40175840,40175865,40170916,40175984,40161698,40170420, 19095690,40170741,40170848,40161809,40161813,45892235,40171114,45892234,37496548,40170660,40172147,40175843,40175898,40175933,40171110, 40175911,40171275,40172704,40171317,40175983,40171135,35201802,40238446,40175899,40227400,40175938,19061053,19112547,43013524,40170475, 40170818,40161805,40167658,1340875,42903998,963757,40171594,37496553,40172160,35201545,40175931,35201783,789889,35201778,40175951,35201548, 40161124,42709317,40161676,40161750,40170521,40161754,40170973,40170979,40170876,40175917) OR ( -concept_name ~* 'rabbit|\ycow\y|\ydog\y|\ycat\y|goose|\yhog\y|\ygland\y|hamster|\yduck|oyster|\yhorse\y|\ylamb|pancreas|brain|kidney|\ybone\y|heart|spleen|lungs|^Pacific|\yfish|\yegg\y|\ypork|shrimp|\yveal|\ytuna|chicken' -AND concept_name ~* 'extract' AND vocabulary_id LIKE 'RxNorm%' -AND standard_concept = 'S' AND concept_class_id = 'Ingredient' -AND concept_id NOT IN (46276144,40170814,40226703,43560374,40227355,42903998,40227484,19086386)) -); +c.concept_name ~* 'rabbit|\ycow\y|\ydog\y|\ycat\y|goose|\yhog\y|\ygland\y|hamster|\yduck|oyster|\yhorse\y|\ylamb|pancreas|brain|kidney|\ybone\y|heart|spleen|lungs|^Pacific|\yfish|\yegg\y|\ypork|shrimp|\yveal|\ytuna|chicken' +AND c.concept_name ~* 'extract' AND c.vocabulary_id LIKE 'RxNorm%' +AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' +AND c.concept_id NOT IN (46276144,40170814,40226703,43560374,40227355,42903998,40227484,19086386)) +); -- 98 -- take descendants of Antiinfectives INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'antiinfectives', - concept_id, - concept_name, - CASE WHEN class_name ~* '^anti-?infectives' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^antiinfectives' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, + concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21605189, 21603552, 21605145, 21601168, 21605188, 21605146) -- Antiinfectives| ANTIINFECTIVES| ANTIINFECTIVES | Antiinfectives | Antiinfectives - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - AND concept_id NOT IN (19044522)-- zinc sulfate - WHERE class_name ~* 'anti-?infectives?' --AND class_name ~* '\yexcl' - AND LENGTH(class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21605189, 21603552, 21605145, 21601168, 21605188, 21605146) -- Antiinfectives| ANTIINFECTIVES| ANTIINFECTIVES | Antiinfectives | Antiinfectives + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN (19044522)-- zinc sulfate + WHERE a.concept_name ~* 'antiinfectives?' --AND class_name ~* '\yexcl' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Cadmium compounds INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'cadmium compounds', - concept_id, - concept_name, - CASE WHEN class_name ~* '^cadmium compounds' THEN 3 ELSE 4 END ::INT AS rnk -- groups don't have primary lateral ings -FROM class_drugs_scraper, concept -WHERE lower(concept_name) LIKE '%cadmium %' -AND concept_class_id = 'Ingredient' -AND vocabulary_id LIKE 'RxNorm%' -AND concept_id <> 45775350 -AND class_name ~* 'cadmium compounds?' --AND class_name ~* '\yexcl' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^cadmium compounds' THEN 3 ELSE 4 END ::INT AS rnk -- groups don't have primary lateral ings +FROM concept_manual a, +concept c +WHERE lower(c.concept_name) LIKE '%cadmium %' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id LIKE 'RxNorm%' +AND c.concept_id <> 45775350 +AND a.concept_name ~* 'cadmium compounds?' --AND class_name ~* '\yexcl' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 4 -- take ingredients indicating Calcium (different salts) INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'calcium (different salts IN combination)', - concept_id, - concept_name, - CASE WHEN class_name ~* '^calcium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept -WHERE concept_name ~* '\ycalcium\y' -AND concept_class_id = 'Ingredient' -AND vocabulary_id LIKE 'RxNorm%' -AND concept_id NOT IN (42903945,43533002,1337191,19007595,43532262,19051475) -- calcium ion|calcium hydride|calcium hydroxide|calcium oxide|calcium peroxide|anhydrous calcium iodide -AND class_name ~* 'calcium' AND class_name ~* '\ysalt' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^calcium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c +WHERE c.concept_name ~* '\ycalcium\y' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id LIKE 'RxNorm%' +AND c.concept_id NOT IN (42903945,43533002,1337191,19007595,43532262,19051475) -- calcium ion|calcium hydride|calcium hydroxide|calcium oxide|calcium peroxide|anhydrous calcium iodide +AND a.concept_name ~* 'calcium' AND a.concept_name ~* '\ysalt' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Calcium compounds INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'calcium compounds', - concept_id, - concept_name, - CASE WHEN class_name ~* '^calcium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept -WHERE concept_name ~* '\ycalcium\y' -AND concept_class_id = 'Ingredient' -AND vocabulary_id LIKE 'RxNorm%' -AND concept_id NOT IN (19014944,42903945) -AND class_name ~* 'calcium' AND class_name ~* '\ycompound' -AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^calcium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c +WHERE c.concept_name ~* '\ycalcium\y' +AND c.concept_class_id = 'Ingredient' +AND c.vocabulary_id LIKE 'RxNorm%' +AND c.concept_id NOT IN (19014944,42903945) +AND a.concept_name ~* 'calcium' AND a.concept_name ~* '\ycompound' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 212 -- take descendants of Laxatives INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'contact laxatives', - concept_id, - concept_name, - CASE WHEN class_name ~* '^contact laxatives' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^contact laxatives' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, +concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21600537) - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' -WHERE class_name ~* 'contact' AND class_name ~* 'laxatives?' -AND LENGTH(class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21600537) + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' +WHERE a.concept_name ~* 'contact' AND a.concept_name ~* 'laxatives?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 22 -- take descendants of Corticosteroids INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'corticosteroids', - concept_id, - concept_name, - CASE WHEN class_name ~* '^corticosteroid|^combinations of corticosteroids?' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^corticosteroids?|^combinations of corticosteroids?' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, +concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21605042, 21605164, 21605200, 21605165, 21605199, 21601607, 975125) - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' -WHERE class_name ~* 'corticosteroid' -AND LENGTH(class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21605042, 21605164, 21605200, 21605165, 21605199, 21601607, 975125) + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' +WHERE a.concept_name ~* 'corticosteroids?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 608 -- take descendants of Cough suppressants INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'cough suppressants', - concept_id, - concept_name, - CASE WHEN class_name ~* '^cough suppressants|^other cough suppressants' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^cough suppressants|^other cough suppressants' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, +concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21603440, 21603366, 21603409, 21603395, 21603436) - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - AND concept_id NOT IN (943191,1139042,1189220,1781321,19008366,19039512,19041843,19050346,19058933,19071861,19088167,19095266,42904041) - WHERE class_name ~* 'cough' AND class_name ~* 'suppressants?' -AND LENGTH(class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21603440, 21603366, 21603409, 21603395, 21603436) + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN (943191,1139042,1189220,1781321,19008366,19039512,19041843,19050346,19058933,19071861,19088167,19095266,42904041) + WHERE a.concept_name ~* 'cough' AND a.concept_name ~* 'suppressants?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Diuretics INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'diuretics', - concept_id, - concept_name, - CASE WHEN class_name ~* '^diuretics' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^diuretics' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id = 21601461 - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - WHERE class_name ~* 'diuretics?' -AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id = 21601461 + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + WHERE a.concept_name ~* 'diuretics?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 2378 -- take descendants of Magnesium (different salts IN combination) INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'magnesium (different salts IN combination)', - concept_id, - concept_name, - CASE WHEN class_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s - JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21600892) - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - WHERE class_name ~* 'magnesium' AND class_name ~* 'salt' -AND LENGTH(class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, -- ATC +concept_ancestor ca + JOIN concept c -- Rx + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21600892) + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + WHERE a.concept_name ~* 'magnesium' AND a.concept_name ~* 'salt' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Magnesium (different salts IN combination) INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'magnesium (different salts IN combination)', - concept_id, - concept_name, - CASE WHEN class_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept c - WHERE class_name ~* 'magnesium' AND class_name ~* 'salt' - AND concept_name ~ 'magnesium' AND standard_concept = 'S' AND concept_class_id = 'Ingredient' -AND LENGTH(class_code)=7 -AND (class_code, concept_id) NOT IN (select class_code, concept_id FROM dev_combo) + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c + WHERE a.concept_name ~* 'magnesium' AND a.concept_name ~* 'salt' + AND c.concept_name ~ 'magnesium' AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' +AND (a.concept_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) AND concept_id NOT IN (43532017, 37498676); -- magnesium cation | magnesium Mg-28 -- take ingredients indicating Multivitamins INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'multivitamins', - concept_id, - concept_name, - CASE WHEN class_name ~* '^multivitamins' THEN 1 ELSE 2 END ::INT AS rnk -FROM class_drugs_scraper, - concept -WHERE concept_id = 36878782 - AND class_name ~* 'multivitamins?' - AND LENGTH (class_code) = 7 ; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^multivitamins' THEN 1 ELSE 2 END ::INT AS rnk +FROM concept_manual a, + concept c +WHERE c.concept_id = 36878782 + AND a.concept_name ~* 'multivitamins?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 7 +------------------------------------------------------------------ -- take descendants of Opium alkaloids WITH morphine INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'opium alkaloids WITH morphine', - concept_id, - concept_name, - CASE WHEN class_name ~* '^opium alkaloids WITH morphine' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^opium alkaloids WITH morphine' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21604255) -- Natural opium alkaloids - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - AND concept_id NOT IN (19112635) - WHERE class_name ~* 'opium alkaloids WITH morphine' - AND LENGTH (class_code) = 7; + AND ca.ancestor_concept_id IN (21604255) -- Natural opium alkaloids + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN (19112635) + WHERE a.concept_name ~* 'opium alkaloids WITH morphine' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Opium derivatives INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'opium derivatives', - concept_id, - concept_name, - CASE WHEN class_name ~* '^opium derivatives' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^opium derivatives' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id = 21603396 - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - AND concept_id NOT IN (19021930, 1201620) - WHERE class_name ~* 'opium derivatives' - AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id = 21603396 + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN (19021930, 1201620) + WHERE a.concept_name ~* 'opium derivatives' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Organic nitrates INSERT INTO dev_combo -SELECT DISTINCT -class_code, class_name, adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'organic nitrates', - concept_id, - concept_name, - CASE WHEN class_name ~* '^organic nitrates' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^organic nitrates' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21600316) - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - WHERE class_name ~* 'organic nitrates' - AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21600316) + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + WHERE a.concept_name ~* 'organic nitrates' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Psycholeptics INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'psycholeptics', - concept_id, - concept_name, - CASE WHEN class_name ~* '^psycholeptics' THEN 3 WHEN CLASS_NAME ~ 'excl\. psycholeptics' THEN 0 ELSE 4 END ::INT AS rnk -- 0 stands for excluded drugs -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^psycholeptics' THEN 3 WHEN a.concept_name ~ 'excl\. psycholeptics' THEN 0 ELSE 4 END ::INT AS rnk -- 0 stands for excluded drugs +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id = 21604489 - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - AND concept_id NOT IN (742594) - WHERE class_name ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' - AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id = 21604489 + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN (742594) + WHERE a.concept_name ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Selenium compounds INSERT INTO dev_combo -SELECT DISTINCT -class_code, class_name, adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'selenium compounds', - concept_id, - concept_name, - CASE WHEN class_name ~* '^selenium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^selenium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21600908) - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - WHERE class_name ~* 'selenium compounds' - AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21600908) + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + WHERE a.concept_name ~* 'selenium compounds' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Silver compounds INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'silver compounds', - concept_id, - concept_name, - CASE WHEN class_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id IN (21602248) - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' - WHERE class_name ~* 'silver compounds' - AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id IN (21602248) + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' + WHERE a.concept_name ~* 'silver compounds' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Silver INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'silver compounds', - concept_id, - concept_name, - CASE WHEN class_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND concept_name ~* 'silver\y' - AND ('silver compounds', concept_id) NOT IN (select class, concept_id FROM dev_combo) -AND class_name ~* 'silver compounds' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.concept_name ~* 'silver\y' + AND ('silver compounds', c.concept_id) NOT IN (select class, concept_id FROM dev_combo) +AND a.concept_name ~* 'silver compounds' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Sulfonylureas INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'sulfonylureas', - concept_id, - concept_name, - CASE WHEN class_name ~* '^sulfonylureas?' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^sulfonylureas?' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id - AND ancestor_concept_id = 21600749 - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' -WHERE class_name ~* 'sulfonylureas?' - AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id + AND ca.ancestor_concept_id = 21600749 + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' +WHERE a.concept_name ~* 'sulfonylureas?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Snake venom antiserum INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'snake venom antiserum', - concept_id, - concept_name, - CASE WHEN class_name ~* '^snake venom antiserum' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'antiserum' AND concept_name ~* 'snake' -AND class_name ~* 'snake venom antiserum' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^snake venom antiserum' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'antiserum' AND c.concept_name ~* 'snake' +AND a.concept_name ~* 'snake venom antiserum' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Aluminium preparations INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'aluminium preparations', - concept_id, - concept_name, - CASE WHEN class_name ~* '^aluminium preparations' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'aluminium|aluminum' - AND class_name ~* 'aluminium preparations' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^aluminium preparations' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'aluminium|aluminum' + AND a.concept_name ~* 'aluminium preparations' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Aluminium compounds INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'aluminium compounds', - concept_id, - concept_name, - CASE WHEN class_name ~* '^aluminium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'aluminium|aluminum' - AND class_name ~* 'aluminium compounds' - AND LENGTH(class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^aluminium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'aluminium|aluminum' + AND a.concept_name ~* 'aluminium compounds' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Lactic acid producing organisms INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'lactic acid producing organisms', - concept_id, - concept_name, - CASE WHEN class_name ~* '^lactic acid producing organisms' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'lactobacil' - AND class_name ~* 'lactic acid producing organisms' - AND LENGTH(class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^lactic acid producing organisms' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'lactobacil' + AND a.concept_name ~* 'lactic acid producing organisms' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Lactobacillus INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'lactobacillus', - concept_id, - concept_name, - CASE WHEN class_name ~* '^lactobacillus' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'lactobacil' - AND class_name ~* 'lactobacillus' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^lactobacillus' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'lactobacil' + AND a.concept_name ~* 'lactobacillus' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Magnesium compounds INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'magnesium compounds', - concept_id, - concept_name, - CASE WHEN class_name ~* '^magnesium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'magnesium' - AND class_name ~* 'magnesium compounds' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^magnesium compounds' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'magnesium' + AND a.concept_name ~* 'magnesium compounds' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Grass pollen INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'grass pollen', - concept_id, - concept_name, - CASE WHEN class_name ~* '^grass pollen' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'grass' AND concept_name ~* 'pollen' - AND class_name ~* 'grass pollen' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^grass pollen' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND a.concept_name ~* 'grass' AND c.concept_name ~* 'pollen' + AND a.concept_name ~* 'grass pollen' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Oil INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'oil', - concept_id, - concept_name, + c.concept_id, + c.concept_name, 3 -- hardcoded - FROM class_drugs_scraper, + FROM concept_manual a, concept c -WHERE vocabulary_id IN ('RxNorm','RxNorm Extension') -AND concept_class_id = 'Ingredient' -AND standard_concept = 'S' -AND concept_name ~* '\yoil\y|\yoleum\y' -AND class_name ~* '^oil$' -AND LENGTH (class_code) = 7; +WHERE c.vocabulary_id IN ('RxNorm','RxNorm Extension') +AND c.concept_class_id = 'Ingredient' +AND c.standard_concept = 'S' +AND c.concept_name ~* '\yoil\y|\yoleum\y' +AND a.concept_name ~* '^oil' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 475 + +--select * from concept_manual where concept_name ~* 'oil'; -- take ingredients indicating Flowers INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'flowers', - concept_id, - concept_name, - CASE WHEN class_name ~* '^flowers' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* '\yflower\y' AND concept_name ~* 'extract' - AND class_name ~* '^flowers' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^flowers' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* '\yflower\y' AND c.concept_name ~* 'extract' + AND a.concept_name ~* '^flowers' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Fumaric acid derivatives INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'fumaric acid derivatives', - concept_id, - concept_name, - CASE WHEN class_name ~* '^fumaric acid derivatives' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'fumarate\y' - AND class_name ~* 'fumaric acid derivatives' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^fumaric acid derivatives' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, + concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'fumarate\y' + AND a.concept_name ~* 'fumaric acid derivatives' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take ingredients indicating Glycerol INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'glycerol', - concept_id, - concept_name, - CASE WHEN class_name ~* '^glycerol' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'glycerol\y' - AND class_name ~* '^glycerol$' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^glycerol' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'glycerol\y' + AND a.concept_name ~* 'glycerol' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Proton pump inhibitors INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'proton pump inhibitors', - concept_id, - concept_name, - CASE WHEN class_name ~* '^proton pump inhibitors' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^proton pump inhibitors' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor s JOIN concept c ON descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND ancestor_concept_id IN (21600095) - WHERE class_name ~* 'proton pump inhibitors?' - AND LENGTH (class_code) = 7; + WHERE a.concept_name ~* 'proton pump inhibitors?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- take descendants of Thiazides INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'thiazides', - concept_id, - concept_name, - CASE WHEN class_name ~* '^thiazides' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper, concept_ancestor s + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^thiazides' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept_ancestor ca JOIN concept c - ON descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - AND ancestor_concept_id IN (21601463) - WHERE class_name ~* 'thiazides' - AND LENGTH (class_code) = 7; + ON ca.descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' + AND ca.ancestor_concept_id IN (21601463) + WHERE a.concept_name ~* 'thiazides' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 160 -- take ingredients indicating Electrolytes INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'electrolytes', - concept_id, - concept_name, + c.concept_id, + c.concept_name, 3 -- hardcoded rank for electrolytes (no 4) - FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* ('^magnesium sulfate|^ammonium chloride|^sodium chloride|^sodium acetate|^magnesium chloride^|potassium lactate|^sodium glycerophosphate|^magnesium phosphate|^potassium chloride|^calcium chloride' + FROM concept_manual a, + concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* ('^magnesium sulfate|^ammonium chloride|^sodium chloride|^sodium acetate|^magnesium chloride^|potassium lactate|^sodium glycerophosphate|^magnesium phosphate|^potassium chloride|^calcium chloride' || '^sodium bicarbonate|^hydrochloric acid|^potassium acetate|^zinc chloride|^sodium phosphate|^potassium bicarbonate|^succinic acid|^sodium lactate|^sodium gluconate|^sodium fumarate') - AND class_name ~* 'electrolytes' - AND LENGTH(class_code) = 7; + AND a.concept_name ~* 'electrolytes' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- bismuth preparations INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'bismuth preparations', - concept_id, - concept_name, + c.concept_id, + c.concept_name, 3 -- hardcoded rank for bismuth preparations (no 4) - FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* ('\ybismuth') - AND class_name ~* 'bismuth preparations' - AND LENGTH (class_code) = 7; + FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* ('\ybismuth') + AND a.concept_name ~* 'bismuth preparations' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- artificial tears INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'artificial tears', - concept_id, - concept_name, - 3 -- hardcoded rank for bismuth preparations (no 4) - FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') --- AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'carboxymethylcellulose$|polyvinyl alcohol$|hydroxypropyl methylcellulose$|^hypromellose$|hydroxypropyl cellulose$|^hyaluronic acid|^hyaluronate' - AND concept_class_id = 'Ingredient' --- AND concept_name ~* 'Ophthalmic Solution' - AND class_name ~* 'artificial tears' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + 3 -- hardcoded rank + FROM concept_manual a,concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.standard_concept = 'S' + AND c.concept_name ~* 'carboxymethylcellulose$|carboxypolymethylene|polyvinyl alcohol$|hydroxypropyl methylcellulose$|^hypromellose$|hydroxypropyl cellulose|^hyaluronate' + AND c.concept_class_id = 'Ingredient' + AND a.concept_name ~* 'artificial tears' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 23 -- potassium-sparing agents INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'potassium-sparing agents', - concept_id, - concept_name, - CASE WHEN class_name ~* '^potassium-sparing agents' THEN 3 ELSE 4 END ::INT AS rnk -FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* '\yAmiloride|Triamterene|Spironolactone|Eplerenone|Finerenone|Canrenone|Canrenoic acid' - AND class_name ~* 'potassium-sparing agents' - AND LENGTH (class_code) = 7; + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^potassium-sparing agents' THEN 3 ELSE 4 END ::INT AS rnk +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* '\yamiloride|triamterene|spironolactone|eplerenone|finerenone|canrenone|canrenoic acid' + AND a.concept_name ~* 'potassium-sparing agents' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + +-- 19050016 4125 ethiodized oil +INSERT INTO dev_combo +SELECT DISTINCT a.concept_code, + a.concept_name, + 'ethyl esters of iodised fatty acids', + c.concept_id, + c.concept_name, + 1 -- hradcoded +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'ethiodized oil' + AND a.concept_name ~* '^ethyl esters of iodised fatty acids' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 1 -- excl\.trimethoprim INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, +SELECT DISTINCT a.concept_code, + a.concept_name, 'excl. trimethoprim', - concept_id, - concept_name, - 0 -- hardcoded rank - FROM class_drugs_scraper,concept c -WHERE vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND concept_class_id = 'Ingredient' - AND standard_concept = 'S' - AND concept_name ~* 'trimethoprim' - AND class_name ~* 'excl. trimethoprim' - AND LENGTH (class_code) = 7; - --- fuzzy match (should be checked before INSERT) -INSERT INTO dev_combo -SELECT DISTINCT a.class_code, - a.class_name, - a.adm_r, - a.class, c.concept_id, c.concept_name, - CASE - WHEN class_code IN ('A12CC30', 'G01AX14', 'V03AE04') THEN 3 - WHEN lower(SUBSTRING(SPLIT_PART(a.class_name,' and ',1),'^...')) = lower(SUBSTRING(c.concept_name,'^...')) THEN 1 - WHEN lower(SUBSTRING(SPLIT_PART(a.class_name,' and ',2),'^...')) = lower(SUBSTRING(c.concept_name,'^...')) THEN 2 - WHEN class = 'arginine' THEN 1 - WHEN class = 'lysine' THEN 2 - WHEN class = 'trastuzumab' THEN 2 - WHEN class_name ~ 'AND iron' THEN 2 - WHEN class_name ~ '^iron' THEN 1 - WHEN class_name ~ 'AND potassium' THEN 2 - ELSE 3 END -FROM dev_combo a, - devv5.concept_synonym b, - concept c -WHERE lower(b.concept_synonym_name) LIKE concat('%',SPLIT_PART(a.class,' (',1),'%') -AND b.concept_id = c.concept_id -AND c.concept_class_id = 'Ingredient' -AND c.standard_concept = 'S' -AND invalid_reason IS NULL -AND c.concept_id NOT IN (40171179, 719174, 19006043,19022417, 43525936, 43013670, 43532256) -- white-tailed deer hair extract |5-hydroxylysine|carbocysteine-lysine -AND (a.class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND class <> ''; - + 0 -- hardcoded rank + FROM concept_manual a, -- ATC + concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'trimethoprim' + AND a.concept_name ~* 'excl. trimethoprim' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + -- fix Vitamin D AND analogues IN combination UPDATE dev_combo SET rnk = 3 @@ -2256,53 +2266,47 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND rnk = 1; --- remove erroneous Ingredient match -DELETE -FROM dev_combo -WHERE class_code = 'V03AE04' -AND concept_name IN ('calcium','magnesium','magnesite'); - DELETE FROM dev_combo WHERE class_code = 'A06AG11' AND class_name = 'sodium lauryl sulfoacetate, incl. combinations' AND concept_name = 'sodium' -AND rnk = 1; +AND rnk = 1; -- 0 DELETE FROM dev_combo WHERE class_code = 'A01AA51' AND class_name = 'sodium fluoride, combinations' AND concept_name = 'sodium' -AND rnk = 1; +AND rnk = 1; --0 DELETE FROM dev_combo WHERE class_code = 'A06AB58' AND class_name = 'sodium picosulfate, combinations' AND concept_name = 'sodium' -AND rnk = 1; +AND rnk = 1; -- 0 DELETE FROM dev_combo WHERE class_code = 'B05XA06' AND class_name = 'potassium phosphate, incl. combinations WITH other potassium salts' AND concept_name = 'potassium' -AND rnk = 1; +AND rnk = 1;--0 DELETE FROM dev_combo WHERE class_code = 'A12BA51' AND class_name = 'potassium chloride, combinations' AND concept_name = 'potassium' -AND rnk = 1; +AND rnk = 1;--0 DELETE FROM dev_combo WHERE class_code = 'C01DA58' AND class_name = 'isosorbide dinitrate, combinations' AND concept_name = 'isosorbide' -AND rnk = 1; +AND rnk = 1; --0 -- fix erroneous rnk of 3 for J07AG53 UPDATE dev_combo @@ -2311,83 +2315,296 @@ WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND rnk = 3; -- 5 +AND rnk = 3; -- 0 --- remove ATC classes other than 5th DELETE FROM dev_combo -WHERE LENGTH(class_code) < 7; +WHERE class_code = 'B03AD01' +AND rnk = 4; -- 0 --- 1300751 105669 polysaccharide iron complex INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, - adm_r, - class_name, - 1300751, - 'polysaccharide iron complex', + 'codeine', + 1189596, + 'dihydrocodeine', 1 FROM dev_combo -WHERE class_code = 'B03AD01'; +WHERE class_code = 'N02AA59'; +-- erroneous map to bismuth DELETE FROM dev_combo -WHERE class_code = 'B03AD01' -AND rnk = 4; +WHERE class_code = 'A02BD08' +AND class = 'bismuth subcitrate, tetracycline' +AND concept_id = 19025138; --0 --- add missing Ingredient -INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, - 'codeine', - 1201620, - 'codeine', - 1 +-- erroneous map to Pentaerythritol Distearate +DELETE FROM dev_combo -WHERE class_code = 'N02AA59'; +WHERE class_code = 'C01DA55' +AND class = '' +AND concept_id = 42903512; -- 0 -INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, - 'codeine', - 1189596, - 'dihydrocodeine', - 1 +-- erroneous map to Potassium +DELETE FROM dev_combo -WHERE class_code = 'N02AA59'; +WHERE class_code = 'B05XA06' +AND class = 'potassium phosphate, incl. combinations with other potassium salts' +AND concept_id = 19049024; -- 0 -INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, - 'paracetamol', - 1125315, - 'acetaminophen', - 1 +-- erroneous map to tenofovir +DELETE FROM dev_combo -WHERE class_code = 'N02BE51'; +WHERE class_code = 'J05AR03' +AND class = 'tenofovir disoproxil' +AND concept_id = 19011093; -- 0 -INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - adm_r, - 'acetylsalicylic acid', - 1112807, - 'aspirin', - 1 +-- remove doubling ingredients with different rank, remaining those which are Primary lateral +DELETE FROM dev_combo -WHERE class_code = 'N02BA51'; +WHERE (class_code,concept_id,rnk) IN (SELECT a.class_code, + a.concept_id, + a.rnk + FROM dev_combo a + JOIN dev_combo b + ON a.class_code = b.class_code + AND a.concept_id = b.concept_id + WHERE a.rnk > 1 + AND b.rnk = 1); -- 20 + +UPDATE dev_combo + SET rnk = 3 +WHERE class_code = 'A07FA51' +AND class_name = 'lactic acid producing organisms, combinations' +AND concept_id = 19136028 +AND concept_name = 'Saccharomyces cerevisiae'; -- 0 + +DELETE +FROM dev_combo +WHERE class_name ~ 'antiinfectives' +AND rnk = 4 +AND concept_id IN (19010309,19136048,1036884,19049024,989878,961145,19018544,917006,914335); -- 13 + +UPDATE dev_combo + SET rnk = 3 +WHERE class_name ~ 'lactic acid producing organisms' +AND rnk = 4 +AND concept_name ~* 'Saccharomyces|Bacillus|Bifidobacterium|Enterococcus|Escherichia|Streptococcus'; -- 9 ---remove vitamin D as a false Ingredient DELETE FROM dev_combo -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE concept_id = 19009405) -AND class_name ~* 'colecalc' -AND class_name !~* 'vitamin D' -AND concept_id = 19009405; +WHERE class_name ~ 'lactic acid producing organisms' +AND rnk = 4; + +UPDATE dev_combo + SET rnk = 3 +WHERE class_name ~ 'opium derivatives' +AND rnk = 1; + +UPDATE dev_combo + SET rnk = 3 +WHERE (class_code,concept_id) IN (SELECT a.class_code, + a.concept_id + FROM dev_combo a + JOIN dev_combo b + ON b.class_code = a.class_code + AND a.rnk <> b.rnk + AND a.concept_id = b.concept_id + WHERE (a.class_Code,a.concept_name) IN (SELECT class_code, + concept_name + FROM dev_combo + GROUP BY class_code, + concept_name + HAVING COUNT(1) > 1) + AND (b.class_Code,b.concept_name) IN (SELECT class_code, + concept_name + FROM dev_combo + GROUP BY class_code, + concept_name + HAVING COUNT(1) > 1) + AND a.class_code = 'S03AA30' + AND a.rnk = 4) +AND rnk = 4; + +DELETE +FROM dev_combo +WHERE class_code = 'R05FB01' +AND class_name = 'cough suppressants and mucolytics' +AND class = 'cough suppressants' +AND concept_id = 19057932 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'R05FB01' +AND class_name = 'cough suppressants and mucolytics' +AND class = 'cough suppressants' +AND concept_id = 19071999 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 3 +WHERE class_code = 'S02AA30' +AND class_name = 'antiinfectives, combinations' +AND class = '' +AND concept_id = 19006842 +AND rnk = 4; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1790868 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1734104 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1748975 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1778162 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1797513 +AND rnk = 3; +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1754994 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1742253 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1707164 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1721543 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 923081 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 19023254 +AND rnk = 3; +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 19024197 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 19037983 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 19070251 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1836948 +AND rnk = 3; +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'J01RA02' +AND class = 'sulfonamides' +AND concept_id = 1702559 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'R05FB02' +AND class = 'cough suppressants' +AND concept_id = 43012226 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'R05FB02' +AND class = 'cough suppressants' +AND concept_id = 912362 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'R05FB02' +AND class = 'cough suppressants' +AND concept_id = 19060831 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 3 +WHERE class_code = 'R05FB02' +AND class = '' +AND concept_id = 1140088 +AND rnk = 4; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'R05FB02' +AND class = 'cough suppressants' +AND concept_id = 19063951 +AND rnk = 3; + +UPDATE dev_combo + SET rnk = 4 +WHERE class_code = 'R05FB02' +AND class = 'cough suppressants' +AND concept_id = 1103137 +AND rnk = 3; +/******************************************* +**** ADD ODDMENTS TO THE INPUT TABLES ***** +********************************************/ -- add links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of OMOP Ingredient names using dev_combo table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) @@ -2468,11 +2685,11 @@ AND concept_class_id = 'Ingredient' AND vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND invalid_reason IS NULL AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); - /*************************************** ******* relationship_to_concept ******** ****************************************/ -- add mappings of ATC Drug Attributes to OMOP Equivalents +TRUNCATE relationship_to_concept; ALTER TABLE relationship_to_concept ALTER COLUMN concept_code_1 TYPE VARCHAR; -- add links between ATC Drug Attributes AND their OMOP equivalents @@ -2485,6 +2702,6 @@ FROM internal_relationship_stage JOIN concept c ON lower(concept_code_2) = lower(c.concept_name) AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - AND c.invalid_reason IS NULL; + AND c.invalid_reason IS NULL; -- run load_interim.sql From c12a6547a08e6ac11c8a364da58bf3dfa8a9d71b Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 3 Aug 2021 10:14:01 +0300 Subject: [PATCH 08/45] Update load_interim.sql amendments and refactoring --- ATC/load_interim.sql | 1789 ++++++++++++++++++++++++++---------------- 1 file changed, 1128 insertions(+), 661 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index ef49d86db..7491d4007 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -1,185 +1,667 @@ +/************************************************************************** +* Copyright 2016 Observational Health Data Sciences AND Informatics (OHDSI) +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may NOT use this file except IN compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to IN writing, software +* distributed under the License is distributed ON an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* Authors: Anna Ostropolets, Polina Talapova +* Date: Jul 2021 +**************************************************************************/ /******************** -***** REFERENCE ***** +***** RX COMBO ***** *********************/ --- create temporary table containing links bwetween ATC Сlass codes and combinations of (ATC_code + Dose Form) from drug_concept_stage -DROP TABLE if exists reference; -CREATE TABLE reference -AS -SELECT DISTINCT class_code, - concept_code -FROM drug_concept_stage - LEFT JOIN class_drugs_scraper ON SPLIT_PART (concept_name,' ',1) = class_code; - -- create a table with aggregated RxE ingredients: rx_combo DROP TABLE IF EXISTS rx_combo; CREATE TABLE rx_combo AS SELECT drug_concept_id, string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo FROM devv5.drug_strength - JOIN concept ON concept_id = drug_concept_id AND + JOIN devv5.concept ON concept_id = drug_concept_id AND concept_class_id IN ('Clinical Drug Form', 'Ingredient') -- 'Clinical Drug Comp' doesn't exist in ATCs GROUP BY drug_concept_id ; +/************************* +***** CLASS TO DRUG ****** +**************************/ +-- assemble a table containing ATC Drug Classes which are hierarchically connected to RxN/RxE Drug Products via 'ATC - RxNorm' relationships (resembles Subsumes). +-- Monocomponent ATC classes +DROP TABLE if exists class_to_drug_new; +CREATE TABLE class_to_drug_new +AS +(WITH t1 +AS +(SELECT * +FROM concept_manual +WHERE invalid_reason IS NULL +AND concept_class_id = 'ATC 5th' +AND concept_code NOT IN (SELECT class_code FROM dev_combo)), +-- get attributes +t2 +AS +(SELECT DISTINCT a.concept_code AS class_code, + a.concept_name AS class_name, + d.concept_code AS ing_name, + d2.concept_code AS df_name +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d + ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient' + LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form') +SELECT DISTINCT class_code, + class_name, + c.*, + 1 AS concept_order, + 'Monocomponent ATC Class' as order_desc +FROM t2 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + AND d_name !~ ' / ' + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' +WHERE class_code NOT IN (SELECT class_code FROM dev_combo)); ---ambiguous_class_ingredient -DROP TABLE IF EXISTS ambiguous_class_ingredient_tst; -CREATE UNLOGGED TABLE ambiguous_class_ingredient_tst +-- add everything - greedy version +INSERT INTO class_to_drug_new +WITH t1 AS -SELECT DISTINCT a.class_code, - a.class_name, - a.concept_name AS concept_code_2, - rnk -FROM dev_combo a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.class_code - JOIN drug_concept_stage c - ON c.concept_code = i.concept_code_2 - AND c.concept_class_id = 'Ingredient' - AND lower (a.concept_name) = lower (c.concept_name); +(SELECT * +FROM concept_manual +WHERE invalid_reason IS NULL +AND concept_class_id = 'ATC 5th'), +-- get attributes +t2 +AS +(SELECT DISTINCT a.concept_code AS class_code, + a.concept_name AS class_name, + d.concept_code AS ing_name, + d2.concept_code AS df_name +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d + ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient' + LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form') +SELECT DISTINCT class_code, + class_name, + c.*, + 2 AS concept_order, + 'Greedy Monocomponent ATC Class' as order_desc +FROM t2 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) + AND class_code NOT IN (SELECT class_code FROM dev_combo); --- add rnk 0 - excluded Ingredients -INSERT INTO ambiguous_class_ingredient_tst +/***** PREPARE ATC COMBO CLASSES ******/ + +-- separate pure ATC Combo Classes with mapping: Primary lateral + Secondary lateral (rnk in (1, 2) in dev_combo) +DROP TABLE IF EXISTS case_1; +CREATE TABLE case_1 +AS SELECT DISTINCT class_code, class_name, - concept_name AS concept_code_2, + concept_id, + concept_name, rnk FROM dev_combo -WHERE rnk = 0; -- 3726 - --- separate pure ATC combinations (Primary lateral + Secondary lateral) -DROP TABLE IF EXISTS pure_combos; -CREATE TABLE pure_combos +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1)-- Primary lateral +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2)-- Secondary lateral +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3)-- exclude Priamry upward +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0); -- exclude Secondary upward + +-- separate pure ATC Combo Classes with 1 defined Ingredient: Primary lateral Ingredients in combination (rnk = 1 in dev_combo) +DROP TABLE if exists case_2; +CREATE TABLE case_2 AS -SELECT DISTINCT class_code, - concept_code_2, +(SELECT * +FROM dev_combo +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) +); + +-- separate Primary lateral in combination with excluded Ingredient (rnk in (1,0) in dev_combo) +DROP TABLE if exists case_2_2; +CREATE TABLE case_2_2 +AS +(SELECT * +FROM dev_combo +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0) +); + +-- separate Primary upward Ingredients (rnk = 3 in dev_combo) +DROP TABLE IF EXISTS case_3; +CREATE TABLE case_3 +AS +(SELECT DISTINCT class_code, class_name, + concept_id, + concept_name, rnk -FROM ambiguous_class_ingredient_tst -WHERE class_code IN (SELECT class_code - FROM ambiguous_class_ingredient_tst - WHERE rnk = 1) -- Primary lateral -AND class_code IN (SELECT class_code - FROM ambiguous_class_ingredient_tst - WHERE rnk = 2) -- Secondary lateral -AND class_code NOT IN (SELECT class_code - FROM ambiguous_class_ingredient_tst - WHERE rnk = 3) -- exclude Priamry upward -AND class_code NOT IN (SELECT class_code - FROM ambiguous_class_ingredient_tst - WHERE rnk = 4); -- exclude Secondary upward - -- 1629 - --- separate Primary upward Ingredients (ATC Groupers mentioned at the beginning of ATC Class name) -- rnk 3 only -DROP TABLE IF EXISTS ing_pr_up; -CREATE TABLE ing_pr_up AS -SELECT DISTINCT class_code, - concept_code_2, +FROM dev_combo +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3)-- include Priamry upward +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1)-- exclude Primary lateral +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2)-- exclude Secondary lateral +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0));-- exclude Secondary upward + +-- add the same Ingredients marked with rnk=1 to create permutations, assume that there will not be more than 3 ingredients in combination +INSERT INTO case_3 -- Primary upward +SELECT class_code, class_name, concept_id, concept_name, 1 + FROM case_3; + +-- separate Primary upward + Secondary upward Ingredients (rnk in (3,4) in dev_combo) +DROP TABLE IF EXISTS case_4; +CREATE TABLE case_4 AS (select distinct +class_code, class_name, + concept_id, +concept_name, rnk -FROM ambiguous_class_ingredient_tst +FROM dev_combo WHERE class_code IN (SELECT class_code - FROM ambiguous_class_ingredient_tst + FROM dev_combo WHERE rnk = 3) -- include Priamry upward AND class_code NOT IN (SELECT class_code - FROM ambiguous_class_ingredient_tst + FROM dev_combo WHERE rnk = 1) -- exclude Primary lateral AND class_code NOT IN (SELECT class_code - FROM ambiguous_class_ingredient_tst + FROM dev_combo WHERE rnk = 2) -- exclude Secondary lateral +AND class_code IN (SELECT class_code + FROM dev_combo + WHERE rnk = 4) +and class_code not in (select class_code from dev_combo where rnk = 0) +);-- exclude Secondary upward + +-- separate Primary upward + Secondary upward with excluded Ingredient (rnk in (3, 4, 0) in dev_combo) +DROP TABLE IF EXISTS case_4_2; +CREATE TABLE case_4_2 AS (select distinct +class_code, + class_name, + concept_id, +concept_name, + rnk +FROM dev_combo +WHERE class_code IN (SELECT class_code + FROM dev_combo + WHERE rnk = 3) -- include Priamry upward +AND class_code NOT IN (SELECT class_code + FROM dev_combo + WHERE rnk = 1) -- exclude Primary lateral AND class_code NOT IN (SELECT class_code - FROM ambiguous_class_ingredient_tst - WHERE rnk = 4);-- exclude Secondary upward - --- add the same Ingredients marked as rnk=1 to create permutations, assume that there will not be more than 3 ingredients in combination -INSERT INTO ing_pr_up -SELECT class_code, concept_code_2,class_name, 1 - FROM ing_pr_up; -- 3186 - --- combine pure ATC combos with ATC groupers -INSERT INTO ambiguous_class_ingredient_tst -SELECT DISTINCT class_code, class_name, concept_code_2, rnk -FROM ing_pr_up -UNION -SELECT DISTINCT class_code, class_name, concept_code_2, rnk -FROM pure_combos -; -- 7920 + FROM dev_combo + WHERE rnk = 2) -- exclude Secondary lateral +AND class_code IN (SELECT class_code + FROM dev_combo + WHERE rnk = 4) +and class_code in (select class_code from dev_combo where rnk = 0) +);-- exclude Secondary upward + +--Note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product + +-- Add ATC Combo Classes with mappings to class_to_drug: +-- Primary lateral + Secondary lateral (order = 3) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT * +FROM concept_manual +WHERE invalid_reason IS NULL +AND concept_class_id = 'ATC 5th'), +-- get attributes +t2 +AS +(SELECT DISTINCT a.concept_code AS class_code, + a.concept_name AS class_name, + d.concept_code AS ing_name, + d2.concept_code AS df_name +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d + ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient' + LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form' +) +SELECT DISTINCT class_code, + class_name, + c.*, + 3 AS concept_order, + 'Primary upward + Secondary upward' as order_desc +FROM t2 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) + AND class_code IN (SELECT class_code FROM case_2); --- create index to make the script faster -CREATE INDEX ambiguous_class_ingredient_test ON ambiguous_class_ingredient_tst (class_code, concept_code_2, rnk); +-- Primary upward only (order = 4) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT * +FROM concept_manual +WHERE invalid_reason IS NULL +AND concept_class_id = 'ATC 5th'), +-- get attributes +t2 +AS +(SELECT DISTINCT a.concept_code AS class_code, + a.concept_name AS class_name, + d.concept_code AS ing_name, + d2.concept_code AS df_name +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d + ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient' + LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form') +SELECT DISTINCT class_code, + class_name, + c.*, + 4 AS concept_order, + 'Primary upward' AS order_desc +FROM t2 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) + AND class_code IN (SELECT class_code FROM case_3) +AND c.concept_name ~ ' / '; + +-- Primary lateral + Secondary lateral, more than 2 ingreds (order = 5) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk +FROM case_1 a + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form' +), +t2 +AS +(SELECT DISTINCT class_code, + class_name, + c.* +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) +JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND rnk = 1), +t3 +AS +(SELECT DISTINCT a.class_code, + a.class_name, + a.ing_name, + c.* +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 2) + SELECT DISTINCT a.*, + 5 AS concept_order, + 'more Pr lat + Sec lat' AS order_desc + FROM t2 a + JOIN t3 b ON b.class_code = a.class_code + JOIN t3 c + ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND b.ing_name <> c.ing_name + AND a.concept_id = b.concept_id + AND a.concept_name ~ ' / '; + +-- Primary lateral + Secondary lateral, 2 ingredients +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk +FROM case_1 a + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form'), +t2 +AS +(SELECT DISTINCT class_code, + class_name, + c.* +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 1), +t3 +AS +(SELECT DISTINCT a.class_code, + a.class_name, + a.ing_name, + c.* +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) +JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND rnk = 2) +SELECT DISTINCT a.*, + 6 AS concept_order, + 'pr lat+sec lat 2 ingreds' as order_desc +FROM t2 a + JOIN t3 b + ON b.class_code = a.class_code + AND a.concept_id = b.concept_id + AND a.concept_name ~ ' / ' + AND a.class_code NOT IN (SELECT class_code FROM class_to_drug_new) + AND a.concept_name !~ ' / .* / '; + +-- Primary lateral only in combination (order = 7) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk +FROM case_2 a + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form') + SELECT DISTINCT class_code, + class_name, + c.*, + 7 AS concept_order, + 'Primary lateral only in comb.' AS order_desc +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND c.concept_name ~ ' / '; + +-- Primary lateral only in combination with an excluded Ingredient (order=8) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk +FROM case_2_2 a + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form'), +t2 AS +(SELECT DISTINCT class_code, + class_name, + c.*, + 2, + b.ing_name +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 1) +SELECT DISTINCT class_code, + class_name, + concept_id, + concept_name, + domain_id, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + valid_start_date, + valid_end_date, + invalid_reason, + 8, + 'Primary lateral in comb with excl' +FROM t2 +WHERE concept_name ~ ' / ' +AND concept_id NOT IN (SELECT d_id + FROM rx_all_combo a + JOIN t1 b + ON LOWER (b.ing_name) = LOWER (a.ing_name) + AND LOWER (b.df_name) = LOWER (a.df_name) + AND b.rnk = 0); + +-- Primary upward only (order= 9) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk +FROM case_3 a + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form') + SELECT DISTINCT class_code, + class_name, + c.*, + 9, + ' Primary upward' AS order_desc +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND c.concept_name ~ ' / '; --- create a table with aggregated ATC ingredients: full_combo (for 3 ingredients only) --- separate direct ingredients -DROP TABLE IF EXISTS ing_pr_lat; -CREATE UNLOGGED TABLE ing_pr_lat +-- Primary upward + Secondary upward (order = 10) +INSERT INTO class_to_drug_new +WITH t1 AS -SELECT DISTINCT a.*, -- distinct is required here - concept_id_2, - 1 AS precedence -- will be changed a bit later for some ATC codes - FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE rnk = 1; -- 4119 +(SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk +FROM case_4 a + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form'), +t2 +AS +(SELECT DISTINCT class_code, + class_name, + c.* +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 3), +t3 +AS +(SELECT DISTINCT a.class_code, + a.class_name, + a.ing_name, + c.* +FROM t1 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 4) +SELECT DISTINCT a.*, 10, 'Primary upward + Secondary upward' +FROM t2 a + JOIN t3 b + ON b.class_code = a.class_code + AND a.concept_id = b.concept_id +WHERE a.concept_name ~ ' / '; +------------------------- +---- GET MORE LINKS ----- +------------------------- +-- separate Primary lateral Ingredients (rnk = 1 in dev_combo) +DROP TABLE IF EXISTS t1; +CREATE UNLOGGED TABLE t1 +AS +SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk + FROM dev_combo WHERE rnk = 1 -- Primary lateral + ; --- separate Secondary lateral ingredients (rnk = 2) -DROP TABLE IF EXISTS ing_sec_lat; -CREATE UNLOGGED TABLE ing_sec_lat +-- separate Secondary lateral Ingredients (rnk = 2 in dev_combo) +DROP TABLE IF EXISTS t2; +CREATE UNLOGGED TABLE t2 AS -SELECT DISTINCT a.*, - concept_id_2, - 1 as precedence -- distinct is required here (can be filled in a futher release) - FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE rnk = 2; -- 1213 +SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk + FROM dev_combo WHERE rnk = 2 -- Secondary lateral + ; --- separate Primary upward ingredients (rnk = 3) -DROP TABLE IF EXISTS ing_pr_up; -CREATE UNLOGGED TABLE ing_pr_up +-- separate Primary upward Ingredients (rnk = 3 in dev_combo) +DROP TABLE IF EXISTS t3; +CREATE UNLOGGED TABLE t3 AS -SELECT DISTINCT a.*, - concept_id_2, - 1 as precedence -FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 -WHERE rnk = 3; -- 4086 - --- separate Secondary upward ingredients (rnk = 4) -DROP TABLE IF EXISTS ing_sec_up; -CREATE UNLOGGED TABLE ing_sec_up -AS -SELECT DISTINCT a.*, -- distinct is required here - concept_id_2, - precedence - FROM ambiguous_class_ingredient_tst a - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk + FROM dev_combo +WHERE rnk = 3 -- Primary upward +; + +-- separate Secondary upward Ingredients (rnk = 4 in dev_combo) +DROP TABLE IF EXISTS t4; +CREATE UNLOGGED TABLE t4 +AS +SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk + FROM dev_combo WHERE rnk = 4; -- Secondary upward --- obtain the full list of all possible combinations of ingredients for a one ATC-combo (48m 46s) +-- create a table with aggregated ATC Ingredients per one ATC Combo Class (no more than 3 ingredients per Class is recommended) +-- add Primary lateral AND (Secondary lateral 1 AND/OR Secondary lateral 2) AND/OR Primary upward AND/OR Secondary upward DROP TABLE if exists full_combo; CREATE UNLOGGED TABLE full_combo -AS +AS( SELECT DISTINCT a.class_code, a.class_name, - a.concept_id_2||COALESCE('-' || b.concept_id_2,'') ||COALESCE('-' || c.concept_id_2,'') ||COALESCE('-' || d.concept_id_2,'') AS i_combo -FROM ing_pr_lat a - JOIN ing_sec_lat b USING (class_code)-- rank 2 - LEFT JOIN ing_pr_up c USING (class_code)-- rank 3 - LEFT JOIN ing_sec_up d USING (class_code)-- rank 4 -ORDER BY class_code; - --- create table with Ingredient permutations + a.concept_id||COALESCE('-' || b.concept_id,'') ||COALESCE('-' || c.concept_id,'') ||COALESCE('-' || c1.concept_id,'') ||COALESCE('-' || d.concept_id,'') AS i_combo +FROM t1 a + JOIN t2 b ON b.class_code = a.class_code-- rank 2 + LEFT JOIN t2 c ON c.class_code = b.class_code-- rank 2 + LEFT JOIN t3 c1 ON c1.class_code = c.class_code-- rank 3 -- is it possible? usually there is no combinations as rnk=1 + rnk=3 + LEFT JOIN t4 d ON d.class_code = c1.class_code-- rank 4 + AND a.concept_id <> b.concept_id + AND a.concept_id <> c.concept_id + AND a.concept_id <> d.concept_id + AND a.concept_id <> c1.concept_id + AND b.concept_id <> c.concept_id + AND b.concept_id <> c1.concept_id + AND b.concept_id <> d.concept_id + AND c.concept_id <> c1.concept_id + AND c1.concept_id <> d.concept_id +ORDER BY class_code); + +-- add Primary lateral AND/OR Secondary lateral AND/OR Primary upward AND/OR Secondary upward +INSERT INTO full_combo +WITH z1 +AS +(SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id||COALESCE('-' || b.concept_id,'') ||COALESCE('-' || c.concept_id,'') ||COALESCE('-' || d.concept_id,'') AS i_combo +FROM t1 a + JOIN t2 b ON b.class_code = a.class_code-- rank 2 + LEFT JOIN t3 c ON c.class_code = b.class_code-- rank 3 + LEFT JOIN t4 d + ON d.class_code = c.class_code-- rank 4 + AND a.concept_id <> b.concept_id + AND a.concept_id <> c.concept_id + AND a.concept_id <> d.concept_id + AND b.concept_id <> c.concept_id + AND b.concept_id <> d.concept_id +ORDER BY class_code +) +SELECT * +FROM z1 +WHERE i_combo NOT IN (SELECT i_combo FROM full_combo); + +-- create temporary table with Ingredient permutations - additional layer of mappings: Primary lateral AND/OR Secondary lateral 1 AND/OR Secondary lateral 2 DROP TABLE IF EXISTS permutations; CREATE UNLOGGED TABLE permutations AS -SELECT distinct a.class_code, a.class_name, a.concept_id_2||COALESCE('-' || b.concept_id_2, '')||COALESCE('-' || c.concept_id_2, '') AS i_combo - FROM ing_pr_lat a -- from Primary lateral -LEFT JOIN ing_sec_lat b ON b.class_code = a.class_code -- to the 1st Secondary lateral -LEFT JOIN ing_sec_lat c ON c.class_code = a.class_code -- and the 2nd Secondary lateral -WHERE b.concept_id_2<>c.concept_id_2 AND b.concept_id_2<>a.concept_id_2;--34262 +SELECT distinct a.class_code, a.class_name, a.concept_id||COALESCE('-' || b.concept_id, '')||COALESCE('-' || c.concept_id, '') AS i_combo + FROM t1 a -- from Primary lateral +LEFT JOIN t2 b ON b.class_code = a.class_code -- to the 1st Secondary lateral +LEFT JOIN t2 c ON c.class_code = a.class_code -- and the 2nd Secondary lateral +WHERE b.concept_id<>c.concept_id AND b.concept_id<>a.concept_id; --- add newly created permutations to the full_combo table in oreder to enrich the set of aggregated ATC ingredients +-- add newly created permutations to the full_combo table in order to enrich the set of aggregated ATC Ingredients INSERT INTO full_combo -SELECT * FROM permutations; -- 34262 - --- Separate 1 Pr lateral in combination -DROP TABLE if exists ing_pr_lat_combo; +SELECT * FROM permutations; +-- Separate Primary lateral in combination (rnk = 1 in dev_combo) +DROP TABLE if exists ing_pr_lat_combo; -- ing_pr_lat_combo CREATE TABLE ing_pr_lat_combo AS (SELECT * @@ -190,12 +672,11 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); --- create temporary table with all possible i_combos for Primary laterla in combinations (with unspecified drugs) +-- create temporary table with all possible i_combos for Primary lateral in combinations (with unspecified drugs) DROP TABLE if exists ing_pr_lat_combo_to_drug; - CREATE TABLE ing_pr_lat_combo_to_drug AS -(WITH t1 +(WITH z1 AS (SELECT drug_concept_id, REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo @@ -205,12 +686,50 @@ WHERE i_combo LIKE '%-%') a.class_name, d.i_combo FROM ing_pr_lat_combo a - JOIN t1 b ON b.i_combo = a.concept_id + JOIN z1 b ON b.i_combo = a.concept_id JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id); --- 2 Pr lat in combinations +-- separate Primary lateral + Secondary upward (rnk in (1,4) in dev_combo) +DROP TABLE if exists ing_pr_lat_sec_up_combo; +CREATE TABLE ing_pr_lat_sec_up_combo +AS +(SELECT * +FROM dev_combo a +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) +); +-- create temporary table with all possible i_combos for Primary lateral + Secondary upward +DROP TABLE if exists ing_pr_lat_sec_up_combo_to_drug; +CREATE TABLE ing_pr_lat_sec_up_combo_to_drug +AS +(WITH z1 +AS +(SELECT drug_concept_id, + REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo +FROM rx_combo +WHERE i_combo LIKE '%-%') +SELECT DISTINCT a.class_code, + a.class_name, + c.i_combo +FROM ing_pr_lat_sec_up_combo a + JOIN z1 b + ON b.i_combo = a.concept_id + AND a.rnk = 1 + JOIN ing_pr_lat_sec_up_combo a1 + ON a1.class_code = a.class_code + AND a1.concept_id <> a.concept_id + JOIN z1 j + ON j.i_combo = a1.concept_id + AND a1.rnk = 4 + JOIN rx_combo c + ON c.drug_concept_id = b.drug_concept_id + AND c.drug_concept_id = j.drug_concept_id); + +-- separate Primary lateral in combination with excluded Ingredient: ing_pr_lat_combo_excl (rnk in (1, 0) in dev_combo)) DROP TABLE if exists ing_pr_lat_combo_excl; - CREATE TABLE ing_pr_lat_combo_excl AS (SELECT * @@ -221,6 +740,7 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); +-- create temporary table with all possible i_combos for Primary lateral in combination with excluded Ingredient DROP TABLE if exists ing_pr_lat_combo_excl_to_drug; CREATE TABLE ing_pr_lat_combo_excl_to_drug AS @@ -229,13 +749,14 @@ AS (SELECT drug_concept_id, REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo FROM rx_combo -WHERE i_combo LIKE '%-%') SELECT DISTINCT a.class_code,a.class_name,d.i_combo FROM ing_pr_lat_combo_excl a JOIN t1 b ON b.i_combo = a.concept_id JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code AND a1.rnk <> a.rnk AND a1.rnk = 0 JOIN t1 f ON f.i_combo = a1.concept_id +WHERE i_combo LIKE '%-%') SELECT DISTINCT a.class_code,a.class_name,d.i_combo FROM ing_pr_lat_combo_excl a JOIN t1 b ON b.i_combo = a.concept_id +JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code AND a1.rnk <> a.rnk AND a1.rnk = 0 JOIN t1 f ON f.i_combo = a1.concept_id -- excluded JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id JOIN rx_combo d1 ON d1.drug_concept_id = f.drug_concept_id -- excluded AND d1.drug_concept_id <> d.drug_concept_id);; --- 3 Pr up+Sec up +-- separate Primary upward + Secondary upward (rnk in (3,4) in dev_combo) DROP TABLE if exists ing_pr_sec_up_combo; CREATE TABLE ing_pr_sec_up_combo AS @@ -248,8 +769,8 @@ AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) ); +-- create a temporary table with all possible i_combos for Primary upward + Secondary upward: ing_pr_sec_up_combo_to_drug DROP TABLE if exists ing_pr_sec_up_combo_to_drug; - CREATE TABLE ing_pr_sec_up_combo_to_drug AS (WITH t1 @@ -275,9 +796,8 @@ FROM ing_pr_sec_up_combo a ON c.drug_concept_id = b.drug_concept_id AND c.drug_concept_id = j.drug_concept_id); --- 4 Pr up+Sec up with excluded ingreds +-- separate Primary upward + Secondary upward with excluded Ingredients (currently no match, but it can change in the future) DROP TABLE if exists ing_pr_sec_up_combo_excl; - CREATE TABLE ing_pr_sec_up_combo_excl AS (SELECT * @@ -288,9 +808,8 @@ AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); --- currently no match, but query works +-- create temporary table with all possible i_combos for Primary upward + Secondary upward with excluded Ingredients (currently no match, but query works ing_pr_sec_up_combo_excl_to_drug) DROP TABLE if exists ing_pr_sec_up_combo_excl_to_drug; - CREATE TABLE ing_pr_sec_up_combo_excl_to_drug AS (WITH t1 @@ -323,6 +842,7 @@ FROM ing_pr_sec_up_combo_excl a AND c1.drug_concept_id = c.drug_concept_id AND c.i_combo !~ a2.concept_id::VARCHAR); +-- add prepared list of aggregated Ingredients of ATC Combo Classes to full_combo INSERT INTO full_combo SELECT * FROM ing_pr_lat_combo_to_drug @@ -334,9 +854,11 @@ SELECT * FROM ing_pr_sec_up_combo_to_drug UNION SELECT * -FROM ing_pr_sec_up_combo_excl_to_drug; -- 10086 +FROM ing_pr_sec_up_combo_excl_to_drug +UNION +SELECT * from ing_pr_lat_sec_up_combo_to_drug; --- create a table full_combo_reodered to order aggregated ATC ingredients by an Ingredient +-- create a table to order aggregated ATC ingredients by an Ingredient DROP TABLE IF EXISTS full_combo_reodered; CREATE UNLOGGED TABLE full_combo_reodered AS SELECT DISTINCT fc.class_code, @@ -347,25 +869,36 @@ CROSS JOIN LATERAL(SELECT STRING_AGG(s0.ing, '-' ORDER BY s0.ing::INT) AS i_comb SELECT UNNEST(STRING_TO_ARRAY(fc.i_combo, '-')) AS ing ) AS s0) l; --- create index to make the script faster -- 30m 7s (ask Timur to check whether these indices are useful) - 55m 38s +-- create index to make the script faster CREATE INDEX i_full_combo_reodered ON full_combo_reodered (class_code, i_combo); --- create full_combo_with_form table containig aggregated ATC ingredients + Dose Form -- 6.03s +--create a table containing aggregated Ingredients + Dose Forms for ATC Combo Classes DROP TABLE full_combo_with_form; CREATE UNLOGGED TABLE full_combo_with_form AS SELECT DISTINCT a.class_code, a.class_name, a.i_combo, - r.concept_id_2::int + r.concept_id_2::int as df_id FROM full_combo_reodered a JOIN internal_relationship_stage i ON class_code = substring (concept_code_1, '\w+') -- cut ATC code before space character JOIN drug_concept_stage b - ON b.concept_code = i.concept_code_2 + ON lower(b.concept_code) = lower(i.concept_code_2) AND b.concept_class_id = 'Dose Form' JOIN relationship_to_concept r ON r.concept_code_1 = i.concept_code_2; - --- add ATC combos without forms from the 'reference' table -- 27 740 577 rows affected +/******************** +***** REFERENCE ***** +*********************/ +-- create a temporary table of reference containing links between ATC source codes and combinations of ATC_codes AND Dose Forms from drug_concept_stage +DROP TABLE if exists reference; +CREATE TABLE reference +AS +SELECT DISTINCT class_code, + concept_code +FROM drug_concept_stage + LEFT JOIN class_drugs_scraper ON SPLIT_PART (concept_name,' ',1) = class_code; + +-- add ATC Combo Classes WO Dose Forms using the 'reference' table INSERT INTO full_combo_with_form (class_code, i_combo) SELECT DISTINCT f.class_code, @@ -373,49 +906,23 @@ SELECT DISTINCT f.class_code, FROM full_combo_reodered f JOIN reference r ON r.class_code = f.class_code WHERE r.concept_code = r.class_code -; -- 17 (7640) - -CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo,concept_id_2); -- 10m 8s -DROP TABLE IF EXISTS combo_not_limited_to_higher_ATC; -CREATE UNLOGGED TABLE combo_not_limited_to_higher_ATC -AS -WITH t1 AS -( - SELECT drug_concept_id, regexp_split_to_table (i_combo, '-')::INT AS i_combo FROM rx_combo -WHERE i_combo LIKE '%-%' -- at least two ingredients -AND drug_concept_id NOT IN (SELECT drug_concept_id FROM ing_excl) -- filter out drugs containig excluded ingredients if any -) -SELECT DISTINCT b.class_code, - d.class_name, - a.drug_concept_id -FROM t1 a-- Pr up - JOIN ing_pr_sec_up b -- Primary and Secondary upward with Ingredients - ON b.concept_id_2 = a.i_combo - JOIN dev_combo d on d.class_code = b.class_code - and d.rnk = 3 - JOIN t1 k -- Sec up - on k.drug_concept_id = a.drug_concept_id - JOIN ing_pr_sec_up c on c.concept_id_2 = k.i_combo - JOIN dev_combo d2 on d2.class_code = c.class_code - and a.i_combo <> k.i_combo - and d2.rnk = 4; - +; +CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo,df_id); /******************************* ******** CLASS TO DRUG ********* ********************************/ --- create table with one-step links between ATC combos of 5th class and Rx/RxN Drug Products using the full macth of an Ingredient and Dose Form -DROP TABLE IF EXISTS class_to_drug; -CREATE TABLE class_to_drug AS --- separate all Rx/RxN combo drugs of 'Clinical Drug Form' concept class -WITH t1 AS -( - SELECT c.concept_id, -- Standard Drug Product +-- add the 2nd portion of multicomponent ATC Class mappings: +-- ATC Combo Classes with Dose Forms using full_combo_with_form and rx_combo (order = 11) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT c.concept_id, -- Standard Drug Product c.concept_name, c.concept_class_id, c.vocabulary_id, - r.concept_id_2, -- Dose Form + r.concept_id_2 AS df_id, -- Dose Form a.i_combo -- combination of Standard Ingredient IDs as a key for join -FROM rx_combo a + FROM rx_combo a JOIN concept c ON c.concept_id = a.drug_concept_id JOIN concept_relationship r ON r.concept_id_1 = c.concept_id WHERE c.concept_class_id = 'Clinical Drug Form' @@ -425,152 +932,51 @@ AND r.relationship_id = 'RxNorm has dose form' AND r.invalid_reason IS NULL ) SELECT DISTINCT f.class_code, -- ATC - f.class_name, - r.concept_id, -- Standard Drug Product - r.concept_name, - r.concept_class_id, - 1 AS ORDER -- specific number for cases with the same order of ids in i_combo? - FROM full_combo_with_form f + c.concept_name as class_name, + d.*, + 11 AS conept_order, + 'Combo with form' + FROM full_combo_with_form f + JOIN concept_manual c on c.concept_code = f.class_code and c.invalid_reason is null and c.concept_class_id = 'ATC 5th' JOIN t1 r ON r.i_combo = f.i_combo -- combination of Standard Ingredient IDs - AND r.concept_id_2 = f.concept_id_2 ; + AND r.df_id = f.df_id + JOIN concept d on d.concept_id = r.concept_id + and f.class_code||r.concept_id not in (select class_code||concept_id from class_to_drug_new); --- add additional links using the table of 'combo_not_limited_to_higher_ATC' (this step gives errors!!!) -INSERT INTO class_to_drug - WITH t1 AS ( - -- separate Rx/RxN combo Drug Forms - SELECT a.concept_id, -- Rx - a.concept_name, - a.concept_class_id, - a.vocabulary_id, - c.concept_id_2, -- Rx Dose Form - r.class_code, -- ATC - r.class_name -FROM combo_not_limited_to_higher_ATC r - JOIN concept a ON r.drug_concept_id = a.concept_id - JOIN concept_relationship c ON c.concept_id_1 = a.concept_id - WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id LIKE 'RxNorm%' - AND a.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason IS NULL - ) -SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.concept_class_id, - 6 -- combo_not_limited_to_higher_ATC -FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.class_code - JOIN relationship_to_concept rtc ON i.concept_code_2 = rtc.concept_code_1 -WHERE a.concept_id_2 = rtc.concept_id_2 -AND (a.class_code, a.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug) -- prevent duplicates -; -- 1416 - --- contraceptive packs -INSERT INTO class_to_drug +-- add mappings of ATC Combo Classes to Contraceptive Packs (order = 12) +INSERT INTO class_to_drug_new SELECT class_code, class_name, - c.concept_id, - c.concept_name, - c.concept_class_id -- was added 17.06 - "order" -FROM class_to_drug ctd + c.*, + 12, + 'contraceptive pack' +FROM class_to_drug_new ctd JOIN concept_ancestor ON ctd.concept_id = ancestor_concept_id JOIN concept c ON descendant_concept_id = c.concept_id -WHERE class_code ~ 'G03FB|G03AB' +WHERE class_code ~ 'G03FB|G03AB' -- the list can be enriched AND c.concept_class_id IN ('Clinical Pack') -AND (ctd.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug); -- 440 +AND (ctd.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); --- insert mono-ATC codes -DROP TABLE IF EXISTS mono_ing; -CREATE UNLOGGED TABLE mono_ing -AS -SELECT DISTINCT a.class_code, - a.class_name, - rtc.concept_id_2 AS ing_id -FROM class_drugs_scraper a - JOIN internal_relationship_stage i ON class_code = SUBSTRING (concept_code_1,'\w+') - JOIN drug_concept_stage d - ON lower (d.concept_code) = lower (i.concept_code_2) - AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 -WHERE LENGTH(class_code) = 7 -AND class_code NOT IN (SELECT class_code FROM dev_combo) -AND class_code NOT IN (SELECT class_code FROM class_drugs_scraper where class_code ~ '^J07|^A10A' AND length(class_code)=7) -AND class_name !~* '\yand\y' -and class_code not in (select class_code from class_to_drug) -; -- 5462 - --- add Forms -DROP TABLE if exists mono_ing_with_form; - -CREATE TABLE mono_ing_with_form -AS -(SELECT class_code, - class_name, - ing_id, - concept_id_2 AS form_id -FROM mono_ing - JOIN internal_relationship_stage i ON class_code = SUBSTRING (i.concept_code_1,'\w+') - JOIN drug_concept_stage d - ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Dose Form' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2);; - -INSERT INTO class_to_drug -with t1 as (SELECT distinct drug_concept_id, i_combo::INT AS i_combo FROM rx_combo -WHERE i_combo NOT LIKE '%-%') -SELECT DISTINCT k.class_code, k.class_name, a.concept_id, a.concept_name, a.concept_class_id,2 -FROM mono_ing_with_form k -- ATC -JOIN t1 r -- Rx -ON r.i_combo = k.ing_id -JOIN concept a ON r.drug_concept_id = a.concept_id -JOIN concept_relationship c ON c.concept_id_1 = a.concept_id AND k.form_id = c.concept_id_2 - WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id LIKE 'RxNorm%' - AND a.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason IS NULL - AND (k.class_code, a.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug); -- 14119 - --- additional links -INSERT INTO class_to_drug -SELECT DISTINCT m.class_code, - m.class_name, - c.concept_id, - c.concept_name, - c.concept_class_id, - 3 -FROM mono_ing m - JOIN internal_relationship_stage i ON class_code = SUBSTRING (concept_code_1,'\w+') - JOIN concept c ON m.ing_id = c.concept_id -WHERE i.concept_code_1 not in (select concept_code_1 from internal_relationship_stage where concept_code_1 LIKE '% %') -- exclude ATC classes with Forms (they contain space inbetween) - AND (m.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug) -;-- 1168 - --- add manual links using concept_relationship_manual -- TO DO: relatioship_id for J07BM J07BM from Subsumes to ATC - RxNorn pr lat --- check in old class_to_drug whether there are additional ingredients -INSERT INTO class_to_drug +-- add manual mappings from concept_relationship_manual (order = 13) +INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, f.concept_name as class_name, -c.concept_id, -c.concept_name, -c.concept_class_id, ---relationship_id, -1 +c.*, +13, +'from crm' from class_drugs_scraper a join concept_relationship_manual b on b.concept_code_1 = a.class_code -join concept_manual f on f.concept_code = a.class_code -AND b.relationship_Id in ('ATC - RxNorm') --'ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up') +join concept_manual f on f.concept_code = a.class_code +AND b.relationship_Id in ('ATC - RxNorm') JOIN concept c on c.concept_code = b.concept_code_2 and c.vocabulary_id = b.vocabulary_id_2 -and c.vocabulary_id in ('RxNorm', 'RxNorm Extension') - AND (a.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug) - and f.invalid_reason is null; -- 4920 - +and c.vocabulary_id in ('RxNorm', 'RxNorm Extension') and c.standard_concept = 'S' + AND (a.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new) + and f.invalid_reason is null and f.concept_class_id = 'ATC 5th' ; + +-- manual clean up for Precise Ingredients and other particular cases (according to the information on the ATC WHO Website) DELETE -FROM class_to_drug +FROM class_to_drug_new WHERE (class_code = 'A02BA07' AND concept_class_id = 'Clinical Drug Form') -- Branded Drug 'Tritec' OR class_code = 'G03GA08' -- choriogonadotropin alfa (no Standard Precise Ingredient) OR class_code='N05AF02' -- clopentixol @@ -580,402 +986,463 @@ OR (class_code = 'B02BD14' AND concept_name LIKE '%Tretten%') -- 2 --catridecac OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog ; -INSERT INTO class_to_drug -SELECT 'B02BD11','catridecacog', concept_id, concept_name, concept_class_id,1 +-- add additional semi-manual mappings based on pattern-matching (order=14) +INSERT INTO class_to_drug_new +with t1 as ( +SELECT 'B02BD11' as class_code,'catridecacog' as class_name, concept_id FROM concept WHERE (vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') - OR concept_id = 35603348 -- the whole hierarchy (35603348 1668002 factor XIII Injection [Tretten] Branded Drug Form) -- 2 -; -INSERT INTO class_to_drug -SELECT 'B02BD14','susoctocog alfa', concept_id, concept_name, concept_class_id,1 + OR concept_id = 35603348 -- the whole hierarchy (factor XIII Injection [Tretten] Branded Drug Form) +UNION ALL +SELECT 'B02BD14','susoctocog alfa', concept_id FROM concept WHERE (vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') OR concept_id IN (35603348, 44109089) -- the whole hierarchy -; -- 3 - -INSERT INTO class_to_drug -SELECT 'A02BA07','ranitidine bismuth citrate', concept_id, concept_name, concept_class_id,1 +UNION ALL +SELECT 'A02BA07','ranitidine bismuth citrate',concept_id FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Tritec%' AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' -; -- 1 - -INSERT INTO class_to_drug -SELECT 'G03GA08','choriogonadotropin alfa', concept_id, concept_name, concept_class_id,1 +UNION ALL +SELECT 'G03GA08','choriogonadotropin alfa', concept_id FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~ 'choriogonadotropin alfa' AND standard_concept = 'S' AND concept_class_id ~ 'Clinical Drug Comp' -; -- 1 - - -INSERT INTO class_to_drug -SELECT 'N05AF02','clopenthixol', concept_id, concept_name, concept_class_id,1 +UNION ALL +SELECT 'N05AF02','clopenthixol', concept_id FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Sordinol|Ciatyl' AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' -; -- 4 - -INSERT INTO class_to_drug -SELECT 'D07AB02','hydrocortisone butyrate', concept_id, concept_name, concept_class_id,1 +UNION ALL +SELECT 'D07AB02','hydrocortisone butyrate',concept_id FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Hydrocortisone butyrate' AND concept_class_id = 'Clinical Drug' AND standard_concept = 'S' -;-- 6 - --- remove pentaerithrityl tetranitrate; oral -DELETE -FROM class_to_drug -WHERE class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral -; -- 7 - -INSERT INTO class_to_drug -SELECT 'C01DA05','pentaerithrityl tetranitrate', concept_id, concept_name, concept_class_id,1 +UNION ALL +SELECT 'C01DA05','pentaerithrityl tetranitrate', concept_id FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ILIKE '%Pentaerythritol Tetranitrate%' and - standard_concept = 'S' AND concept_class_id = 'Clinical Drug Comp' -- forms do not exist -; -- 9 + standard_concept = 'S' AND concept_class_id = 'Clinical Drug Comp' + ) +SELECT DISTINCT a.class_code, + d.concept_name, + c.*, + 14, + 'point fix' +FROM t1 a + JOIN concept c ON c.concept_id = a.concept_id + JOIN concept_manual d + ON d.concept_code = a.class_code + AND d.concept_class_id = 'ATC 5th' + AND d.invalid_reason IS NULL; +; -- clean up erroneous amount of ingredients DELETE -FROM class_to_drug +--select * +FROM class_to_drug_new WHERE class_name LIKE '%,%and%' AND class_name NOT LIKE '%,%,%and%' AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT LIKE '% / % / %'; -- 135 - --- process Packs, which should be added to the ATC hiearchy in parallel (to be shown in Athena as well) -DROP TABLE IF EXISTS pack_a; - create table pack_a AS ( - SELECT DISTINCT concept_id_1, - drug_concept_id, - string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo, - count(drug_concept_id) over (partition by concept_id_1) as cnt - FROM drug_strength - JOIN concept_relationship r - on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL - JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' - GROUP BY drug_concept_id,concept_id_1); - -DROP TABLE IF EXISTS pack_b; - create table pack_b as - (SELECT DISTINCT class_code, class_name, r.concept_code_1, r.concept_id_2 - FROM dev_combo - JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 - WHERE class_name LIKE '% and %'); - -DROP TABLE IF EXISTS pack_c; - create table pack_c AS (SELECT DISTINCT class_code, r.concept_code_1, r.concept_id_2 - FROM dev_combo - JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 - WHERE class_name LIKE '% and %'); - -DROP TABLE IF EXISTS pack_all; -CREATE TABLE pack_all as - SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name - FROM pack_a a - JOIN pack_a aa on aa.concept_id_1 = a.concept_id_1 - JOIN concept cc on concept_id = a.concept_id_1 - JOIN pack_b b on cast(b.concept_id_2 AS varchar) = aa.i_combo - JOIN pack_c c on cast(c.concept_id_2 AS varchar) = a.i_combo - WHERE a.drug_concept_id != aa.drug_concept_id - AND b.class_code = c.class_code - AND b.concept_code_1 != c.concept_code_1 - AND a.cnt = 2; - --- errnoues pack match -DELETE -FROM pack_all - WHERE (class_code='A02BD11' AND concept_id=42731634) - OR (class_code='R03AL08' AND concept_id=43045404); - -DROP TABLE IF EXISTS pack_temp; -CREATE TABLE pack_temp -as -with a AS (SELECT concept_id_1, - string_agg(ingredient_concept_id::varchar, '-' ORDER BY ingredient_concept_id) AS i_combo - FROM drug_strength - JOIN concept_relationship r on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL - JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' - GROUP BY concept_id_1), - b AS ( -SELECT DISTINCT class_code, class_name, r.concept_code_1, r.concept_id_2 -FROM dev_combo -JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code -JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' -JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 -WHERE class_name LIKE '%, combinations') -SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name -FROM a -JOIN b on i_combo ~ cast(b.concept_id_2 AS varchar) -and i_combo!=cast(b.concept_id_2 AS varchar) -JOIN concept cc on concept_id_1 = concept_id + AND concept_name NOT LIKE '% / % / %'; + +--- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug(order=15) +INSERT INTO class_to_drug_new +SELECT DISTINCT b.concept_code AS class_code, + b.concept_name AS class_name, + a.concept_id, + a.concept_name, + a.concept_class_id, + 14, + 'from sources.c_t_d' +FROM sources.class_to_drug a + JOIN concept_manual b ON b.concept_code = a.class_code + JOIN concept c + ON a.concept_id = c.concept_id + AND c.standard_concept = 'S' + AND b.invalid_reason IS NULL +WHERE (class_code) NOT IN (SELECT class_code FROM class_to_drug_new) +AND class_code NOT IN ('S02CA01','V03AB05','P03AC54','S02CA03','S02CA03') +AND (class_code,c.concept_id) NOT IN ( + SELECT 'A06AA02', 40031558 -- oral - otic + UNION ALL + SELECT 'A06AA02', 40031561 -- oral - rectal + UNION ALL + SELECT 'A06AA02', 40031561 -- oral - enema + UNION ALL + SELECT 'A06AA02', 40723180 -- oral - enema + UNION ALL + SELECT 'A06AA02',41080219 + UNION ALL + SELECT 'A06AA02',41205788 + UNION ALL + SELECT 'A06AA02',43158334 + UNION ALL + SELECT 'A06AA02',40036796); +/********************** +****** ADD PACKS ****** +***********************/ +-- add packs of Primary lateral only in combination (order 16) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + j.*, + 16 as concept_order, + 'Pack: Pr lat in combo' as order_desc +FROM class_to_drug_new a + JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + JOIN concept d + ON d.concept_Id = r.concept_id_2 + AND d.concept_class_id = 'Clinical Drug' + JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id + JOIN concept j ON j.concept_id = r2.concept_id_2 +WHERE a.concept_class_id = 'Clinical Drug Form' +AND r.invalid_reason IS NULL +AND r2.invalid_reason IS NULL +AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') +AND j.standard_concept = 'S' +AND a.class_code IN (SELECT class_code + FROM dev_combo + WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) +AND j.concept_name ~ ' / ' -- combos only +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + +-- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 17) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + j.*, + 17, + 'Pack: Pr lat in combo additional' +FROM class_to_drug_new a + JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id + JOIN concept j ON j.concept_id = r2.concept_id_2 +WHERE r2.invalid_reason IS NULL +AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack','Branded Pack Box') +AND j.standard_concept = 'S' +AND a.class_code IN (SELECT class_code + FROM dev_combo + WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) +AND j.concept_name ~ ' / ' -- combos only +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + +-- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 18) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + j.*, + 18, + 'Pack: Pr lat + Sec lat' +FROM class_to_drug_new a + JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + JOIN concept d + ON d.concept_Id = r.concept_id_2 + AND d.concept_class_id = 'Clinical Drug' + JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id + JOIN concept j ON j.concept_id = r2.concept_id_2 +WHERE a.concept_class_id = 'Clinical Drug Form' +AND r.invalid_reason IS NULL +AND r2.invalid_reason IS NULL +AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') +AND j.standard_concept = 'S' +AND a.class_code IN (SELECT class_code + FROM dev_combo + WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) + AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) +AND j.concept_name ~ ' / ' +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + +-- add Branded packs of Primary lateral + Secondary lateral (order = 19) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + j.*, + 19, + 'Pack: Pr lat + Sec lat Branded' +FROM class_to_drug_new a + JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id + JOIN concept j ON j.concept_id = r2.concept_id_2 +WHERE r2.invalid_reason IS NULL +AND j.concept_class_id IN ('Branded Pack') +-- Boxes are suspicious +AND j.standard_concept = 'S' +AND a.class_code IN (SELECT class_code + FROM dev_combo + WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) + AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) +AND j.concept_name ~ ' / ' +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + +-- add packs of Primary upward and Secondary upward (Class C AND Class D, order = 20) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + j.*, + 20, + 'Pack: Pr up + Sec up' +FROM class_to_drug_new a + JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + JOIN concept d + ON d.concept_Id = r.concept_id_2 + AND d.concept_class_id = 'Clinical Drug' + JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id + JOIN concept j ON j.concept_id = r2.concept_id_2 +WHERE a.concept_class_id = 'Clinical Drug Form' +AND r.invalid_reason IS NULL +AND r2.invalid_reason IS NULL +AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') +AND j.standard_concept = 'S' +AND a.class_code IN (SELECT class_code + FROM dev_combo + WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) + AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) + AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) +AND j.concept_name ~ ' / '; + +-- add packs of Primary lateral and Secondary upward (Class A + Class D, order = 21) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + j.*, + 21, + 'Pack: Pr lat + Sec up' +FROM class_to_drug_new a + JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + JOIN concept d + ON d.concept_Id = r.concept_id_2 + AND d.concept_class_id = 'Clinical Drug' + JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id + JOIN concept j ON j.concept_id = r2.concept_id_2 +WHERE a.concept_class_id = 'Clinical Drug Form' +AND r.invalid_reason IS NULL +AND r2.invalid_reason IS NULL +AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') +AND j.standard_concept = 'S' +AND a.class_code IN (SELECT class_code + FROM dev_combo + WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) + AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) +AND j.concept_name ~ ' / ' +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + +-- add additional packs of Primary lateral and Secondary upward (Class A + Class D, order = 22) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + j.*, + 22, + 'Pack: Pr lat + Sec up additional' +FROM class_to_drug_new a + JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id + JOIN concept j ON j.concept_id = r2.concept_id_2 +WHERE r2.invalid_reason IS NULL +AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') +-- Boxes are suspicious +AND j.standard_concept = 'S' +AND a.class_code IN (SELECT class_code + FROM dev_combo + WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) + AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) +AND j.concept_name ~ ' / ' +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + +--- add missing Packs using previous version of class_to_drug +INSERT INTO class_to_drug_new +SELECT DISTINCT b.concept_code AS class_code, + b.concept_name AS class_name, + c.*, + 23, + 'packs: from sources.c_t_d' +FROM sources.class_to_drug a + JOIN concept_manual b ON b.concept_code = a.class_code + JOIN concept c + ON a.concept_id = c.concept_id + AND c.standard_concept = 'S' + AND b.invalid_reason IS NULL +WHERE (class_code,a.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new) +AND a.concept_class_id ~ 'Pack' +AND a.class_code NOT IN ('G01AF55','S03CA04','S01CA03','S02CA03')-- gives packs with erroneous forms +AND a.class_code NOT IN ('B03AE01','C07BB52','D01AC52','C10AD52') -- wrong ing combo ; -INSERT INTO pack_all -with a AS ( - SELECT p.concept_id, p.concept_name, p.concept_class_id, class_code,class_name, r.concept_id_2 - FROM pack_temp p - JOIN internal_relationship_stage ON substring(concept_code_1, '\w+') = class_code - JOIN drug_concept_stage d ON concept_code_2 = concept_code AND d.concept_class_id = 'Dose Form' - JOIN relationship_to_concept r ON r.concept_code_1 = concept_code_2), - b AS (SELECT p.*, r2.concept_id_2 - FROM pack_temp p - JOIN devv5.concept_ancestor r on concept_id = descendant_concept_id - JOIN concept_relationship r2 - ON r2.concept_id_1 = ancestor_concept_id AND r2.invalid_reason IS NULL - AND relationship_id = 'RxNorm has dose form') -SELECT a.concept_id, a.concept_name, a.concept_class_id, a.class_code, a.class_name -FROM a -JOIN b USING (concept_id, concept_id_2) -WHERE (concept_id, a.class_code) NOT IN (SELECT concept_id, class_code FROM pack_all) -UNION -SELECT p.* -FROM pack_temp p -JOIN reference r on concept_code = p.class_code -WHERE (concept_id, p.class_code) NOT IN (SELECT concept_id, class_code FROM pack_all) -; -- 3945 - -INSERT INTO class_to_drug -( - class_code, - class_name, - concept_id, - concept_name, - concept_class_id, - "order" -) -SELECT DISTINCT class_code, - class_name, - concept_id, - concept_name, - concept_class_id, - 8 -FROM pack_all; -- 4882 - --- 4.11 fix packs -INSERT INTO class_to_drug +-- Enrich pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' (they are always used as packs) +INSERT INTO class_to_drug_new SELECT DISTINCT class_code, class_name, - c.concept_id, - c.concept_name, - c.concept_class_id, - 8 -FROM class_to_drug f + c.*, + 24, + 'Pack: semi-manual' +FROM class_to_drug_new f JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) JOIN devv5.concept c ON c.concept_id = descendant_concept_id AND c.concept_class_id LIKE '%Pack%' -WHERE f.class_code ~ 'G03FB|G03AB' -- packs -; -- 1809 - --- get rid of all other forms except Packs --- 4.12 Progestogens and estrogens -DELETE -FROM class_to_drug -WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens -AND concept_class_id IN ('Clinical Drug Form','Ingredient'); -- 68 - --- 4.14 Solution for the first run: for inambiguous ATC classes (those that classify an ingredient through only one class) --- we relate this ATC class to the entire group of drugs that have this ingredient. -DROP TABLE IF EXISTS interim; -CREATE TABLE interim -AS -WITH a AS -( - SELECT DISTINCT class_code, - class_name, - c.concept_id, - c.concept_name - FROM class_to_drug crd - JOIN devv5.concept_ancestor ON crd.concept_id = descendant_concept_id - JOIN concept c - ON c.concept_id = ancestor_concept_id - AND c.concept_class_id = 'Ingredient' -) -SELECT * -FROM (SELECT COUNT(*) OVER (PARTITION BY TRIM(REGEXP_REPLACE(class_name,'\(.*\)',''))) AS cnt, - class_code, - class_name, - a.concept_id, - a.concept_name -- regexp for (vit C) - FROM a) a -WHERE cnt = 1 -AND class_code NOT IN (SELECT class_code FROM class_drugs_scraper -WHERE class_code ~ '^J07|^A10A' -AND length(class_code)=7) ---and class_code NOT IN ('G02BB02') +WHERE f.class_code ~ 'G03FB|G03AB' ; - -DELETE -FROM interim -WHERE class_name IN ( - SELECT class_name - FROM (SELECT DISTINCT class_code, class_name FROM class_drugs_scraper) a - GROUP BY class_name - HAVING count(1) > 1); -- 58 - +-- get rid of all other concept_class_ids except Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' DELETE -FROM interim -WHERE concept_id IN ( - SELECT a.concept_id - FROM interim a - JOIN interim b ON a.concept_id = b.concept_id - WHERE a.class_code != b.class_code); -- 105 - -DROP TABLE IF EXISTS interim_2; -CREATE TABLE interim_2 AS -WITH a AS ( -SELECT DISTINCT class_code, class_name, c.concept_id,c.concept_name -FROM class_to_drug crd -JOIN devv5.concept_ancestor on crd.concept_id = descendant_concept_id -JOIN concept c on c.concept_id = ancestor_concept_id AND c.concept_class_id = 'Ingredient' - ), -b AS (SELECT concept_id FROM a -GROUP BY concept_id having count(1)=1), -c AS (SELECT a.class_code, a.class_name, a.concept_id FROM a JOIN b USING(concept_id) - ) -SELECT c.* -FROM c -JOIN interim USING(class_code) -WHERE class_code NOT IN (SELECT class_code FROM class_to_drug_manual) -; +FROM class_to_drug_new +WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens +AND concept_class_id !~ 'Pack'; -DELETE -FROM interim_2 -WHERE class_name IN ( - SELECT class_name - FROM (SELECT DISTINCT class_code, class_name FROM class_drugs_scraper) a - GROUP BY class_name - HAVING count(1) > 1); --0 - --- -- inambiguous only for forms -DELETE FROM class_to_drug -WHERE class_code IN (SELECT class_code FROM interim) -; -- 6988 (9196) - --- add missing forms for the monocomponent ATC Drug Classes -INSERT INTO class_to_drug -- ingredients are partially inambiguous -(class_code, class_name, concept_id, concept_name, concept_class_id, "order") -SELECT class_code, class_name, c.concept_id, c.concept_name, c.concept_class_id, 2 -FROM interim i -JOIN devv5.concept_ancestor ca on i.concept_id = ancestor_concept_id -JOIN concept c on descendant_concept_id = c.concept_id -WHERE c.concept_class_id = 'Clinical Drug Form' AND c.concept_name NOT LIKE '% / %' -and (class_code, c.concept_id) not in (select class_code, concept_id from class_to_drug) -; -- 8653 - --- remove absolutely inambiguous Ingredient -DELETE FROM class_to_drug -WHERE class_code IN(SELECT class_code FROM interim_2) -; -- 4019 --- add their maps -INSERT INTO class_to_drug -- ingredients are absolutely inambiguous -(class_code, class_name, concept_id, concept_name, concept_class_id, "order") -SELECT DISTINCT class_code, class_name, concept_id, c.concept_name, c.concept_class_id, 3 -FROM interim_2 i -JOIN concept c USING (concept_id) -where (class_code, c.concept_id) not in (select class_code, concept_id from class_to_drug) -; -- 1415 - --- also adding those that don't have forms, but are unique -with ing AS ( +-- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique +INSERT INTO class_to_drug_new + WITH ing AS ( SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt FROM ( -SELECT DISTINCT substring(i.concept_code_1,'\w+'), r.concept_code_1, concept_id_2 + SELECT DISTINCT substring(i.concept_code_1,'\w+'), r.concept_code_1, concept_id_2 FROM relationship_to_concept r JOIN internal_relationship_stage i on i.concept_code_2 = r.concept_code_1 JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' ) a GROUP BY concept_id_2,concept_code_1 ), -drug AS ( -SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 + drug AS ( + SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 FROM internal_relationship_stage i WHERE not exists (SELECT 1 FROM internal_relationship_stage i2 WHERE i.concept_code_2=i2.concept_code_2 AND substring(i.concept_code_1,'\w+')!=substring(i2.concept_code_1,'\w+')) ), - drug_name AS ( + drug_name AS ( SELECT DISTINCT class_code,class_name, concept_id_2 FROM ing -JOIN drug ON concept_code_2=concept_code_1 +JOIN drug ON concept_code_2 = concept_code_1 JOIN class_drugs_scraper ON code = class_code WHERE cnt=1 AND class_name = concept_code_1 -and code NOT IN (SELECT class_code FROM class_to_drug) - ), +AND code NOT IN (SELECT class_code FROM class_to_drug_new) +), all_drug AS ( -SELECT class_code,class_name, concept_id,concept_name, concept_class_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt -FROM drug_name -JOIN concept on concept_id_2 = concept_id +SELECT class_code, class_name, concept_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt +FROM drug_name a +JOIN concept c on c.concept_id = a.concept_id_2 ) -INSERT INTO class_to_drug -SELECT class_code, class_name, concept_id, concept_name,concept_class_id,3 -FROM all_drug -WHERE cnt=1 -and (class_code, concept_id) not in (select class_code, concept_id from class_to_drug) -; -- 25 - -UPDATE class_to_drug -SET "order" = 5 -WHERE "order" in (98,99); -- 649 - -UPDATE class_to_drug -SET "order" = 4 -WHERE class_name ~ ' and ' -AND NOT class_name ~ 'adrenergics|analgesics|antispasmodics|antibacterials|antiflatulents|imidazoles|minerals|polyfructosans|triazoles|natural phospholipids|lactic acid producing organisms|antiflatulents|beta-lactamase inhibitor|sulfonylureas|antibiotics|antiinfectives|antiseptics|artificial tears|contact laxatives|corticosteroids|ordinary salt combinations|imidazoles/triazoles|cough suppressants|diuretics|drugs for obstructive airway diseases|expectorants|belladonna alkaloids|mucolytics|mydriatics|non-opioid analgesics|organic nitrates|potassium-sparing agents|proton pump inhibitors|psycholeptics|thiazides|snake venom antiserum|fat emulsions|amino acids|acid preparations|sulfur compounds|stramoni preparations|protamines|penicillins|comt inhibitor|cannabinoids|decarboxylase inhibitor|edetates|barbiturates|excl|combinations of|derivate|with' -AND "order" in (98,99); -- 89 - --- calcium compounds -UPDATE class_to_drug -SET "order" = 7 -WHERE class_name ~ 'compounds' -AND "order" = 2; -- 2 - --- working with duplicates to remove a and b vs a/b, comb -DELETE FROM class_to_drug -WHERE (class_code,concept_id) IN( -SELECT b.class_code, b.concept_id FROM class_to_drug a -JOIN class_to_drug b on a.concept_id = b.concept_id -WHERE a."order" = 4 AND b."order" in (6,7)) -; -- 0 - --- 'Progestogens and estrogens, sequential preparations' should be Clinical Packs only --- check manual links for this!!! -- fix concept_class_ids -DELETE -FROM class_to_drug -WHERE class_code ~ 'G03FB|G03AB' -AND concept_class_id NOT IN ('Clinical Pack') -; -- 1371 +SELECT DISTINCT a.class_code, + a.class_name, + c.*, + 25, + 'ATC Classes WO Dose Forms to unique Drug Product' +FROM all_drug a +join concept c on c.concept_id = a.concept_id +WHERE cnt = 1 +AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); -update class_to_drug -set "order" = 99 -where "order" is null; -- 440 +-- add links from ATC Monocomponent Classes which do not have Dose Forms to Ingredients +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT * +FROM concept_manual +WHERE concept_code NOT IN (SELECT class_code FROM class_to_drug_new) +AND concept_class_Id = 'ATC 5th' +AND invalid_reason IS NULL +AND concept_code NOT IN (SELECT class_code FROM atc_inexistent)) +SELECT DISTINCT a.concept_code, + a.concept_name, + c.*, + 26, + 'Monocomponent ATC WO Dose Forms' +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (concept_code_1,'\w+') = a.concept_code + JOIN relationship_to_concept r ON r.concept_code_1 = i.concept_code_2 + JOIN concept c + ON c.concept_id = r.concept_id_2 + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND a.concept_code NOT IN (SELECT class_code FROM dev_combo); + +-- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (do we need them?) +INSERT INTO class_to_drug_new WITH t1 +AS +(SELECT * +FROM concept_manual +WHERE concept_code NOT IN (SELECT class_code FROM class_to_drug_new) +AND concept_class_Id = 'ATC 5th' +AND invalid_reason IS NULL +AND concept_code NOT IN (SELECT class_code FROM atc_inexistent) --- add missing alive OLD links(to solve) -insert into class_to_drug -WITH t1 AS -( - SELECT atc_code, - concept_id, - concept_code, - concept_name - FROM dev_atc.atc_all_relationships - WHERE vocabulary_id ~ 'Rx' and relationship_id = 'ATC - RxNorm' - and concept_class_id = 'Clinical Drug Form' -EXCEPT - SELECT atc_code,concept_id,concept_code,concept_name FROM dev_atc.atc_all_new_relationships_2 -WHERE vocabulary_id ~ 'Rx' and relationship_id = 'ATC - RxNorm' - and concept_class_id = 'Clinical Drug Form' ) -SELECT distinct atc_code, d.concept_name as class_name, c.concept_id, c.concept_name, c.concept_class_id, 23 +SELECT DISTINCT a.concept_code, + a.concept_name, + c.*, + 27, + 'Multicomp ATC Class to Ingredient' FROM t1 a + JOIN dev_combo k + ON k.class_code = a.concept_code + AND rnk IN (1, 2) JOIN concept c - ON c.concept_id = a.concept_id - JOIN concept_manual d on d.concept_code = a.atc_code - AND c.standard_concept = 'S' - AND d.invalid_reason is null - where (atc_code, c.concept_id) not in (select class_code, concept_id from class_to_drug); -- 878 (855) + ON c.concept_id = k.concept_id + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S'; + +-- remove excessive links to children among Packs +WITH t1 AS +( + SELECT * + FROM class_to_drug_new + WHERE class_code IN (SELECT class_code + FROM class_to_drug_new + GROUP BY class_code + HAVING COUNT(1) >= 2) + and class_code in (select class_code from class_to_drug_new where concept_class_id ~ 'Pack') + and class_code not in (select class_code from class_to_drug_new where concept_class_id !~ 'Pack') +) +delete from class_to_drug_new where class_code||concept_id in ( +SELECT a1.class_code|| + a2.concept_id +FROM t1 a1 -- papa + JOIN t1 a2 -- child +ON a1.class_code = a2.class_code + JOIN concept_ancestor ca + ON a1.concept_id = ca.ancestor_concept_id + AND a2.concept_id = ca.descendant_concept_id + AND a1.concept_id <> a2.concept_id + and a2.concept_class_id ~ 'Pack'); + +-- remove excessive links to children among unpacked Drug Products :) +WITH t1 AS +( + SELECT * + FROM class_to_drug_new + WHERE class_code IN (SELECT class_code + FROM class_to_drug_new + GROUP BY class_code + HAVING COUNT(1) >= 2) + and class_code not in (select class_code from class_to_drug_new where concept_class_id ~ 'Pack') +) +delete from class_to_drug_new where class_code||concept_id in ( +SELECT a1.class_code|| + a2.concept_id -- child mapping +FROM t1 a1 -- papa + JOIN t1 a2 -- child +ON a1.class_code = a2.class_code + JOIN concept_ancestor ca + ON a1.concept_id = ca.ancestor_concept_id + AND a2.concept_id = ca.descendant_concept_id + AND a1.concept_id <> a2.concept_id + and a2.concept_class_id !~ 'Pack'); + +-- run load_stage.sql From 9e4c8c796ff40a63321d6eeb0df92642f9be77f2 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 2 Sep 2021 16:37:17 +0300 Subject: [PATCH 09/45] Update load_input.sql code improvement --- ATC/load_input.sql | 1432 +++++++++++++------------------------------- 1 file changed, 421 insertions(+), 1011 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 170d7ef7b..31625025b 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -16,10 +16,11 @@ * Authors: Anna Ostropolets, Polina Talapova * Date: Jul 2021 **************************************************************************/ --- in ATC we do not use ds_stage AND pc_stage + DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; -DROP TABLE IF EXISTS relationship_to_concept CASCADE;; +DROP TABLE IF EXISTS relationship_to_concept CASCADE; +-- ds_stage AND pc_stage are not used in the ATC deployment CREATE TABLE drug_concept_stage ( concept_name VARCHAR(255), @@ -45,7 +46,7 @@ CREATE TABLE relationship_to_concept precedence INT, conversion_factor FLOAT); ---create indexes AND constraints +-- create indexes AND constraints DROP INDEX if exists irs_concept_code_1; DROP INDEX if exists irs_concept_code_2; DROP INDEX if exists dcs_concept_code; @@ -67,16 +68,19 @@ CREATE INDEX irs_unique_concept_code /************************************* ***** internal_relationship_stage **** **************************************/ +---------------- +-- Dose Forms -- +---------------- -- increase the LENGTH for concept_code_1 AND concept_code_2 fields to infinity ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_1 TYPE VARCHAR; ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_2 TYPE VARCHAR; ------------------------ --- oral formulations -- ------------------------- --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_oral; -CREATE TABLE dev_oral -AS + +-- create a temporary table WITH all ATC-related RxN/RxE Dose Forms +DROP TABLE if exists dev_form; +CREATE TABLE dev_form +AS ( +with dev_oral -- 1 - Oral forms +as (SELECT DISTINCT d.* FROM concept_relationship r JOIN concept c ON c.concept_Id = r.concept_id_1 @@ -84,72 +88,11 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217214 -- Oral Product (Dose Form Group) +WHERE concept_id_1 IN (36217214, 36244020, 36217223) -- Oral Product | Buccal Product | Paste product (Dose Form Group) AND relationship_id = 'RxNorm inverse is a' -AND d.concept_name !~* 'sublingual'); -- will be processed separately - --- add links between Oral ATC Drug Classes AND RxN/RxE Dose Forms into the internal_relationship_stage -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT -a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_oral b -WHERE a.concept_name ~* 'oral|systemic|chewing gum' -- respective ATC dose forms (preliminarily converted from adm_r and added to the names in concept_manual) -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; -- indicates alive ATC code - --- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_oral b -WHERE a.concept_name ~* 'oral|systemic|chewing gum' -AND a.invalid_reason IS NULL -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code - FROM t1 a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) -- remove all unnecessary information after the semicolon - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_oral b -WHERE a.concept_name ~* 'oral|systemic|chewing gum' -AND a.invalid_reason IS NULL -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code - FROM t1 a - JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = upper(TRIM(REGEXP_REPLACE(a.concept_name, ';.*$', ''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); ------------------------------ --- sublingual formulations -- ------------------------------ --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_sub; -CREATE TABLE dev_sub -AS +),-- sublingual route is included as well despite the fact it is processed separately +dev_sub -- 2 - Sublingual forms +AS (SELECT DISTINCT d.* FROM concept_relationship r JOIN concept c ON c.concept_Id = r.concept_id_1 @@ -157,70 +100,11 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217214 +WHERE concept_id_1 in (36217214, 36244020) -- Sublingual Product | Buccal Product (Dose Form Group) AND relationship_id = 'RxNorm inverse is a' -AND d.concept_name ~* 'sublingual'); -- should be separated FROM oral forms in the ATC vocabulary. - --- add links between Sublingual ATC Drug Classes and Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_sub b -WHERE a.concept_name ~* 'sublingual' -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; -- 507 - --- add links between Sublingual ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_sub b -WHERE a.concept_name ~* 'sublingual' -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -FROM t1 a -JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Sublingual ATC Drug Classes AND Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_sub b -WHERE a.concept_name ~* 'sublingual' -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); ------------------------------ --- parenteral formulations -- ------------------------------ --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_parenteral; -CREATE TABLE dev_parenteral +AND d.concept_name ~* 'sublingual' +), -- should be separated FROM oral forms in the ATC vocabulary. + dev_parenteral -- 3 - Parenteral forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -229,70 +113,9 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217210 -- Injectable Product -AND relationship_id = 'RxNorm inverse is a'); -- returns all children of Injectable Product - --- add links between Injectable ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_parenteral b -WHERE a.concept_name ~* 'parenteral|systemic' -- respective ATC routes -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; - --- add links between Parenteral ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_parenteral b -WHERE a.concept_name ~* 'parenteral|systemic' -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept c - ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Parenteral ATC Drug Classes AND Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_parenteral b -WHERE a.concept_name ~* 'parenteral|systemic' -AND a.invalid_reason IS NULL - ) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); ------------------------- --- nasal formulations -- ------------------------- --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_nasal; -CREATE TABLE dev_nasal +WHERE concept_id_1 in ( 36217210, 36217222) -- Injectable Product | Irrigation Product +AND relationship_id = 'RxNorm inverse is a'), -- returns all children of Injectable Product + dev_nasal -- 4 - Nasal forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -302,72 +125,9 @@ FROM concept_relationship r AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' WHERE concept_id_1 = 36217213 -- Nasal Product -AND relationship_id = 'RxNorm inverse is a'); -- returns all children of Nasal Product - --- add those which are out of ancestry (Nasal Pin) -INSERT INTO dev_nasal -SELECT * FROM concept WHERE concept_id = 43563498; - --- add links between Nasal ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_nasal b -WHERE a.concept_name ~* 'nasal' -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; - --- add links between Nasal ATC Drug Classes AND Standars Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_nasal b -WHERE a.concept_name ~* 'nasal' -AND a.invalid_reason IS NULL -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Nasal ATC Drug Classes AND Standars Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_nasal b -WHERE a.concept_name ~* 'nasal' -AND a.invalid_reason IS NULL -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); --------------------------- --- topical formulations -- --------------------------- --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE IF EXISTS dev_topic; -CREATE TABLE dev_topic +AND relationship_id = 'RxNorm inverse is a' +), -- returns all children of Nasal Product +dev_topic -- 5 - Topical forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -378,127 +138,9 @@ FROM concept_relationship r AND d.concept_class_id = 'Dose Form' WHERE concept_id_1 IN ( 36217206,36244040,36244034,36217219,-- Topical Product|Soap Product|Shampoo Product|Drug Implant Product - 36217223,36217212,36217224) -- Paste Product|Mucosal Product|Prefilled Applicator Product -AND relationship_id = 'RxNorm inverse is a'); -- returns all children of Topical Product - --- add those which are out of Dose Form ancestry -INSERT INTO dev_topic -SELECT * FROM concept WHERE concept_id IN (43126087); -- Medicated Nail Polish - --- add links between 1) genuine Topical ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_topic b -WHERE a.concept_name ~* 'topical' -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; -- exclude transdermal systems - --- add links between 2) Transdermal or Implantable ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_topic b -WHERE a.concept_name ~* 'topical' -AND a.invalid_reason IS NULL; - --- add links between Topical ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_topic b -WHERE a.concept_name ~* 'topical' -AND a.invalid_reason IS NULL -) -- respective ATC routes -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Ingredient name AS a code -FROM t1 a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Topical ATC Drug Classes AND Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_topic b -WHERE a.concept_name ~* 'topical' -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Transdermal or Implantable ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_topic b -WHERE a.concept_name ~* 'transdermal|implant' -- respective ATC routes -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2-- Standard Ingredient name AS a code -FROM t1 a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Transdermal or Implantable ATC Drug Classes AND Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_topic b -WHERE a.concept_name ~* 'transdermal|implant' -- respective ATC routes -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); ------------------------------ --- mouthwash formulations -- ------------------------------ --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_mouth; -CREATE TABLE dev_mouth + 36217223,36217212,36217224, 19016586) -- Paste Product|Mucosal Product|Prefilled Applicator Product|Irrigation Solution +AND relationship_id = 'RxNorm inverse is a'), -- returns all children of Topical Product +dev_mouth -- 6 - Local oral forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -507,47 +149,10 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36244022 -- Mouthwash Product (Dose Form Group) +WHERE concept_id_1 in (36244022, 36217223) -- Mouthwash Product | Paste Product (Dose Form Group) AND relationship_id = 'RxNorm inverse is a' -AND d.concept_name ~* 'mouthwash'); - --- add links between Local Oral ATC Drug Classes AND Rx Mouthwash formulations -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_mouth b -WHERE a.concept_name ~* 'local oral' -- respective ATC route -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; -- 21 - --- add links between Local Oral ATC Drug Classes AND Ingredients using the concept table (up to date no need to use tne concept_synonym but you may check) -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_mouth b -WHERE a.concept_name ~* 'local oral' -- respective ATC routes -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -FROM t1 a -JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); -------------------------- --- rectal formulations -- -------------------------- --- create a temporary table WITH all related Dose Forms -DROP TABLE IF EXISTS dev_rectal; -CREATE TABLE dev_rectal +AND d.concept_name ~* 'mouthwash'), +dev_rectal -- 7 - Rectal forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -557,69 +162,9 @@ FROM concept_relationship r AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' WHERE concept_id_1 = 36217211 -- Rectal Product -AND relationship_id = 'RxNorm inverse is a'); - --- add links between Rectal ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_rectal b -WHERE a.concept_name ~* 'rectal' -- respective ATC route -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; -- 1150 - --- add links between Rectal ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_rectal b -WHERE a.concept_name ~* 'rectal' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Rectal ATC Drug Classes AND Standard Ingredients using the concept_synonym synonym -INSERT INTO internal_relationship_stage -( concept_code_1, - concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_rectal b -WHERE a.concept_name ~* 'rectal' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); --------------------------- --- vaginal formulations -- --------------------------- --- create a temporary table WITH all related Dose Forms -DROP TABLE IF EXISTS dev_vaginal; -CREATE TABLE dev_vaginal +AND relationship_id = 'RxNorm inverse is a' +), +dev_vaginal -- 8 - Vaginal forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -630,69 +175,8 @@ FROM concept_relationship r AND d.concept_class_id = 'Dose Form' AND d.invalid_reason IS NULL WHERE concept_id_1 = 36217209 -AND relationship_id = 'RxNorm inverse is a'); -- Vaginal Product - --- add links between Vaginal ATC Drug Classes and Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_vaginal b -WHERE a.concept_name ~* 'vaginal' -- respective ATC routes -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; - --- add links between Vaginal ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_vaginal b -WHERE a.concept_name ~* 'vaginal' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Vaginal ATC Drug Classes AND Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_vaginal b -WHERE a.concept_name ~* 'vaginal' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); ---------------------------- --- urethral formulations -- ---------------------------- --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE IF EXISTS dev_urethral; -CREATE TABLE dev_urethral -AS +AND relationship_id = 'RxNorm inverse is a'), -- Vaginal Product + dev_urethral AS -- 9 - Urethral forms (SELECT DISTINCT d.* FROM concept_relationship r JOIN concept c ON c.concept_Id = r.concept_id_1 @@ -701,46 +185,9 @@ FROM concept_relationship r AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' WHERE concept_id_1 = 36217225 -- Urethral Product -AND relationship_id = 'RxNorm inverse is a'); - --- add links between Urethral ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_urethral b -WHERE a.concept_name ~* 'urethral' -- respective ATC route -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; - --- add links between Urethral ATC Drug Classes AND Standard Ingredients using the concept table (up to date no need to use tne concept_synonym but you may check) -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) - WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_urethral b -WHERE a.concept_name ~* 'urethral' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code -c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); ------------------------------- ---- ophtalmic formulations --- ------------------------------- --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_opht; -CREATE TABLE dev_opht -AS +AND relationship_id = 'RxNorm inverse is a'), + dev_opht -- 10 - Ophthalmic forms +AS (SELECT DISTINCT d.* FROM concept_relationship r JOIN concept c ON c.concept_Id = r.concept_id_1 @@ -748,70 +195,10 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217218 -- Ophthalmic Product (Dose Form Group) +WHERE concept_id_1 in (36217218, 36217224) -- Ophthalmic Product | Prefilled Applicator (Dose Form Group) AND relationship_id = 'RxNorm inverse is a' -AND d.concept_name ~* 'ophthalmic'); - --- add links between Ophthalmic ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_opht b -WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; - --- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_opht b -WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -FROM t1 a -JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_opht b -WHERE a.concept_name ~* 'ophthalmic' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - ------------------------- ---- otic formulations --- --------------------------- --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_otic; -CREATE TABLE dev_otic +AND d.concept_name ~* 'ophthalmic'), +dev_otic -- 11 - Otic forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -822,68 +209,8 @@ FROM concept_relationship r AND d.concept_class_id = 'Dose Form' WHERE concept_id_1 = 36217217 -- Otic Product (Dose Form Group) AND relationship_id = 'RxNorm inverse is a' -); - --- add links between Ophthalmic ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_otic b -WHERE a.concept_name ~* '\yotic' -- respective ATC route -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; - --- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_otic b -WHERE a.concept_name ~* '\yotic' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -FROM t1 a -JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Ophthalmic ATC Drug Classes AND Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_otic b -WHERE a.concept_name ~* '\yotic' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code -FROM t1 a - JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); ---------------------------- --- inhalant formulations -- ---------------------------- --- create a temporary table WITH all related RxN/RxE Dose Forms -DROP TABLE if exists dev_inhal; -CREATE TABLE dev_inhal +), +dev_inhal -- 12 - Inhalation forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -892,63 +219,89 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 IN (36217207, 36244037) -- Inhalant Product| Oral Spray Product -AND relationship_id = 'RxNorm inverse is a'); +WHERE concept_id_1 IN (36217207, 36244037) -- Inhalant Product| Oral Spray Product +AND relationship_id = 'RxNorm inverse is a') +select * from ( +select *, 'dev_oral' as df from dev_oral -- 1 +UNION ALL +select *, 'dev_sub' from dev_sub -- 2 +UNION ALL +select *, 'dev_parenteral' from dev_parenteral -- 3 +UNION ALL +select *, 'dev_nasal' from dev_nasal -- 4 +UNION ALL +select *, 'dev_topic' from dev_topic -- 5 +UNION ALL +select *, 'dev_mouth' from dev_mouth -- 6 +UNION ALL +select *,'dev_rectal' from dev_rectal -- 7 +UNION ALL +select *, 'dev_vaginal' from dev_vaginal -- 8 +UNION ALL +select *, 'dev_urethral' from dev_urethral -- 9 +UNION ALL +select *, 'dev_opht' from dev_opht -- 10 +UNION ALL +select *, 'dev_otic' from dev_otic -- 11 +UNION ALL +select *, 'dev_inhal' from dev_inhal -- 12 +)l); --- add links between Inhalant ATC Drug Classes AND Rx Dose Forms -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name -- OMOP Dose Form name treated AS a code +-- connect all existing RxN/RxE forms of interest from dev_form to the ATC +DROP TABLE if exists atc_to_form; +CREATE TABLE atc_to_form AS +SELECT DISTINCT +a.concept_name, a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code FROM concept_manual a, - dev_inhal b -WHERE a.concept_name ~* 'inhalant' -- respective ATC route -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; + dev_form b +WHERE a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th' +AND +(( a.concept_name ~* 'oral|systemic|chewing gum' and b.df = 'dev_oral') -- 1 +OR (a.concept_name ~* 'sublingual' and b.df = 'dev_sub') -- 2 +OR (a.concept_name ~* 'parenteral|systemic|irrigat' and b.df = 'dev_parenteral') -- 3 +OR (a.concept_name ~* 'nasal'and b.df = 'dev_nasal') -- 4 +OR (a.concept_name ~* 'topical' AND b.df = 'dev_topic') -- 5 +OR (a.concept_name ~* 'transdermal|implant|systemic' AND b.df = 'dev_topical' and b.concept_name ~* 'transdermal|Drug Implant') +OR (a.concept_name ~* 'local oral' AND b.df = 'dev_mouth') -- 7 +OR (a.concept_name ~* 'rectal' AND b.df = 'dev_rectal') +OR (a.concept_name ~* 'vaginal' AND b.df = 'dev_vaginal') +OR (a.concept_name ~* 'urethral' AND b.df = 'dev_urethral') +OR (a.concept_name ~* 'ophthalmic' AND b.df = 'dev_opht') +OR (a.concept_name ~* '\yotic' AND b.df = 'dev_otic') +OR (a.concept_name ~* 'inhalant|systemic' AND b.df = 'dev_inhal')) +; --- add links between Inhalant ATC Drug Classes AND Standard Ingredients using the concept table +-- add links between Oral ATC Drug Classes AND RxN/RxE Dose Forms into the internal_relationship_stage +TRUNCATE internal_relationship_stage; INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_inhal b -WHERE a.concept_name ~* 'inhalant' -- respective ATC route -AND a.invalid_reason IS NULL -) +SELECT DISTINCT concept_code_1, concept_code_2 from atc_to_form +WHERE (concept_code_1, concept_code_2) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +INSERT INTO internal_relationship_stage SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -FROM t1 a - JOIN concept c ON UPPER (c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code + FROM atc_to_form a + JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) -- remove all unnecessary information after the semicolon AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between Inhalant ATC Drug Classes AND Standard Ingredients using the concept_synonym table +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + +-- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept_synonym table INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.concept_name -- ATC name to be used AS a key for JOIN -FROM concept_manual a, - dev_inhal b -WHERE a.concept_name ~* 'inhalant' -- respective ATC route -AND a.invalid_reason IS NULL -) -SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -FROM t1 a - JOIN concept_synonym b ON UPPER (b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) + SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code + FROM atc_to_form a + JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = upper(TRIM(REGEXP_REPLACE(a.concept_name, ';.*$', ''))) JOIN concept c ON c.concept_id = b.concept_id AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + AND c.invalid_reason IS NULL +WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); -------------------------------- -- Ingredients W/O Dose Forms -- -------------------------------- @@ -958,7 +311,8 @@ INSERT INTO internal_relationship_stage SELECT DISTINCT a.concept_code AS concept_code_1, -- for such Drug Classes use ATC code only c.concept_name AS concept_code_2 FROM concept_manual a - JOIN concept c ON TRIM(UPPER (REGEXP_REPLACE (c.concept_name,'\s+|\W+','','g'))) = TRIM( UPPER (REGEXP_REPLACE (a.concept_name,'\s+|\W+| \(.*\)|, combinations.*|;.*$','','g'))) + JOIN concept c + ON TRIM(UPPER (REGEXP_REPLACE (c.concept_name,'\s+|\W+','','g'))) = TRIM( UPPER (REGEXP_REPLACE (a.concept_name,'\s+|\W+| \(.*\)|, combinations.*|;.*$','','g'))) -- to neglect spaces, non-word characters, additional information and dose forms AND a.concept_class_id = 'ATC 5th' AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' @@ -978,11 +332,13 @@ FROM concept_manual a AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.invalid_reason IS NULL - WHERE (a.concept_code, c.concept_name) not IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); -/***************************** -**** combined ATC Classes **** -******************************/ --- separate all ATC Combo Classes + WHERE (a.concept_code, c.concept_name) not IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); + +-- note, name matching with Non-standard OMOP drugs and cross-walk to Standard via concept_relationship gives a lot of errors (clean up is required). That is why this step is ignored here. +/************************** +**** ATC Combo Classes **** +***************************/ +-- separate all ATC Combo Classes DROP TABLE if exists combo_pull; CREATE TABLE combo_pull AS @@ -992,12 +348,12 @@ AS concept_name AS class_name, SPLIT_PART(concept_name,';','1') AS nm FROM concept_manual -WHERE SPLIT_PART(concept_name,';','1') ~ ' and |combinat|preparations|acids|animals|antiinfectives|compounds|lytes\y|flowers|^glycerol|grass pollen|bacillus|^oil|alkaloids|\/|antiserum|organisms|antiseptics' +WHERE SPLIT_PART(concept_name,';','1') ~* ' and |combinat|preparations|acids|animals|antiinfectives|compounds|lytes\y|flowers|^glycerol|grass pollen|bacillus|^oil|alkaloids|\/|antiserum|organisms|antiseptics' AND invalid_reason IS NULL AND concept_class_id = 'ATC 5th' AND concept_name !~* 'varicella/zoster|tositumomab/iodine') SELECT*FROM t1); --- create a table for ATC Combo Class mappings, start with the 1st ATC Combo Ingredient using the concept table and full name match +-- obtain 1st ATC Combo Ingredient using the concept table and full name match DROP TABLE if exists dev_combo; CREATE TABLE dev_combo AS @@ -1013,7 +369,7 @@ WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx'); --- obtain 1st ATC Combo Ingredient using the concept table and partial name match +-- obtain 1st ATC Combo Ingredient using the concept table and full name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -1028,7 +384,7 @@ AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); --- obtain 1st ATC Combo Ingredient using the concept_synonym table and name match +-- obtain 1st ATC Combo Ingredient using the concept_synonym table and full name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -1042,9 +398,9 @@ JOIN concept d ON d.concept_id = cs.concept_id WHERE d.standard_concept = 'S' AND d.concept_class_id = 'Ingredient' AND d.vocabulary_id ~ 'Rx' -AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); -- 15 +AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); --- obtain 1st ATC Combo Ingredient using the concept table and partial name match +-- last hope INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -1057,7 +413,7 @@ FROM combo_pull a WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' -AND (class_code) NOT IN (select class_code FROM dev_combo); -- 3 +AND (class_code) NOT IN (select class_code FROM dev_combo); -- obtain 2nd ATC Combo Ingredient using the concept table and full name match INSERT INTO dev_combo @@ -1071,7 +427,7 @@ FROM combo_pull a JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (a.nm,' and ', 2))) WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx'; -- 249 +AND c.vocabulary_id ~ 'Rx'; -- obtain 2nd ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo @@ -1087,7 +443,7 @@ WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND c.concept_id NOT IN (19049024, 19136048); -- 4 +AND c.concept_id NOT IN (19049024, 19136048); -- obtain 2nd ATC Combo Ingredient using the concept_synonym table and partial name match INSERT INTO dev_combo @@ -1103,7 +459,7 @@ JOIN concept d ON d.concept_id = cs.concept_id WHERE d.standard_concept = 'S' AND d.concept_class_id = 'Ingredient' AND d.vocabulary_id ~ '^Rx' -AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); -- 1 +AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); -- last hope for the 2nd one INSERT INTO dev_combo @@ -1120,7 +476,7 @@ AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) AND c.concept_id NOT IN (19049024, 19136048) -and class_code in ('R03AK10', 'R03AL01', 'R03AL03', 'R03AL04', 'R03AL05', 'R03AL06', 'R03AL07', 'R03AL10') ; -- 4 +and class_code in ('R03AK10', 'R03AL01', 'R03AL03', 'R03AL04', 'R03AL05', 'R03AL06', 'R03AL07', 'R03AL10'); -- add manual mappings for ATC Combos using concept_relationship_manual INSERT INTO dev_combo @@ -1140,8 +496,9 @@ JOIN concept_relationship_manual r ON r.concept_code_1= a.class_code JOIN concept c ON c.concept_code = r.concept_code_2 AND c.vocabulary_id ~ 'Rx' AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' WHERE (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND r.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up'); -- 17075 +AND r.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up'); +-- add mappings to those Ingredients, which have problems with name matching -- add Acetylsalicylic acid INSERT INTO dev_combo SELECT class_code, @@ -1152,9 +509,9 @@ SELECT class_code, CASE WHEN class_name ~ '^acetylsalicylic' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1112807 FROM dev_combo WHERE class_name ~* 'acetylsalicylic') -AND class_name ~* 'acetylsalicylic'; -- 276 +AND class_name ~* 'acetylsalicylic'; --- ethinylestradiol +-- add Ethinylestradiol INSERT INTO dev_combo SELECT class_code, class_name, @@ -1164,59 +521,59 @@ SELECT class_code, CASE WHEN class_name ~* '^ethinylestradiol' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1549786 FROM dev_combo WHERE class_name ~* 'ethinylestradiol') -AND class_name ~* 'ethinylestradiol'; -- 44 +AND class_name ~* 'ethinylestradiol'; --- estrogen +-- add Estrogen INSERT INTO dev_combo SELECT class_code, class_name, - 'ethinylestradiol', + 'estrogens', 19049228, 'estrogens', - CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END + CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 19049228 FROM dev_combo WHERE class_name ~* 'estrogen') -AND class_name ~ 'estrogen' +AND split_part(class_name, ';', 1) ~ 'estrogen' UNION ALL SELECT class_code, class_name, - 'estrogens, conjugated (USP)', + 'estrogens', 1549080, 'estrogens, conjugated (USP)', - CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END + CASE WHEN SPLIT_PART(class_name,';',1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1549080 FROM dev_combo WHERE class_name ~* 'estrogen') -AND class_name ~ 'estrogen' +AND split_part(class_name, ';', 1) ~ 'estrogen' UNION ALL SELECT class_code, class_name, - 'estrogens, esterified (USP)', + 'estrogens', 1551673, 'estrogens, esterified (USP)', - CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END + CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1551673 FROM dev_combo WHERE class_name ~* 'estrogen') -AND class_name ~ 'estrogen' +AND split_part(class_name, ';', 1) ~ 'estrogen' UNION ALL SELECT class_code, class_name, - 'synthetic conjugated estrogens, A', + 'estrogens', 1596779, 'synthetic conjugated estrogens, A', - CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END + CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1596779 FROM dev_combo WHERE class_name ~* 'estrogen') -AND class_name ~* 'estrogen' +AND split_part(class_name, ';', 1) ~* 'estrogen' UNION ALL SELECT class_code, class_name, - 'synthetic conjugated estrogens, B' AS class, + 'estrogens' AS class, 1586808, 'synthetic conjugated estrogens, B', - CASE WHEN class_name ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END + CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1586808 FROM dev_combo WHERE class_name ~* 'estrogen') -AND class_name ~* 'estrogen'; -- 1235 +AND split_part(class_name, ';', 1) ~* 'estrogen'; -- remove erroneous automap DELETE FROM dev_combo WHERE class_code = 'M05BX53' AND concept_id = 19000815; -- strontium @@ -1229,9 +586,9 @@ AS (SELECT DISTINCT a.class_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code a.concept_id FROM dev_combo a, - dev_oral b, + dev_form b, concept_manual c -WHERE c.concept_name ~* 'oral|systemic|chewing gum' +WHERE c.concept_name ~* 'oral|systemic|chewing gum' AND b.df = 'dev_oral' AND c.concept_code = a.class_code ) SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code @@ -1251,9 +608,9 @@ AS a.class_name, a.concept_id FROM dev_combo a, - dev_parenteral b, + dev_form b, concept_manual c -WHERE c.concept_name ~ 'parenteral|systemic' +WHERE c.concept_name ~ 'parenteral|systemic' AND b.df = 'dev_parenteral' AND c.concept_code = a.class_code ) -- Oral formulations SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code @@ -1273,9 +630,9 @@ AS a.class_name, a.concept_id FROM dev_combo a, - dev_vaginal b, + dev_form b, concept_manual c -WHERE c.concept_name ~* 'vaginal' +WHERE c.concept_name ~* 'vaginal' and b.df = 'dev_vaginal' AND c.concept_code = a.class_code ) SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code @@ -1304,31 +661,27 @@ WHERE class_name !~';' JOIN concept c ON c.concept_id = a.concept_id AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); -- 1198 - +WHERE (concept_code_1, c.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); /****************************** ******* manual mapping ******** *******************************/ -- add manually mapped ATC Drug Classes to Standard Ingredients using concept_relationship_manual INSERT INTO internal_relationship_stage ( concept_code_1, concept_code_2) -WITH t1 AS (select distinct a.concept_code_1, a.relationship_id, c.concept_name +WITH t1 AS (select distinct a.concept_code_1, a.relationship_id, c.concept_name as concept_code_2 FROM concept_relationship_manual a JOIN concept_manual b - ON b.concept_code = a.concept_code_1 + ON b.concept_code = a.concept_code_1 AND a.invalid_reason is null JOIN concept c ON c.concept_code = a.concept_code_2 AND c.vocabulary_id = a.vocabulary_id_2 AND c.standard_concept = 'S' AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient') + AND c.concept_class_id = 'Ingredient' + and a.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up')) -- use ATC-specific relationships only SELECT DISTINCT concept_code_1, - a.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code, -FROM t1 a - JOIN concept_manual b ON b.concept_code = a.concept_code_1 AND b.invalid_reason is null and b.concept_class_id = 'ATC 5th' - AND a.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up') - AND (concept_code_1, a.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1),concept_code_2 - FROM internal_relationship_stage); -- 7208 - + concept_code_2 -- OMOP Ingredient name AS an ATC Drug Attribute code, +FROM t1 WHERE (concept_code_1, concept_code_2) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1),concept_code_2 + FROM internal_relationship_stage); /********************************** ******* drug_concept_stage ******** ***********************************/ @@ -1389,7 +742,7 @@ AND concept_class_id = 'Dose Form' AND vocabulary_id ~ 'RxNorm' AND invalid_reason IS NULL; --- add ATC Drug Attributes IN the form of Standard Ingredient names using the internal_relationship_stage table +-- add ATC Drug Attributes in the form of Standard Ingredient names using the internal_relationship_stage table INSERT INTO drug_concept_stage ( concept_name, vocabulary_id, @@ -1447,7 +800,7 @@ AND class_code NOT IN (SELECT SPLIT_PART(concept_code_1,' ',1) AND c.concept_class_id = 'Ingredient') AND LENGTH(class_code) = 7) OR - -- with absent Ingredient IN drug_relationship_stage + -- with absent Ingredient in drug_relationship_stage (class_code IN (SELECT SPLIT_PART(concept_code_1,' ',1) FROM internal_relationship_stage GROUP BY concept_code_1 @@ -1459,7 +812,7 @@ HAVING COUNT(1) = 1) AND class_code NOT IN ( AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S')) ), --- fuzzy macth 1 using name similarity +-- fuzzy macth using name similarity t2 AS (SELECT a.*, c.* @@ -1513,11 +866,11 @@ WHERE (class_code,concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage) AND concept_id <> 43013482; -- butyl ester of methyl vinyl ether-maleic anhydride copolymer (125 kD) -/***************************************** -*** FUTHER WORK WITH ATC COMBO CLASSES *** -******************************************/ +/********************************** +*** FUTHER WORK WITH ATC COMBOS *** +***********************************/ -- assemble mappings for ATC Classes indicating Ingredient Groups using the the concept_ancestor AND/OR concept tables along WITH word pattern matching --- take descendants of Acid preparations +-- add descendants of Acid preparations INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -1532,10 +885,10 @@ FROM concept_manual a, -- ATC AND ca.ancestor_concept_id = 21600704-- ATC code of Acid preparations AND vocabulary_id LIKE 'RxNorm%' AND concept_class_id = 'Ingredient' -WHERE a.concept_name ~* 'acid preparations' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 8 +WHERE SPLIT_PART(a.concept_name,';',1) ~* 'acid preparations' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- Sulfonamides +-- add descendants of Sulfonamides INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -1552,17 +905,17 @@ FROM concept_manual a, -- ATC AND c.concept_class_id = 'Ingredient' JOIN concept_relationship b ON b.concept_id_1 = ancestor_concept_id AND b.invalid_reason is null AND b.relationship_id = 'ATC - RxNorm pr lat' -WHERE a.concept_name ~* 'sulfonamides' AND a.concept_name !~* '^short-acting sulfonamides|^intermediate-acting sulfonamides|^long-acting sulfonamides' +WHERE SPLIT_PART(a.concept_name,';',1) ~* 'sulfonamides' AND SPLIT_PART(a.concept_name,';',1) !~* '^short-acting sulfonamides|^intermediate-acting sulfonamides|^long-acting sulfonamides' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Amino acids +-- add descendants of Amino acids INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'amino acids', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^amino acids' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1)~* '^amino acids' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, -- ATC concept_ancestor ca JOIN concept c @@ -1570,18 +923,18 @@ FROM concept_manual a, -- ATC AND ca.ancestor_concept_id IN (21601215, 21601034) -- 21601215 B05XB Amino acids| 21601034 B02AA Amino acids AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' - WHERE a.concept_name ~* 'amino\s*acid' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'amino\s*acid' AND a.concept_code <> 'B03AD01' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- ferrous amino acid complex --- take descendants of Analgesics +-- add descendants of Analgesics INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'analgesics', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^analgesics' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1)~* '^analgesics' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id @@ -1589,20 +942,20 @@ FROM concept_manual a, concept_ancestor ca AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' AND c.concept_id NOT IN (939506, 950435, 964407) -- sodium bicarbonate|citric acid|salicylic acid - WHERE a.concept_name ~* 'analgesics?' AND a.concept_name !~* '\yexcl' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'analgesics?' AND SPLIT_PART(a.concept_name,';',1) !~* '\yexcl' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Animals +-- add ingredients indicating Animals INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'animals', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^animals' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^animals' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c -WHERE a.concept_name ~* 'Animals' +WHERE SPLIT_PART(a.concept_name,';',1) ~* 'Animals' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' AND (c.concept_id IN (19091701,19056189,40170543,40170448,40170341,40170416,40175840,40175865,40170916,40175984,40161698,40170420, 19095690,40170741,40170848,40161809,40161813,45892235,40171114,45892234,37496548,40170660,40172147,40175843,40175898,40175933,40171110, @@ -1614,16 +967,16 @@ c.concept_name ~* 'rabbit|\ycow\y|\ydog\y|\ycat\y|goose|\yhog\y|\ygland\y|hamste AND c.concept_name ~* 'extract' AND c.vocabulary_id LIKE 'RxNorm%' AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.concept_id NOT IN (46276144,40170814,40226703,43560374,40227355,42903998,40227484,19086386)) -); -- 98 +); --- take descendants of Antiinfectives +-- add descendants of Antiinfectives INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'antiinfectives', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^antiinfectives' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^antiinfectives' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c @@ -1632,66 +985,66 @@ FROM concept_manual a, AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' AND c.concept_id NOT IN (19044522)-- zinc sulfate - WHERE a.concept_name ~* 'antiinfectives?' --AND class_name ~* '\yexcl' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'antiinfectives?' --AND class_name ~* '\yexcl' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - --- take ingredients indicating Cadmium compounds + +-- add ingredients indicating Cadmium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'cadmium compounds', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^cadmium compounds' THEN 3 ELSE 4 END ::INT AS rnk -- groups don't have primary lateral ings + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^cadmium compounds' THEN 3 ELSE 4 END ::INT AS rnk -- groups don't have primary lateral ings FROM concept_manual a, concept c WHERE lower(c.concept_name) LIKE '%cadmium %' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_id <> 45775350 -AND a.concept_name ~* 'cadmium compounds?' --AND class_name ~* '\yexcl' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 4 +AND SPLIT_PART(a.concept_name,';',1) ~* 'cadmium compounds?' --AND class_name ~* '\yexcl' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Calcium (different salts) +-- add ingredients indicating Calcium (different salts) INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'calcium (different salts IN combination)', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^calcium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^calcium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.concept_name ~* '\ycalcium\y' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_id NOT IN (42903945,43533002,1337191,19007595,43532262,19051475) -- calcium ion|calcium hydride|calcium hydroxide|calcium oxide|calcium peroxide|anhydrous calcium iodide -AND a.concept_name ~* 'calcium' AND a.concept_name ~* '\ysalt' +AND SPLIT_PART(a.concept_name,';',1) ~* 'calcium' AND SPLIT_PART(a.concept_name,';',1) ~* '\ysalt' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Calcium compounds +-- add ingredients indicating Calcium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'calcium compounds', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^calcium compounds' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^calcium compounds' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.concept_name ~* '\ycalcium\y' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_id NOT IN (19014944,42903945) -AND a.concept_name ~* 'calcium' AND a.concept_name ~* '\ycompound' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 212 +AND SPLIT_PART(a.concept_name,';',1) ~* 'calcium' AND SPLIT_PART(a.concept_name,';',1) ~* '\ycompound' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Laxatives +-- add descendants of Laxatives INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'contact laxatives', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^contact laxatives' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^contact laxatives' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c @@ -1699,17 +1052,17 @@ concept_ancestor ca AND ca.ancestor_concept_id IN (21600537) AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' -WHERE a.concept_name ~* 'contact' AND a.concept_name ~* 'laxatives?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 22 +WHERE SPLIT_PART(a.concept_name,';',1) ~* 'contact' AND SPLIT_PART(a.concept_name,';',1) ~* 'laxatives?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Corticosteroids +-- add descendants of Corticosteroids INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'corticosteroids', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^corticosteroids?|^combinations of corticosteroids?' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^corticosteroids?|^combinations of corticosteroids?' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c @@ -1717,17 +1070,17 @@ concept_ancestor ca AND ca.ancestor_concept_id IN (21605042, 21605164, 21605200, 21605165, 21605199, 21601607, 975125) AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' -WHERE a.concept_name ~* 'corticosteroids?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 608 +WHERE SPLIT_PART(a.concept_name,';',1) ~* 'corticosteroids?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Cough suppressants +-- add descendants of Cough suppressants INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'cough suppressants', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^cough suppressants|^other cough suppressants' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^cough suppressants|^other cough suppressants' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c @@ -1736,34 +1089,34 @@ concept_ancestor ca AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' AND c.concept_id NOT IN (943191,1139042,1189220,1781321,19008366,19039512,19041843,19050346,19058933,19071861,19088167,19095266,42904041) - WHERE a.concept_name ~* 'cough' AND a.concept_name ~* 'suppressants?' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'cough' AND SPLIT_PART(a.concept_name,';',1) ~* 'suppressants?' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Diuretics +-- add descendants of Diuretics INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'diuretics', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^diuretics' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^diuretics' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id AND ca.ancestor_concept_id = 21601461 AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' - WHERE a.concept_name ~* 'diuretics?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 2378 + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'diuretics?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Magnesium (different salts IN combination) +-- add descendants of Magnesium (different salts IN combination) INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'magnesium (different salts IN combination)', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, -- ATC concept_ancestor ca JOIN concept c -- Rx @@ -1771,47 +1124,46 @@ concept_ancestor ca AND ca.ancestor_concept_id IN (21600892) AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' - WHERE a.concept_name ~* 'magnesium' AND a.concept_name ~* 'salt' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'magnesium' AND SPLIT_PART(a.concept_name,';',1) ~* 'salt' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Magnesium (different salts IN combination) +-- add ingredients indicating Magnesium (different salts IN combination) INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'magnesium (different salts IN combination)', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c - WHERE a.concept_name ~* 'magnesium' AND a.concept_name ~* 'salt' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'magnesium' AND SPLIT_PART(a.concept_name,';',1) ~* 'salt' AND c.concept_name ~ 'magnesium' AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' AND (a.concept_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) AND concept_id NOT IN (43532017, 37498676); -- magnesium cation | magnesium Mg-28 --- take ingredients indicating Multivitamins +-- add ingredients indicating Multivitamins INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'multivitamins', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^multivitamins' THEN 1 ELSE 2 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^multivitamins' THEN 1 ELSE 2 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.concept_id = 36878782 - AND a.concept_name ~* 'multivitamins?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 7 ------------------------------------------------------------------- + AND SPLIT_PART(a.concept_name,';',1) ~* 'multivitamins?' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Opium alkaloids WITH morphine +-- add descendants of Opium alkaloids WITH morphine INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'opium alkaloids WITH morphine', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^opium alkaloids WITH morphine' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^opium alkaloids WITH morphine' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON descendant_concept_id = c.concept_id @@ -1819,17 +1171,17 @@ FROM concept_manual a, concept_ancestor ca AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' AND c.concept_id NOT IN (19112635) - WHERE a.concept_name ~* 'opium alkaloids WITH morphine' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'opium alkaloids WITH morphine' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Opium derivatives +-- add descendants of Opium derivatives INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'opium derivatives', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^opium derivatives' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^opium derivatives' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id @@ -1837,34 +1189,36 @@ FROM concept_manual a, concept_ancestor ca AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' AND c.concept_id NOT IN (19021930, 1201620) - WHERE a.concept_name ~* 'opium derivatives' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'opium derivatives' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Organic nitrates +-- add descendants of Organic nitrates INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'organic nitrates', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^organic nitrates' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^organic nitrates' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id AND ca.ancestor_concept_id IN (21600316) AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' - WHERE a.concept_name ~* 'organic nitrates' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'organic nitrates' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Psycholeptics +-- add descendants of Psycholeptics INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'psycholeptics', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^psycholeptics' THEN 3 WHEN a.concept_name ~ 'excl\. psycholeptics' THEN 0 ELSE 4 END ::INT AS rnk -- 0 stands for excluded drugs + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^psycholeptics' THEN 3 + WHEN SPLIT_PART(a.concept_name,';',1) ~ 'excl\. psycholeptics' THEN 0 + ELSE 4 END ::INT AS rnk -- 0 stands for excluded drugs FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id @@ -1872,189 +1226,189 @@ FROM concept_manual a, concept_ancestor ca AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' AND c.concept_id NOT IN (742594) - WHERE a.concept_name ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Selenium compounds +-- add descendants of Selenium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'selenium compounds', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^selenium compounds' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^selenium compounds' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id AND ca.ancestor_concept_id IN (21600908) AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' - WHERE a.concept_name ~* 'selenium compounds' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'selenium compounds' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Silver compounds +-- add descendants of Silver compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'silver compounds', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id AND ca.ancestor_concept_id IN (21602248) AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' - WHERE a.concept_name ~* 'silver compounds' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'silver compounds' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Silver +-- add ingredients indicating Silver INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'silver compounds', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.concept_name ~* 'silver\y' AND ('silver compounds', c.concept_id) NOT IN (select class, concept_id FROM dev_combo) -AND a.concept_name ~* 'silver compounds' +AND SPLIT_PART(a.concept_name,';',1) ~* 'silver compounds' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Sulfonylureas +-- add descendants of Sulfonylureas INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'sulfonylureas', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^sulfonylureas?' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^sulfonylureas?' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id AND ca.ancestor_concept_id = 21600749 AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' -WHERE a.concept_name ~* 'sulfonylureas?' +WHERE SPLIT_PART(a.concept_name,';',1) ~* 'sulfonylureas?' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Snake venom antiserum +-- add ingredients indicating Snake venom antiserum INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'snake venom antiserum', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^snake venom antiserum' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^snake venom antiserum' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a,concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'antiserum' AND c.concept_name ~* 'snake' -AND a.concept_name ~* 'snake venom antiserum' +AND SPLIT_PART(a.concept_name,';',1) ~* 'snake venom antiserum' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Aluminium preparations +-- add ingredients indicating Aluminium preparations INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'aluminium preparations', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^aluminium preparations' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^aluminium preparations' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a,concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'aluminium|aluminum' - AND a.concept_name ~* 'aluminium preparations' + AND SPLIT_PART(a.concept_name,';',1) ~* 'aluminium preparations' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Aluminium compounds +-- add ingredients indicating Aluminium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'aluminium compounds', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^aluminium compounds' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^aluminium compounds' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a,concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'aluminium|aluminum' - AND a.concept_name ~* 'aluminium compounds' + AND SPLIT_PART(a.concept_name,';',1) ~* 'aluminium compounds' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Lactic acid producing organisms +-- add ingredients indicating Lactic acid producing organisms INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'lactic acid producing organisms', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^lactic acid producing organisms' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^lactic acid producing organisms' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'lactobacil' - AND a.concept_name ~* 'lactic acid producing organisms' + AND SPLIT_PART(a.concept_name,';',1) ~* 'lactic acid producing organisms' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Lactobacillus +-- add ingredients indicating Lactobacillus INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'lactobacillus', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^lactobacillus' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^lactobacillus' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'lactobacil' - AND a.concept_name ~* 'lactobacillus' + AND SPLIT_PART(a.concept_name,';',1) ~* 'lactobacillus' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Magnesium compounds +-- add ingredients indicating Magnesium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'magnesium compounds', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^magnesium compounds' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^magnesium compounds' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a,concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'magnesium' - AND a.concept_name ~* 'magnesium compounds' + AND SPLIT_PART(a.concept_name,';',1) ~* 'magnesium compounds' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Grass pollen +-- add ingredients indicating Grass pollen INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'grass pollen', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^grass pollen' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^grass pollen' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a,concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - AND a.concept_name ~* 'grass' AND c.concept_name ~* 'pollen' - AND a.concept_name ~* 'grass pollen' + AND c.concept_name ~* 'grass' AND c.concept_name ~* 'pollen' + AND SPLIT_PART(a.concept_name,';',1) ~* 'grass pollen' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Oil +-- add ingredients indicating Oil INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -2068,91 +1422,89 @@ WHERE c.vocabulary_id IN ('RxNorm','RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* '\yoil\y|\yoleum\y' -AND a.concept_name ~* '^oil' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 475 - ---select * from concept_manual where concept_name ~* 'oil'; +AND SPLIT_PART(a.concept_name,';',1) ~* '^oil' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Flowers +-- add ingredients indicating Flowers INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'flowers', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^flowers' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^flowers' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* '\yflower\y' AND c.concept_name ~* 'extract' - AND a.concept_name ~* '^flowers' + AND SPLIT_PART(a.concept_name,';',1) ~* '^flowers' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Fumaric acid derivatives +-- add ingredients indicating Fumaric acid derivatives INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'fumaric acid derivatives', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^fumaric acid derivatives' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^fumaric acid derivatives' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'fumarate\y' - AND a.concept_name ~* 'fumaric acid derivatives' + AND SPLIT_PART(a.concept_name,';',1) ~* 'fumaric acid derivatives' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Glycerol +-- add ingredients indicating Glycerol INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'glycerol', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^glycerol' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN a.concept_name ~* '^glycerol' THEN 1 ELSE 2 END ::INT AS rnk FROM concept_manual a,concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - AND c.concept_name ~* 'glycerol\y' - AND a.concept_name ~* 'glycerol' + AND c.concept_name ~* 'glycerol\y' and SPLIT_PART(a.concept_name,';',1) !~ 'rectal' + AND SPLIT_PART(a.concept_name,';',1) ~* 'glycerol' and SPLIT_PART(a.concept_name,';',1) !~ 'phenylbutyrate' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Proton pump inhibitors +-- add descendants of Proton pump inhibitors INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'proton pump inhibitors', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^proton pump inhibitors' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^proton pump inhibitors' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor s JOIN concept c ON descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND ancestor_concept_id IN (21600095) - WHERE a.concept_name ~* 'proton pump inhibitors?' + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'proton pump inhibitors?' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take descendants of Thiazides +-- add descendants of Thiazides INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'thiazides', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^thiazides' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^thiazides' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND ca.ancestor_concept_id IN (21601463) - WHERE a.concept_name ~* 'thiazides' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 160 + WHERE SPLIT_PART(a.concept_name,';',1) ~* 'thiazides' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- take ingredients indicating Electrolytes +-- add ingredients indicating Electrolytes INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -2167,10 +1519,10 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.standard_concept = 'S' AND c.concept_name ~* ('^magnesium sulfate|^ammonium chloride|^sodium chloride|^sodium acetate|^magnesium chloride^|potassium lactate|^sodium glycerophosphate|^magnesium phosphate|^potassium chloride|^calcium chloride' || '^sodium bicarbonate|^hydrochloric acid|^potassium acetate|^zinc chloride|^sodium phosphate|^potassium bicarbonate|^succinic acid|^sodium lactate|^sodium gluconate|^sodium fumarate') - AND a.concept_name ~* 'electrolytes' + AND SPLIT_PART(a.concept_name,';',1) ~* 'electrolytes' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- bismuth preparations +-- add ingredients indicating bismuth preparations INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -2183,10 +1535,10 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* ('\ybismuth') - AND a.concept_name ~* 'bismuth preparations' + AND SPLIT_PART(a.concept_name,';',1) ~* 'bismuth preparations' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- artificial tears +-- add ingredients indicating Artificial Tears INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -2199,26 +1551,26 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.standard_concept = 'S' AND c.concept_name ~* 'carboxymethylcellulose$|carboxypolymethylene|polyvinyl alcohol$|hydroxypropyl methylcellulose$|^hypromellose$|hydroxypropyl cellulose|^hyaluronate' AND c.concept_class_id = 'Ingredient' - AND a.concept_name ~* 'artificial tears' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 23 + AND SPLIT_PART(a.concept_name,';',1) ~* 'artificial tears' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- potassium-sparing agents +-- add ingredients indicating Potassium-sparing agents INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, 'potassium-sparing agents', c.concept_id, c.concept_name, - CASE WHEN a.concept_name ~* '^potassium-sparing agents' THEN 3 ELSE 4 END ::INT AS rnk + CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^potassium-sparing agents' THEN 3 ELSE 4 END ::INT AS rnk FROM concept_manual a, concept c WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* '\yamiloride|triamterene|spironolactone|eplerenone|finerenone|canrenone|canrenoic acid' - AND a.concept_name ~* 'potassium-sparing agents' + AND SPLIT_PART(a.concept_name,';',1) ~* 'potassium-sparing agents' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- 19050016 4125 ethiodized oil +-- add ingredients indicating ethiodized oil INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -2231,10 +1583,50 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'ethiodized oil' - AND a.concept_name ~* '^ethyl esters of iodised fatty acids' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- 1 - --- excl\.trimethoprim + AND SPLIT_PART(a.concept_name,';',1) ~* '^ethyl esters of iodised fatty acids' +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + +-- add ingredients indicating Ophthalmic Antibiotics +INSERT INTO dev_combo +SELECT DISTINCT a.concept_code, + a.concept_name, + 'antibiotics ophthalmic', + c.concept_id, + c.concept_name, + 3 -- hradcoded +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND SPLIT_PART(a.concept_name,';',1) ~* 'antibiotics' and SPLIT_PART(a.concept_name,';',1) ~* 'combination' and SPLIT_PART(a.concept_name,';',1) ~* 'ophthalmic' +AND lower (substring(c.concept_name, '\w+')) in ( +'azithromycin','bacitracin','besifloxacin','ciprofloxacin','erythromycin','gatifloxacin','gentamicin','levofloxacin', +'moxifloxacin','ofloxacin','sulfacetamide','tobramycin','polymyxin B','trimethoprim','sulfacetamide','neomycin','gramicidin' + ) +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; + +-- add ingredients indicating Topical Antibiotics +INSERT INTO dev_combo +SELECT DISTINCT a.concept_code, + a.concept_name, + 'antibiotics topical', + c.concept_id, + c.concept_name, + 4 -- hradcoded +FROM concept_manual a, concept c +WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND SPLIT_PART(a.concept_name,';',1) ~* 'antibiotics' and SPLIT_PART(a.concept_name,';',1) ~* 'combination' and SPLIT_PART(a.concept_name,';',1) ~* 'topical' +AND lower(c.concept_name)in ( +'mupirocin','sulfacetamide','retapamulin','silver sulfadiazine','polymyxin b','bacitracin', +'neomycin','ozenoxacin','erythromycin','mafenide','gentamicin','demeclocycline','retapamulin', +'chlortetracycline','virginiamycin','chloramphenicol','oxytetracycline','tetracycline') +AND a.invalid_reason IS NULL +AND a.concept_class_id = 'ATC 5th'; + +-- add excluded Trimethoprim INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -2248,9 +1640,10 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND c.concept_name ~* 'trimethoprim' - AND a.concept_name ~* 'excl. trimethoprim' + AND SPLIT_PART(a.concept_name,';',1) ~* 'excl. trimethoprim' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - + +-- perform dev_combo cleanup -- fix Vitamin D AND analogues IN combination UPDATE dev_combo SET rnk = 3 @@ -2271,42 +1664,42 @@ FROM dev_combo WHERE class_code = 'A06AG11' AND class_name = 'sodium lauryl sulfoacetate, incl. combinations' AND concept_name = 'sodium' -AND rnk = 1; -- 0 +AND rnk = 1; DELETE FROM dev_combo WHERE class_code = 'A01AA51' AND class_name = 'sodium fluoride, combinations' AND concept_name = 'sodium' -AND rnk = 1; --0 +AND rnk = 1; DELETE FROM dev_combo WHERE class_code = 'A06AB58' AND class_name = 'sodium picosulfate, combinations' AND concept_name = 'sodium' -AND rnk = 1; -- 0 +AND rnk = 1; DELETE FROM dev_combo WHERE class_code = 'B05XA06' AND class_name = 'potassium phosphate, incl. combinations WITH other potassium salts' AND concept_name = 'potassium' -AND rnk = 1;--0 +AND rnk = 1; DELETE FROM dev_combo WHERE class_code = 'A12BA51' AND class_name = 'potassium chloride, combinations' AND concept_name = 'potassium' -AND rnk = 1;--0 +AND rnk = 1; DELETE FROM dev_combo WHERE class_code = 'C01DA58' AND class_name = 'isosorbide dinitrate, combinations' AND concept_name = 'isosorbide' -AND rnk = 1; --0 +AND rnk = 1; -- fix erroneous rnk of 3 for J07AG53 UPDATE dev_combo @@ -2315,12 +1708,12 @@ WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND rnk = 3; -- 0 +AND rnk = 3; DELETE FROM dev_combo WHERE class_code = 'B03AD01' -AND rnk = 4; -- 0 +AND rnk = 4; INSERT INTO dev_combo SELECT DISTINCT class_code, @@ -2337,28 +1730,27 @@ DELETE FROM dev_combo WHERE class_code = 'A02BD08' AND class = 'bismuth subcitrate, tetracycline' -AND concept_id = 19025138; --0 - +AND concept_id = 19025138; -- erroneous map to Pentaerythritol Distearate DELETE FROM dev_combo WHERE class_code = 'C01DA55' AND class = '' -AND concept_id = 42903512; -- 0 +AND concept_id = 42903512; -- erroneous map to Potassium DELETE FROM dev_combo WHERE class_code = 'B05XA06' AND class = 'potassium phosphate, incl. combinations with other potassium salts' -AND concept_id = 19049024; -- 0 +AND concept_id = 19049024; -- erroneous map to tenofovir DELETE FROM dev_combo WHERE class_code = 'J05AR03' AND class = 'tenofovir disoproxil' -AND concept_id = 19011093; -- 0 +AND concept_id = 19011093; -- remove doubling ingredients with different rank, remaining those which are Primary lateral DELETE @@ -2371,26 +1763,26 @@ WHERE (class_code,concept_id,rnk) IN (SELECT a.class_code, ON a.class_code = b.class_code AND a.concept_id = b.concept_id WHERE a.rnk > 1 - AND b.rnk = 1); -- 20 + AND b.rnk = 1); UPDATE dev_combo SET rnk = 3 WHERE class_code = 'A07FA51' AND class_name = 'lactic acid producing organisms, combinations' AND concept_id = 19136028 -AND concept_name = 'Saccharomyces cerevisiae'; -- 0 +AND concept_name = 'Saccharomyces cerevisiae'; DELETE FROM dev_combo WHERE class_name ~ 'antiinfectives' AND rnk = 4 -AND concept_id IN (19010309,19136048,1036884,19049024,989878,961145,19018544,917006,914335); -- 13 +AND concept_id IN (19010309,19136048,1036884,19049024,989878,961145,19018544,917006,914335); UPDATE dev_combo SET rnk = 3 WHERE class_name ~ 'lactic acid producing organisms' AND rnk = 4 -AND concept_name ~* 'Saccharomyces|Bacillus|Bifidobacterium|Enterococcus|Escherichia|Streptococcus'; -- 9 +AND concept_name ~* 'Saccharomyces|Bacillus|Bifidobacterium|Enterococcus|Escherichia|Streptococcus'; DELETE FROM dev_combo @@ -2402,31 +1794,6 @@ UPDATE dev_combo WHERE class_name ~ 'opium derivatives' AND rnk = 1; -UPDATE dev_combo - SET rnk = 3 -WHERE (class_code,concept_id) IN (SELECT a.class_code, - a.concept_id - FROM dev_combo a - JOIN dev_combo b - ON b.class_code = a.class_code - AND a.rnk <> b.rnk - AND a.concept_id = b.concept_id - WHERE (a.class_Code,a.concept_name) IN (SELECT class_code, - concept_name - FROM dev_combo - GROUP BY class_code, - concept_name - HAVING COUNT(1) > 1) - AND (b.class_Code,b.concept_name) IN (SELECT class_code, - concept_name - FROM dev_combo - GROUP BY class_code, - concept_name - HAVING COUNT(1) > 1) - AND a.class_code = 'S03AA30' - AND a.rnk = 4) -AND rnk = 4; - DELETE FROM dev_combo WHERE class_code = 'R05FB01' @@ -2434,6 +1801,13 @@ AND class_name = 'cough suppressants and mucolytics' AND class = 'cough suppressants' AND concept_id = 19057932 AND rnk = 3; +DELETE +FROM dev_combo +WHERE class_code = 'R05FB01' +AND class_name = 'cough suppressants and mucolytics' +AND class = '' +AND concept_id = 19057932 +AND rnk = 4; UPDATE dev_combo SET rnk = 4 @@ -2443,6 +1817,14 @@ AND class = 'cough suppressants' AND concept_id = 19071999 AND rnk = 3; +UPDATE dev_combo + SET rnk = 3 +WHERE class_code = 'S02AA30' +AND class_name = 'antiinfectives, combinations' +AND class = '' +AND concept_id = 963742 +AND rnk = 4; + UPDATE dev_combo SET rnk = 3 WHERE class_code = 'S02AA30' @@ -2456,21 +1838,21 @@ UPDATE dev_combo WHERE class_code = 'J01RA02' AND class = 'sulfonamides' AND concept_id = 1790868 -AND rnk = 3; +AND rnk = 3; -- 1 UPDATE dev_combo SET rnk = 4 WHERE class_code = 'J01RA02' AND class = 'sulfonamides' AND concept_id = 1734104 -AND rnk = 3; +AND rnk = 3; -- 1 UPDATE dev_combo SET rnk = 4 WHERE class_code = 'J01RA02' AND class = 'sulfonamides' AND concept_id = 1748975 -AND rnk = 3; +AND rnk = 3; -- 1 UPDATE dev_combo SET rnk = 4 @@ -2527,6 +1909,7 @@ WHERE class_code = 'J01RA02' AND class = 'sulfonamides' AND concept_id = 19023254 AND rnk = 3; + UPDATE dev_combo SET rnk = 4 WHERE class_code = 'J01RA02' @@ -2554,6 +1937,7 @@ WHERE class_code = 'J01RA02' AND class = 'sulfonamides' AND concept_id = 1836948 AND rnk = 3; + UPDATE dev_combo SET rnk = 4 WHERE class_code = 'J01RA02' @@ -2601,7 +1985,7 @@ UPDATE dev_combo WHERE class_code = 'R05FB02' AND class = 'cough suppressants' AND concept_id = 1103137 -AND rnk = 3; +AND rnk = 3; /******************************************* **** ADD ODDMENTS TO THE INPUT TABLES ***** ********************************************/ @@ -2685,6 +2069,32 @@ AND concept_class_id = 'Ingredient' AND vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND invalid_reason IS NULL AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); + +-- remove dead deprecated or updated codes +DELETE +FROM internal_relationship_stage +WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT concept_code + FROM concept_manual + WHERE invalid_reason IS NOT NULL); + +DELETE +FROM drug_concept_stage +WHERE concept_code IN (SELECT concept_code + FROM concept_manual + WHERE invalid_reason IS NOT NULL); + +-- remove inexistent drug mapping (old and wrong) +DELETE +FROM drug_concept_stage +WHERE concept_code IN (SELECT class_code FROM atc_inexistent) +AND concept_code NOT IN (SELECT class_code FROM dev_combo); + +DELETE +FROM internal_relationship_stage +WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT class_code FROM atc_inexistent) +AND SUBSTRING(concept_code_1,'\w+') NOT IN (SELECT class_code FROM dev_combo) +AND concept_code_1 !~ '\s+'; + /*************************************** ******* relationship_to_concept ******** ****************************************/ From 370c0e345ba3e135196c6c1317af99a405c3e203 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 2 Sep 2021 17:12:09 +0300 Subject: [PATCH 10/45] Update load_interim.sql code improvement --- ATC/load_interim.sql | 728 ++++++++++++++++++++++++++++++++----------- 1 file changed, 551 insertions(+), 177 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 7491d4007..e1db91872 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -27,16 +27,72 @@ SELECT drug_concept_id, FROM devv5.drug_strength JOIN devv5.concept ON concept_id = drug_concept_id AND concept_class_id IN ('Clinical Drug Form', 'Ingredient') -- 'Clinical Drug Comp' doesn't exist in ATCs -GROUP BY drug_concept_id ; +GROUP BY drug_concept_id; + +-- additional reference +-- Rx combo - ing - df +DROP TABLE rx_all_combo; +CREATE TABLE rx_all_combo +AS +(SELECT DISTINCT c.concept_id AS d_id, + c.concept_name AS d_name, + d.concept_id AS ing_id, + d.concept_name AS ing_name, + k.concept_id AS df_id, + k.concept_name AS df_name +FROM concept c + JOIN concept_relationship r ON r.concept_id_1 = c.concept_id + JOIN concept d ON d.concept_id = r.concept_id_2 +--df + JOIN concept_relationship r2 ON r2.concept_id_1 = c.concept_id + JOIN concept k ON k.concept_id = r2.concept_id_2 +WHERE c.concept_class_id = 'Clinical Drug Form' +AND c.standard_concept = 'S' +AND d.standard_concept = 'S' +AND d.concept_class_id = 'Ingredient' +AND r.relationship_id = 'RxNorm has ing' +AND r2.relationship_id = 'RxNorm has dose form' +AND k.concept_class_id = 'Dose Form' +AND k.invalid_reason IS NULL +AND r.invalid_reason IS NULL +AND r2.invalid_reason IS NULL); + +-- additional reference for ATC +DROP TABLE IF EXISTS atc_all_combo; +CREATE TABLE atc_all_combo +AS +(SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id AS ing_id, + a.concept_name AS ing_name, + c.concept_id AS df_id, + c.concept_name AS df_name, + rnk +FROM dev_combo a + JOIN internal_relationship_stage b ON SPLIT_PART (b.concept_code_1,' ',1) = a.class_code + JOIN drug_concept_stage d + ON d.concept_code = b.concept_code_2 + AND d.concept_class_id = 'Dose Form' + JOIN concept c + ON c.concept_name = d.concept_name + AND c.invalid_reason IS NULL); + +DROP INDEX IF EXISTS idx_atc; +CREATE INDEX idx_atc ON atc_all_combo (class_code, df_id, ing_id); +ANALYZE atc_all_combo; +DROP INDEX IF EXISTS idx_rx; +CREATE INDEX idx_rx ON rx_all_combo (ing_id, df_id); +ANALYZE rx_all_combo; /************************* ***** CLASS TO DRUG ****** **************************/ -- assemble a table containing ATC Drug Classes which are hierarchically connected to RxN/RxE Drug Products via 'ATC - RxNorm' relationships (resembles Subsumes). --- Monocomponent ATC classes +-- add mappings from Monocomponent ATC Classes to respective Monocomponent RxN/RxE Drug Products DROP TABLE if exists class_to_drug_new; CREATE TABLE class_to_drug_new AS -(WITH t1 +( +WITH t1 AS (SELECT * FROM concept_manual @@ -56,14 +112,14 @@ FROM t1 a ON d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d2 + LEFT JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 AND d2.concept_class_id = 'Dose Form') SELECT DISTINCT class_code, class_name, c.*, 1 AS concept_order, - 'Monocomponent ATC Class' as order_desc + 'ATC Monocomp Class' as order_desc FROM t2 a JOIN rx_all_combo b ON lower (b.ing_name) = lower (a.ing_name) @@ -72,9 +128,10 @@ FROM t2 a JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' -WHERE class_code NOT IN (SELECT class_code FROM dev_combo)); +WHERE class_code NOT IN (SELECT class_code FROM dev_combo) +); --- add everything - greedy version +-- add mappings from Monocomponent ATC Classes to associated Polycomponent RxN/RxE Drug Products ("greedy" logic) INSERT INTO class_to_drug_new WITH t1 AS @@ -94,15 +151,15 @@ FROM t1 a JOIN drug_concept_stage d ON d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' - LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d2 + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 AND d2.concept_class_id = 'Dose Form') SELECT DISTINCT class_code, class_name, c.*, 2 AS concept_order, - 'Greedy Monocomponent ATC Class' as order_desc + 'Greedy ATC Monocomp Class' as order_desc FROM t2 a JOIN rx_all_combo b ON lower (b.ing_name) = lower (a.ing_name) @@ -112,9 +169,9 @@ FROM t2 a AND c.standard_concept = 'S' AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) AND class_code NOT IN (SELECT class_code FROM dev_combo); - -/***** PREPARE ATC COMBO CLASSES ******/ - +/************************************ +***** PREPARE ATC COMBO CLASSES ***** +*************************************/ -- separate pure ATC Combo Classes with mapping: Primary lateral + Secondary lateral (rnk in (1, 2) in dev_combo) DROP TABLE IF EXISTS case_1; CREATE TABLE case_1 @@ -225,8 +282,7 @@ AND class_code IN (SELECT class_code WHERE rnk = 4) and class_code in (select class_code from dev_combo where rnk = 0) );-- exclude Secondary upward - ---Note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product +-- Note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product -- Add ATC Combo Classes with mappings to class_to_drug: -- Primary lateral + Secondary lateral (order = 3) @@ -258,7 +314,7 @@ SELECT DISTINCT class_code, class_name, c.*, 3 AS concept_order, - 'Primary upward + Secondary upward' as order_desc + 'ATC Combo Class: Primary upward + Secondary upward' as order_desc FROM t2 a JOIN rx_all_combo b ON lower (b.ing_name) = lower (a.ing_name) @@ -289,15 +345,15 @@ FROM t1 a JOIN drug_concept_stage d ON d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' - LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d2 + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 AND d2.concept_class_id = 'Dose Form') SELECT DISTINCT class_code, class_name, c.*, 4 AS concept_order, - 'Primary upward' AS order_desc + 'ATC Combo Class: Primary upward only' AS order_desc FROM t2 a JOIN rx_all_combo b ON lower (b.ing_name) = lower (a.ing_name) @@ -350,7 +406,7 @@ FROM t1 a AND rnk = 2) SELECT DISTINCT a.*, 5 AS concept_order, - 'more Pr lat + Sec lat' AS order_desc + 'ATC Combo Class: Primary lateral + Secondary lateral, more than 2 Ingredients' AS order_desc FROM t2 a JOIN t3 b ON b.class_code = a.class_code JOIN t3 c @@ -400,7 +456,7 @@ FROM t1 a JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND rnk = 2) SELECT DISTINCT a.*, 6 AS concept_order, - 'pr lat+sec lat 2 ingreds' as order_desc + 'ATC Combo Class: Primary lateral + Secondary lateral, 2 Ingredients' as order_desc FROM t2 a JOIN t3 b ON b.class_code = a.class_code @@ -423,11 +479,11 @@ FROM case_2 a JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 AND d2.concept_class_id = 'Dose Form') - SELECT DISTINCT class_code, - class_name, + SELECT DISTINCT a.class_code, + a.class_name, c.*, 7 AS concept_order, - 'Primary lateral only in comb.' AS order_desc + 'ATC Combo Class: Primary lateral only in combination' AS order_desc FROM t1 a JOIN rx_all_combo b ON lower (b.ing_name) = lower (a.ing_name) @@ -435,9 +491,10 @@ FROM t1 a JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' - AND c.concept_name ~ ' / '; + AND c.concept_name ~ ' / ' + AND (a.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); --- Primary lateral only in combination with an excluded Ingredient (order=8) +-- Primary lateral only in combination with an excluded Ingredient (order = 8) INSERT INTO class_to_drug_new WITH t1 AS @@ -477,8 +534,8 @@ SELECT DISTINCT class_code, valid_start_date, valid_end_date, invalid_reason, - 8, - 'Primary lateral in comb with excl' + 8 as concept_order, + 'ATC Combo Class: Primary lateral in combination with excluded Ingredient' FROM t2 WHERE concept_name ~ ' / ' AND concept_id NOT IN (SELECT d_id @@ -488,7 +545,7 @@ AND concept_id NOT IN (SELECT d_id AND LOWER (b.df_name) = LOWER (a.df_name) AND b.rnk = 0); --- Primary upward only (order= 9) +-- Primary upward only (order = 9) INSERT INTO class_to_drug_new WITH t1 AS @@ -505,8 +562,8 @@ FROM case_3 a SELECT DISTINCT class_code, class_name, c.*, - 9, - ' Primary upward' AS order_desc + 9 as concept_order, + 'ATC Combo Class: more Primary upward' AS order_desc FROM t1 a JOIN rx_all_combo b ON lower (b.ing_name) = lower (a.ing_name) @@ -514,7 +571,8 @@ FROM t1 a JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' - AND c.concept_name ~ ' / '; + AND c.concept_name ~ ' / ' + AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new); -- Primary upward + Secondary upward (order = 10) INSERT INTO class_to_drug_new @@ -557,7 +615,9 @@ FROM t1 a ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND rnk = 4) -SELECT DISTINCT a.*, 10, 'Primary upward + Secondary upward' +SELECT DISTINCT a.*, + 10 AS concept_order, + 'ATC Combo Class: Primary upward + Secondary upward' FROM t2 a JOIN t3 b ON b.class_code = a.class_code @@ -571,16 +631,14 @@ DROP TABLE IF EXISTS t1; CREATE UNLOGGED TABLE t1 AS SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk - FROM dev_combo WHERE rnk = 1 -- Primary lateral - ; + FROM dev_combo WHERE rnk = 1; -- Primary lateral -- separate Secondary lateral Ingredients (rnk = 2 in dev_combo) DROP TABLE IF EXISTS t2; CREATE UNLOGGED TABLE t2 AS SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk - FROM dev_combo WHERE rnk = 2 -- Secondary lateral - ; + FROM dev_combo WHERE rnk = 2; -- Secondary lateral -- separate Primary upward Ingredients (rnk = 3 in dev_combo) DROP TABLE IF EXISTS t3; @@ -588,8 +646,7 @@ CREATE UNLOGGED TABLE t3 AS SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk FROM dev_combo -WHERE rnk = 3 -- Primary upward -; +WHERE rnk = 3; -- Primary upward -- separate Secondary upward Ingredients (rnk = 4 in dev_combo) DROP TABLE IF EXISTS t4; @@ -599,7 +656,7 @@ SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk FROM dev_combo WHERE rnk = 4; -- Secondary upward --- create a table with aggregated ATC Ingredients per one ATC Combo Class (no more than 3 ingredients per Class is recommended) +-- create table with aggregated ATC Ingredients per one ATC Combo Class (no more than 3 ingredients per Class is recommended) -- add Primary lateral AND (Secondary lateral 1 AND/OR Secondary lateral 2) AND/OR Primary upward AND/OR Secondary upward DROP TABLE if exists full_combo; CREATE UNLOGGED TABLE full_combo @@ -660,7 +717,7 @@ WHERE b.concept_id<>c.concept_id AND b.concept_id<>a.concept_id; INSERT INTO full_combo SELECT * FROM permutations; --- Separate Primary lateral in combination (rnk = 1 in dev_combo) +-- separate Primary lateral in combination (rnk = 1 in dev_combo) DROP TABLE if exists ing_pr_lat_combo; -- ing_pr_lat_combo CREATE TABLE ing_pr_lat_combo AS @@ -738,7 +795,8 @@ WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0) +); -- create temporary table with all possible i_combos for Primary lateral in combination with excluded Ingredient DROP TABLE if exists ing_pr_lat_combo_excl_to_drug; @@ -754,7 +812,7 @@ JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code AND a1.rnk <> a.rn -- excluded JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id JOIN rx_combo d1 ON d1.drug_concept_id = f.drug_concept_id -- excluded -AND d1.drug_concept_id <> d.drug_concept_id);; +AND d1.drug_concept_id <> d.drug_concept_id); -- separate Primary upward + Secondary upward (rnk in (3,4) in dev_combo) DROP TABLE if exists ing_pr_sec_up_combo; @@ -935,7 +993,7 @@ AND r.invalid_reason IS NULL c.concept_name as class_name, d.*, 11 AS conept_order, - 'Combo with form' + 'ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations' FROM full_combo_with_form f JOIN concept_manual c on c.concept_code = f.class_code and c.invalid_reason is null and c.concept_class_id = 'ATC 5th' JOIN t1 r @@ -944,35 +1002,26 @@ AND r.invalid_reason IS NULL JOIN concept d on d.concept_id = r.concept_id and f.class_code||r.concept_id not in (select class_code||concept_id from class_to_drug_new); --- add mappings of ATC Combo Classes to Contraceptive Packs (order = 12) -INSERT INTO class_to_drug_new -SELECT class_code, - class_name, - c.*, - 12, - 'contraceptive pack' -FROM class_to_drug_new ctd - JOIN concept_ancestor ON ctd.concept_id = ancestor_concept_id - JOIN concept c ON descendant_concept_id = c.concept_id -WHERE class_code ~ 'G03FB|G03AB' -- the list can be enriched -AND c.concept_class_id IN ('Clinical Pack') -AND (ctd.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); - --- add manual mappings from concept_relationship_manual (order = 13) +-- add manual mappings from concept_relationship_manual (order = 12) INSERT INTO class_to_drug_new -SELECT DISTINCT a.class_code, -f.concept_name as class_name, -c.*, -13, -'from crm' -from class_drugs_scraper a -join concept_relationship_manual b on b.concept_code_1 = a.class_code -join concept_manual f on f.concept_code = a.class_code -AND b.relationship_Id in ('ATC - RxNorm') -JOIN concept c on c.concept_code = b.concept_code_2 and c.vocabulary_id = b.vocabulary_id_2 -and c.vocabulary_id in ('RxNorm', 'RxNorm Extension') and c.standard_concept = 'S' - AND (a.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new) - and f.invalid_reason is null and f.concept_class_id = 'ATC 5th' ; +SELECT DISTINCT a.class_code, + f.concept_name AS class_name, + c.*, + 12 AS concept_order, + 'ATC Class to Drug Product from concept_relationship_manual' +FROM class_drugs_scraper a + JOIN concept_relationship_manual b ON b.concept_code_1 = a.class_code + JOIN concept_manual f + ON f.concept_code = a.class_code + AND b.relationship_Id IN ('ATC - RxNorm') + JOIN concept c + ON c.concept_code = b.concept_code_2 + AND c.vocabulary_id = b.vocabulary_id_2 + AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') + AND c.standard_concept = 'S' + AND (a.class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) +AND f.invalid_reason IS NULL +AND f.concept_class_id = 'ATC 5th'; -- manual clean up for Precise Ingredients and other particular cases (according to the information on the ATC WHO Website) DELETE @@ -983,17 +1032,17 @@ OR class_code='N05AF02' -- clopentixol OR class_code IN ('D07AB02','D07BB04') -- hydrocortisone butyrate + combo that so far doesn't exist OR class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral OR (class_code = 'B02BD14' AND concept_name LIKE '%Tretten%') -- 2 --catridecacog -OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog -; +OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient');-- susoctocog alfa | catridecacog --- add additional semi-manual mappings based on pattern-matching (order=14) + +-- add additional semi-manual mappings based on pattern-matching (order = 13) INSERT INTO class_to_drug_new with t1 as ( SELECT 'B02BD11' as class_code,'catridecacog' as class_name, concept_id FROM concept WHERE (vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') - OR concept_id = 35603348 -- the whole hierarchy (factor XIII Injection [Tretten] Branded Drug Form) + OR concept_id = 35603348 -- the whole hierarchy (factor XIII Injection [Tretten] Branded Drug Form) UNION ALL SELECT 'B02BD14','susoctocog alfa', concept_id FROM concept @@ -1029,16 +1078,16 @@ WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ILIKE '%Pentaerythritol Tetr SELECT DISTINCT a.class_code, d.concept_name, c.*, - 14, - 'point fix' + 13 as concept_order, + 'ATC Class with semi-manual point fix' FROM t1 a JOIN concept c ON c.concept_id = a.concept_id JOIN concept_manual d ON d.concept_code = a.class_code AND d.concept_class_id = 'ATC 5th' - AND d.invalid_reason IS NULL; -; - + AND d.invalid_reason IS NULL +AND a.class_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); + -- clean up erroneous amount of ingredients DELETE --select * @@ -1048,15 +1097,13 @@ WHERE class_name LIKE '%,%and%' AND NOT class_name ~* 'comb|other|whole root|selective' AND concept_name NOT LIKE '% / % / %'; ---- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug(order=15) +--- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug (order = 14) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, - a.concept_id, - a.concept_name, - a.concept_class_id, - 14, - 'from sources.c_t_d' + c.*, + 14 as concept_order, + 'ATC Class from old class_to_drug' FROM sources.class_to_drug a JOIN concept_manual b ON b.concept_code = a.class_code JOIN concept c @@ -1081,16 +1128,17 @@ AND (class_code,c.concept_id) NOT IN ( SELECT 'A06AA02',43158334 UNION ALL SELECT 'A06AA02',40036796); + /********************** ****** ADD PACKS ****** ***********************/ --- add packs of Primary lateral only in combination (order 16) +-- add packs of Primary lateral only in combination (order = 15) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 16 as concept_order, - 'Pack: Pr lat in combo' as order_desc + 15 as concept_order, + 'Pack: Primary lateral in combo' as order_desc FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id JOIN concept d @@ -1113,13 +1161,13 @@ AND a.class_code IN (SELECT class_code AND j.concept_name ~ ' / ' -- combos only AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); --- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 17) +-- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 16) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 17, - 'Pack: Pr lat in combo additional' + 16 as concept_order, + 'Pack: Primary lateral in combo additional' FROM class_to_drug_new a JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id JOIN concept j ON j.concept_id = r2.concept_id_2 @@ -1136,13 +1184,13 @@ AND a.class_code IN (SELECT class_code AND j.concept_name ~ ' / ' -- combos only AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); --- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 18) +-- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 17) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 18, - 'Pack: Pr lat + Sec lat' + 17 as concept_order, + 'Pack: Primary lateral + Secondary lateral' FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id JOIN concept d @@ -1165,13 +1213,13 @@ AND a.class_code IN (SELECT class_code AND j.concept_name ~ ' / ' AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); --- add Branded packs of Primary lateral + Secondary lateral (order = 19) +-- add Branded packs of Primary lateral + Secondary lateral (order = 18) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 19, - 'Pack: Pr lat + Sec lat Branded' + 18 as concept_order, + 'Pack: Primary lateral + Secondary lateral Branded' FROM class_to_drug_new a JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id JOIN concept j ON j.concept_id = r2.concept_id_2 @@ -1189,13 +1237,13 @@ AND a.class_code IN (SELECT class_code AND j.concept_name ~ ' / ' AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); --- add packs of Primary upward and Secondary upward (Class C AND Class D, order = 20) +-- add packs of Primary upward and Secondary upward (Class C AND Class D, order = 19) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 20, - 'Pack: Pr up + Sec up' + 19 as concept_order, + 'Pack: Primary upward + Secondary upward' FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id JOIN concept d @@ -1217,13 +1265,13 @@ AND a.class_code IN (SELECT class_code AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) AND j.concept_name ~ ' / '; --- add packs of Primary lateral and Secondary upward (Class A + Class D, order = 21) +-- add packs of Primary lateral and Secondary upward (Class A + Class D, order = 20) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 21, - 'Pack: Pr lat + Sec up' + 20 as concept_order, + 'Pack: Primary lateral + Secondary upward' FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id JOIN concept d @@ -1246,13 +1294,13 @@ AND a.class_code IN (SELECT class_code AND j.concept_name ~ ' / ' AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); --- add additional packs of Primary lateral and Secondary upward (Class A + Class D, order = 22) +-- add additional packs of Primary lateral and Secondary upward (Class A + Class D, order = 21) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 22, - 'Pack: Pr lat + Sec up additional' + 21 as concept_order, + 'Pack: Primary lateral + Secondary upward additional' FROM class_to_drug_new a JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id JOIN concept j ON j.concept_id = r2.concept_id_2 @@ -1270,13 +1318,13 @@ AND a.class_code IN (SELECT class_code AND j.concept_name ~ ' / ' AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); ---- add missing Packs using previous version of class_to_drug +--- add missing Packs using previous version of class_to_drug (order = 22) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, c.*, - 23, - 'packs: from sources.c_t_d' + 22 as concept_order, + 'Pack: from old c_t_d' FROM sources.class_to_drug a JOIN concept_manual b ON b.concept_code = a.class_code JOIN concept c @@ -1289,29 +1337,45 @@ AND a.class_code NOT IN ('G01AF55','S03CA04','S01CA03','S02CA03')-- gives pack AND a.class_code NOT IN ('B03AE01','C07BB52','D01AC52','C10AD52') -- wrong ing combo ; --- Enrich pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' (they are always used as packs) +-- add mappings of ATC Combo Classes to Contraceptive Packs (order = 23) +INSERT INTO class_to_drug_new +SELECT class_code, + class_name, + c.*, + 23 as concept_order, + 'Pack: semi-manual contraceptive' +FROM class_to_drug_new ctd + JOIN concept_ancestor ON ctd.concept_id = ancestor_concept_id + JOIN concept c ON descendant_concept_id = c.concept_id +WHERE class_code ~ 'G03FB|G03AB' -- the list can be enriched +AND c.concept_class_id IN ('Clinical Pack') +AND (ctd.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + +-- еnrich pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' +-- they are always used as packs (order = 24) INSERT INTO class_to_drug_new SELECT DISTINCT class_code, class_name, c.*, - 24, - 'Pack: semi-manual' + 24 as concept_order, + 'Pack: semi-manual contraceptive' FROM class_to_drug_new f JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) JOIN devv5.concept c ON c.concept_id = descendant_concept_id AND c.concept_class_id LIKE '%Pack%' WHERE f.class_code ~ 'G03FB|G03AB' -; +AND (f.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + -- get rid of all other concept_class_ids except Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' DELETE FROM class_to_drug_new WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens -AND concept_class_id !~ 'Pack'; +AND concept_class_id !~ 'Pack'; -- 68 --- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique +-- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 25) INSERT INTO class_to_drug_new - WITH ing AS ( + WITH ing AS ( SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt FROM ( SELECT DISTINCT substring(i.concept_code_1,'\w+'), r.concept_code_1, concept_id_2 @@ -1319,14 +1383,14 @@ FROM relationship_to_concept r JOIN internal_relationship_stage i on i.concept_code_2 = r.concept_code_1 JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' ) a GROUP BY concept_id_2,concept_code_1 ), - drug AS ( + drug AS ( SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 FROM internal_relationship_stage i WHERE not exists (SELECT 1 FROM internal_relationship_stage i2 WHERE i.concept_code_2=i2.concept_code_2 AND substring(i.concept_code_1,'\w+')!=substring(i2.concept_code_1,'\w+')) ), - drug_name AS ( + drug_name AS ( SELECT DISTINCT class_code,class_name, concept_id_2 FROM ing JOIN drug ON concept_code_2 = concept_code_1 @@ -1334,7 +1398,7 @@ JOIN class_drugs_scraper ON code = class_code WHERE cnt=1 AND class_name = concept_code_1 AND code NOT IN (SELECT class_code FROM class_to_drug_new) ), -all_drug AS ( + all_drug AS ( SELECT class_code, class_name, concept_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt FROM drug_name a JOIN concept c on c.concept_id = a.concept_id_2 @@ -1342,14 +1406,349 @@ JOIN concept c on c.concept_id = a.concept_id_2 SELECT DISTINCT a.class_code, a.class_name, c.*, - 25, - 'ATC Classes WO Dose Forms to unique Drug Product' + 25 as concept_order, + 'ATC Class WO Dose Form to unique Drug Product' FROM all_drug a join concept c on c.concept_id = a.concept_id WHERE cnt = 1 AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); --- add links from ATC Monocomponent Classes which do not have Dose Forms to Ingredients +-- add mappings of Monocomponent ATC Classes using additional reference (order = 26) +INSERT INTO class_to_drug_new +SELECT DISTINCT class_code, + class_name, + c.*, + 26, + 'ATC Monocomp: from additional reference' +FROM atc_all_combo a + JOIN rx_all_combo b + ON b.ing_id = a.ing_id + AND a.df_id = b.df_id + JOIN concept c + ON c.concept_id = b.d_id + AND a.class_code||b.d_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) + AND a.class_code IN (SELECT class_code FROM ing_pr_lat_combo_to_drug) +AND c.concept_name ~ '/'; + +-- add mappings of ATC Combo Classes using additional reference (order = 27) +INSERT INTO class_to_drug_new +SELECT DISTINCT class_code, + class_name, + c.*, + 27, + 'ATC Combo: Pr upward only from additional reference' +FROM atc_all_combo a + JOIN rx_all_combo b + ON b.ing_id = a.ing_id + AND a.df_id = b.df_id + JOIN concept c + ON c.concept_id = b.d_id + AND a.class_code||b.d_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) + AND a.class_code IN (SELECT class_code FROM case_3) +AND c.concept_name ~ '/'; + +-- add those which are absent in the drug hierarchy +-- step 1 +DROP TABLE if exists no_atc_1; +CREATE TABLE no_atc_1 +AS +(SELECT descendant_concept_id +FROM concept_ancestor + JOIN dev_atc.class_to_drug_new n ON ancestor_concept_id = n.concept_id); + +-- step 2 +DROP TABLE if exists no_atc_2; +CREATE TABLE no_atc_2 +AS +(SELECT DISTINCT c.* +FROM concept c +WHERE domain_id = 'Drug' +AND concept_id NOT IN (SELECT descendant_concept_id FROM no_atc_1) +AND c.vocabulary_id IN ('RxNorm','RxNorm Extension') +AND c.concept_class_id != 'Ingredient' +AND c.standard_concept = 'S'); + +-- step 3 +DROP TABLE if exists no_atc_1_with_form; +CREATE TABLE no_atc_1_with_form +AS +(SELECT DISTINCT a.concept_id, + a.concept_name, + a.concept_class_id, + a.vocabulary_id, + b.ingredient_concept_id AS ing_id, + d.concept_name AS ing_nm, + d.standard_concept, + g.concept_id AS df_id, + g.concept_name AS df_nm +FROM no_atc_2 a + JOIN drug_strength b ON b.drug_concept_id = a.concept_id + JOIN concept d ON d.concept_id = b.ingredient_concept_id + JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + JOIN concept g + ON g.concept_id = r.concept_id_2 + AND g.concept_class_id = 'Dose Form'); + +-- add additional mappings for hierarchical absentees (order = 28) +INSERT INTO class_to_drug_new +SELECT DISTINCT k.concept_code AS class_code, + k.concept_name AS class_name, + p.*, + 28 as concept_order, + 'ATC Monocomp Class to Drug Product which is out of hierarchy' +FROM no_atc_1_with_form a + JOIN internal_relationship_stage i ON lower (i.concept_code_2) = lower (a.ing_nm) + JOIN internal_relationship_stage i2 + ON lower (i2.concept_code_2) = lower (a.df_nm) + AND i.concept_code_1 = i2.concept_code_1 + JOIN concept_manual k ON k.concept_code = SUBSTRING (i.concept_code_1,'\w+') + JOIN concept p + ON p.concept_id = a.concept_id + AND p.standard_concept = 'S' +WHERE a.concept_name !~ ' / ' +AND k.invalid_reason IS NULL +AND k.concept_class_id = 'ATC 5th' +AND k.concept_code NOT IN (SELECT class_code FROM dev_combo) +AND k.concept_code||p.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); + +-- obtain more ATC Combo classes +DROP TABLE no_atc_full_combo; +CREATE TABLE no_atc_full_combo +AS +SELECT DISTINCT concept_id, + concept_name, + df_id, + STRING_AGG(ing_id::VARCHAR,'-' ORDER BY ing_id::INT) AS i_combo +FROM no_atc_1_with_form +GROUP BY concept_id, + concept_name, + df_id; + +DROP TABLE IF EXISTS no_atc_reodered; +CREATE UNLOGGED TABLE no_atc_reodered AS +SELECT DISTINCT fc.concept_id, -- missing Rx Drug + fc.concept_name, + df_id, + l.i_combo +FROM no_atc_full_combo fc +CROSS JOIN LATERAL(SELECT STRING_AGG(s0.ing, '-' ORDER BY s0.ing::INT) AS i_combo FROM ( + SELECT UNNEST(STRING_TO_ARRAY(fc.i_combo, '-')) AS ing + ) AS s0) l; + +INSERT INTO no_atc_full_combo +(concept_id, i_combo, df_id) +SELECT DISTINCT concept_id, + i_combo, df_id + FROM no_atc_reodered ; +; +CREATE INDEX i_no_atc_full_combo ON no_atc_full_combo (concept_id, i_combo,df_id); + +-- add mappings of ATC Combo Class to Drug Product which is out of drug hierarchy (order = 29) +INSERT INTO class_to_drug_new + SELECT DISTINCT k.concept_code as class_code, -- ATC + k.concept_name as class_name, + d.*, + 29 AS conept_order, + 'ATC Combo Class to Drug Product which is out of drug hierarchy' + FROM no_atc_full_combo f + JOIN full_combo_with_form c on c.i_combo = f.i_combo and c.df_id = f.df_id is null + join concept_manual k on k.concept_code = c.class_code and k.concept_class_id = 'ATC 5th' and k.invalid_reason is null + JOIN concept d on d.concept_id = f.concept_id + and c.class_code||d.concept_id not in (select class_code||concept_id from class_to_drug_new); + +-- add additional manual mapping (order = 30) => to crm +INSERT INTO class_to_drug_new +SELECT DISTINCT b.concept_code, + b.concept_name, + c.*, + 33, + 'gentle manual introduction of a bit fishy RxE/RxN mappings which are out of the hierarchy' +FROM no_atc_manual a + JOIN concept c ON c.concept_id = a.concept_id + JOIN concept_manual b + ON b.concept_code = a.class_code + AND b.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); + +-- remove excessive links to children among Packs +WITH t1 AS +( + SELECT * + FROM class_to_drug_new + where concept_class_id ~ 'Pack') +delete from class_to_drug_new where class_code||concept_id in ( +SELECT a1.class_code|| + a2.concept_id +FROM t1 a1-- papa + JOIN t1 a2 -- child +ON a1.class_code = a2.class_code + JOIN concept_ancestor ca + ON a1.concept_id = ca.ancestor_concept_id + AND a2.concept_id = ca.descendant_concept_id + AND a1.concept_id <> a2.concept_id + and a2.concept_class_id ~ 'Pack' + AND a1.class_code not in ('B02BD14', 'G03FB01', 'J07BK01') )-- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic); -- 2558 + ; + +-- remove excessive links to children among unpacked Drug Products +WITH t0 AS +( + SELECT * + FROM class_to_drug_new + WHERE concept_class_id !~ 'Pack' +), +t1 as ( +select * from t0 where class_code IN (SELECT class_code + FROM class_to_drug_new + GROUP BY class_code + HAVING COUNT(1) >= 2)) +DELETE +FROM class_to_drug_new where class_code||concept_id IN ( +SELECT a1.class_code|| + a2.concept_id -- child mapping +FROM t1 a1-- papa + JOIN t1 a2 -- child +ON a1.class_code = a2.class_code + JOIN concept_ancestor ca + ON a1.concept_id = ca.ancestor_concept_id + AND a2.concept_id = ca.descendant_concept_id + AND a1.concept_id <> a2.concept_id + and a2.concept_class_id !~ 'Pack' + AND a1.class_code NOT IN ('B02BD11', 'B02BD14', 'G03FB01', 'J07BK01', 'J07BK02')) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic + ; +/*************************** +******** DF CLEAN UP ******* +****************************/ +-- clean up oral forms +DROP TABLE IF EXISTS wrong_df; +CREATE TABLE wrong_df as ( +SELECT *, 'oral mismatch' as issue_desc +FROM class_to_drug_new +WHERE SPLIT_PART(class_name,';',2) ~ 'oral' +AND concept_name !~* 'oral|chew|tooth|mouth|elixir|Extended Release Suspension|buccal|Sublingual|Paste|Mucosal|Prefilled Syringe|\...' +AND class_name !~ 'rectal|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|nasal' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' +AND class_code NOT IN ('A01AA04','A01AA02','G04BD08') +); + +-- clean up rectal forms +INSERT INTO wrong_df +SELECT *, 'rectal mismatch' +FROM class_to_drug_new +WHERE class_name ~ 'rectal' +AND concept_name !~* 'rectal|topical|mucosal|enema' +AND class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|systemic' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + +-- clean up topical forms +INSERT INTO wrong_df +SELECT *, 'topical mismatch' +FROM class_to_drug_new +WHERE class_name ~ 'topical' +AND concept_name !~* 'topical|mucosal|Drug Implant|Prefilled Applicator|Shampoo|Paste|Medicated Pad|Transdermal System|Soap|Powder Spray|Medicated Patch|Douche|vaginal|\yNail\y|Intrauterine System|Mouthwash' +AND concept_name !~* '\yStick|Rectal Foam|Medicated Tape|Medicated Guaze|Paint|Rectal|Nasal|Otic|Ophthalmic Solution|Dry Powder|Urethral Suppository|Intrauterine|irrigation|Cement|Cream|ointment|spray|gum|enema|Ophthalmic' +AND class_name !~* 'oral|vaginal|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + +-- clean up local oral forms +INSERT INTO wrong_df +SELECT *, 'local oral mismatch' +FROM class_to_drug_new +WHERE class_name ~ 'local oral' and class_name !~ 'oral, local oral' +AND concept_name !~* 'mouth|topical|paste|irrig|Lozenge|gum|buccal|Suspension|solution|subl|Gel|spray|Disintegrating Oral Tablet|Effervescent Oral Tablet|Oral Powder|Chewable Tablet|Oral Film|Oral Granules' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + +-- clean up inhalant +INSERT INTO wrong_df +SELECT *, 'inhalant mismatch' +FROM class_to_drug_new +WHERE class_name ~* 'inhalant' +AND concept_name !~* 'inhal|nasal|Powder Spray' +AND class_name !~ 'oral|local oral|vaginal|parenteral|transdermal|ophthalmic|otic|rectal|systemic|topical' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + +-- clean up parenteral +INSERT INTO wrong_df +SELECT *, 'parenteral mismatch' +FROM class_to_drug_new +WHERE class_name ~* 'parenteral' +AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant' +AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|ophthalmic|otic|nasal|rectal|systemic|urethral' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; +-- clean up otic|ophthalmic +INSERT INTO wrong_df +SELECT *, 'otic|ophthalmic mismatch' +FROM class_to_drug_new +WHERE class_name ~* 'otic|ophthalmic' +AND concept_name !~* 'otic|ophthalmic|topical|Prefilled Applicator' +AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|nasal|rectal|systemic|urethral|parenteral' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + +-- clean up systemic forms +INSERT INTO wrong_df +SELECT *, 'systemic mismatch' +FROM class_to_drug_new +WHERE class_name ~* 'systemic' +AND concept_name ~* 'nasal' +AND concept_name !~ 'metered' +AND class_name !~* 'rectal|nasal|vaginal|topical' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' +AND concept_name ~* 'caine|azoline|thrombin|sodium chloride|glycerol|acetylcysteine|chlorhexidine|amlexanox|ammonia|Chlorobutanol|phenylephrine|dexamethasone|Ephedrine|Histamine|Hydrocortisone|Peppermint oil|pyrilamine|dexpanthenol|Phentolamine|Sodium Chloride|Cellulose|glycerin|pantenol|tetrahydrozoline|triamcinolone' +; + +-- clean up systemic forms 2 +INSERT INTO wrong_df +SELECT *, 'systemic mismatch 2' +FROM class_to_drug_new +WHERE class_name ~* 'systemic' +AND concept_name !~* '\.\.\.$|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' +AND class_name !~* 'rectal|nasal|vaginal|topical' +AND concept_name !~* 'Alfentanil|scopolamine|amyl nitrite|Heroin|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + +-- clean up instillation packs +INSERT INTO wrong_df +SELECT *, 'instill pack mismatch' +FROM class_to_drug_new +WHERE class_name ~ 'instill' +AND concept_name !~* 'instil|irrig' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' +AND concept_name !~ 'Intratracheal|Phospholipids / Soybean Oil'; + +-- clean up parenteral packs +INSERT INTO wrong_df +SELECT *, 'parenteral pack mismatch' +FROM class_to_drug_new +WHERE class_name ~* 'parenteral' +AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant' +AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|ophthalmic|otic|nasal|rectal|systemic|urethral' +AND concept_class_id ~ 'Pack'; + +-- clean up systemic packs +INSERT INTO wrong_df +SELECT *, 'systemic pack mismatch 2' +FROM class_to_drug_new +WHERE class_name ~* 'systemic' +AND concept_name !~* '\...$|\...\) \} Pack$|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' +AND class_name !~* 'rectal|nasal|vaginal|topical' +AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' +AND concept_class_id ~ 'Pack'; + +-- clean up systemic packs 2 - check This next time +INSERT INTO wrong_df +SELECT * , 'systemic pack mismatch 3' +FROM class_to_drug_new +WHERE class_name ~* 'systemic' +AND concept_name !~* '\.\.\.$|\.\.\.\) \} Pack|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' +AND class_name !~* 'rectal|nasal|vaginal|topical' +AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' +AND concept_class_id ~ 'Pack'; + +-- look at them and remove from class_to_drug_new +DELETE +FROM class_to_drug_new +WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM wrong_df); + +-- add links from ATC Monocomponent Classes which do not have Dose Forms to Ingredients (in the future this concepts could be processed manually, some of them have mappings) INSERT INTO class_to_drug_new WITH t1 AS @@ -1362,8 +1761,8 @@ AND concept_code NOT IN (SELECT class_code FROM atc_inexistent)) SELECT DISTINCT a.concept_code, a.concept_name, c.*, - 26, - 'Monocomponent ATC WO Dose Forms' + 31 as concept_order, + 'ATC Monocomp Class WO Dose Form to Ingredient' FROM t1 a JOIN internal_relationship_stage i ON SUBSTRING (concept_code_1,'\w+') = a.concept_code JOIN relationship_to_concept r ON r.concept_code_1 = i.concept_code_2 @@ -1371,8 +1770,9 @@ FROM t1 a ON c.concept_id = r.concept_id_2 AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - AND a.concept_code NOT IN (SELECT class_code FROM dev_combo); - + AND a.concept_code NOT IN (SELECT class_code FROM dev_combo) + and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new); + -- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (do we need them?) INSERT INTO class_to_drug_new WITH t1 AS @@ -1387,8 +1787,8 @@ AND concept_code NOT IN (SELECT class_code FROM atc_inexistent) SELECT DISTINCT a.concept_code, a.concept_name, c.*, - 27, - 'Multicomp ATC Class to Ingredient' + 32 as concept_order, + 'ATC Combo Class to Ingredient' FROM t1 a JOIN dev_combo k ON k.class_code = a.concept_code @@ -1396,53 +1796,27 @@ FROM t1 a JOIN concept c ON c.concept_id = k.concept_id AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S'; + AND c.standard_concept = 'S' + and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new); + +-- remove suspicious mapping of inexistent drugs (this table should be checked before) +DELETE +FROM class_to_drug_new +WHERE class_code IN (SELECT class_code FROM atc_inexistent); --- remove excessive links to children among Packs -WITH t1 AS -( - SELECT * - FROM class_to_drug_new - WHERE class_code IN (SELECT class_code - FROM class_to_drug_new - GROUP BY class_code - HAVING COUNT(1) >= 2) - and class_code in (select class_code from class_to_drug_new where concept_class_id ~ 'Pack') - and class_code not in (select class_code from class_to_drug_new where concept_class_id !~ 'Pack') -) -delete from class_to_drug_new where class_code||concept_id in ( -SELECT a1.class_code|| - a2.concept_id -FROM t1 a1 -- papa - JOIN t1 a2 -- child -ON a1.class_code = a2.class_code - JOIN concept_ancestor ca - ON a1.concept_id = ca.ancestor_concept_id - AND a2.concept_id = ca.descendant_concept_id - AND a1.concept_id <> a2.concept_id - and a2.concept_class_id ~ 'Pack'); - --- remove excessive links to children among unpacked Drug Products :) -WITH t1 AS -( - SELECT * - FROM class_to_drug_new - WHERE class_code IN (SELECT class_code - FROM class_to_drug_new - GROUP BY class_code - HAVING COUNT(1) >= 2) - and class_code not in (select class_code from class_to_drug_new where concept_class_id ~ 'Pack') -) -delete from class_to_drug_new where class_code||concept_id in ( -SELECT a1.class_code|| - a2.concept_id -- child mapping -FROM t1 a1 -- papa - JOIN t1 a2 -- child -ON a1.class_code = a2.class_code - JOIN concept_ancestor ca - ON a1.concept_id = ca.ancestor_concept_id - AND a2.concept_id = ca.descendant_concept_id - AND a1.concept_id <> a2.concept_id - and a2.concept_class_id !~ 'Pack'); +-- remove dead ATC Classes if any +DELETE +FROM class_to_drug_new +WHERE class_code IN (SELECT concept_code + FROM concept_manual + WHERE invalid_reason IS NOT NULL); + +-- remove duplicates if any (however they should not be there) +DELETE +FROM class_to_drug_new +WHERE CTID NOT IN (SELECT MIN(CTID) + FROM class_to_drug_new + GROUP BY class_code, + concept_id); -- run load_stage.sql From 76b3985da2180220ea32b90db8c68f274fea1f9b Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 2 Sep 2021 20:09:16 +0300 Subject: [PATCH 11/45] refactoring --- ATC/load_input.sql | 129 ++++++--------------------------------------- 1 file changed, 15 insertions(+), 114 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 31625025b..f1eb9a62d 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -1659,62 +1659,7 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND rnk = 1; -DELETE -FROM dev_combo -WHERE class_code = 'A06AG11' -AND class_name = 'sodium lauryl sulfoacetate, incl. combinations' -AND concept_name = 'sodium' -AND rnk = 1; - -DELETE -FROM dev_combo -WHERE class_code = 'A01AA51' -AND class_name = 'sodium fluoride, combinations' -AND concept_name = 'sodium' -AND rnk = 1; - -DELETE -FROM dev_combo -WHERE class_code = 'A06AB58' -AND class_name = 'sodium picosulfate, combinations' -AND concept_name = 'sodium' -AND rnk = 1; - -DELETE -FROM dev_combo -WHERE class_code = 'B05XA06' -AND class_name = 'potassium phosphate, incl. combinations WITH other potassium salts' -AND concept_name = 'potassium' -AND rnk = 1; - -DELETE -FROM dev_combo -WHERE class_code = 'A12BA51' -AND class_name = 'potassium chloride, combinations' -AND concept_name = 'potassium' -AND rnk = 1; - -DELETE -FROM dev_combo -WHERE class_code = 'C01DA58' -AND class_name = 'isosorbide dinitrate, combinations' -AND concept_name = 'isosorbide' -AND rnk = 1; - --- fix erroneous rnk of 3 for J07AG53 -UPDATE dev_combo - SET rnk = 1 -WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND rnk = 3; - -DELETE -FROM dev_combo -WHERE class_code = 'B03AD01' -AND rnk = 4; - +-- add missing codeine INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -1725,33 +1670,6 @@ SELECT DISTINCT class_code, FROM dev_combo WHERE class_code = 'N02AA59'; --- erroneous map to bismuth -DELETE -FROM dev_combo -WHERE class_code = 'A02BD08' -AND class = 'bismuth subcitrate, tetracycline' -AND concept_id = 19025138; --- erroneous map to Pentaerythritol Distearate -DELETE -FROM dev_combo -WHERE class_code = 'C01DA55' -AND class = '' -AND concept_id = 42903512; - --- erroneous map to Potassium -DELETE -FROM dev_combo -WHERE class_code = 'B05XA06' -AND class = 'potassium phosphate, incl. combinations with other potassium salts' -AND concept_id = 19049024; - --- erroneous map to tenofovir -DELETE -FROM dev_combo -WHERE class_code = 'J05AR03' -AND class = 'tenofovir disoproxil' -AND concept_id = 19011093; - -- remove doubling ingredients with different rank, remaining those which are Primary lateral DELETE FROM dev_combo @@ -1764,13 +1682,6 @@ WHERE (class_code,concept_id,rnk) IN (SELECT a.class_code, AND a.concept_id = b.concept_id WHERE a.rnk > 1 AND b.rnk = 1); - -UPDATE dev_combo - SET rnk = 3 -WHERE class_code = 'A07FA51' -AND class_name = 'lactic acid producing organisms, combinations' -AND concept_id = 19136028 -AND concept_name = 'Saccharomyces cerevisiae'; DELETE FROM dev_combo @@ -1801,13 +1712,6 @@ AND class_name = 'cough suppressants and mucolytics' AND class = 'cough suppressants' AND concept_id = 19057932 AND rnk = 3; -DELETE -FROM dev_combo -WHERE class_code = 'R05FB01' -AND class_name = 'cough suppressants and mucolytics' -AND class = '' -AND concept_id = 19057932 -AND rnk = 4; UPDATE dev_combo SET rnk = 4 @@ -1817,22 +1721,6 @@ AND class = 'cough suppressants' AND concept_id = 19071999 AND rnk = 3; -UPDATE dev_combo - SET rnk = 3 -WHERE class_code = 'S02AA30' -AND class_name = 'antiinfectives, combinations' -AND class = '' -AND concept_id = 963742 -AND rnk = 4; - -UPDATE dev_combo - SET rnk = 3 -WHERE class_code = 'S02AA30' -AND class_name = 'antiinfectives, combinations' -AND class = '' -AND concept_id = 19006842 -AND rnk = 4; - UPDATE dev_combo SET rnk = 4 WHERE class_code = 'J01RA02' @@ -1985,7 +1873,8 @@ UPDATE dev_combo WHERE class_code = 'R05FB02' AND class = 'cough suppressants' AND concept_id = 1103137 -AND rnk = 3; +AND rnk = 3; + /******************************************* **** ADD ODDMENTS TO THE INPUT TABLES ***** ********************************************/ @@ -2095,6 +1984,18 @@ WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT class_code FROM atc_inexistent) AND SUBSTRING(concept_code_1,'\w+') NOT IN (SELECT class_code FROM dev_combo) AND concept_code_1 !~ '\s+'; +-- pentaerithrityl tetranitrate, combinations +DELETE +FROM drug_concept_stage +WHERE concept_code ~ 'C01DA55'; + +DELETE +FROM internal_relationship_stage +WHERE concept_code_1 ~ 'C01DA55'; + +DELETE +FROM dev_combo +WHERE class_code = 'C01DA55'; /*************************************** ******* relationship_to_concept ******** ****************************************/ From 3d77852690832c2b87913e916000bace262980b4 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 16 Sep 2021 19:54:32 +0300 Subject: [PATCH 12/45] code optimization --- ATC/load_interim.sql | 981 +++++++++++++++++++++++-------------------- 1 file changed, 530 insertions(+), 451 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index e1db91872..7d3011242 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -42,8 +42,7 @@ AS k.concept_name AS df_name FROM concept c JOIN concept_relationship r ON r.concept_id_1 = c.concept_id - JOIN concept d ON d.concept_id = r.concept_id_2 ---df + JOIN concept d ON d.concept_id = r.concept_id_2 --df JOIN concept_relationship r2 ON r2.concept_id_1 = c.concept_id JOIN concept k ON k.concept_id = r2.concept_id_2 WHERE c.concept_class_id = 'Clinical Drug Form' @@ -77,98 +76,32 @@ FROM dev_combo a ON c.concept_name = d.concept_name AND c.invalid_reason IS NULL); -DROP INDEX IF EXISTS idx_atc; -CREATE INDEX idx_atc ON atc_all_combo (class_code, df_id, ing_id); -ANALYZE atc_all_combo; -DROP INDEX IF EXISTS idx_rx; -CREATE INDEX idx_rx ON rx_all_combo (ing_id, df_id); -ANALYZE rx_all_combo; -/************************* -***** CLASS TO DRUG ****** -**************************/ --- assemble a table containing ATC Drug Classes which are hierarchically connected to RxN/RxE Drug Products via 'ATC - RxNorm' relationships (resembles Subsumes). --- add mappings from Monocomponent ATC Classes to respective Monocomponent RxN/RxE Drug Products -DROP TABLE if exists class_to_drug_new; -CREATE TABLE class_to_drug_new +DROP TABLE if exists atc_all_mono; -- 8m 22s +CREATE TABLE atc_all_mono AS ( -WITH t1 -AS -(SELECT * -FROM concept_manual -WHERE invalid_reason IS NULL -AND concept_class_id = 'ATC 5th' -AND concept_code NOT IN (SELECT class_code FROM dev_combo)), --- get attributes -t2 -AS -(SELECT DISTINCT a.concept_code AS class_code, - a.concept_name AS class_name, - d.concept_code AS ing_name, - d2.concept_code AS df_name -FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d - ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient' - LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - LEFT JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form') -SELECT DISTINCT class_code, - class_name, - c.*, - 1 AS concept_order, - 'ATC Monocomp Class' as order_desc -FROM t2 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - AND d_name !~ ' / ' - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' -WHERE class_code NOT IN (SELECT class_code FROM dev_combo) -); - --- add mappings from Monocomponent ATC Classes to associated Polycomponent RxN/RxE Drug Products ("greedy" logic) -INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT * -FROM concept_manual -WHERE invalid_reason IS NULL -AND concept_class_id = 'ATC 5th'), --- get attributes -t2 -AS -(SELECT DISTINCT a.concept_code AS class_code, - a.concept_name AS class_name, - d.concept_code AS ing_name, - d2.concept_code AS df_name -FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d - ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient' - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form') -SELECT DISTINCT class_code, - class_name, - c.*, - 2 AS concept_order, - 'Greedy ATC Monocomp Class' as order_desc -FROM t2 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) - AND class_code NOT IN (SELECT class_code FROM dev_combo); +SELECT DISTINCT a1.concept_code as class_code, + a1.concept_name as class_name, + c.concept_id AS ing_id, + c.concept_name AS ing_name, + d.concept_id AS df_id, + d.concept_name AS df_name, + 1 as rnk +FROM concept_manual a1 + JOIN internal_relationship_stage b1 ON substring (b1.concept_code_1, '\w+') = a1.concept_code + JOIN drug_concept_stage d1 + ON d1.concept_code = b1.concept_code_2 + AND d1.concept_class_id = 'Ingredient' + JOIN concept c + ON lower(c.concept_name) = lower( d1.concept_code) and c.standard_concept = 'S' and c.domain_id = 'Drug' + JOIN concept_manual a2 on a2.concept_code = a1.concept_code and a2.invalid_reason is null + JOIN internal_relationship_stage b2 ON substring (b2.concept_code_1, '\w+') = a2.concept_code + JOIN drug_concept_stage d2 ON d2.concept_code = b2.concept_code_2 + AND d2.concept_class_id = 'Dose Form' + JOIN concept d on lower(d.concept_name) = lower(d2.concept_code) + AND d2.invalid_reason IS NULL and d.domain_id = 'Drug' + AND a1.concept_code not in (select class_code from dev_combo) + ); /************************************ ***** PREPARE ATC COMBO CLASSES ***** *************************************/ @@ -237,14 +170,14 @@ SELECT class_code, class_name, concept_id, concept_name, 1 -- separate Primary upward + Secondary upward Ingredients (rnk in (3,4) in dev_combo) DROP TABLE IF EXISTS case_4; -CREATE TABLE case_4 AS (select distinct -class_code, +CREATE TABLE case_4 AS ( +SELECT DISTINCT class_code, class_name, - concept_id, -concept_name, + concept_id, + concept_name, rnk FROM dev_combo -WHERE class_code IN (SELECT class_code +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -- include Priamry upward AND class_code NOT IN (SELECT class_code @@ -284,217 +217,405 @@ and class_code in (select class_code from dev_combo where rnk = 0) );-- exclude Secondary upward -- Note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product --- Add ATC Combo Classes with mappings to class_to_drug: --- Primary lateral + Secondary lateral (order = 3) +DROP INDEX IF EXISTS idx_atc; +CREATE INDEX idx_atc ON atc_all_combo (class_code, df_id, ing_id); +ANALYZE atc_all_combo; +DROP INDEX IF EXISTS idx_rx; +CREATE INDEX idx_rx ON rx_all_combo (ing_id, df_id); +ANALYZE rx_all_combo; +DROP INDEX IF EXISTS idx_rx_0; +CREATE INDEX idx_rx_0 ON rx_combo (i_combo); +ANALYZE rx_combo; +DROP INDEX IF EXISTS idx_rx_1; +CREATE INDEX idx_rx_1 ON rx_combo (drug_concept_id, i_combo); +ANALYZE rx_combo; +DROP INDEX IF EXISTS idx_rx_3; +CREATE INDEX idx_rx_3 ON rx_all_combo (lower(ing_name), lower(df_name)); +ANALYZE rx_all_combo; +DROP INDEX IF EXISTS idx_rx_4; +CREATE INDEX idx_rx_4 ON atc_all_combo (lower(ing_name), lower(df_name)); +ANALYZE atc_all_combo; +DROP INDEX IF EXISTS irs_x; +CREATE INDEX irs_x ON internal_relationship_stage (SUBSTRING (concept_code_1,'\w+')); +ANALYZE internal_relationship_stage; + +/************************* +***** CLASS TO DRUG ****** +**************************/ +-- assemble a table containing ATC Drug Classes which are hierarchically connected to RxN/RxE Drug Products via 'ATC - RxNorm' relationships (resembles Subsumes). +-- add mappings from Monocomponent ATC Classes to respective Monocomponent RxN/RxE Drug Products (order = 1) +-- 5m 40s +DROP TABLE if exists class_to_drug_new; +CREATE TABLE class_to_drug_new +AS +(WITH atc +AS +(SELECT DISTINCT a.class_code, + a.class_name, + df_id, + ARRAY_AGG(a.ing_id) OVER (PARTITION BY a.class_code) ings +FROM dev_atc.atc_all_mono a),rx AS (SELECT DISTINCT r.d_id, r.d_name, r.df_id, ARRAY_AGG(r.ing_id) OVER (PARTITION BY r.d_id) ings + FROM dev_atc.rx_all_combo r) + SELECT DISTINCT atc.class_code,atc.class_name,c.*,1 AS concept_order,'ATC Monocomp Class' AS order_desc + FROM atc + JOIN rx USING (df_id) + JOIN concept c ON c.concept_id = rx.d_id +WHERE atc.ings @> rx.ings +AND atc.ings <@ rx.ings); + +-- add greedy ATC Monocomponent Classes (order = 2) INSERT INTO class_to_drug_new WITH t1 AS -(SELECT * -FROM concept_manual -WHERE invalid_reason IS NULL -AND concept_class_id = 'ATC 5th'), --- get attributes -t2 +(SELECT i.class_code,i.class_name,i.ing_id,i.ing_name,i.df_id,i.df_name,i.rnk,i.d_id,i.d_name +FROM (SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + a.rnk, + r.d_id, + r.d_name, + ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, + ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks + FROM dev_atc.atc_all_mono a + LEFT JOIN dev_atc.rx_all_combo r USING (ing_id,df_id) + ) i +WHERE i.d_id IS NOT NULL) + SELECT DISTINCT + class_code, + class_name, + c.*, + 2 AS concept_order, + 'Greedy ATC Monocomp Class' AS order_desc + FROM t1 a + JOIN concept c ON c.concept_id = a.d_id + WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 68243 + +-- add Primary lateral in combination (order = 3) +INSERT INTO class_to_drug_new +WITH t1 AS -(SELECT DISTINCT a.concept_code AS class_code, - a.concept_name AS class_name, - d.concept_code AS ing_name, - d2.concept_code AS df_name -FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d - ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient' - LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form' -) -SELECT DISTINCT class_code, +(SELECT i.class_code,i.class_name,i.ing_id,i.ing_name,i.df_id,i.df_name,i.rnk,i.d_id,i.d_name +FROM (SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + a.rnk, + r.d_id, + r.d_name, + ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, + ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks + FROM dev_atc.atc_all_combo a + LEFT JOIN dev_atc.rx_all_combo r USING (ing_id,df_id) + WHERE class_code IN (SELECT class_code FROM case_2)) i +WHERE i.d_id IS NOT NULL) +SELECT DISTINCT + class_code, class_name, c.*, 3 AS concept_order, - 'ATC Combo Class: Primary upward + Secondary upward' as order_desc -FROM t2 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) + 'ATC Combo Class: Primary lateral in combination' AS order_desc +FROM t1 a JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) - AND class_code IN (SELECT class_code FROM case_2); + ON c.concept_id = a.d_id + AND c.concept_name ~ ' / ' +WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 8577 --- Primary upward only (order = 4) +-- add Primary upward only (order = 4) INSERT INTO class_to_drug_new WITH t1 AS -(SELECT * -FROM concept_manual -WHERE invalid_reason IS NULL -AND concept_class_id = 'ATC 5th'), --- get attributes -t2 -AS -(SELECT DISTINCT a.concept_code AS class_code, - a.concept_name AS class_name, - d.concept_code AS ing_name, - d2.concept_code AS df_name -FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d - ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient' - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form') -SELECT DISTINCT class_code, +(SELECT i.class_code,i.class_name,i.ing_id,i.ing_name,i.df_id,i.df_name,i.rnk,i.d_id,i.d_name +FROM (SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + a.rnk, + r.d_id, + r.d_name, + ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, + ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks + FROM dev_atc.atc_all_combo a + LEFT JOIN dev_atc.rx_all_combo r USING (ing_id,df_id) + WHERE class_code IN (SELECT class_code FROM case_3)) i +WHERE i.d_id IS NOT NULL) +SELECT DISTINCT + class_code, class_name, c.*, 4 AS concept_order, - 'ATC Combo Class: Primary upward only' AS order_desc -FROM t2 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) + 'ATC Combo Class: Primary upward in combination' AS order_desc +FROM t1 a JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) - AND class_code IN (SELECT class_code FROM case_3) -AND c.concept_name ~ ' / '; - --- Primary lateral + Secondary lateral, more than 2 ingreds (order = 5) + ON c.concept_id = a.d_id + AND c.concept_name ~ ' / ' +WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 12768 + +-- add ATC Combo Class: Primary lateral + Secondary lateral, 4 ingreds (order = 5) INSERT INTO class_to_drug_new WITH t1 AS -(SELECT DISTINCT a.class_code AS class_code, - a.class_name AS class_name, - a.concept_name AS ing_name, - d2.concept_code AS df_name, +(SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, rnk -FROM case_1 a - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form' -), -t2 -AS -(SELECT DISTINCT class_code, - class_name, - c.* +FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id +WHERE a.class_code IN (SELECT class_code FROM case_1)) +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 5, + 'ATC Combo Class: Primary lateral + Secondary lateral, 4 ingreds' AS order_desc FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) -JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND rnk = 1), -t3 -AS -(SELECT DISTINCT a.class_code, + JOIN t1 b + ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id + JOIN t1 c + ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id + JOIN t1 d + ON d.class_code = a.class_code + AND d.concept_id = b.concept_id + AND a.ing_id <> d.ing_id + AND d.ing_id <> c.ing_id + AND d.ing_id <> b.ing_id +WHERE a.rnk = 1 +AND b.rnk <> 1 +AND c.rnk <> 1 +AND d.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id +FROM class_to_drug_new); -- 33 + +-- add ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds (order = 6) +INSERT INTO class_to_drug_new +WITH t1 AS +( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id + WHERE a.class_code IN (SELECT class_code FROM case_1) +) +SELECT DISTINCT a.class_code, a.class_name, - a.ing_name, - c.* + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 6, + 'ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds' AS order_desc FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND rnk = 2) - SELECT DISTINCT a.*, - 5 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary lateral, more than 2 Ingredients' AS order_desc - FROM t2 a - JOIN t3 b ON b.class_code = a.class_code - JOIN t3 c + JOIN t1 b + ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id + JOIN t1 c ON c.class_code = a.class_code AND c.concept_id = b.concept_id - AND b.ing_name <> c.ing_name - AND a.concept_id = b.concept_id - AND a.concept_name ~ ' / '; - --- Primary lateral + Secondary lateral, 2 ingredients + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id +WHERE a.rnk = 1 +AND b.rnk <> 1 +AND c.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 261 + +-- 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' (order = 7) +INSERT INTO class_to_drug_new +WITH t1 AS +( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id + WHERE a.class_code IN (SELECT class_code FROM case_1) +) +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 7, + 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' AS order_desc +FROM t1 a + JOIN t1 b + ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id +WHERE a.rnk = 1 +AND b.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2426 + +-- add ATC Combo Class: Primary upward + Secondary upward, 4 ing (order = 8) INSERT INTO class_to_drug_new WITH t1 AS -(SELECT DISTINCT a.class_code AS class_code, - a.class_name AS class_name, - a.concept_name AS ing_name, - d2.concept_code AS df_name, +(SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, rnk -FROM case_1 a - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form'), -t2 -AS -(SELECT DISTINCT class_code, - class_name, - c.* +FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id +WHERE a.class_code IN (SELECT class_code FROM case_4)) +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 8, + 'ATC Combo Class: Primary upward + Secondary upward, 4 ing' AS order_desc FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND rnk = 1), -t3 -AS -(SELECT DISTINCT a.class_code, + JOIN t1 b + ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id + JOIN t1 c + ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id + JOIN t1 d + ON d.class_code = a.class_code + AND d.concept_id = b.concept_id + AND a.ing_id <> d.ing_id + AND d.ing_id <> c.ing_id + AND d.ing_id <> b.ing_id +WHERE a.rnk = 3 +AND b.rnk <> 3 +AND c.rnk <> 3 +AND d.rnk <> 3 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 10 + +-- add ATC Combo Class: Primary upward + Secondary upward, 3 ing (order = 9) +INSERT INTO class_to_drug_new +WITH t1 AS +( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id + WHERE a.class_code IN (SELECT class_code FROM case_4) +) +SELECT DISTINCT a.class_code, a.class_name, - a.ing_name, - c.* + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 9, + 'ATC Combo Class: Primary upward + Secondary upward, 3 ing' AS order_desc FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) -JOIN concept c ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND rnk = 2) -SELECT DISTINCT a.*, - 6 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary lateral, 2 Ingredients' as order_desc -FROM t2 a - JOIN t3 b + JOIN t1 b ON b.class_code = a.class_code - AND a.concept_id = b.concept_id - AND a.concept_name ~ ' / ' - AND a.class_code NOT IN (SELECT class_code FROM class_to_drug_new) - AND a.concept_name !~ ' / .* / '; - --- Primary lateral only in combination (order = 7) + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id + JOIN t1 c + ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id +WHERE a.rnk = 3 +AND b.rnk <> 3 +AND c.rnk <> 3 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 30 + +-- add ATC Combo Class: Primary upward + Secondary upward, 2 ing (order = 10) INSERT INTO class_to_drug_new WITH t1 AS -(SELECT DISTINCT a.class_code AS class_code, - a.class_name AS class_name, - a.concept_name AS ing_name, - d2.concept_code AS df_name, - rnk -FROM case_2 a - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form') - SELECT DISTINCT a.class_code, +(SELECT a.class_code, a.class_name, + a.ing_id, c.*, - 7 AS concept_order, - 'ATC Combo Class: Primary lateral only in combination' AS order_desc + rnk +FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id +WHERE a.class_code IN (SELECT class_code FROM case_4)) +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 10, + 'ATC Combo Class: Primary upward + Secondary upward, 2 ing' AS order_desc FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND c.concept_name ~ ' / ' - AND (a.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); + JOIN t1 b + ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id +WHERE a.rnk = 3 +AND b.rnk <> 3 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 285 --- Primary lateral only in combination with an excluded Ingredient (order = 8) +-- add Primary lateral only in combination with an excluded Ingredient (order = 11) INSERT INTO class_to_drug_new WITH t1 AS @@ -512,7 +633,7 @@ t2 AS (SELECT DISTINCT class_code, class_name, c.*, - 2, + 11, b.ing_name FROM t1 a JOIN rx_all_combo b @@ -534,7 +655,7 @@ SELECT DISTINCT class_code, valid_start_date, valid_end_date, invalid_reason, - 8 as concept_order, + 11 as concept_order, 'ATC Combo Class: Primary lateral in combination with excluded Ingredient' FROM t2 WHERE concept_name ~ ' / ' @@ -543,9 +664,59 @@ AND concept_id NOT IN (SELECT d_id JOIN t1 b ON LOWER (b.ing_name) = LOWER (a.ing_name) AND LOWER (b.df_name) = LOWER (a.df_name) - AND b.rnk = 0); + AND b.rnk = 0) +AND class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 300 + +-- add 'ATC Combo Class: Primary upward + Secondary upward with excluded ingredient' (currently, no such, that is why the same order with previous one) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + c.*, + rnk +FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id +WHERE a.class_code IN (SELECT class_code FROM case_4_2)), +t2 as ( +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 11, + 'ATC Combo Class: Primary upward + Secondary upward with excluded ingredient' AS order_desc +FROM t1 a + JOIN t1 b + ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id +WHERE a.rnk = 3 +AND b.rnk <> 3) + +select DISTINCT * from t2 +WHERE concept_name ~ ' / ' +AND concept_id NOT IN (SELECT d_id + FROM rx_all_combo a + JOIN t1 b + ON LOWER (b.ing_name) = LOWER (a.ing_name) + AND LOWER (b.df_name) = LOWER (a.df_name) + AND b.rnk = 0) +AND class_code||concept_id::varchar NOT IN (SELECT class_code||concept_id::varchar FROM class_to_drug_new); --0 --- Primary upward only (order = 9) +-- add more Primary upward in combination (order = 12) INSERT INTO class_to_drug_new WITH t1 AS @@ -562,7 +733,7 @@ FROM case_3 a SELECT DISTINCT class_code, class_name, c.*, - 9 as concept_order, + 12 as concept_order, 'ATC Combo Class: more Primary upward' AS order_desc FROM t1 a JOIN rx_all_combo b @@ -572,9 +743,9 @@ FROM t1 a ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND c.concept_name ~ ' / ' - AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new); + AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new); -- 174 --- Primary upward + Secondary upward (order = 10) +-- add Primary upward + Secondary upward (order = 12) INSERT INTO class_to_drug_new WITH t1 AS @@ -616,13 +787,13 @@ FROM t1 a AND c.standard_concept = 'S' AND rnk = 4) SELECT DISTINCT a.*, - 10 AS concept_order, + 12 AS concept_order, 'ATC Combo Class: Primary upward + Secondary upward' FROM t2 a JOIN t3 b ON b.class_code = a.class_code AND a.concept_id = b.concept_id -WHERE a.concept_name ~ ' / '; +WHERE a.concept_name ~ ' / '; -- 325 ------------------------- ---- GET MORE LINKS ----- ------------------------- @@ -963,14 +1134,14 @@ SELECT DISTINCT f.class_code, i_combo FROM full_combo_reodered f JOIN reference r ON r.class_code = f.class_code -WHERE r.concept_code = r.class_code -; +WHERE r.concept_code = r.class_code; + CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo,df_id); /******************************* ******** CLASS TO DRUG ********* ********************************/ -- add the 2nd portion of multicomponent ATC Class mappings: --- ATC Combo Classes with Dose Forms using full_combo_with_form and rx_combo (order = 11) +-- ATC Combo Classes with Dose Forms using full_combo_with_form and rx_combo (order = 13) INSERT INTO class_to_drug_new WITH t1 AS @@ -992,7 +1163,7 @@ AND r.invalid_reason IS NULL SELECT DISTINCT f.class_code, -- ATC c.concept_name as class_name, d.*, - 11 AS conept_order, + 13 AS conept_order, 'ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations' FROM full_combo_with_form f JOIN concept_manual c on c.concept_code = f.class_code and c.invalid_reason is null and c.concept_class_id = 'ATC 5th' @@ -1000,14 +1171,14 @@ AND r.invalid_reason IS NULL ON r.i_combo = f.i_combo -- combination of Standard Ingredient IDs AND r.df_id = f.df_id JOIN concept d on d.concept_id = r.concept_id - and f.class_code||r.concept_id not in (select class_code||concept_id from class_to_drug_new); + and f.class_code||r.concept_id not in (select class_code||concept_id from class_to_drug_new); -- 2590 --- add manual mappings from concept_relationship_manual (order = 12) +-- add manual mappings from concept_relationship_manual (order = 14) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, f.concept_name AS class_name, c.*, - 12 AS concept_order, + 14 AS concept_order, 'ATC Class to Drug Product from concept_relationship_manual' FROM class_drugs_scraper a JOIN concept_relationship_manual b ON b.concept_code_1 = a.class_code @@ -1021,7 +1192,7 @@ FROM class_drugs_scraper a AND c.standard_concept = 'S' AND (a.class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) AND f.invalid_reason IS NULL -AND f.concept_class_id = 'ATC 5th'; +AND f.concept_class_id = 'ATC 5th'; -- 7535 -- manual clean up for Precise Ingredients and other particular cases (according to the information on the ATC WHO Website) DELETE @@ -1032,10 +1203,10 @@ OR class_code='N05AF02' -- clopentixol OR class_code IN ('D07AB02','D07BB04') -- hydrocortisone butyrate + combo that so far doesn't exist OR class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral OR (class_code = 'B02BD14' AND concept_name LIKE '%Tretten%') -- 2 --catridecacog -OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient');-- susoctocog alfa | catridecacog - +OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog +; -- 309 --- add additional semi-manual mappings based on pattern-matching (order = 13) +-- add additional semi-manual mappings based on pattern-matching (order = 15) INSERT INTO class_to_drug_new with t1 as ( SELECT 'B02BD11' as class_code,'catridecacog' as class_name, concept_id @@ -1078,7 +1249,7 @@ WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ILIKE '%Pentaerythritol Tetr SELECT DISTINCT a.class_code, d.concept_name, c.*, - 13 as concept_order, + 15 as concept_order, 'ATC Class with semi-manual point fix' FROM t1 a JOIN concept c ON c.concept_id = a.concept_id @@ -1086,23 +1257,23 @@ FROM t1 a ON d.concept_code = a.class_code AND d.concept_class_id = 'ATC 5th' AND d.invalid_reason IS NULL -AND a.class_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); +AND a.class_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);--22 -- clean up erroneous amount of ingredients DELETE ---select * +--SELECT * FROM class_to_drug_new WHERE class_name LIKE '%,%and%' AND class_name NOT LIKE '%,%,%and%' AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT LIKE '% / % / %'; + AND concept_name NOT LIKE '% / % / %'; -- 149 ---- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug (order = 14) +--- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug (order = 16) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, c.*, - 14 as concept_order, + 16 as concept_order, 'ATC Class from old class_to_drug' FROM sources.class_to_drug a JOIN concept_manual b ON b.concept_code = a.class_code @@ -1127,17 +1298,16 @@ AND (class_code,c.concept_id) NOT IN ( UNION ALL SELECT 'A06AA02',43158334 UNION ALL - SELECT 'A06AA02',40036796); - + SELECT 'A06AA02',40036796); -- 29 /********************** ****** ADD PACKS ****** ***********************/ --- add packs of Primary lateral only in combination (order = 15) +-- add packs of Primary lateral only in combination (order = 17) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 15 as concept_order, + 17 as concept_order, 'Pack: Primary lateral in combo' as order_desc FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id @@ -1159,14 +1329,14 @@ AND a.class_code IN (SELECT class_code AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) AND j.concept_name ~ ' / ' -- combos only -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 1166 --- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 16) +-- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 18) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 16 as concept_order, + 18 as concept_order, 'Pack: Primary lateral in combo additional' FROM class_to_drug_new a JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id @@ -1182,14 +1352,14 @@ AND a.class_code IN (SELECT class_code AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) AND j.concept_name ~ ' / ' -- combos only -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 262 --- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 17) +-- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 19) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 17 as concept_order, + 19 as concept_order, 'Pack: Primary lateral + Secondary lateral' FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id @@ -1211,38 +1381,14 @@ AND a.class_code IN (SELECT class_code AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) AND j.concept_name ~ ' / ' -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); - --- add Branded packs of Primary lateral + Secondary lateral (order = 18) -INSERT INTO class_to_drug_new -SELECT DISTINCT a.class_code, - a.class_name, - j.*, - 18 as concept_order, - 'Pack: Primary lateral + Secondary lateral Branded' -FROM class_to_drug_new a - JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id - JOIN concept j ON j.concept_id = r2.concept_id_2 -WHERE r2.invalid_reason IS NULL -AND j.concept_class_id IN ('Branded Pack') --- Boxes are suspicious -AND j.standard_concept = 'S' -AND a.class_code IN (SELECT class_code - FROM dev_combo - WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) - AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) -AND j.concept_name ~ ' / ' -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 3649 --- add packs of Primary upward and Secondary upward (Class C AND Class D, order = 19) +/*-- add packs of Primary upward and Secondary upward (Class C AND Class D, order = 20) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 19 as concept_order, + 20 as concept_order, 'Pack: Primary upward + Secondary upward' FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id @@ -1263,7 +1409,7 @@ AND a.class_code IN (SELECT class_code AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) -AND j.concept_name ~ ' / '; +AND j.concept_name ~ ' / '; -- 53 */ -- to exclude, will be deleted later -- add packs of Primary lateral and Secondary upward (Class A + Class D, order = 20) INSERT INTO class_to_drug_new @@ -1292,39 +1438,15 @@ AND a.class_code IN (SELECT class_code AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) AND j.concept_name ~ ' / ' -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); - --- add additional packs of Primary lateral and Secondary upward (Class A + Class D, order = 21) -INSERT INTO class_to_drug_new -SELECT DISTINCT a.class_code, - a.class_name, - j.*, - 21 as concept_order, - 'Pack: Primary lateral + Secondary upward additional' -FROM class_to_drug_new a - JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id - JOIN concept j ON j.concept_id = r2.concept_id_2 -WHERE r2.invalid_reason IS NULL -AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') --- Boxes are suspicious -AND j.standard_concept = 'S' -AND a.class_code IN (SELECT class_code - FROM dev_combo - WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) - AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) -AND j.concept_name ~ ' / ' -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 218 ---- add missing Packs using previous version of class_to_drug (order = 22) +--- add missing Packs using previous version of class_to_drug (order = 21) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, c.*, - 22 as concept_order, - 'Pack: from old c_t_d' + 21 as concept_order, + 'Additional Pack: from old c_t_d' FROM sources.class_to_drug a JOIN concept_manual b ON b.concept_code = a.class_code JOIN concept c @@ -1335,45 +1457,31 @@ WHERE (class_code,a.concept_id) NOT IN (SELECT class_code, concept_id FROM class AND a.concept_class_id ~ 'Pack' AND a.class_code NOT IN ('G01AF55','S03CA04','S01CA03','S02CA03')-- gives packs with erroneous forms AND a.class_code NOT IN ('B03AE01','C07BB52','D01AC52','C10AD52') -- wrong ing combo -; - --- add mappings of ATC Combo Classes to Contraceptive Packs (order = 23) -INSERT INTO class_to_drug_new -SELECT class_code, - class_name, - c.*, - 23 as concept_order, - 'Pack: semi-manual contraceptive' -FROM class_to_drug_new ctd - JOIN concept_ancestor ON ctd.concept_id = ancestor_concept_id - JOIN concept c ON descendant_concept_id = c.concept_id -WHERE class_code ~ 'G03FB|G03AB' -- the list can be enriched -AND c.concept_class_id IN ('Clinical Pack') -AND (ctd.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); +; -- 281 -- еnrich pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' --- they are always used as packs (order = 24) +-- they are always used as packs (order = 22) INSERT INTO class_to_drug_new SELECT DISTINCT class_code, class_name, c.*, - 24 as concept_order, - 'Pack: semi-manual contraceptive' + 22 as concept_order, + 'Additional Pack: semi-manual contraceptive' FROM class_to_drug_new f JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) JOIN devv5.concept c ON c.concept_id = descendant_concept_id AND c.concept_class_id LIKE '%Pack%' WHERE f.class_code ~ 'G03FB|G03AB' -AND (f.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); +AND (f.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 264 -- get rid of all other concept_class_ids except Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' DELETE FROM class_to_drug_new WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens -AND concept_class_id !~ 'Pack'; -- 68 +AND concept_class_id !~ 'Pack'; -- 81 --- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 25) +-- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 23) INSERT INTO class_to_drug_new WITH ing AS ( SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt @@ -1406,46 +1514,12 @@ JOIN concept c on c.concept_id = a.concept_id_2 SELECT DISTINCT a.class_code, a.class_name, c.*, - 25 as concept_order, + 23 as concept_order, 'ATC Class WO Dose Form to unique Drug Product' FROM all_drug a join concept c on c.concept_id = a.concept_id WHERE cnt = 1 -AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); - --- add mappings of Monocomponent ATC Classes using additional reference (order = 26) -INSERT INTO class_to_drug_new -SELECT DISTINCT class_code, - class_name, - c.*, - 26, - 'ATC Monocomp: from additional reference' -FROM atc_all_combo a - JOIN rx_all_combo b - ON b.ing_id = a.ing_id - AND a.df_id = b.df_id - JOIN concept c - ON c.concept_id = b.d_id - AND a.class_code||b.d_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) - AND a.class_code IN (SELECT class_code FROM ing_pr_lat_combo_to_drug) -AND c.concept_name ~ '/'; - --- add mappings of ATC Combo Classes using additional reference (order = 27) -INSERT INTO class_to_drug_new -SELECT DISTINCT class_code, - class_name, - c.*, - 27, - 'ATC Combo: Pr upward only from additional reference' -FROM atc_all_combo a - JOIN rx_all_combo b - ON b.ing_id = a.ing_id - AND a.df_id = b.df_id - JOIN concept c - ON c.concept_id = b.d_id - AND a.class_code||b.d_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) - AND a.class_code IN (SELECT class_code FROM case_3) -AND c.concept_name ~ '/'; +AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); -- 5 -- add those which are absent in the drug hierarchy -- step 1 @@ -1489,12 +1563,12 @@ FROM no_atc_2 a ON g.concept_id = r.concept_id_2 AND g.concept_class_id = 'Dose Form'); --- add additional mappings for hierarchical absentees (order = 28) +-- add additional mappings for hierarchical absentees (order = 24) INSERT INTO class_to_drug_new SELECT DISTINCT k.concept_code AS class_code, k.concept_name AS class_name, p.*, - 28 as concept_order, + 24 as concept_order, 'ATC Monocomp Class to Drug Product which is out of hierarchy' FROM no_atc_1_with_form a JOIN internal_relationship_stage i ON lower (i.concept_code_2) = lower (a.ing_nm) @@ -1509,7 +1583,7 @@ WHERE a.concept_name !~ ' / ' AND k.invalid_reason IS NULL AND k.concept_class_id = 'ATC 5th' AND k.concept_code NOT IN (SELECT class_code FROM dev_combo) -AND k.concept_code||p.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); +AND k.concept_code||p.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);--6107 -- obtain more ATC Combo classes DROP TABLE no_atc_full_combo; @@ -1540,34 +1614,21 @@ INSERT INTO no_atc_full_combo SELECT DISTINCT concept_id, i_combo, df_id FROM no_atc_reodered ; -; + CREATE INDEX i_no_atc_full_combo ON no_atc_full_combo (concept_id, i_combo,df_id); --- add mappings of ATC Combo Class to Drug Product which is out of drug hierarchy (order = 29) +-- add mappings of ATC Combo Class to Drug Product which is out of drug hierarchy (order = 25) INSERT INTO class_to_drug_new SELECT DISTINCT k.concept_code as class_code, -- ATC k.concept_name as class_name, d.*, - 29 AS conept_order, + 25 AS conept_order, 'ATC Combo Class to Drug Product which is out of drug hierarchy' FROM no_atc_full_combo f JOIN full_combo_with_form c on c.i_combo = f.i_combo and c.df_id = f.df_id is null join concept_manual k on k.concept_code = c.class_code and k.concept_class_id = 'ATC 5th' and k.invalid_reason is null JOIN concept d on d.concept_id = f.concept_id - and c.class_code||d.concept_id not in (select class_code||concept_id from class_to_drug_new); - --- add additional manual mapping (order = 30) => to crm -INSERT INTO class_to_drug_new -SELECT DISTINCT b.concept_code, - b.concept_name, - c.*, - 33, - 'gentle manual introduction of a bit fishy RxE/RxN mappings which are out of the hierarchy' -FROM no_atc_manual a - JOIN concept c ON c.concept_id = a.concept_id - JOIN concept_manual b - ON b.concept_code = a.class_code - AND b.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); + and c.class_code||d.concept_id not in (select class_code||concept_id from class_to_drug_new); -- 6188 -- remove excessive links to children among Packs WITH t1 AS @@ -1587,7 +1648,7 @@ ON a1.class_code = a2.class_code AND a1.concept_id <> a2.concept_id and a2.concept_class_id ~ 'Pack' AND a1.class_code not in ('B02BD14', 'G03FB01', 'J07BK01') )-- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic); -- 2558 - ; + ; -- 3920 -- remove excessive links to children among unpacked Drug Products WITH t0 AS @@ -1614,7 +1675,7 @@ ON a1.class_code = a2.class_code AND a1.concept_id <> a2.concept_id and a2.concept_class_id !~ 'Pack' AND a1.class_code NOT IN ('B02BD11', 'B02BD14', 'G03FB01', 'J07BK01', 'J07BK02')) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic - ; + ;-- 10432 /*************************** ******** DF CLEAN UP ******* ****************************/ @@ -1630,6 +1691,12 @@ AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' AND class_code NOT IN ('A01AA04','A01AA02','G04BD08') ); +INSERT INTO wrong_df +select *, 'vaginal mismatch' from class_to_drug_new where class_name ~ 'vaginal' +and concept_name !~* 'vaginal|topical|mucosal|Drug Implant|Douche|Irrigation Solution' +and class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' +and concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; -- 0 + -- clean up rectal forms INSERT INTO wrong_df SELECT *, 'rectal mismatch' @@ -1741,14 +1808,15 @@ WHERE class_name ~* 'systemic' AND concept_name !~* '\.\.\.$|\.\.\.\) \} Pack|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' AND class_name !~* 'rectal|nasal|vaginal|topical' AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' -AND concept_class_id ~ 'Pack'; +AND concept_class_id ~ 'Pack'; -- look at them and remove from class_to_drug_new DELETE FROM class_to_drug_new -WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM wrong_df); +WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM wrong_df); -- 891 -- add links from ATC Monocomponent Classes which do not have Dose Forms to Ingredients (in the future this concepts could be processed manually, some of them have mappings) +-- (order = 26) INSERT INTO class_to_drug_new WITH t1 AS @@ -1761,7 +1829,7 @@ AND concept_code NOT IN (SELECT class_code FROM atc_inexistent)) SELECT DISTINCT a.concept_code, a.concept_name, c.*, - 31 as concept_order, + 26 as concept_order, 'ATC Monocomp Class WO Dose Form to Ingredient' FROM t1 a JOIN internal_relationship_stage i ON SUBSTRING (concept_code_1,'\w+') = a.concept_code @@ -1771,10 +1839,11 @@ FROM t1 a AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' AND a.concept_code NOT IN (SELECT class_code FROM dev_combo) - and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new); + and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new); -- 78 -- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (do we need them?) -INSERT INTO class_to_drug_new WITH t1 +INSERT INTO class_to_drug_new +WITH t1 AS (SELECT * FROM concept_manual @@ -1787,7 +1856,7 @@ AND concept_code NOT IN (SELECT class_code FROM atc_inexistent) SELECT DISTINCT a.concept_code, a.concept_name, c.*, - 32 as concept_order, + 27 as concept_order, 'ATC Combo Class to Ingredient' FROM t1 a JOIN dev_combo k @@ -1797,26 +1866,36 @@ FROM t1 a ON c.concept_id = k.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new); + and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new);--314 -- remove suspicious mapping of inexistent drugs (this table should be checked before) DELETE +--select * FROM class_to_drug_new -WHERE class_code IN (SELECT class_code FROM atc_inexistent); +WHERE class_code IN (SELECT class_code FROM atc_inexistent); -- 16 +-- remove wrong mappings +DELETE +--select * +FROM class_to_drug_new +WHERE class_code||concept_code IN (SELECT concept_code_1||concept_code_2 + FROM concept_relationship_manual + WHERE invalid_reason IS NOT NULL);-- 3 -- remove dead ATC Classes if any DELETE FROM class_to_drug_new WHERE class_code IN (SELECT concept_code FROM concept_manual - WHERE invalid_reason IS NOT NULL); + WHERE invalid_reason IS NOT NULL); -- 0 --- remove duplicates if any (however they should not be there) +-- remove duplicates if any (however they should not be there - can be solved in the future) +--SELECT * DELETE FROM class_to_drug_new -WHERE CTID NOT IN (SELECT MIN(CTID) +WHERE CTID NOT IN (SELECT MAX(CTID) FROM class_to_drug_new GROUP BY class_code, - concept_id); + concept_id) + ;-- 345 -- run load_stage.sql From ae011257226cb4628d937f179081e061da73cded Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sat, 18 Sep 2021 11:41:42 +0300 Subject: [PATCH 13/45] refactoring --- ATC/load_input.sql | 54 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index f1eb9a62d..5f95ceca3 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -16,7 +16,6 @@ * Authors: Anna Ostropolets, Polina Talapova * Date: Jul 2021 **************************************************************************/ - DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; DROP TABLE IF EXISTS relationship_to_concept CASCADE; @@ -1984,18 +1983,55 @@ WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT class_code FROM atc_inexistent) AND SUBSTRING(concept_code_1,'\w+') NOT IN (SELECT class_code FROM dev_combo) AND concept_code_1 !~ '\s+'; --- pentaerithrityl tetranitrate, combinations +-- remove mappings which have been deprecated in crm +WITH t1 AS +( + SELECT * + FROM concept_relationship_manual + WHERE relationship_id IN ('ATC - RxNorm pr lat','ATC - RxNorm sec up','ATC - RxNorm pr up','ATC - RxNorm sec lat') +) DELETE +FROM dev_combo +WHERE class_code||concept_id IN (SELECT DISTINCT a.class_code||c.concept_id + FROM dev_combo a + JOIN concept c ON c.concept_id = a.concept_id + JOIN t1 r + ON r.concept_code_1 = a.class_code + AND r.concept_code_2 = c.concept_code + AND r.invalid_reason IS NOT NULL); + +-- clean up sulfonamides DELETE -FROM drug_concept_stage -WHERE concept_code ~ 'C01DA55'; +FROM dev_combo +WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id + FROM dev_combo + WHERE class_name ~ 'sulfonamides' + AND (concept_id IN (SELECT concept_id FROM dev_combo WHERE class_code = 'J01EB20') OR concept_name ~* '^sulfa') + AND rnk = 3) +AND class_name ~ 'sulfonamides' +AND rnk = 3; -DELETE -FROM internal_relationship_stage -WHERE concept_code_1 ~ 'C01DA55'; +INSERT INTO dev_combo +WITH t1 +AS +(SELECT concept_id, + concept_name, + rnk +FROM dev_combo +WHERE class_name ~ 'sulfonamides' +AND (concept_id IN (SELECT concept_id FROM dev_combo WHERE class_code = 'J01EB20') OR concept_name ~* '^sulfa') +AND rnk = 3) SELECT DISTINCT class_code,class_name,'sulfonamides',b.*FROM dev_combo a,t1 b WHERE class_name ~ 'sulfonamides' +AND class_code||b.concept_id NOT IN (SELECT class_code||concept_id FROM dev_combo); +-- non-combo: glycerol DELETE FROM dev_combo -WHERE class_code = 'C01DA55'; +WHERE class_code = 'A16AX09'; + +UPDATE dev_combo + SET rnk = 2 +WHERE concept_id = 529303 +AND class_code = 'J07AM51'; + /*************************************** ******* relationship_to_concept ******** ****************************************/ @@ -2015,4 +2051,4 @@ FROM internal_relationship_stage AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND c.invalid_reason IS NULL; --- run load_interim.sql + -- run load_interim.sql From d6583e0fbcad219b04d0c49ed5f31e968094bf0d Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 28 Sep 2021 12:59:01 +0300 Subject: [PATCH 14/45] Update load_stage.sql code refactoring --- ATC/load_stage.sql | 623 +++++++++++++++++++++++++++------------------ 1 file changed, 372 insertions(+), 251 deletions(-) diff --git a/ATC/load_stage.sql b/ATC/load_stage.sql index c89b37216..3ee2ab7ae 100644 --- a/ATC/load_stage.sql +++ b/ATC/load_stage.sql @@ -16,49 +16,48 @@ * Authors: Anna Ostropolets, Polina Talapova, Timur Vakhitov * Date: Jul 2021 **************************************************************************/ --- Update latest_UPDATE field to new date DO $_$ BEGIN PERFORM VOCABULARY_PACK.SetLatestUpdate( - pVocabularyName => 'ATC', - pVocabularyDate => (SELECT vocabulary_date FROM sources.rxnatomarchive LIMIT 1), - pVocabularyVersion => (SELECT vocabulary_version FROM sources.rxnatomarchive LIMIT 1), - pVocabularyDevSchema => 'DEV_ATC' + pVocabularyName => 'ATC', + pVocabularyDate => (SELECT vocabulary_date FROM sources.rxnatomarchive LIMIT 1), + pVocabularyVersion => (SELECT vocabulary_version FROM sources.rxnatomarchive LIMIT 1), + pVocabularyDevSchema => 'DEV_ATC' ); END $_$; --- Truncate all working tables AND remove indices +-- truncate all working tables TRUNCATE TABLE concept_stage; TRUNCATE TABLE concept_relationship_stage; TRUNCATE TABLE concept_synonym_stage; TRUNCATE TABLE pack_content_stage; TRUNCATE TABLE drug_strength_stage; --- Add ATC codes using concept_manual (processed inside function below) +-- add all ATC codes to staging tables using the function which processes the concept_manual table DO $_$ BEGIN PERFORM VOCABULARY_PACK.ProcessManualConcepts(); END $_$; ---Add manual relationships +-- add manually created relationships using the function which processes the concept_relationship_manual table DO $_$ BEGIN PERFORM VOCABULARY_PACK.ProcessManualRelationships(); END $_$; ---5. Manual synonyms +-- add manually created synonyms using the function processing the concept_synonym_manual DO $_$ BEGIN PERFORM VOCABULARY_PACK.ProcessManualSynonyms(); END $_$; --- Add 1) crosslinks 'SNOMED - ATC eq' between SNOMED drugs and higher ATC classes (not 5th) and 2) internal 'Is a' relationships using mrconso +-- add 1) 'SNOMED - ATC eq' relationships between SNOMED Drugs and Higher ATC Classes (excl. 5th) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, - relationship_id, vocabulary_id_1, vocabulary_id_2, + relationship_id, valid_start_date, valid_end_date, invalid_reason @@ -66,9 +65,9 @@ INSERT INTO concept_relationship_stage ( -- crosslinks between SNOMED Drug Class AND ATC Classes (not ATC 5th) SELECT DISTINCT d.concept_code AS concept_code_1, e.concept_code AS concept_code_2, - 'SNOMED - ATC eq' AS relationship_id, 'SNOMED' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, + 'SNOMED - ATC eq' AS relationship_id, d.valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason @@ -85,12 +84,12 @@ JOIN concept_manual e ON r2.code = e.concept_code WHERE d.vocabulary_id = 'SNOMED' AND d.invalid_reason IS NULL UNION ALL --- Hierarchy inside ATC +-- 2) 'Is a' relationships between ATC Classes using mrconso (internal ATC hierarchy) SELECT uppr.concept_code AS concept_code_1, lowr.concept_code AS concept_code_2, - 'Is a' AS relationship_id, 'ATC' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, + 'Is a' AS relationship_id, v.latest_UPDATE AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason @@ -106,7 +105,7 @@ OR (LENGTH(uppr.concept_code) IN (3,7) AND lowr.concept_code = SUBSTR(uppr.conce AND lowr.vocabulary_id = 'ATC' AND v.vocabulary_id = 'ATC'; --6493 --- Add new 'ATC - RxNorm' links between ATC classes and RxN/RxE using class_to_drug table +-- add 'ATC - RxNorm' relationships between ATC Classes and RxN/RxE Drug Products using class_to_drug table INSERT INTO concept_relationship_stage ( concept_code_1, @@ -126,8 +125,8 @@ SELECT DISTINCT cs.class_code AS concept_code_1, CURRENT_DATE AS valid_start_date, TO_DATE('20991231','yyyymmdd') AS valid_end_date, NULL AS invalid_reason -FROM class_to_drug cs ---manual source table +FROM class_to_drug cs -- manually curated table with ATC Class to Rx Drug Product links +JOIN concept_manual k ON k.concept_code = cs.class_code AND k.invalid_reason IS NULL JOIN concept c ON c.concept_id = cs.concept_id WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs @@ -135,13 +134,11 @@ WHERE NOT EXISTS (SELECT 1 AND crs.vocabulary_id_1 = 'ATC' AND crs.concept_code_2 = c.concept_code AND crs.vocabulary_id_2 = c.vocabulary_id - AND crs.relationship_id = 'ATC - RxNorm') -AND c.concept_class_id != 'Ingredient' -AND (cs.class_code,c.concept_code) NOT IN (SELECT concept_code_1, - concept_code_2 - FROM concept_relationship_stage); + AND crs.relationship_id = 'ATC - RxNorm' -- and crs.invalid_reason is null + ) +AND c.concept_class_id != 'Ingredient'; -- 107533 --- Add 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between an ATC class and RxN/RxE drug using input tables and ambiguous_class_ingredient_tst +-- add 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between ATC Classes and RxN/RxE Drug Products (using input tables and dev_combo populated during previous Steps) INSERT INTO concept_relationship_stage ( concept_code_1, @@ -163,23 +160,28 @@ SELECT DISTINCT SUBSTRING(irs.concept_code_1,'\w+') AS concept_code_1, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason FROM internal_relationship_stage irs - JOIN relationship_to_concept rtc ON irs.concept_code_2 = rtc.concept_code_1 + JOIN relationship_to_concept rtc ON lower (irs.concept_code_2) = lower (rtc.concept_code_1) JOIN concept c ON concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' -WHERE - NOT EXISTS (SELECT 1 -FROM dev_combo t -WHERE lower(t.concept_name) = lower(rtc.concept_code_1) -AND t.class_code = SUBSTRING(irs.concept_code_1,'\w+') -AND t.rnk in (2,3,4) -)) -select * from t1 -where (concept_code_1, concept_code_2) NOT IN (SELECT concept_code_1, - concept_code_2 - FROM concept_relationship_stage); + JOIN concept_manual k + ON k.concept_code = SUBSTRING (irs.concept_code_1,'\w+') + AND k.invalid_reason IS NULL +WHERE NOT EXISTS (SELECT 1 + FROM dev_combo t + WHERE LOWER(t.concept_name) = LOWER(rtc.concept_code_1) + AND t.class_code = SUBSTRING(irs.concept_code_1,'\w+'))) +SELECT * FROM t1 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = 'ATC - RxNorm pr lat' -- and crs.invalid_reason is null + ); -- 3292 --- Add more 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between an ATC class and RxN/RxE drug using rtc and ambiguous_class_ingredient_tst +-- add 'ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up' for ATC Combo Classes using dev_combo INSERT INTO concept_relationship_stage ( concept_code_1, @@ -191,31 +193,36 @@ INSERT INTO concept_relationship_stage valid_end_date, invalid_reason ) -SELECT DISTINCT class_code AS concept_code_1, - c.concept_code AS concept_code_2, +with t1 as ( +select distinct class_code, class_name, concept_id, rnk from dev_combo where rnk in (1,2,3)), +t2 as ( +SELECT a.class_code AS concept_code_1, + c.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr lat' AS relationship_id, +case rnk when 1 then 'ATC - RxNorm pr lat' +when 2 then 'ATC - RxNorm sec lat' +when 3 then 'ATC - RxNorm pr up' +--when 4 then 'ATC - RxNorm sec up' +end AS relationship_id, CURRENT_DATE AS valid_start_date, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason -FROM dev_combo a - JOIN relationship_to_concept rtc - ON lower(a.concept_name) = lower(rtc.concept_code_1) - AND rnk = 1 - JOIN concept c - ON concept_id_2 = c.concept_id - AND c.concept_class_id = 'Ingredient' -WHERE - (class_code,concept_code,'ATC - RxNorm pr lat') NOT IN (SELECT concept_code_1, - concept_code_2, - relationship_id -FROM concept_relationship_stage) AND class_name != 'combinations' -; -- 1 - ---select * from concept_relationship_stage where concept_code_1 = 'N02AA59'; +FROM t1 a +JOIN concept c on c.concept_id = a.concept_id +JOIN concept_manual k ON k.concept_code = a.class_code AND k.invalid_reason IS NULL +AND c.standard_concept = 'S') +SELECT DISTINCT * FROM t2 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = cs.relationship_id -- and crs.invalid_reason is null + ); -- 3670 --- add 'ATC - RxNorm sec lat' relationships indicating Secondary unambiguous links between an ATC class and RxN/RxE drug using rtc and ambiguous_class_ingredient_tst +-- add 'ATC - RxNorm sec up' relationships for Primary lateral in combination INSERT INTO concept_relationship_stage ( concept_code_1, @@ -227,59 +234,61 @@ INSERT INTO concept_relationship_stage valid_end_date, invalid_reason ) -SELECT DISTINCT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM dev_combo a - JOIN relationship_to_concept rtc - ON lower(a.concept_name) = lower(rtc.concept_code_1) - AND rnk = 2 - JOIN concept c - ON concept_id_2 = c.concept_id - AND c.concept_class_id = 'Ingredient' -WHERE - (class_code,concept_code) NOT IN (SELECT concept_code_1, - concept_code_2 -FROM concept_relationship_stage) -AND (class_code, - concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage); -- 381 (171 (9817 (2755)_ ) - --- add 'ATC - RxNorm pr up' relationships meaning Primary ambiguous links between an ATC class and RxN/RxE drug using rtc and ambiguous_class_ingredient_tst -INSERT INTO concept_relationship_stage - (concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason - ) -SELECT DISTINCT class_code AS concept_code_1, - c.concept_code AS concept_code_2, +WITH t1 AS -- the list of the main Ingredients +(SELECT class_code, + class_name, + concept_id +FROM ing_pr_lat_sec_up WHERE rnk = 1 +UNION ALL +SELECT class_code, +class_name, concept_id FROM ing_pr_up_sec_up WHERE rnk = 3 +UNION ALL +SELECT class_code, +class_name, concept_id FROM ing_pr_up_combo WHERE rnk = 3 +UNION ALL +SELECT class_code, +class_name, concept_id FROM ing_pr_lat_combo WHERE rnk = 1 +UNION ALL +SELECT class_code, +class_name, concept_id FROM Ing_pr_lat_combo_excl WHERE rnk = 1 +UNION ALL +SELECT class_code, +class_name, concept_id FROM ing_pr_up_sec_up_excl WHERE rnk = 3 +), +t2 as ( +SELECT DISTINCT c.class_code AS concept_code_1, + cc.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr up' AS relationship_id, + cc.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec up' AS relationship_id, CURRENT_DATE AS valid_start_date, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason -FROM dev_combo a - JOIN relationship_to_concept rtc - ON lower (a.concept_name) = lower (rtc.concept_code_1) - AND rnk = 3 - JOIN concept c - ON concept_id_2 = c.concept_id - AND c.concept_class_id = 'Ingredient' -WHERE (class_code,concept_code) NOT IN (SELECT concept_code_1, - concept_code_2 - FROM concept_relationship_stage); -- 3382 +FROM class_to_drug c + JOIN concept_ancestor ON descendant_concept_id = c.concept_id + JOIN concept cc + ON cc.concept_id = ancestor_concept_id + AND cc.standard_concept = 'S' + AND cc.concept_class_id = 'Ingredient' + AND cc.vocabulary_id IN ('RxNorm', 'RxNorm Extension') + JOIN concept_manual k + ON k.concept_code = c.class_code + AND k.invalid_reason IS NULL + AND k.concept_code IN (SELECT class_code FROM dev_combo) +WHERE (c.class_code,cc.concept_id) NOT IN (SELECT class_code, concept_id FROM t1)-- exclude main Ingredients +) +SELECT * FROM t2 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = 'ATC - RxNorm sec up' + AND crs.invalid_reason = cs.invalid_reason + ); -- 23477 --- add 'ATC - RxNorm sec up' relationships indicating Secondary ambiguous links between ATC classes and RxN/RxE drugs using rtc and ambiguous_class_ingredient_tst +-- deprecate links between ATC classes and dead RxN/RxE INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -290,72 +299,33 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT DISTINCT class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM dev_combo a - JOIN relationship_to_concept rtc ON lower(a.concept_name) = lower(rtc.concept_code_1) - JOIN concept c - ON concept_id_2 = c.concept_id - AND c.concept_class_id = 'Ingredient' -WHERE -a.rnk = 4 -AND (class_code,concept_code) NOT IN (SELECT concept_code_1, - concept_code_2 - FROM concept_relationship_stage); -- 7881 - --- add 'ATC - RxNorm sec up' relationships for 'x, combinations' -INSERT INTO concept_relationship_stage -( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason -) -WITH main_ing AS -( - SELECT class_code,class_name, - concept_id_2 - FROM class_to_drug c - JOIN internal_relationship_stage i ON c.class_code = SUBSTRING (i.concept_code_1,'\w+') - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 - WHERE (class_name ~ '(, combinations)' - AND NOT class_name ~ '\ywith|and thiazides|and other diuretics') - OR class_name ~ '(in combination with other drugs)' -- gives errors INC07BB52 AND C07CB53 if removed -) -SELECT DISTINCT class_code AS concept_code_1, + with t1 as ( +SELECT c.concept_code AS concept_code_1, cc.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_1, cc.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM class_to_drug c - JOIN devv5.concept_ancestor ON descendant_concept_id = c.concept_id - JOIN concept cc - ON cc.concept_id = ancestor_concept_id - AND cc.standard_concept = 'S' - AND cc.concept_class_id = 'Ingredient' - AND cc.vocabulary_id LIKE 'Rx%' -WHERE (c.class_code,cc.concept_id) NOT IN (SELECT class_code, concept_id_2 FROM main_ing) -AND (class_code,cc.concept_code) NOT IN (SELECT concept_code_1, - concept_code_2 - FROM concept_relationship_stage) -AND ((class_name ~ '(, combinations)' AND NOT class_name ~ '\ywith|and thiazides|and other diuretics') -OR class_name ~ '(in combination with other drugs)') -;-- 336 + relationship_id AS relationship_id, + cr.valid_start_date AS valid_start_date, + CURRENT_DATE AS valid_end_date, + 'D' AS invalid_reason +FROM concept_relationship cr + JOIN concept c ON concept_id_1 = c.concept_id + JOIN concept cc ON concept_id_2 = cc.concept_id +AND cc.standard_concept is null -- non-standard +WHERE c.vocabulary_id = 'ATC' +AND cc.vocabulary_id LIKE 'RxNorm%' +AND cr.invalid_reason IS NULL) +SELECT * FROM t1 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = cs.relationship_id + and crs.invalid_reason = cs.invalid_reason); -- 8365 --- deprecate links between ATC classes and dead RxN/RxE (should we kill links from updated ATC to RxN/RxE?) +-- Deprecate accessory links for invalid codes INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -366,7 +336,10 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT c.concept_code AS concept_code_1, +WITH t1 AS + ( +SELECT + c.concept_code AS concept_code_1, cc.concept_code AS concept_code_2, c.vocabulary_id AS vocabulary_id_1, cc.vocabulary_id AS vocabulary_id_2, @@ -377,19 +350,23 @@ SELECT c.concept_code AS concept_code_1, FROM concept_relationship cr JOIN concept c ON concept_id_1 = c.concept_id JOIN concept cc ON concept_id_2 = cc.concept_id -AND cc.standard_concept is null -- non-standard + JOIN Concept_manual k on k.concept_code = c.concept_code + and k.invalid_reason is not null +--AND cc.standard_concept is null -- non-standard WHERE c.vocabulary_id = 'ATC' -AND cc.vocabulary_id LIKE 'RxNorm%' -AND relationship_id ~ 'ATC - RxNorm' -- to cover all types of ATC links -AND cr.invalid_reason IS NULL -AND (c.concept_code,relationship_id,cc.concept_code) NOT IN ( -SELECT concept_code_1, - relationship_id, - concept_code_2 -FROM concept_relationship_stage) -; -- 4994 (245) => 239 => 254 +AND cr.invalid_reason IS NULL) +SELECT * FROM t1 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = cs.relationship_id + AND crs.invalid_reason = cs.invalid_reason + ); -- 1215 --- add 'Maps to' for 'ATC - RxNorm pr lat' for monocomponent ATC classes which do not have doubling Standard ingredients (1-to-many mapping is permissive only for real combo ATC classes) +-- add mirroring 'Maps to' for 'ATC - RxNorm pr lat' for monocomponent ATC Classes, which do not have doubling Standard ingredients (1-to-many mappings are permissive only for ATC Combo Classes) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -400,26 +377,45 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT DISTINCT concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - 'Maps to', - valid_start_date, - valid_end_date, - invalid_reason -FROM ( -SELECT *, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt -FROM concept_relationship_stage -WHERE relationship_id IN ('ATC - RxNorm pr lat') -and invalid_reason IS NULL - -- exclude Maps to CVX - ) a -WHERE cnt = 1-- can be just one , old version - cnt<25 -and (concept_code_1, 'Maps to') - NOT IN (SELECT concept_code_1, relationship_id FROM concept_relationship_stage); -- 4608 - --- more than one +WITH t1 AS +( + SELECT * + FROM concept_relationship_stage + WHERE relationship_id = 'ATC - RxNorm pr lat' + AND invalid_reason IS NULL +), +t2 AS ( +SELECT DISTINCT a.concept_code_1, --k.concept_name, + a.concept_code_2,-- d.concept_name, + a.vocabulary_id_1, + a.vocabulary_id_2, + 'Maps to' as relationship_id, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason +FROM (SELECT *, + COUNT(concept_code_1) OVER (PARTITION BY concept_code_1) AS cnt + FROM t1) a + JOIN concept_manual k + ON k.concept_code = a.concept_code_1 + AND k.invalid_reason IS NULL + JOIN concept d + ON d.concept_code = a.concept_code_2 + AND d.vocabulary_id = a.vocabulary_id_2 + AND d.standard_concept = 'S' +WHERE cnt = 1 -- can be just one +) +SELECT * FROM t2 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = cs.relationship_id -- and crs.invalid_reason is null + ); -- 4374 + +-- add mirroring 'Maps to' of 'ATC - RxNorm sec lat' relationships for ATC Combo Classes (1-to-1) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -430,26 +426,45 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -with t1 as ( -SELECT distinct a.*, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt -FROM concept_relationship_stage a -JOIN atc_one_to_many_excl b on b.atc_code = a.concept_code_1 -JOIN concept c on c.concept_code = a.concept_code_2 and c.vocabulary_id = a.vocabulary_id_2 and c.standard_concept = 'S' -and b.concept_code <> a.concept_code_2 -WHERE a.relationship_id IN ('ATC - RxNorm pr lat') -and a.invalid_reason IS NULL) - select DISTINCT concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - 'Maps to', - valid_start_date, - valid_end_date, - invalid_reason from t1 where concept_code_1 in (select concept_code_1 from t1 group by concept_code_1 having count (1)=1) - and (concept_code_1, 'Maps to') NOT IN (SELECT concept_code_1, relationship_id FROM concept_relationship_stage) - ; -- 66 +WITH T1 AS +( + SELECT class_code, + class_name, + a.concept_id, + c.concept_code, + a.concept_name, + c.vocabulary_id + FROM dev_combo a + JOIN concept c ON c.concept_id = a.concept_id + WHERE rnk = 2 +), +t2 as ( +SELECT DISTINCT class_code as concept_code_1, + -- class_name, + concept_code as concept_code_2, + -- concept_name, + 'ATC' as vocabulary_id_1, + vocabulary_id as vocabulary_id_2, + 'Maps to' as relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM t1 +WHERE class_code IN (SELECT class_code + FROM t1 + GROUP BY class_code + HAVING COUNT(1) = 1) + ) +SELECT * FROM t2 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = 'Maps to' -- and crs.invalid_reason is null + ); -- 209 --- add 'Maps to' to Standard Ingredients for polycomponent ATC classes having 'ATC - RxNorm sec lat' INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -460,33 +475,49 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) - with t1 as ( -SELECT *, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt -FROM concept_relationship_stage -WHERE relationship_id IN('ATC - RxNorm sec lat') -- 'Maps to', 'Drug class of drug', -and invalid_reason IS NULL +WITH T1 AS +( + SELECT class_code, + class_name, + a.concept_id, + c.concept_code, + a.concept_name, + c.vocabulary_id + FROM dev_combo a + JOIN concept c ON c.concept_id = a.concept_id + WHERE rnk = 2 +), +t2 as ( +SELECT DISTINCT class_code as concept_code_1, + -- class_name, + concept_code as concept_code_2, + -- concept_name, + 'ATC' as vocabulary_id_1, + vocabulary_id as vocabulary_id_2, + 'Maps to' as relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM t1 a +WHERE class_code IN (SELECT class_code + FROM t1 + GROUP BY class_code + HAVING COUNT(1) <= 3) -- Combo Classes with COUNT(1) > 3 were added from concept_relationship_manual +and not exists (select 1 from atc_one_to_many_excl b where b.atc_code = a.class_code and a.concept_id = b.concept_id) ) -select distinct a.concept_code_1, ---b.class_name, - a.concept_code_2, - -- c.concept_name, - a.vocabulary_id_1, - a.vocabulary_id_2, - 'Maps to', - a.valid_start_date, - a.valid_end_date, - a.invalid_reason from t1 a -join class_drugs_scraper b on a.concept_code_1 = b.class_code -join concept c on c.concept_code = a.concept_code_2 and c.vocabulary_id = a.vocabulary_id_2 -where class_name !~ 'thiazides|agents|combinations' -and cnt<10 -and (b.class_code, c.concept_id) not in (select atc_code, concept_id from atc_one_to_many_excl) -and concept_code_2 not in ('OMOP995053','142141','4100', '117466') -AND (concept_code_1||concept_code_2) not in ('G03FA10'||'4083') -and (concept_code_1, vocabulary_id_1, 'Maps to', concept_code_2,vocabulary_id_2) - NOT IN (SELECT concept_code_1, vocabulary_id_1, relationship_id, concept_code_2,vocabulary_id_2 FROM concept_relationship_stage);-- 448 - --- Add synonyms to concept_synonym stage for each of the rxcui/code combinations in atc_tmp_table (can we concept_synonym manual?) +SELECT * FROM t2 cs +WHERE NOT EXISTS (SELECT 1 + FROM concept_relationship_stage crs + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = 'ATC' + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = 'Maps to' -- and crs.invalid_reason is null + ) +AND concept_code_1 not in ('P01BF05', 'J07AG52', 'J07BD51') -- artenimol and piperaquine|hemophilus influenzae B, combinations with pertussis and toxoids; systemic|measles, combinations with mumps, live attenuated; systemic +; -- 193 + +-- Add synonyms to concept_synonym stage for each of the rxcui/code combinations INSERT INTO concept_synonym_stage ( synonym_concept_code, synonym_name, @@ -502,13 +533,9 @@ JOIN sources.rxnconso r ON dv.concept_code = r.code AND r.code != 'NOCODE' AND r.lat = 'ENG' AND r.sab = 'ATC' - AND r.tty IN ( - 'PT', - 'IN' - ) - ; -- 7105 (6440) -- to compare with old script using except + AND r.tty IN ('PT','IN'); --- perform mapping replacement using function below (only to Standard concepts?) +-- perform mapping replacement using function below DO $_$ BEGIN PERFORM VOCABULARY_PACK.CheckReplacementMappings(); @@ -520,10 +547,10 @@ BEGIN PERFORM VOCABULARY_PACK.AddFreshMAPSTO(); END $_$; --- Deprecate 'Maps to' mappings to deprecated AND upgraded concepts (the step of such deprecation can be deleted) but what should we do with ancestor? +-- Deprecate 'Maps to' mappings to deprecated AND upgraded concepts DO $_$ BEGIN - PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO(); + PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO(); END $_$; -- DELETE ambiguous 'Maps to' mappings @@ -531,3 +558,97 @@ DO $_$ BEGIN PERFORM VOCABULARY_PACK.DELETEAmbiguousMAPSTO(); END $_$; + +--16. Build reverse relationship. This is necessary for next point +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT DISTINCT crs.concept_code_2, + crs.concept_code_1, + crs.vocabulary_id_2, + crs.vocabulary_id_1, + r.reverse_relationship_id, + crs.valid_start_date, + crs.valid_end_date, + crs.invalid_reason +FROM concept_relationship_stage crs +JOIN relationship r ON r.relationship_id = crs.relationship_id +WHERE NOT EXISTS ( + -- the inverse record + SELECT 1 + FROM concept_relationship_stage i + WHERE crs.concept_code_1 = i.concept_code_2 + AND crs.concept_code_2 = i.concept_code_1 + AND crs.vocabulary_id_1 = i.vocabulary_id_2 + AND crs.vocabulary_id_2 = i.vocabulary_id_1 + AND r.reverse_relationship_id = i.relationship_id + ); + +--17. Deprecate all relationships in concept_relationship that do not exist in concept_relationship_stage +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT DISTINCT + a.concept_code, --a.concept_name, + b.concept_code, --b.concept_name, + a.vocabulary_id, + b.vocabulary_id, + relationship_id, + r.valid_start_date, + CURRENT_DATE, + 'D' +FROM concept a +JOIN concept_relationship r ON a.concept_id = concept_id_1 + AND r.invalid_reason IS NULL + AND r.relationship_id NOT IN ( + 'Concept replaced by', + 'Concept replaces','Drug has drug class', 'Drug class of drug', 'Subsumes', + 'Is a', 'ATC - SNOMED eq', 'SNOMED - ATC eq', 'VA Class to ATC eq', 'ATC to VA Class eq', 'ATC to NDFRT eq', 'NDFRT to ATC eq' + ) +JOIN concept b ON b.concept_id = concept_id_2 +WHERE 'ATC' IN ( + a.vocabulary_id, + b.vocabulary_id + ) + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = a.concept_code + AND crs_int.concept_code_2 = b.concept_code + AND crs_int.vocabulary_id_1 = a.vocabulary_id + AND crs_int.vocabulary_id_2 = b.vocabulary_id + AND crs_int.relationship_id = r.relationship_id + ); -- 5418 + +-- remove suspicious replacement mapping for OLD codes +DELETE FROM concept_relationship_stage +WHERE concept_code_1 IN ('C10AA55','J05AE06','C10AA52','C10AA53','C10AA51','N02AX52', 'H01BA06') +AND concept_code_2 IN ('17767','85762','7393','1191','161', '11149') +AND invalid_reason IS NULL; -- 6 + +--delete duplicate (to do: define its origin) +DELETE +FROM concept_relationship_stage +WHERE ctid NOT IN (SELECT MIN(ctid) + FROM concept_relationship_stage + + GROUP BY concept_code_1, + concept_code_2, + relationship_id, + invalid_reason + ) + and concept_code_1 = 'H01BA06';-- 1 From f68161e4a5ecd164d9e0bfe88e9c9645996aeac7 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Thu, 30 Sep 2021 00:01:37 +0300 Subject: [PATCH 15/45] Update load_input.sql refactoring --- ATC/load_input.sql | 208 ++++++++++++++++++++++++++++----------------- 1 file changed, 128 insertions(+), 80 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 5f95ceca3..b28001534 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -15,6 +15,7 @@ * * Authors: Anna Ostropolets, Polina Talapova * Date: Jul 2021 +* Total script execution time: 9m 51s **************************************************************************/ DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; @@ -64,9 +65,9 @@ CREATE UNIQUE INDEX dcs_unique_concept_code ON drug_concept_stage (concept_code); CREATE INDEX irs_unique_concept_code ON internal_relationship_stage (concept_code_1, concept_code_2); -/************************************* -***** internal_relationship_stage **** -**************************************/ +/************************************************* +***** Mono ATC to internal_relationship_stage **** +**************************************************/ ---------------- -- Dose Forms -- ---------------- @@ -74,7 +75,7 @@ CREATE INDEX irs_unique_concept_code ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_1 TYPE VARCHAR; ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_2 TYPE VARCHAR; --- create a temporary table WITH all ATC-related RxN/RxE Dose Forms +-- create a temporary table WITH all ATC-related RxN/RxE Dose Forms (using Dose Form Groups) DROP TABLE if exists dev_form; CREATE TABLE dev_form AS ( @@ -87,7 +88,8 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 IN (36217214, 36244020, 36217223) -- Oral Product | Buccal Product | Paste product (Dose Form Group) +WHERE concept_id_1 IN (36217214, 36244020, 36217223, 36244035, 36217215, 36244032, 36244021, 36244031, -- Oral Product |Buccal Product |Paste product |Chewable Product|Dental Product|Disintegrating Oral Product|Lozenge Product|Wafer Product + 36244030 , 36244029, 36217220, 36244027, 36244036, 36244033, 36217216) -- Oral Powder Product|Oral Paste Product|Oral Liquid Product|Granule Product|Flake Product|Pellet Product|Pill AND relationship_id = 'RxNorm inverse is a' ),-- sublingual route is included as well despite the fact it is processed separately dev_sub -- 2 - Sublingual forms @@ -99,7 +101,7 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 in (36217214, 36244020) -- Sublingual Product | Buccal Product (Dose Form Group) +WHERE concept_id_1 in (36217214, 36244020) -- Sublingual Product|Buccal Product AND relationship_id = 'RxNorm inverse is a' AND d.concept_name ~* 'sublingual' ), -- should be separated FROM oral forms in the ATC vocabulary. @@ -112,7 +114,7 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 in ( 36217210, 36217222) -- Injectable Product | Irrigation Product +WHERE concept_id_1 in ( 36217210, 36248213, 36217221) -- Injectable Product|Intratracheal Product|Intraperitoneal Product AND relationship_id = 'RxNorm inverse is a'), -- returns all children of Injectable Product dev_nasal -- 4 - Nasal forms AS @@ -136,10 +138,10 @@ FROM concept_relationship r AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' WHERE concept_id_1 IN ( - 36217206,36244040,36244034,36217219,-- Topical Product|Soap Product|Shampoo Product|Drug Implant Product - 36217223,36217212,36217224, 19016586) -- Paste Product|Mucosal Product|Prefilled Applicator Product|Irrigation Solution -AND relationship_id = 'RxNorm inverse is a'), -- returns all children of Topical Product -dev_mouth -- 6 - Local oral forms + 36217206,36244040,36244034,36217219, 36217222, 36217208, -- Topical Product|Soap Product|Shampoo Product|Drug Implant Product|Irrigation Product |Medicated Pad or Tape + 36217223,36217212,36217224,36217225,1146249, 36217221) -- Paste Product|Mucosal Product|Prefilled Applicator Product|Urethral Product|Pyelocalyceal Product|Intraperitoneal Product +AND relationship_id = 'RxNorm inverse is a'), + dev_mouth -- 6 - Local oral forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -148,10 +150,11 @@ FROM concept_relationship r ON d.concept_id = r.concept_id_2 AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 in (36244022, 36217223) -- Mouthwash Product | Paste Product (Dose Form Group) +WHERE concept_id_1 in (36244022, 36217223, 36217214, 36244020, 36217215, 36244021, 36244026,-- Mouthwash Product|Paste Product|Sublingual Product|Buccal Product|Dental Product|Lozenge Product|Toothpaste Product + 36244037, 36244023, 36244041, 37498345, 36244028, 36244024) -- Oral Spray Product|Oral Ointment Product|Oral Gel Product |Oral Film Product| Oral Foam Product| Oral Cream Product AND relationship_id = 'RxNorm inverse is a' AND d.concept_name ~* 'mouthwash'), -dev_rectal -- 7 - Rectal forms + dev_rectal -- 7 - Rectal forms AS (SELECT DISTINCT d.* FROM concept_relationship r @@ -219,31 +222,46 @@ FROM concept_relationship r AND d.invalid_reason IS NULL AND d.concept_class_id = 'Dose Form' WHERE concept_id_1 IN (36217207, 36244037) -- Inhalant Product| Oral Spray Product -AND relationship_id = 'RxNorm inverse is a') -select * from ( -select *, 'dev_oral' as df from dev_oral -- 1 +AND relationship_id = 'RxNorm inverse is a' +), +dev_irrig +AS +(SELECT DISTINCT d.* +FROM concept_relationship r + JOIN concept c ON c.concept_Id = r.concept_id_1 + JOIN concept d + ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' +WHERE concept_id_1 = '36217222' -- Irrigation Product +AND relationship_id = 'RxNorm inverse is a' +) +SELECT * FROM ( +SELECT *, 'dev_oral' as df FROM dev_oral -- 1 UNION ALL -select *, 'dev_sub' from dev_sub -- 2 +SELECT *, 'dev_sub' FROM dev_sub -- 2 UNION ALL -select *, 'dev_parenteral' from dev_parenteral -- 3 +SELECT *, 'dev_parenteral' FROM dev_parenteral -- 3 UNION ALL -select *, 'dev_nasal' from dev_nasal -- 4 +SELECT *, 'dev_nasal' FROM dev_nasal -- 4 UNION ALL -select *, 'dev_topic' from dev_topic -- 5 +SELECT *, 'dev_topic' FROM dev_topic -- 5 UNION ALL -select *, 'dev_mouth' from dev_mouth -- 6 +SELECT *, 'dev_mouth' FROM dev_mouth -- 6 UNION ALL -select *,'dev_rectal' from dev_rectal -- 7 +SELECT *,'dev_rectal' FROM dev_rectal -- 7 UNION ALL -select *, 'dev_vaginal' from dev_vaginal -- 8 +SELECT *, 'dev_vaginal' FROM dev_vaginal -- 8 UNION ALL -select *, 'dev_urethral' from dev_urethral -- 9 +SELECT *, 'dev_urethral' FROM dev_urethral -- 9 UNION ALL -select *, 'dev_opht' from dev_opht -- 10 +SELECT *, 'dev_opht' FROM dev_opht -- 10 UNION ALL -select *, 'dev_otic' from dev_otic -- 11 +SELECT *, 'dev_otic' FROM dev_otic -- 11 UNION ALL -select *, 'dev_inhal' from dev_inhal -- 12 +SELECT *, 'dev_inhal' FROM dev_inhal -- 12 +UNION ALL +SELECT *, 'dev_irrig' FROM dev_irrig -- 13 )l); -- connect all existing RxN/RxE forms of interest from dev_form to the ATC @@ -259,7 +277,7 @@ AND a.concept_class_id = 'ATC 5th' AND (( a.concept_name ~* 'oral|systemic|chewing gum' and b.df = 'dev_oral') -- 1 OR (a.concept_name ~* 'sublingual' and b.df = 'dev_sub') -- 2 -OR (a.concept_name ~* 'parenteral|systemic|irrigat' and b.df = 'dev_parenteral') -- 3 +OR (a.concept_name ~* 'parenteral|systemic|instillat' and b.df = 'dev_parenteral') -- 3 OR (a.concept_name ~* 'nasal'and b.df = 'dev_nasal') -- 4 OR (a.concept_name ~* 'topical' AND b.df = 'dev_topic') -- 5 OR (a.concept_name ~* 'transdermal|implant|systemic' AND b.df = 'dev_topical' and b.concept_name ~* 'transdermal|Drug Implant') @@ -269,16 +287,18 @@ OR (a.concept_name ~* 'vaginal' AND b.df = 'dev_vaginal') OR (a.concept_name ~* 'urethral' AND b.df = 'dev_urethral') OR (a.concept_name ~* 'ophthalmic' AND b.df = 'dev_opht') OR (a.concept_name ~* '\yotic' AND b.df = 'dev_otic') -OR (a.concept_name ~* 'inhalant|systemic' AND b.df = 'dev_inhal')) -; +OR (a.concept_name ~* 'inhalant|systemic' AND b.df = 'dev_inhal') +OR (a.concept_name ~* '\yirrigat' AND b.df = 'dev_irrig') +); --- add links between Oral ATC Drug Classes AND RxN/RxE Dose Forms into the internal_relationship_stage +-- add links connecting ATC Drug Classes with the specified administration route AND RxN/RxE Dose Forms using atc_to_form TRUNCATE internal_relationship_stage; INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) -SELECT DISTINCT concept_code_1, concept_code_2 from atc_to_form -WHERE (concept_code_1, concept_code_2) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +SELECT DISTINCT concept_code_1, concept_code_2 FROM atc_to_form +WHERE (concept_code_1, concept_code_2) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +-- add links connecting ATC Drug Classes with the specified administration route AND Standard Ingredients using the concept_synonym table INSERT INTO internal_relationship_stage SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code @@ -289,7 +309,7 @@ SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code AND c.invalid_reason IS NULL WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); --- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept_synonym table +-- add links between ATC Drug Classes AND Standard Ingredients using the concept_synonym table INSERT INTO internal_relationship_stage SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code @@ -301,10 +321,10 @@ INSERT INTO internal_relationship_stage AND c.concept_class_id = 'Ingredient' AND c.invalid_reason IS NULL WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); --------------------------------- --- Ingredients W/O Dose Forms -- --------------------------------- --- add links between ATC Drug Classes, which do not have Dose Forms, AND Standard Ingredients using the concept table +----------------------------- +-- Mono ATC W/O Dose Forms -- +----------------------------- +-- add IRS links connecting ATC Drug Classes W/O Dose Forms and Standard Ingredients using the concept table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) SELECT DISTINCT a.concept_code AS concept_code_1, -- for such Drug Classes use ATC code only @@ -318,7 +338,7 @@ FROM concept_manual a AND c.invalid_reason IS NULL WHERE (a.concept_code, c.concept_name) not in (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); --- add links between ATC Drug Classes, which do not have Dose Forms, AND Standard Ingredients using the concept_synonym table +-- add links connecting ATC Drug Classes W/O Dose Forms and Standard Ingredients using the concept_synonym table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) SELECT DISTINCT a.concept_code AS concept_code_1, -- for such Drug Classes use ATC code only @@ -337,7 +357,7 @@ FROM concept_manual a /************************** **** ATC Combo Classes **** ***************************/ --- separate all ATC Combo Classes +-- assemble all Multicomponent ATC Classes into one table DROP TABLE if exists combo_pull; CREATE TABLE combo_pull AS @@ -347,12 +367,19 @@ AS concept_name AS class_name, SPLIT_PART(concept_name,';','1') AS nm FROM concept_manual -WHERE SPLIT_PART(concept_name,';','1') ~* ' and |combinat|preparations|acids|animals|antiinfectives|compounds|lytes\y|flowers|^glycerol|grass pollen|bacillus|^oil|alkaloids|\/|antiserum|organisms|antiseptics' +WHERE ( +SPLIT_PART(concept_name,';','1') ~* ' and |\ywith|\ycomb|preparations|acids|animals|antiinfectives|compounds|lytes\y|flowers|grass pollen|\yresins|tree pollen|dust mites|multienzymes' +OR SPLIT_PART(concept_name,';','1') ~* 'organisms|antiseptics|feather|diastase|emulsions|immunoglobulins|substitutes|glycosides|cannabinoids|typhoid-|\ydrugs|\/|antiserum' +OR SPLIT_PART(concept_name,';','1') ~* 'diphtheria-|carbohydrates|diphtheria-hepatitis B|edetates|insects|medicated shampoos|phospholipids|various|bacillus|^oil|alkaloids' + ) AND invalid_reason IS NULL AND concept_class_id = 'ATC 5th' -AND concept_name !~* 'varicella/zoster|tositumomab/iodine') SELECT*FROM t1); +AND concept_name !~* 'varicella/zoster|tositumomab/iodine|\yIUD\y|ferric oxide polymaltose' +AND concept_code <> 'G02BB02' -- vaginal ring with progestogen; vaginal +) +SELECT*FROM t1); --- obtain 1st ATC Combo Ingredient using the concept table and full name match +-- create a table for Multicomponent ATC Class mappings to Standard Ingredient, start with the 1st ATC Combo Ingredient using the concept table and full name match DROP TABLE if exists dev_combo; CREATE TABLE dev_combo AS @@ -368,7 +395,7 @@ WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx'); --- obtain 1st ATC Combo Ingredient using the concept table and full name match +-- add the 1st ATC Combo Ingredient using the concept table and full name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -383,7 +410,7 @@ AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); --- obtain 1st ATC Combo Ingredient using the concept_synonym table and full name match +-- add the 1st ATC Combo Ingredient using the concept_synonym table and full name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -399,7 +426,7 @@ AND d.concept_class_id = 'Ingredient' AND d.vocabulary_id ~ 'Rx' AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); --- last hope +-- add the the 1st ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -414,7 +441,7 @@ AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx' AND (class_code) NOT IN (select class_code FROM dev_combo); --- obtain 2nd ATC Combo Ingredient using the concept table and full name match +-- add the 2nd ATC Combo Ingredient using the concept table and full name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -428,7 +455,7 @@ WHERE c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' AND c.vocabulary_id ~ 'Rx'; --- obtain 2nd ATC Combo Ingredient using the concept table and partial name match +-- add the 2nd ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -444,7 +471,7 @@ AND c.vocabulary_id ~ 'Rx' AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) AND c.concept_id NOT IN (19049024, 19136048); --- obtain 2nd ATC Combo Ingredient using the concept_synonym table and partial name match +-- add the 2nd ATC Combo Ingredient using the concept_synonym table and partial name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -460,7 +487,7 @@ AND d.concept_class_id = 'Ingredient' AND d.vocabulary_id ~ '^Rx' AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); --- last hope for the 2nd one +-- add to dev_combo the 2nd ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo SELECT DISTINCT class_code, class_name, @@ -497,10 +524,10 @@ AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' WHERE (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) AND r.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up'); --- add mappings to those Ingredients, which have problems with name matching +-- add mappings to those Ingredients, which have problems with name matching using the hardcoding -- add Acetylsalicylic acid INSERT INTO dev_combo -SELECT class_code, +SELECT DISTINCT class_code, class_name, 'acetylsalicylic acid', 1112807, @@ -512,7 +539,7 @@ AND class_name ~* 'acetylsalicylic'; -- add Ethinylestradiol INSERT INTO dev_combo -SELECT class_code, +SELECT DISTINCT class_code, class_name, 'ethinylestradiol', 1549786, @@ -533,7 +560,7 @@ SELECT class_code, FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 19049228 FROM dev_combo WHERE class_name ~* 'estrogen') AND split_part(class_name, ';', 1) ~ 'estrogen' - UNION ALL + UNION SELECT class_code, class_name, 'estrogens', @@ -543,7 +570,7 @@ SELECT class_code, FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1549080 FROM dev_combo WHERE class_name ~* 'estrogen') AND split_part(class_name, ';', 1) ~ 'estrogen' - UNION ALL + UNION SELECT class_code, class_name, 'estrogens', @@ -553,7 +580,7 @@ SELECT class_code, FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1551673 FROM dev_combo WHERE class_name ~* 'estrogen') AND split_part(class_name, ';', 1) ~ 'estrogen' - UNION ALL + UNION SELECT class_code, class_name, 'estrogens', @@ -563,7 +590,7 @@ SELECT class_code, FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1596779 FROM dev_combo WHERE class_name ~* 'estrogen') AND split_part(class_name, ';', 1) ~* 'estrogen' - UNION ALL + UNION SELECT class_code, class_name, 'estrogens' AS class, @@ -574,10 +601,13 @@ FROM dev_combo WHERE (class_code, concept_id) NOT IN ( select class_code, 1586808 FROM dev_combo WHERE class_name ~* 'estrogen') AND split_part(class_name, ';', 1) ~* 'estrogen'; --- remove erroneous automap +-- remove erroneous mapping of strontium ranelate to strontium DELETE FROM dev_combo WHERE class_code = 'M05BX53' AND concept_id = 19000815; -- strontium --- add links between Oral ATC Drug Classes AND Standard Ingredients using the concept table +/********************************************** +**** Combo to internal_realtionship_stage **** +***********************************************/ +-- add links between Multicomponent Oral ATC Drug Classes AND Standard Ingredients using the tables of dev_combo and concept table. INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) WITH t1 @@ -598,7 +628,7 @@ AND c.concept_code = a.class_code AND c.concept_class_id = 'Ingredient' WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); --- add links between Oral ATC Drug Classes AND Ingredients using the concept table +-- add links between Multicomponent Parenteral ATC Drug Classes AND Standard Ingredients using the tables of dev_combo and concept table. INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) WITH t1 @@ -611,7 +641,7 @@ FROM dev_combo a, concept_manual c WHERE c.concept_name ~ 'parenteral|systemic' AND b.df = 'dev_parenteral' AND c.concept_code = a.class_code -) -- Oral formulations +) SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code FROM t1 a @@ -620,7 +650,7 @@ AND c.concept_code = a.class_code AND c.concept_class_id = 'Ingredient' WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); --- add links between Vaginal ATC Drug Classes AND Standard Ingredients using the concept table +-- add links between Multicomponent Vaginal ATC Drug Classes AND Standard Ingredients using the tables of dev_combo and concept table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) WITH t1 @@ -641,8 +671,10 @@ AND c.concept_code = a.class_code AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); - --- add links between ATC Drug Classes W/O Dose Forms AND Standard Ingredients using the concept table + ------------------------------ +-- Combo ATC W/O Dose Forms --- +------------------------------- +-- add links between Multicomponent ATC Drug Classes W/O Dose Forms AND Standard Ingredients using the tables of dev_combo and concept table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) WITH t1 @@ -664,7 +696,7 @@ WHERE (concept_code_1, c.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, /****************************** ******* manual mapping ******** *******************************/ --- add manually mapped ATC Drug Classes to Standard Ingredients using concept_relationship_manual +-- add manually created maps between ATC Classes and equivalent Standard Ingredients using crm INSERT INTO internal_relationship_stage ( concept_code_1, concept_code_2) WITH t1 AS (select distinct a.concept_code_1, a.relationship_id, c.concept_name as concept_code_2 @@ -1221,12 +1253,29 @@ SELECT DISTINCT a.concept_code, FROM concept_manual a, concept_ancestor ca JOIN concept c ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id = 21604489 - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - AND c.concept_id NOT IN (742594) + AND (ca.ancestor_concept_id = 21604489 -- -- Crotarbital|butabarbital|Phenobarbital|butalbital +OR c.concept_name ~* ('Butalbital|Lorazepam|Ethchlorvynol|Ziprasidone|Talbutal|Pentobarbital|Olanzapine|Clobazam|'|| +'Clozapine|Meprobamate|Sulpiride|Eszopiclone|Alprazolam|Loxapine|Remoxipride|Secobarbital|' || +'Promazine|Zolpidem|Prochlorperazine|Droperidol|Methohexital|Chlordiazepoxide|' || +'Chlorpromazine|Buspirone|Haloperidol|Triflupromazine|Adinazolam|Hydroxyzine|' || +'Thiopental|Fluphenazine|Dexmedetomidine|Thioridazine|Midazolam|Flurazepam|' || +'Risperidone|Propiomazine|Primidone|Halazepam|Diazepam|Trifluoperazine|Oxazepam|' || +'Methylphenobarbital|Perphenazine|Flupentixol|Triazolam|Mesoridazine|Zaleplon|' || +'Ramelteon|Acetophenazine|Melatonin|Pimozide|Methyprylon|Thiamylal|' || +'Phenobarbital|Zopiclone|Estazolam|Quetiapine|Aripiprazole|Chlorprothixene|'|| +'Paliperidone|Amobarbital|Aprobarbital|Butobarbital|Heptabarbital|Hexobarbital|'|| +'Methotrimeprazine|Glutethimide|Barbital|Camazepam|Dichloralphenazone|Flunitrazepam|'|| +'Ethyl loflazepate|Cloxazolam|Bromazepam|Clotiazepam|Chloral hydrate|Fludiazepam|'|| +'Ketazolam|Prazepam|Quazepam|Cinolazepam|Nitrazepam|Periciazine|Acepromazine|Molindone|Pipotiazine|'|| +'Thioproperazine|Thiothixene|Zuclopenthixol|Methaqualone|Fluspirilene|Iloperidone|'|| +'Cariprazine|Sertindole|Asenapine|Amisulpride|Clomethiazole|Triclofos|Mebutamate|Tofisopam')::text +) + AND c.concept_id NOT IN (742594) WHERE SPLIT_PART(a.concept_name,';',1) ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id = 'Ingredient' and c.standard_concept = 'S'; + + -- add descendants of Selenium compounds INSERT INTO dev_combo @@ -1457,7 +1506,7 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND SPLIT_PART(a.concept_name,';',1) ~* 'fumaric acid derivatives' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- add ingredients indicating Glycerol +/*-- add ingredients indicating Glycerol INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -1471,7 +1520,7 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.standard_concept = 'S' AND c.concept_name ~* 'glycerol\y' and SPLIT_PART(a.concept_name,';',1) !~ 'rectal' AND SPLIT_PART(a.concept_name,';',1) ~* 'glycerol' and SPLIT_PART(a.concept_name,';',1) !~ 'phenylbutyrate' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; */ -- add descendants of Proton pump inhibitors INSERT INTO dev_combo @@ -2008,7 +2057,7 @@ WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id AND (concept_id IN (SELECT concept_id FROM dev_combo WHERE class_code = 'J01EB20') OR concept_name ~* '^sulfa') AND rnk = 3) AND class_name ~ 'sulfonamides' -AND rnk = 3; +AND rnk = 3; -- 16 INSERT INTO dev_combo WITH t1 @@ -2020,18 +2069,17 @@ FROM dev_combo WHERE class_name ~ 'sulfonamides' AND (concept_id IN (SELECT concept_id FROM dev_combo WHERE class_code = 'J01EB20') OR concept_name ~* '^sulfa') AND rnk = 3) SELECT DISTINCT class_code,class_name,'sulfonamides',b.*FROM dev_combo a,t1 b WHERE class_name ~ 'sulfonamides' -AND class_code||b.concept_id NOT IN (SELECT class_code||concept_id FROM dev_combo); - --- non-combo: glycerol -DELETE -FROM dev_combo -WHERE class_code = 'A16AX09'; +AND class_code||b.concept_id NOT IN (SELECT class_code||concept_id FROM dev_combo); --40 +-- assign correct rnk 529303 798304 diphtheria toxoid vaccine, inactivated UPDATE dev_combo SET rnk = 2 WHERE concept_id = 529303 AND class_code = 'J07AM51'; +DELETE from dev_combo where class_code = 'B03AD02' +and rnk = 2; + /*************************************** ******* relationship_to_concept ******** ****************************************/ @@ -2049,6 +2097,6 @@ FROM internal_relationship_stage JOIN concept c ON lower(concept_code_2) = lower(c.concept_name) AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - AND c.invalid_reason IS NULL; + AND c.invalid_reason IS NULL; -- 5701 -- run load_interim.sql From 5ed575dc64471f5af9f79a77d00b741775355f4b Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 01:33:03 +0300 Subject: [PATCH 16/45] Update readme.md --- ATC/readme.md | 53 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/ATC/readme.md b/ATC/readme.md index bd04a2bc6..36fb76b52 100644 --- a/ATC/readme.md +++ b/ATC/readme.md @@ -1,11 +1,48 @@ -Update of ATC +### ATC Refresh Process ### -Prerequisites: -- Schema DevV5 with copies of tables concept, concept_relationship and concept_synonym from ProdV5, fully indexed. -- SNOMED and RxNorm must be loaded first with the full release cycle. -- Working directory ATC. +#### Prerequisites: #### +* Basic knowledge of the [ATC vocabulary as a part of OMOP CDM](https://www.ohdsi.org/web/wiki/doku.php?id=documentation:vocabulary:atc) +* Schema DEVV5 with copies of tables concept, concept_relationship and concept_synonym from PRODV5, fully indexed +* SNOMED, RxNorm and RxNorm Extension must be loaded first with the full release cycle. +* Working directory, e.g. *dev_atc*. -1. Run load_stage.sql -2. Run generic_update: devv5.GenericUpdate(); +##### I - Manual work ##### +1. Create a backup of the **class_drugs_scraper** table +2. There are 2 options: +**Scenario A** - addendum of new ATC codes +Insert them manually into the **class_drugs_scraper** table, populating the following fields: class_code (varchar), class_name (varchar), ddd (varchar), u (varchar), adm_r (varchar), note (varchar), valid_start_date (date), valid_end_date (date), change_type (varchar). The query example: +```sql +INSERT INTO class_drugs_scraper +SELECT id, + atc_code AS class_code, + atc_name AS class_name, + ddd, + u, + adm_r, + note, + TO_DATE('2021-01-01','YYYYMMDD') AS valid_start_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, + 'A' AS change_type +FROM atc_addendum_MMYY; +``` +**Scenario B** - changes of the existing ATC codes +Insert them as new entities into the class_drugs_scraper table assigning them unique ids and preserving old versions of ATC codes. +3. Using class_drugs_scraper, add new codes with OMOPized names and their row variants to the concept_manual and concept_synonym_manual tables respectively. - \ No newline at end of file +##### II - Machinery ##### +1. Run load_input.sql, which populates the **input tables** of *drug_concept_stage, internal_relationship_stage, relationship_to_concept* and ATC-specific *dev_combo*. +2. Run load_interim.sql, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products +3. Run load_stage.sql which populates the **staging tables** of *concept_stage, concept_relationship_stage*, and *concept_synonym_stage* +4. Run generic_update.sql +5. Perform post-processing: +```sql +DO $_$ +BEGIN + PERFORM VOCABULARY_PACK.pConceptAncestor(IS_SMALL=>TRUE); +END $_$; +``` +1. Run load_stage.sql +2. Run generic_update: +```sql +SELECT devv5.genericupdate(); +```sql From 901ac8230286d13d6d7ae9f7e98b06ac437512f2 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 01:33:37 +0300 Subject: [PATCH 17/45] Update readme.md --- ATC/readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ATC/readme.md b/ATC/readme.md index 36fb76b52..e14601074 100644 --- a/ATC/readme.md +++ b/ATC/readme.md @@ -6,7 +6,7 @@ * SNOMED, RxNorm and RxNorm Extension must be loaded first with the full release cycle. * Working directory, e.g. *dev_atc*. -##### I - Manual work ##### +#### I - Manual work #### 1. Create a backup of the **class_drugs_scraper** table 2. There are 2 options: **Scenario A** - addendum of new ATC codes @@ -29,7 +29,7 @@ FROM atc_addendum_MMYY; Insert them as new entities into the class_drugs_scraper table assigning them unique ids and preserving old versions of ATC codes. 3. Using class_drugs_scraper, add new codes with OMOPized names and their row variants to the concept_manual and concept_synonym_manual tables respectively. -##### II - Machinery ##### +#### II - Machinery #### 1. Run load_input.sql, which populates the **input tables** of *drug_concept_stage, internal_relationship_stage, relationship_to_concept* and ATC-specific *dev_combo*. 2. Run load_interim.sql, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products 3. Run load_stage.sql which populates the **staging tables** of *concept_stage, concept_relationship_stage*, and *concept_synonym_stage* @@ -45,4 +45,4 @@ END $_$; 2. Run generic_update: ```sql SELECT devv5.genericupdate(); -```sql +```sql From d5789bbc14ca122f7f010bf053d4af523c1e9800 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 01:35:50 +0300 Subject: [PATCH 18/45] Update readme.md --- ATC/readme.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ATC/readme.md b/ATC/readme.md index e14601074..d90f01561 100644 --- a/ATC/readme.md +++ b/ATC/readme.md @@ -10,7 +10,7 @@ 1. Create a backup of the **class_drugs_scraper** table 2. There are 2 options: **Scenario A** - addendum of new ATC codes -Insert them manually into the **class_drugs_scraper** table, populating the following fields: class_code (varchar), class_name (varchar), ddd (varchar), u (varchar), adm_r (varchar), note (varchar), valid_start_date (date), valid_end_date (date), change_type (varchar). The query example: +- Insert them manually into the **class_drugs_scraper** table, populating the following fields: class_code (varchar), class_name (varchar), ddd (varchar), u (varchar), adm_r (varchar), note (varchar), valid_start_date (date), valid_end_date (date), change_type (varchar). The query example: ```sql INSERT INTO class_drugs_scraper SELECT id, @@ -26,14 +26,14 @@ SELECT id, FROM atc_addendum_MMYY; ``` **Scenario B** - changes of the existing ATC codes -Insert them as new entities into the class_drugs_scraper table assigning them unique ids and preserving old versions of ATC codes. -3. Using class_drugs_scraper, add new codes with OMOPized names and their row variants to the concept_manual and concept_synonym_manual tables respectively. +- Insert them as new entities into the **class_drugs_scraper** table assigning them unique *ids* and preserving old versions of ATC codes. +3. Using the **class_drugs_scraper table**, add new codes with OMOPized names and their row variants to the *concept_manual* and *concept_synonym_manual* tables respectively. #### II - Machinery #### -1. Run load_input.sql, which populates the **input tables** of *drug_concept_stage, internal_relationship_stage, relationship_to_concept* and ATC-specific *dev_combo*. -2. Run load_interim.sql, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products -3. Run load_stage.sql which populates the **staging tables** of *concept_stage, concept_relationship_stage*, and *concept_synonym_stage* -4. Run generic_update.sql +1. Run *load_input.sql*, which populates the **input tables** of *drug_concept_stage, internal_relationship_stage, relationship_to_concept* and ATC-specific *dev_combo*. +2. Run *load_interim.sql*, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products +3. Run *load_stage.sql* which populates the **staging tables** of *concept_stage, concept_relationship_stage*, and *concept_synonym_stage* +4. Run *generic_update.sql* 5. Perform post-processing: ```sql DO $_$ From 77c287682864ed17888e5f25b5d692e4d813a1ca Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 01:42:51 +0300 Subject: [PATCH 19/45] Update readme.md --- ATC/readme.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ATC/readme.md b/ATC/readme.md index d90f01561..aac7d2a8d 100644 --- a/ATC/readme.md +++ b/ATC/readme.md @@ -8,7 +8,7 @@ #### I - Manual work #### 1. Create a backup of the **class_drugs_scraper** table -2. There are 2 options: +2. There are 3 options: **Scenario A** - addendum of new ATC codes - Insert them manually into the **class_drugs_scraper** table, populating the following fields: class_code (varchar), class_name (varchar), ddd (varchar), u (varchar), adm_r (varchar), note (varchar), valid_start_date (date), valid_end_date (date), change_type (varchar). The query example: ```sql @@ -27,22 +27,23 @@ FROM atc_addendum_MMYY; ``` **Scenario B** - changes of the existing ATC codes - Insert them as new entities into the **class_drugs_scraper** table assigning them unique *ids* and preserving old versions of ATC codes. -3. Using the **class_drugs_scraper table**, add new codes with OMOPized names and their row variants to the *concept_manual* and *concept_synonym_manual* tables respectively. +**Scenario C** - use Scenario A together with Scenario B if both the addendum and changes are required. + +3. Using the **class_drugs_scraper** table, add new codes with _OMOPized names_ and their *original versions* to the **concept_manual** and **concept_synonym_manual** tables respectively. #### II - Machinery #### -1. Run *load_input.sql*, which populates the **input tables** of *drug_concept_stage, internal_relationship_stage, relationship_to_concept* and ATC-specific *dev_combo*. +1. Run *load_input.sql*, which populates the **input tables** of **drug_concept_stage, internal_relationship_stage, relationship_to_concept** and ATC-specific **dev_combo**. 2. Run *load_interim.sql*, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products -3. Run *load_stage.sql* which populates the **staging tables** of *concept_stage, concept_relationship_stage*, and *concept_synonym_stage* +3. Run *load_stage.sql* which populates the **staging tables** of **concept_stage, concept_relationship_stage**, and **concept_synonym_stage** 4. Run *generic_update.sql* -5. Perform post-processing: +5. ```sql +SELECT devv5.genericupdate(); +```sql +6. Perform **post-processing**: ```sql DO $_$ BEGIN PERFORM VOCABULARY_PACK.pConceptAncestor(IS_SMALL=>TRUE); END $_$; ``` -1. Run load_stage.sql -2. Run generic_update: -```sql -SELECT devv5.genericupdate(); -```sql +7. enjoy! <3 From 10c8a0990ffa31ff309ea5ac862f0ddcee7fa9e4 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 01:54:34 +0300 Subject: [PATCH 20/45] Update readme.md --- ATC/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ATC/readme.md b/ATC/readme.md index aac7d2a8d..21c55794d 100644 --- a/ATC/readme.md +++ b/ATC/readme.md @@ -36,7 +36,7 @@ FROM atc_addendum_MMYY; 2. Run *load_interim.sql*, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products 3. Run *load_stage.sql* which populates the **staging tables** of **concept_stage, concept_relationship_stage**, and **concept_synonym_stage** 4. Run *generic_update.sql* -5. ```sql +```sql SELECT devv5.genericupdate(); ```sql 6. Perform **post-processing**: From 44dad02f8050df80bd6a5aa531b993db13e04578 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:35:45 +0300 Subject: [PATCH 21/45] refactoring --- ATC/load_input.sql | 64 ++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index b28001534..645c31d57 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -603,7 +603,6 @@ AND split_part(class_name, ';', 1) ~* 'estrogen'; -- remove erroneous mapping of strontium ranelate to strontium DELETE FROM dev_combo WHERE class_code = 'M05BX53' AND concept_id = 19000815; -- strontium - /********************************************** **** Combo to internal_realtionship_stage **** ***********************************************/ @@ -720,7 +719,7 @@ TRUNCATE drug_concept_stage; -- change length of concept_code field ALTER TABLE drug_concept_stage ALTER COLUMN concept_code TYPE VARCHAR; --- add all ATC Drug Classes using the internal_relationship_stage table +-- add to DCS ATC Drug Classes as Drug Products using the internal_relationship_stage table INSERT INTO drug_concept_stage ( concept_name, @@ -802,7 +801,7 @@ AND standard_concept = 'S' AND invalid_reason IS NULL AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); --- obtain additional ingredients for those ATC codes which are still unmapped using fuzzy match +-- obtain additional Ingredients for those ATC Classes which are still unmapped using fuzzy match INSERT INTO internal_relationship_stage WITH t1 AS ( @@ -900,7 +899,7 @@ WHERE (class_code,concept_name) NOT IN (SELECT concept_code_1, /********************************** *** FUTHER WORK WITH ATC COMBOS *** ***********************************/ --- assemble mappings for ATC Classes indicating Ingredient Groups using the the concept_ancestor AND/OR concept tables along WITH word pattern matching +-- assemble mappings for ATC Classes indicating Ingredient Groups using the concept_ancestor AND/OR concept tables along WITH word pattern matching -- add descendants of Acid preparations INSERT INTO dev_combo SELECT DISTINCT a.concept_code, @@ -1269,14 +1268,12 @@ OR c.concept_name ~* ('Butalbital|Lorazepam|Ethchlorvynol|Ziprasidone|Talbutal| 'Ketazolam|Prazepam|Quazepam|Cinolazepam|Nitrazepam|Periciazine|Acepromazine|Molindone|Pipotiazine|'|| 'Thioproperazine|Thiothixene|Zuclopenthixol|Methaqualone|Fluspirilene|Iloperidone|'|| 'Cariprazine|Sertindole|Asenapine|Amisulpride|Clomethiazole|Triclofos|Mebutamate|Tofisopam')::text -) +) -- obtained from https://go.drugbank.com/categories/DBCAT002185 AND c.concept_id NOT IN (742594) WHERE SPLIT_PART(a.concept_name,';',1) ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' AND c.vocabulary_id LIKE 'RxNorm%' AND c.concept_class_id = 'Ingredient' and c.standard_concept = 'S'; - - -- add descendants of Selenium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, @@ -1505,22 +1502,6 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND c.concept_name ~* 'fumarate\y' AND SPLIT_PART(a.concept_name,';',1) ~* 'fumaric acid derivatives' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - -/*-- add ingredients indicating Glycerol -INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'glycerol', - c.concept_id, - c.concept_name, - CASE WHEN a.concept_name ~* '^glycerol' THEN 1 ELSE 2 END ::INT AS rnk -FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'glycerol\y' and SPLIT_PART(a.concept_name,';',1) !~ 'rectal' - AND SPLIT_PART(a.concept_name,';',1) ~* 'glycerol' and SPLIT_PART(a.concept_name,';',1) !~ 'phenylbutyrate' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; */ -- add descendants of Proton pump inhibitors INSERT INTO dev_combo @@ -1618,7 +1599,7 @@ WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') AND SPLIT_PART(a.concept_name,';',1) ~* 'potassium-sparing agents' AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; --- add ingredients indicating ethiodized oil +-- add ingredients indicating Ethiodized oil INSERT INTO dev_combo SELECT DISTINCT a.concept_code, a.concept_name, @@ -1926,7 +1907,7 @@ AND rnk = 3; /******************************************* **** ADD ODDMENTS TO THE INPUT TABLES ***** ********************************************/ --- add links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of OMOP Ingredient names using dev_combo table +-- add to links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of RxN/RxE Standard Ingredient names using the dev_combo table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) SELECT DISTINCT class_code, -- ATC @@ -1940,7 +1921,7 @@ FROM internal_relationship_stage) WHERE LENGTH (class_code) = 7 AND rnk <> 0; --- add more links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of OMOP Ingredient names using dev_combo table +-- add more links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of RxN/RxE Standard Ingredient names using the dev_combo table INSERT INTO internal_relationship_stage (concept_code_1, concept_code_2) SELECT DISTINCT class_code, -- ATC @@ -1955,7 +1936,7 @@ FROM internal_relationship_stage) WHERE LENGTH (class_code) = 7 AND rnk <> 0; --- add ATC Groupers to DCS AS Drug Products +-- add ATC Classes of Ingredient Groups AS Drug Products using the internal_relationship_stage and class_drugs_scraper tables INSERT INTO drug_concept_stage ( concept_name, vocabulary_id, @@ -1967,20 +1948,20 @@ INSERT INTO drug_concept_stage valid_start_date, valid_end_date ) -SELECT DISTINCT b.class_name, -- ATC code+name - 'ATC', - 'Drug Product', - NULL, - concept_code_1, - NULL, - 'Drug', +SELECT DISTINCT b.class_name AS concept_name, + 'ATC' AS vocabulary_id, + 'Drug Product' AS concept_class_id, + NULL AS standard_concept, + concept_code_1 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, TO_DATE('20991231','YYYYMMDD') AS valid_end_date FROM internal_relationship_stage a JOIN class_drugs_scraper b ON b.class_code = SPLIT_PART(a.concept_code_1, ' ', 1) WHERE concept_code_1 NOT IN (select concept_code FROM drug_concept_stage); --- add ATC Drug Attributes IN the form of Standard Ingredient names using internal_relationship_stage +-- add ATC Drug Attributes IN the form of Standard Ingredient names using the internal_relationship_stage and concept tables INSERT INTO drug_concept_stage ( concept_name, vocabulary_id, @@ -2007,7 +1988,7 @@ AND vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND invalid_reason IS NULL AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); --- remove dead deprecated or updated codes +-- remove dead deprecated or updated ATC codes DELETE FROM internal_relationship_stage WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT concept_code @@ -2020,7 +2001,7 @@ WHERE concept_code IN (SELECT concept_code FROM concept_manual WHERE invalid_reason IS NOT NULL); --- remove inexistent drug mapping (old and wrong) +-- remove mappings of ATC Classes which are nonexistent in OMOP (old and wrong) DELETE FROM drug_concept_stage WHERE concept_code IN (SELECT class_code FROM atc_inexistent) @@ -2032,7 +2013,7 @@ WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT class_code FROM atc_inexistent) AND SUBSTRING(concept_code_1,'\w+') NOT IN (SELECT class_code FROM dev_combo) AND concept_code_1 !~ '\s+'; --- remove mappings which have been deprecated in crm +-- remove mappings which have been deprecated in concept_relationship_manual WITH t1 AS ( SELECT * @@ -2059,6 +2040,7 @@ WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id AND class_name ~ 'sulfonamides' AND rnk = 3; -- 16 +-- add missing sulfonamides INSERT INTO dev_combo WITH t1 AS @@ -2077,17 +2059,15 @@ UPDATE dev_combo WHERE concept_id = 529303 AND class_code = 'J07AM51'; +-- remove wrong rank for the Mono ATC of B03AD02 ferrous fumarate, combinations DELETE from dev_combo where class_code = 'B03AD02' and rnk = 2; - /*************************************** ******* relationship_to_concept ******** ****************************************/ --- add mappings of ATC Drug Attributes to OMOP Equivalents +-- add mappings of ATC Drug Attributes to OMOP Equivalents (Standard for the Ingredients and valid for the Dose Forms) TRUNCATE relationship_to_concept; ALTER TABLE relationship_to_concept ALTER COLUMN concept_code_1 TYPE VARCHAR; - --- add links between ATC Drug Attributes AND their OMOP equivalents INSERT INTO relationship_to_concept ( concept_code_1, vocabulary_id_1, concept_id_2) SELECT DISTINCT concept_code_2 AS concept_code_1, -- ATC attribute IN the form of OMOP Dose Form OR Ingredient name From a39d6477fa34a39e7c286c4a9ad84bdb825af65a Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:36:37 +0300 Subject: [PATCH 22/45] Update load_input.sql --- ATC/load_input.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 645c31d57..200bb5b9d 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -15,7 +15,7 @@ * * Authors: Anna Ostropolets, Polina Talapova * Date: Jul 2021 -* Total script execution time: 9m 51s +* Total script execution time: 21m 27s **************************************************************************/ DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; From f629e57dc2f72ffe04331a7dcd2753dcf215d23e Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 16:35:39 +0300 Subject: [PATCH 23/45] Update load_interim.sql --- ATC/load_interim.sql | 882 +++++++++++++++++++++++-------------------- 1 file changed, 482 insertions(+), 400 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 7d3011242..780aac239 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. * -* Authors: Anna Ostropolets, Polina Talapova -* Date: Jul 2021 +* Authors: Anna Ostropolets, Polina Talapova, Timur Vakhitov +* Date: Sep 2021 +* Total script execution time: 33m 56s **************************************************************************/ /******************** ***** RX COMBO ***** *********************/ --- create a table with aggregated RxE ingredients: rx_combo +-- create an interim table of rx_combo with aggregated concept_ids of RxN/RxN Standard Ingredients DROP TABLE IF EXISTS rx_combo; CREATE TABLE rx_combo AS SELECT drug_concept_id, @@ -29,8 +30,7 @@ FROM devv5.drug_strength concept_class_id IN ('Clinical Drug Form', 'Ingredient') -- 'Clinical Drug Comp' doesn't exist in ATCs GROUP BY drug_concept_id; --- additional reference --- Rx combo - ing - df +-- create an interim table of rx_all_combo as the assemblage of Multicomponent RxN/RxE Drug Products, RxN/RxE Standard Ingredients and RxN/RxE Dose Forms DROP TABLE rx_all_combo; CREATE TABLE rx_all_combo AS @@ -42,7 +42,7 @@ AS k.concept_name AS df_name FROM concept c JOIN concept_relationship r ON r.concept_id_1 = c.concept_id - JOIN concept d ON d.concept_id = r.concept_id_2 --df + JOIN concept d ON d.concept_id = r.concept_id_2--df JOIN concept_relationship r2 ON r2.concept_id_1 = c.concept_id JOIN concept k ON k.concept_id = r2.concept_id_2 WHERE c.concept_class_id = 'Clinical Drug Form' @@ -56,7 +56,7 @@ AND k.invalid_reason IS NULL AND r.invalid_reason IS NULL AND r2.invalid_reason IS NULL); --- additional reference for ATC +-- create an interim table of atc_all_combo as the assemblage of Multicomponent ATC Classes, RxN/RxE Standard Ingredients and RxN/RxE Dose Forms DROP TABLE IF EXISTS atc_all_combo; CREATE TABLE atc_all_combo AS @@ -75,8 +75,9 @@ FROM dev_combo a JOIN concept c ON c.concept_name = d.concept_name AND c.invalid_reason IS NULL); - -DROP TABLE if exists atc_all_mono; -- 8m 22s + +-- create an interim table of atc_all_combo as the assemblage of Monocomponent ATC Classes, RxN/RxE Standard Ingredients and RxN/RxE Dose Forms +drop table if exists atc_all_mono; CREATE TABLE atc_all_mono AS ( @@ -105,9 +106,9 @@ FROM concept_manual a1 /************************************ ***** PREPARE ATC COMBO CLASSES ***** *************************************/ --- separate pure ATC Combo Classes with mapping: Primary lateral + Secondary lateral (rnk in (1, 2) in dev_combo) -DROP TABLE IF EXISTS case_1; -CREATE TABLE case_1 +-- create an interim table which contains pure Multicomponent ATC Classes with semantic links of 'Primary lateral' + 'Secondary lateral' (ranks of 1 and 2 in dev_combo) +DROP TABLE IF EXISTS ing_pr_lat_sec_lat; +CREATE TABLE ing_pr_lat_sec_lat AS SELECT DISTINCT class_code, class_name, @@ -121,9 +122,22 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3)-- exclu AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0); -- exclude Secondary upward --- separate pure ATC Combo Classes with 1 defined Ingredient: Primary lateral Ingredients in combination (rnk = 1 in dev_combo) -DROP TABLE if exists case_2; -CREATE TABLE case_2 +-- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary lateral' + 'Secondary upward' (ranks of 1 and 4 in dev_combo) +DROP TABLE if exists ing_pr_lat_sec_up; +CREATE TABLE ing_pr_lat_sec_up +AS +(SELECT * +FROM dev_combo a +WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) +AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) +AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) +); + +-- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary lateral' in combination with other drugs (rank of 1 in dev_combo) +DROP TABLE if exists ing_pr_lat_combo; +CREATE TABLE ing_pr_lat_combo AS (SELECT * FROM dev_combo @@ -134,9 +148,10 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) ); --- separate Primary lateral in combination with excluded Ingredient (rnk in (1,0) in dev_combo) -DROP TABLE if exists case_2_2; -CREATE TABLE case_2_2 +-- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary lateral' in combination with other drugs and excluded Ingredient +-- (ranks of 1 and 0 in dev_combo) +DROP TABLE if exists ing_pr_lat_combo_excl; +CREATE TABLE ing_pr_lat_combo_excl AS (SELECT * FROM dev_combo @@ -147,9 +162,9 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0) ); --- separate Primary upward Ingredients (rnk = 3 in dev_combo) -DROP TABLE IF EXISTS case_3; -CREATE TABLE case_3 +-- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary upward' in combination with other drugs (rank of 3 in dev_combo) +DROP TABLE IF EXISTS ing_pr_up_combo; +CREATE TABLE ing_pr_up_combo AS (SELECT DISTINCT class_code, class_name, @@ -163,14 +178,14 @@ AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2)-- exclu AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0));-- exclude Secondary upward --- add the same Ingredients marked with rnk=1 to create permutations, assume that there will not be more than 3 ingredients in combination -INSERT INTO case_3 -- Primary upward +-- add the same Ingredients marked with rank of 1 to create permutation +INSERT INTO ing_pr_up_combo -- Primary upward SELECT class_code, class_name, concept_id, concept_name, 1 - FROM case_3; + FROM ing_pr_up_combo; --- separate Primary upward + Secondary upward Ingredients (rnk in (3,4) in dev_combo) -DROP TABLE IF EXISTS case_4; -CREATE TABLE case_4 AS ( +-- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary upward' + 'Secondary upward' (ranks of 3 and 4 in dev_combo) +DROP TABLE IF EXISTS ing_pr_up_sec_up; +CREATE TABLE ing_pr_up_sec_up AS ( SELECT DISTINCT class_code, class_name, concept_id, @@ -188,18 +203,19 @@ AND class_code NOT IN (SELECT class_code WHERE rnk = 2) -- exclude Secondary lateral AND class_code IN (SELECT class_code FROM dev_combo - WHERE rnk = 4) -and class_code not in (select class_code from dev_combo where rnk = 0) -);-- exclude Secondary upward + WHERE rnk = 4) +and class_code not in (select class_code from dev_combo where rnk = 0) -- exclude Excluded +); --- separate Primary upward + Secondary upward with excluded Ingredient (rnk in (3, 4, 0) in dev_combo) -DROP TABLE IF EXISTS case_4_2; -CREATE TABLE case_4_2 AS (select distinct -class_code, - class_name, - concept_id, -concept_name, - rnk +-- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary upward' + 'Secondary upward' with excluded Ingredient +-- ranks of 3, 4 and 0 in dev_combo +DROP TABLE IF EXISTS ing_pr_up_sec_up_excl; +CREATE TABLE ing_pr_up_sec_up_excl AS ( +SELECT distinct class_code, + class_name, + concept_id, + concept_name, + rnk FROM dev_combo WHERE class_code IN (SELECT class_code FROM dev_combo @@ -213,10 +229,12 @@ AND class_code NOT IN (SELECT class_code AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -and class_code in (select class_code from dev_combo where rnk = 0) -);-- exclude Secondary upward --- Note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product +AND class_code IN (SELECT class_code + FROM dev_combo + WHERE rnk = 0) +);-- Note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product +-- create indices DROP INDEX IF EXISTS idx_atc; CREATE INDEX idx_atc ON atc_all_combo (class_code, df_id, ing_id); ANALYZE atc_all_combo; @@ -238,13 +256,11 @@ ANALYZE atc_all_combo; DROP INDEX IF EXISTS irs_x; CREATE INDEX irs_x ON internal_relationship_stage (SUBSTRING (concept_code_1,'\w+')); ANALYZE internal_relationship_stage; - /************************* ***** CLASS TO DRUG ****** **************************/ --- assemble a table containing ATC Drug Classes which are hierarchically connected to RxN/RxE Drug Products via 'ATC - RxNorm' relationships (resembles Subsumes). +-- assemble a table containing ATC Drug Classes which are as ancestors hierarchically connected to RxN/RxE Drug Products (the first line target - Clinical Drug Form) -- add mappings from Monocomponent ATC Classes to respective Monocomponent RxN/RxE Drug Products (order = 1) --- 5m 40s DROP TABLE if exists class_to_drug_new; CREATE TABLE class_to_drug_new AS @@ -254,16 +270,61 @@ AS a.class_name, df_id, ARRAY_AGG(a.ing_id) OVER (PARTITION BY a.class_code) ings -FROM dev_atc.atc_all_mono a),rx AS (SELECT DISTINCT r.d_id, r.d_name, r.df_id, ARRAY_AGG(r.ing_id) OVER (PARTITION BY r.d_id) ings - FROM dev_atc.rx_all_combo r) - SELECT DISTINCT atc.class_code,atc.class_name,c.*,1 AS concept_order,'ATC Monocomp Class' AS order_desc - FROM atc +FROM atc_all_mono a),rx AS (SELECT DISTINCT r.d_id, r.d_name, r.df_id, ARRAY_AGG(r.ing_id) OVER (PARTITION BY r.d_id) ings + FROM rx_all_combo r) + SELECT DISTINCT atc.class_code, + atc.class_name, + c.*, + 1 AS concept_order, + 'ATC Monocomp Class' AS order_desc +FROM atc JOIN rx USING (df_id) JOIN concept c ON c.concept_id = rx.d_id WHERE atc.ings @> rx.ings AND atc.ings <@ rx.ings); --- add greedy ATC Monocomponent Classes (order = 2) +-- add more mappings from Monocomponent ATC Classes to respective Monocomponent RxN/RxE Drug Products (order = 1) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT * +FROM concept_manual +WHERE invalid_reason IS NULL +AND concept_class_id = 'ATC 5th' +AND concept_code NOT IN (SELECT class_code FROM dev_combo)), +-- get attributes +t2 +AS +(SELECT DISTINCT a.concept_code AS class_code, + a.concept_name AS class_name, + d.concept_code AS ing_name, + d2.concept_code AS df_name +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d + ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient' + LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + LEFT JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form') +SELECT DISTINCT class_code, + class_name, + c.*, + 1 AS concept_order, + 'ATC Monocomp Class' as order_desc +FROM t2 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + AND d_name !~ ' / ' -- exclude Multicomponent + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' +WHERE class_code NOT IN (SELECT class_code FROM dev_combo) + AND class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);-- 1085 + +-- add mappings of "greedy" ATC Monocomponent Classes, which are matched with Multicomponent RxN/RxE Drug Products (order = 2) INSERT INTO class_to_drug_new WITH t1 AS @@ -279,8 +340,8 @@ FROM (SELECT a.class_code, r.d_name, ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks - FROM dev_atc.atc_all_mono a - LEFT JOIN dev_atc.rx_all_combo r USING (ing_id,df_id) + FROM atc_all_mono a + LEFT JOIN rx_all_combo r USING (ing_id,df_id) ) i WHERE i.d_id IS NOT NULL) SELECT DISTINCT @@ -291,9 +352,48 @@ WHERE i.d_id IS NOT NULL) 'Greedy ATC Monocomp Class' AS order_desc FROM t1 a JOIN concept c ON c.concept_id = a.d_id - WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 68243 + WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 60223 --- add Primary lateral in combination (order = 3) +-- add more mappings of "greedy" ATC Monocomponent Classes, which are matched with Multicomponent RxN/RxE Drug Products (order = 2) +INSERT INTO class_to_drug_new +WITH t1 +AS +(SELECT * +FROM concept_manual +WHERE invalid_reason IS NULL +AND concept_class_id = 'ATC 5th'), +-- get attributes +t2 +AS +(SELECT DISTINCT a.concept_code AS class_code, + a.concept_name AS class_name, + d.concept_code AS ing_name, + d2.concept_code AS df_name +FROM t1 a + JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d + ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient' + JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code + JOIN drug_concept_stage d2 + ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form') +SELECT DISTINCT class_code, + class_name, + c.*, + 2 AS concept_order, + 'Greedy ATC Monocomp Class' as order_desc +FROM t2 a + JOIN rx_all_combo b + ON lower (b.ing_name) = lower (a.ing_name) + AND lower (b.df_name) = lower (a.df_name) + JOIN concept c + ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) + AND class_code NOT IN (SELECT class_code FROM dev_combo); + +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' in combination (order = 3) INSERT INTO class_to_drug_new WITH t1 AS @@ -309,9 +409,9 @@ FROM (SELECT a.class_code, r.d_name, ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks - FROM dev_atc.atc_all_combo a - LEFT JOIN dev_atc.rx_all_combo r USING (ing_id,df_id) - WHERE class_code IN (SELECT class_code FROM case_2)) i + FROM atc_all_combo a + LEFT JOIN rx_all_combo r USING (ing_id,df_id) + WHERE class_code IN (SELECT class_code FROM ing_pr_lat_combo)) i WHERE i.d_id IS NOT NULL) SELECT DISTINCT class_code, @@ -323,9 +423,9 @@ FROM t1 a JOIN concept c ON c.concept_id = a.d_id AND c.concept_name ~ ' / ' -WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 8577 +WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 9201 --- add Primary upward only (order = 4) +-- add mappings of Multicomponent ACT Classes of 'Primary upward' in combination with other drugs (order = 4) INSERT INTO class_to_drug_new WITH t1 AS @@ -341,9 +441,9 @@ FROM (SELECT a.class_code, r.d_name, ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks - FROM dev_atc.atc_all_combo a - LEFT JOIN dev_atc.rx_all_combo r USING (ing_id,df_id) - WHERE class_code IN (SELECT class_code FROM case_3)) i + FROM atc_all_combo a + LEFT JOIN rx_all_combo r USING (ing_id,df_id) + WHERE class_code IN (SELECT class_code FROM ing_pr_up_combo)) i WHERE i.d_id IS NOT NULL) SELECT DISTINCT class_code, @@ -355,9 +455,9 @@ FROM t1 a JOIN concept c ON c.concept_id = a.d_id AND c.concept_name ~ ' / ' -WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 12768 +WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 14307 --- add ATC Combo Class: Primary lateral + Secondary lateral, 4 ingreds (order = 5) +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary lateral, 4 ingreds' (order = 5) INSERT INTO class_to_drug_new WITH t1 AS @@ -369,7 +469,7 @@ AS FROM atc_all_combo a JOIN rx_all_combo b USING (ing_id,df_id) JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM case_1)) +WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_lat)) SELECT DISTINCT a.class_code, a.class_name, a.concept_id, @@ -405,22 +505,22 @@ AND b.rnk <> 1 AND c.rnk <> 1 AND d.rnk <> 1 AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id -FROM class_to_drug_new); -- 33 - --- add ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds (order = 6) +FROM class_to_drug_new) +and a.class_code !~ '^J07'; -- 269 + +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary upward', 4 ingreds (order = 50) INSERT INTO class_to_drug_new -WITH t1 AS -( - SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk - FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id - WHERE a.class_code IN (SELECT class_code FROM case_1) -) +WITH t1 +AS +(SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk +FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id +WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_up)) SELECT DISTINCT a.class_code, a.class_name, a.concept_id, @@ -433,8 +533,8 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 6, - 'ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds' AS order_desc + 50, + 'ATC Combo Class: Primary lateral + Secondary upward, 4 ingreds' AS order_desc FROM t1 a JOIN t1 b ON b.class_code = a.class_code @@ -445,12 +545,20 @@ FROM t1 a AND c.concept_id = b.concept_id AND a.ing_id <> c.ing_id AND b.ing_id <> c.ing_id + JOIN t1 d + ON d.class_code = a.class_code + AND d.concept_id = b.concept_id + AND a.ing_id <> d.ing_id + AND d.ing_id <> c.ing_id + AND d.ing_id <> b.ing_id WHERE a.rnk = 1 AND b.rnk <> 1 AND c.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 261 - --- 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' (order = 7) +AND d.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) +; -- 141 + +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary lateral', 3 ingreds (order = 6) INSERT INTO class_to_drug_new WITH t1 AS ( @@ -462,9 +570,9 @@ WITH t1 AS FROM atc_all_combo a JOIN rx_all_combo b USING (ing_id,df_id) JOIN concept c ON c.concept_id = d_id - WHERE a.class_code IN (SELECT class_code FROM case_1) + WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_lat) ) -SELECT DISTINCT a.class_code, +SELECT DISTINCT a.class_code, a.class_name, a.concept_id, a.concept_name, @@ -476,18 +584,24 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 7, - 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' AS order_desc + 6, + 'ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds' AS order_desc FROM t1 a JOIN t1 b ON b.class_code = a.class_code AND b.concept_id = a.concept_id AND a.ing_id <> b.ing_id + JOIN t1 c + ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id WHERE a.rnk = 1 AND b.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2426 - --- add ATC Combo Class: Primary upward + Secondary upward, 4 ing (order = 8) +AND c.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 261 + +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary upward', 3 ingreds (order = 60) INSERT INTO class_to_drug_new WITH t1 AS @@ -499,7 +613,7 @@ AS FROM atc_all_combo a JOIN rx_all_combo b USING (ing_id,df_id) JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM case_4)) +WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_up)) SELECT DISTINCT a.class_code, a.class_name, a.concept_id, @@ -512,8 +626,8 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 8, - 'ATC Combo Class: Primary upward + Secondary upward, 4 ing' AS order_desc + 60, + 'ATC Combo Class: Primary lateral + Secondary upward, 3 ingreds' AS order_desc FROM t1 a JOIN t1 b ON b.class_code = a.class_code @@ -524,19 +638,12 @@ FROM t1 a AND c.concept_id = b.concept_id AND a.ing_id <> c.ing_id AND b.ing_id <> c.ing_id - JOIN t1 d - ON d.class_code = a.class_code - AND d.concept_id = b.concept_id - AND a.ing_id <> d.ing_id - AND d.ing_id <> c.ing_id - AND d.ing_id <> b.ing_id -WHERE a.rnk = 3 -AND b.rnk <> 3 -AND c.rnk <> 3 -AND d.rnk <> 3 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 10 +WHERE a.rnk = 1 +AND b.rnk <> 1 +AND c.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2 --- add ATC Combo Class: Primary upward + Secondary upward, 3 ing (order = 9) +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary lateral', 2 ingreds' (order = 7) INSERT INTO class_to_drug_new WITH t1 AS ( @@ -548,7 +655,7 @@ WITH t1 AS FROM atc_all_combo a JOIN rx_all_combo b USING (ing_id,df_id) JOIN concept c ON c.concept_id = d_id - WHERE a.class_code IN (SELECT class_code FROM case_4) + WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_lat) ) SELECT DISTINCT a.class_code, a.class_name, @@ -562,37 +669,34 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 9, - 'ATC Combo Class: Primary upward + Secondary upward, 3 ing' AS order_desc + 7, + 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' AS order_desc FROM t1 a JOIN t1 b ON b.class_code = a.class_code AND b.concept_id = a.concept_id AND a.ing_id <> b.ing_id - JOIN t1 c - ON c.class_code = a.class_code - AND c.concept_id = b.concept_id - AND a.ing_id <> c.ing_id - AND b.ing_id <> c.ing_id -WHERE a.rnk = 3 -AND b.rnk <> 3 -AND c.rnk <> 3 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 30 +WHERE a.rnk = 1 +AND b.rnk <> 1 +AND a.class_code NOT IN (SELECT class_code FROM class_to_drug_new) +AND a.class_name NOT LIKE '%,%and%' AND a.class_code !~ '^J07' +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2426 --- add ATC Combo Class: Primary upward + Secondary upward, 2 ing (order = 10) +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary upward', 2 ingreds' (order = 70) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk -FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM case_4)) -SELECT DISTINCT a.class_code, +WITH t1 AS +( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id,df_id) + JOIN concept c ON c.concept_id = d_id + WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_up) +) +SELECT DISTINCT a.class_code, a.class_name, a.concept_id, a.concept_name, @@ -604,18 +708,18 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 10, - 'ATC Combo Class: Primary upward + Secondary upward, 2 ing' AS order_desc + 70, + 'ATC Combo Class: Primary lateral + Secondary upward, 2 ingreds' AS order_desc FROM t1 a JOIN t1 b ON b.class_code = a.class_code AND b.concept_id = a.concept_id AND a.ing_id <> b.ing_id -WHERE a.rnk = 3 -AND b.rnk <> 3 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 285 +WHERE a.rnk = 1 +AND b.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 0 --- add Primary lateral only in combination with an excluded Ingredient (order = 11) +-- add mappings of Multicomponent ACT Classes of 'Primary lateral' in combination with other drugs and an excluded Ingredient (order = 8) INSERT INTO class_to_drug_new WITH t1 AS @@ -624,7 +728,7 @@ AS a.concept_name AS ing_name, d2.concept_code AS df_name, rnk -FROM case_2_2 a +FROM ing_pr_lat_combo_excl a JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 @@ -655,7 +759,7 @@ SELECT DISTINCT class_code, valid_start_date, valid_end_date, invalid_reason, - 11 as concept_order, + 8 as concept_order, 'ATC Combo Class: Primary lateral in combination with excluded Ingredient' FROM t2 WHERE concept_name ~ ' / ' @@ -665,9 +769,10 @@ AND concept_id NOT IN (SELECT d_id ON LOWER (b.ing_name) = LOWER (a.ing_name) AND LOWER (b.df_name) = LOWER (a.df_name) AND b.rnk = 0) -AND class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 300 +AND class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 0 --- add 'ATC Combo Class: Primary upward + Secondary upward with excluded ingredient' (currently, no such, that is why the same order with previous one) +-- add mappings of Multicomponent ACT Classes of 'Primary upward' + 'Secondary upward' with excluded ingredient' +-- currently, no such (order = 0) INSERT INTO class_to_drug_new WITH t1 AS @@ -682,7 +787,7 @@ AS FROM atc_all_combo a JOIN rx_all_combo b USING (ing_id,df_id) JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM case_4_2)), +WHERE a.class_code IN (SELECT class_code FROM ing_pr_up_sec_up_excl)), t2 as ( SELECT DISTINCT a.class_code, a.class_name, @@ -696,7 +801,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 11, + 0, -- no match up to now, but it can happen in the future 'ATC Combo Class: Primary upward + Secondary upward with excluded ingredient' AS order_desc FROM t1 a JOIN t1 b @@ -705,8 +810,7 @@ FROM t1 a AND a.ing_id <> b.ing_id WHERE a.rnk = 3 AND b.rnk <> 3) - -select DISTINCT * from t2 +SELECT DISTINCT * from t2 WHERE concept_name ~ ' / ' AND concept_id NOT IN (SELECT d_id FROM rx_all_combo a @@ -716,7 +820,7 @@ AND concept_id NOT IN (SELECT d_id AND b.rnk = 0) AND class_code||concept_id::varchar NOT IN (SELECT class_code||concept_id::varchar FROM class_to_drug_new); --0 --- add more Primary upward in combination (order = 12) +-- add mappings of Multicomponent ACT Classes of 'Primary upward' + 'Secondary upward' (order = 9) INSERT INTO class_to_drug_new WITH t1 AS @@ -725,36 +829,7 @@ AS a.concept_name AS ing_name, d2.concept_code AS df_name, rnk -FROM case_3 a - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form') - SELECT DISTINCT class_code, - class_name, - c.*, - 12 as concept_order, - 'ATC Combo Class: more Primary upward' AS order_desc -FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND c.concept_name ~ ' / ' - AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new); -- 174 - --- add Primary upward + Secondary upward (order = 12) -INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT DISTINCT a.class_code AS class_code, - a.class_name AS class_name, - a.concept_name AS ing_name, - d2.concept_code AS df_name, - rnk -FROM case_4 a +FROM ing_pr_up_sec_up a JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 @@ -787,7 +862,7 @@ FROM t1 a AND c.standard_concept = 'S' AND rnk = 4) SELECT DISTINCT a.*, - 12 AS concept_order, + 9 AS concept_order, 'ATC Combo Class: Primary upward + Secondary upward' FROM t2 a JOIN t3 b @@ -797,21 +872,21 @@ WHERE a.concept_name ~ ' / '; -- 325 ------------------------- ---- GET MORE LINKS ----- ------------------------- --- separate Primary lateral Ingredients (rnk = 1 in dev_combo) +-- create an interim table with all Primary lateral Multicomponent ATC Classes (rnk = 1 in dev_combo) DROP TABLE IF EXISTS t1; CREATE UNLOGGED TABLE t1 AS SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk FROM dev_combo WHERE rnk = 1; -- Primary lateral --- separate Secondary lateral Ingredients (rnk = 2 in dev_combo) +-- create an interim table with all Secondary lateral Multicomponent ATC Classes (rnk = 2 in dev_combo) DROP TABLE IF EXISTS t2; CREATE UNLOGGED TABLE t2 AS SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk FROM dev_combo WHERE rnk = 2; -- Secondary lateral --- separate Primary upward Ingredients (rnk = 3 in dev_combo) +-- create an interim table with all Primary upward Multicomponent ATC Classes DROP TABLE IF EXISTS t3; CREATE UNLOGGED TABLE t3 AS @@ -819,7 +894,7 @@ SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk FROM dev_combo WHERE rnk = 3; -- Primary upward --- separate Secondary upward Ingredients (rnk = 4 in dev_combo) +-- create an interim table with all Secondary upward Multicomponent ATC DROP TABLE IF EXISTS t4; CREATE UNLOGGED TABLE t4 AS @@ -827,8 +902,8 @@ SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk FROM dev_combo WHERE rnk = 4; -- Secondary upward --- create table with aggregated ATC Ingredients per one ATC Combo Class (no more than 3 ingredients per Class is recommended) --- add Primary lateral AND (Secondary lateral 1 AND/OR Secondary lateral 2) AND/OR Primary upward AND/OR Secondary upward +-- create an interim table with aggregated ATC Ingredients per one Multicomponent ATC Class (no more than 3 ingredients per Class is recommended) +-- add Primary lateral AND (Secondary lateral 1 AND/OR Secondary lateral 2) AND/OR Primary upward AND/OR Secondary upward Multicomponent ATC Classes DROP TABLE if exists full_combo; CREATE UNLOGGED TABLE full_combo AS( @@ -874,7 +949,7 @@ SELECT * FROM z1 WHERE i_combo NOT IN (SELECT i_combo FROM full_combo); --- create temporary table with Ingredient permutations - additional layer of mappings: Primary lateral AND/OR Secondary lateral 1 AND/OR Secondary lateral 2 +-- create a temporary table with Ingredient permutations - additional layer of mappings: Primary lateral AND/OR Secondary lateral 1 AND/OR Secondary lateral 2 DROP TABLE IF EXISTS permutations; CREATE UNLOGGED TABLE permutations AS @@ -888,18 +963,6 @@ WHERE b.concept_id<>c.concept_id AND b.concept_id<>a.concept_id; INSERT INTO full_combo SELECT * FROM permutations; --- separate Primary lateral in combination (rnk = 1 in dev_combo) -DROP TABLE if exists ing_pr_lat_combo; -- ing_pr_lat_combo -CREATE TABLE ing_pr_lat_combo -AS -(SELECT * -FROM dev_combo a -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); - -- create temporary table with all possible i_combos for Primary lateral in combinations (with unspecified drugs) DROP TABLE if exists ing_pr_lat_combo_to_drug; CREATE TABLE ing_pr_lat_combo_to_drug @@ -917,18 +980,6 @@ FROM ing_pr_lat_combo a JOIN z1 b ON b.i_combo = a.concept_id JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id); --- separate Primary lateral + Secondary upward (rnk in (1,4) in dev_combo) -DROP TABLE if exists ing_pr_lat_sec_up_combo; -CREATE TABLE ing_pr_lat_sec_up_combo -AS -(SELECT * -FROM dev_combo a -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) -); -- create temporary table with all possible i_combos for Primary lateral + Secondary upward DROP TABLE if exists ing_pr_lat_sec_up_combo_to_drug; CREATE TABLE ing_pr_lat_sec_up_combo_to_drug @@ -942,11 +993,11 @@ WHERE i_combo LIKE '%-%') SELECT DISTINCT a.class_code, a.class_name, c.i_combo -FROM ing_pr_lat_sec_up_combo a +FROM ing_pr_lat_sec_up a JOIN z1 b ON b.i_combo = a.concept_id AND a.rnk = 1 - JOIN ing_pr_lat_sec_up_combo a1 + JOIN ing_pr_lat_sec_up a1 ON a1.class_code = a.class_code AND a1.concept_id <> a.concept_id JOIN z1 j @@ -955,19 +1006,6 @@ FROM ing_pr_lat_sec_up_combo a JOIN rx_combo c ON c.drug_concept_id = b.drug_concept_id AND c.drug_concept_id = j.drug_concept_id); - --- separate Primary lateral in combination with excluded Ingredient: ing_pr_lat_combo_excl (rnk in (1, 0) in dev_combo)) -DROP TABLE if exists ing_pr_lat_combo_excl; -CREATE TABLE ing_pr_lat_combo_excl -AS -(SELECT * -FROM dev_combo a -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0) -); -- create temporary table with all possible i_combos for Primary lateral in combination with excluded Ingredient DROP TABLE if exists ing_pr_lat_combo_excl_to_drug; @@ -985,19 +1023,6 @@ JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id JOIN rx_combo d1 ON d1. -- excluded AND d1.drug_concept_id <> d.drug_concept_id); --- separate Primary upward + Secondary upward (rnk in (3,4) in dev_combo) -DROP TABLE if exists ing_pr_sec_up_combo; -CREATE TABLE ing_pr_sec_up_combo -AS -(SELECT * -FROM dev_combo a -WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) -); - -- create a temporary table with all possible i_combos for Primary upward + Secondary upward: ing_pr_sec_up_combo_to_drug DROP TABLE if exists ing_pr_sec_up_combo_to_drug; CREATE TABLE ing_pr_sec_up_combo_to_drug @@ -1011,11 +1036,11 @@ WHERE i_combo LIKE '%-%') SELECT DISTINCT a.class_code, a.class_name, c.i_combo -FROM ing_pr_sec_up_combo a +FROM ing_pr_up_sec_up a JOIN t1 b ON b.i_combo = a.concept_id AND a.rnk = 3 - JOIN ing_pr_sec_up_combo a1 + JOIN ing_pr_up_sec_up a1 ON a1.class_code = a.class_code AND a1.concept_id <> a.concept_id JOIN t1 j @@ -1024,20 +1049,9 @@ FROM ing_pr_sec_up_combo a JOIN rx_combo c ON c.drug_concept_id = b.drug_concept_id AND c.drug_concept_id = j.drug_concept_id); - --- separate Primary upward + Secondary upward with excluded Ingredients (currently no match, but it can change in the future) -DROP TABLE if exists ing_pr_sec_up_combo_excl; -CREATE TABLE ing_pr_sec_up_combo_excl -AS -(SELECT * -FROM dev_combo a -WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0)); - --- create temporary table with all possible i_combos for Primary upward + Secondary upward with excluded Ingredients (currently no match, but query works ing_pr_sec_up_combo_excl_to_drug) + +-- create temporary table with all possible i_combos for Primary upward + Secondary upward with excluded Ingredients +-- currently no match, but query works ing_pr_sec_up_combo_excl_to_drug DROP TABLE if exists ing_pr_sec_up_combo_excl_to_drug; CREATE TABLE ing_pr_sec_up_combo_excl_to_drug AS @@ -1140,8 +1154,8 @@ CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo /******************************* ******** CLASS TO DRUG ********* ********************************/ --- add the 2nd portion of multicomponent ATC Class mappings: --- ATC Combo Classes with Dose Forms using full_combo_with_form and rx_combo (order = 13) +-- add the 2nd portion of mappings of Multicomponent ATC Classes +-- ATC Combo Classes with Dose Forms using full_combo_with_form and rx_combo (order = 10) INSERT INTO class_to_drug_new WITH t1 AS @@ -1163,7 +1177,7 @@ AND r.invalid_reason IS NULL SELECT DISTINCT f.class_code, -- ATC c.concept_name as class_name, d.*, - 13 AS conept_order, + 10 AS conept_order, 'ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations' FROM full_combo_with_form f JOIN concept_manual c on c.concept_code = f.class_code and c.invalid_reason is null and c.concept_class_id = 'ATC 5th' @@ -1171,14 +1185,15 @@ AND r.invalid_reason IS NULL ON r.i_combo = f.i_combo -- combination of Standard Ingredient IDs AND r.df_id = f.df_id JOIN concept d on d.concept_id = r.concept_id - and f.class_code||r.concept_id not in (select class_code||concept_id from class_to_drug_new); -- 2590 + and f.class_code||r.concept_id not in (select class_code||concept_id from class_to_drug_new) + and f.class_code in ( 'J07CA10', 'G03FB09', 'G03FB06', 'G03FB05', 'G03FA12', 'G03FA11','G03FA01', 'G03EA03','A11GB01'); -- 63 --- add manual mappings from concept_relationship_manual (order = 14) +-- add manual mappings from concept_relationship_manual (order = 11) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, f.concept_name AS class_name, c.*, - 14 AS concept_order, + 11 AS concept_order, 'ATC Class to Drug Product from concept_relationship_manual' FROM class_drugs_scraper a JOIN concept_relationship_manual b ON b.concept_code_1 = a.class_code @@ -1191,22 +1206,21 @@ FROM class_drugs_scraper a AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND c.standard_concept = 'S' AND (a.class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) -AND f.invalid_reason IS NULL -AND f.concept_class_id = 'ATC 5th'; -- 7535 +AND f.invalid_reason IS NULL AND b.invalid_reason is null +AND f.concept_class_id = 'ATC 5th'; -- 7558 -- manual clean up for Precise Ingredients and other particular cases (according to the information on the ATC WHO Website) DELETE FROM class_to_drug_new WHERE (class_code = 'A02BA07' AND concept_class_id = 'Clinical Drug Form') -- Branded Drug 'Tritec' -OR class_code = 'G03GA08' -- choriogonadotropin alfa (no Standard Precise Ingredient) -OR class_code='N05AF02' -- clopentixol -OR class_code IN ('D07AB02','D07BB04') -- hydrocortisone butyrate + combo that so far doesn't exist +OR class_code = 'N05AF02' -- clopentixol +OR class_code IN ('D07AB02','D07BB04') -- hydrocortisone butyrate + combo that so far doesn't exist OR class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral OR (class_code = 'B02BD14' AND concept_name LIKE '%Tretten%') -- 2 --catridecacog OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog -; -- 309 +; -- 310 --- add additional semi-manual mappings based on pattern-matching (order = 15) +-- add additional semi-manual mappings based on pattern-matching (order = 12) INSERT INTO class_to_drug_new with t1 as ( SELECT 'B02BD11' as class_code,'catridecacog' as class_name, concept_id @@ -1226,11 +1240,6 @@ FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Tritec%' AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' UNION ALL -SELECT 'G03GA08','choriogonadotropin alfa', concept_id -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~ 'choriogonadotropin alfa' - AND standard_concept = 'S' AND concept_class_id ~ 'Clinical Drug Comp' -UNION ALL SELECT 'N05AF02','clopenthixol', concept_id FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Sordinol|Ciatyl' @@ -1240,16 +1249,11 @@ SELECT 'D07AB02','hydrocortisone butyrate',concept_id FROM concept WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Hydrocortisone butyrate' AND concept_class_id = 'Clinical Drug' AND standard_concept = 'S' -UNION ALL -SELECT 'C01DA05','pentaerithrityl tetranitrate', concept_id -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ILIKE '%Pentaerythritol Tetranitrate%' and - standard_concept = 'S' AND concept_class_id = 'Clinical Drug Comp' ) SELECT DISTINCT a.class_code, d.concept_name, c.*, - 15 as concept_order, + 12 as concept_order, 'ATC Class with semi-manual point fix' FROM t1 a JOIN concept c ON c.concept_id = a.concept_id @@ -1257,7 +1261,7 @@ FROM t1 a ON d.concept_code = a.class_code AND d.concept_class_id = 'ATC 5th' AND d.invalid_reason IS NULL -AND a.class_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);--22 +AND a.class_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 13 -- clean up erroneous amount of ingredients DELETE @@ -1266,20 +1270,20 @@ FROM class_to_drug_new WHERE class_name LIKE '%,%and%' AND class_name NOT LIKE '%,%,%and%' AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT LIKE '% / % / %'; -- 149 - ---- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug (order = 16) + AND concept_name NOT LIKE '% / % / %'; -- 14 + +--- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug (order = 13) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, c.*, - 16 as concept_order, + 13 as concept_order, 'ATC Class from old class_to_drug' -FROM sources.class_to_drug a +FROM sources_class_to_drug_160921 a JOIN concept_manual b ON b.concept_code = a.class_code JOIN concept c ON a.concept_id = c.concept_id - AND c.standard_concept = 'S' + AND c.standard_concept = 'S' and c.concept_class_id !~ 'Pack|Ingredient' AND b.invalid_reason IS NULL WHERE (class_code) NOT IN (SELECT class_code FROM class_to_drug_new) AND class_code NOT IN ('S02CA01','V03AB05','P03AC54','S02CA03','S02CA03') @@ -1298,16 +1302,16 @@ AND (class_code,c.concept_id) NOT IN ( UNION ALL SELECT 'A06AA02',43158334 UNION ALL - SELECT 'A06AA02',40036796); -- 29 + SELECT 'A06AA02',40036796); -- 235 /********************** ****** ADD PACKS ****** ***********************/ --- add packs of Primary lateral only in combination (order = 17) +-- add packs of Primary lateral only in combination (order = 14) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 17 as concept_order, + 14 as concept_order, 'Pack: Primary lateral in combo' as order_desc FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id @@ -1322,21 +1326,22 @@ AND r2.invalid_reason IS NULL AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') AND j.standard_concept = 'S' AND a.class_code IN (SELECT class_code - FROM dev_combo + FROM ing_pr_lat_combo) + /*dev_combo WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0))*/ AND j.concept_name ~ ' / ' -- combos only -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 1166 +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 1174 --- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 18) +-- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 15) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 18 as concept_order, + 15 as concept_order, 'Pack: Primary lateral in combo additional' FROM class_to_drug_new a JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id @@ -1345,21 +1350,22 @@ WHERE r2.invalid_reason IS NULL AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack','Branded Pack Box') AND j.standard_concept = 'S' AND a.class_code IN (SELECT class_code - FROM dev_combo + FROM ing_pr_lat_combo) + /*dev_combo WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) + AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) */ AND j.concept_name ~ ' / ' -- combos only AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 262 --- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 19) +-- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 16) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 19 as concept_order, + 16 as concept_order, 'Pack: Primary lateral + Secondary lateral' FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id @@ -1374,49 +1380,16 @@ AND r2.invalid_reason IS NULL AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') AND j.standard_concept = 'S' AND a.class_code IN (SELECT class_code - FROM dev_combo - WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) - AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) + FROM ing_pr_lat_sec_lat) AND j.concept_name ~ ' / ' -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 3649 +AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 3830 -/*-- add packs of Primary upward and Secondary upward (Class C AND Class D, order = 20) +-- add packs of Primary lateral and Secondary upward (Class A + Class D, order = 17) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 20 as concept_order, - 'Pack: Primary upward + Secondary upward' -FROM class_to_drug_new a - JOIN concept_relationship r ON r.concept_id_1 = a.concept_id - JOIN concept d - ON d.concept_Id = r.concept_id_2 - AND d.concept_class_id = 'Clinical Drug' - JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id - JOIN concept j ON j.concept_id = r2.concept_id_2 -WHERE a.concept_class_id = 'Clinical Drug Form' -AND r.invalid_reason IS NULL -AND r2.invalid_reason IS NULL -AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') -AND j.standard_concept = 'S' -AND a.class_code IN (SELECT class_code - FROM dev_combo - WHERE class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) - AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) - AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) -AND j.concept_name ~ ' / '; -- 53 */ -- to exclude, will be deleted later - --- add packs of Primary lateral and Secondary upward (Class A + Class D, order = 20) -INSERT INTO class_to_drug_new -SELECT DISTINCT a.class_code, - a.class_name, - j.*, - 20 as concept_order, + 17 as concept_order, 'Pack: Primary lateral + Secondary upward' FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id @@ -1431,23 +1404,18 @@ AND r2.invalid_reason IS NULL AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') AND j.standard_concept = 'S' AND a.class_code IN (SELECT class_code - FROM dev_combo - WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) - AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) + FROM ing_pr_lat_sec_up) AND j.concept_name ~ ' / ' AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 218 ---- add missing Packs using previous version of class_to_drug (order = 21) +--- add missing Packs using previous version of class_to_drug (order = 18) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, c.*, - 21 as concept_order, + 18 as concept_order, 'Additional Pack: from old c_t_d' -FROM sources.class_to_drug a +FROM sources_class_to_drug_160921 a JOIN concept_manual b ON b.concept_code = a.class_code JOIN concept c ON a.concept_id = c.concept_id @@ -1457,15 +1425,15 @@ WHERE (class_code,a.concept_id) NOT IN (SELECT class_code, concept_id FROM class AND a.concept_class_id ~ 'Pack' AND a.class_code NOT IN ('G01AF55','S03CA04','S01CA03','S02CA03')-- gives packs with erroneous forms AND a.class_code NOT IN ('B03AE01','C07BB52','D01AC52','C10AD52') -- wrong ing combo -; -- 281 +; -- 289 -- еnrich pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' --- they are always used as packs (order = 22) +-- they are always used as packs (order = 19) INSERT INTO class_to_drug_new SELECT DISTINCT class_code, class_name, c.*, - 22 as concept_order, + 19 as concept_order, 'Additional Pack: semi-manual contraceptive' FROM class_to_drug_new f JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) @@ -1478,10 +1446,10 @@ AND (f.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM cla -- get rid of all other concept_class_ids except Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' DELETE FROM class_to_drug_new -WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens -AND concept_class_id !~ 'Pack'; -- 81 +WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens +AND concept_class_id !~ 'Pack'; -- 81 --- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 23) +-- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 20) INSERT INTO class_to_drug_new WITH ing AS ( SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt @@ -1491,14 +1459,14 @@ FROM relationship_to_concept r JOIN internal_relationship_stage i on i.concept_code_2 = r.concept_code_1 JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' ) a GROUP BY concept_id_2,concept_code_1 ), - drug AS ( +drug AS ( SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 FROM internal_relationship_stage i WHERE not exists (SELECT 1 FROM internal_relationship_stage i2 WHERE i.concept_code_2=i2.concept_code_2 AND substring(i.concept_code_1,'\w+')!=substring(i2.concept_code_1,'\w+')) ), - drug_name AS ( +drug_name AS ( SELECT DISTINCT class_code,class_name, concept_id_2 FROM ing JOIN drug ON concept_code_2 = concept_code_1 @@ -1506,20 +1474,22 @@ JOIN class_drugs_scraper ON code = class_code WHERE cnt=1 AND class_name = concept_code_1 AND code NOT IN (SELECT class_code FROM class_to_drug_new) ), - all_drug AS ( +all_drug AS ( SELECT class_code, class_name, concept_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt FROM drug_name a JOIN concept c on c.concept_id = a.concept_id_2 ) SELECT DISTINCT a.class_code, - a.class_name, + b.concept_name, c.*, - 23 as concept_order, - 'ATC Class WO Dose Form to unique Drug Product' + 20 as concept_order, + 'ATC Mono WO Dose Form to Ingredient' FROM all_drug a +join concept_manual b on b.concept_code = a.class_code and b.invalid_reason is null join concept c on c.concept_id = a.concept_id WHERE cnt = 1 -AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); -- 5 +AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); -- 5 */ -- to check + -- add those which are absent in the drug hierarchy -- step 1 @@ -1563,12 +1533,12 @@ FROM no_atc_2 a ON g.concept_id = r.concept_id_2 AND g.concept_class_id = 'Dose Form'); --- add additional mappings for hierarchical absentees (order = 24) +-- add additional mappings for hierarchical absentees (order = 21) INSERT INTO class_to_drug_new SELECT DISTINCT k.concept_code AS class_code, k.concept_name AS class_name, p.*, - 24 as concept_order, + 21 as concept_order, 'ATC Monocomp Class to Drug Product which is out of hierarchy' FROM no_atc_1_with_form a JOIN internal_relationship_stage i ON lower (i.concept_code_2) = lower (a.ing_nm) @@ -1617,18 +1587,28 @@ SELECT DISTINCT concept_id, CREATE INDEX i_no_atc_full_combo ON no_atc_full_combo (concept_id, i_combo,df_id); --- add mappings of ATC Combo Class to Drug Product which is out of drug hierarchy (order = 25) +-- add mappings of ATC Combo Class to Drug Product which is out of drug hierarchy (order = 22) INSERT INTO class_to_drug_new SELECT DISTINCT k.concept_code as class_code, -- ATC k.concept_name as class_name, d.*, - 25 AS conept_order, + 22 AS conept_order, 'ATC Combo Class to Drug Product which is out of drug hierarchy' FROM no_atc_full_combo f JOIN full_combo_with_form c on c.i_combo = f.i_combo and c.df_id = f.df_id is null join concept_manual k on k.concept_code = c.class_code and k.concept_class_id = 'ATC 5th' and k.invalid_reason is null JOIN concept d on d.concept_id = f.concept_id and c.class_code||d.concept_id not in (select class_code||concept_id from class_to_drug_new); -- 6188 + +-- ATC Combo Class to Drug Product which is out of drug hierarchy (order = 23) +INSERT INTO class_to_drug_new + SELECT DISTINCT +atc_code,atc_name,c.*, + 23 AS conept_order, + 'ATC Combo Class to Drug Product which is out of drug hierarchy' + FROM MISSING a + join concept c using (concept_id) + where atc_code||concept_id not in (select class_code||concept_id from class_to_drug_new); -- 2890 -- remove excessive links to children among Packs WITH t1 AS @@ -1647,7 +1627,7 @@ ON a1.class_code = a2.class_code AND a2.concept_id = ca.descendant_concept_id AND a1.concept_id <> a2.concept_id and a2.concept_class_id ~ 'Pack' - AND a1.class_code not in ('B02BD14', 'G03FB01', 'J07BK01') )-- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic); -- 2558 + AND a1.class_code not in ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02' ))-- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic); -- 2558 ; -- 3920 -- remove excessive links to children among unpacked Drug Products @@ -1674,8 +1654,66 @@ ON a1.class_code = a2.class_code AND a2.concept_id = ca.descendant_concept_id AND a1.concept_id <> a2.concept_id and a2.concept_class_id !~ 'Pack' - AND a1.class_code NOT IN ('B02BD11', 'B02BD14', 'G03FB01', 'J07BK01', 'J07BK02')) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic + AND a1.class_code NOT IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02')) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic ;-- 10432 + +-- clean up the same issue among exclusions +WITH t0 AS +( + SELECT * + FROM class_to_drug_new + WHERE concept_class_id !~ 'Pack' + AND class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') +), +t1 as ( +select * from t0 where class_code IN (SELECT class_code + FROM class_to_drug_new + GROUP BY class_code + HAVING COUNT(1) >= 2)) +DELETE +FROM class_to_drug_new where class_code||concept_id IN ( +SELECT a1.class_code|| + a1.concept_id -- papa mapping +FROM t1 a1-- papa + JOIN t1 a2 -- child +ON a1.class_code = a2.class_code + JOIN concept_ancestor ca + ON a1.concept_id = ca.ancestor_concept_id + AND a2.concept_id = ca.descendant_concept_id + AND a1.concept_id <> a2.concept_id + and a2.concept_class_id !~ 'Pack' + AND a1.class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') + ) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic + ; + +-- clean up the same issue among exclusions - Packs +WITH t0 AS +( + SELECT * + FROM class_to_drug_new + WHERE concept_class_id ~ 'Pack' + AND class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') +), +t1 as ( +select * from t0 where class_code IN (SELECT class_code + FROM class_to_drug_new + GROUP BY class_code + HAVING COUNT(1) >= 2)) +DELETE +FROM class_to_drug_new where class_code||concept_id IN ( +SELECT a1.class_code|| + a1.concept_id -- papa mapping +FROM t1 a1-- papa + JOIN t1 a2 -- child +ON a1.class_code = a2.class_code + JOIN concept_ancestor ca + ON a1.concept_id = ca.ancestor_concept_id + AND a2.concept_id = ca.descendant_concept_id + AND a1.concept_id <> a2.concept_id + and a2.concept_class_id ~ 'Pack' + AND a1.class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') + ) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic + ;-- 19 /*************************** ******** DF CLEAN UP ******* ****************************/ @@ -1685,7 +1723,7 @@ CREATE TABLE wrong_df as ( SELECT *, 'oral mismatch' as issue_desc FROM class_to_drug_new WHERE SPLIT_PART(class_name,';',2) ~ 'oral' -AND concept_name !~* 'oral|chew|tooth|mouth|elixir|Extended Release Suspension|buccal|Sublingual|Paste|Mucosal|Prefilled Syringe|\...' +AND concept_name !~* 'oral|chew|tooth|mouth|elixir|Extended Release Suspension|buccal|Sublingual|Paste|Mucosal|Prefilled Syringe|\...|Oral Ointment|Oral Cream' AND class_name !~ 'rectal|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|nasal' AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' AND class_code NOT IN ('A01AA04','A01AA02','G04BD08') @@ -1695,7 +1733,7 @@ INSERT INTO wrong_df select *, 'vaginal mismatch' from class_to_drug_new where class_name ~ 'vaginal' and concept_name !~* 'vaginal|topical|mucosal|Drug Implant|Douche|Irrigation Solution' and class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' -and concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; -- 0 +and concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; -- 1 -- clean up rectal forms INSERT INTO wrong_df @@ -1711,7 +1749,7 @@ INSERT INTO wrong_df SELECT *, 'topical mismatch' FROM class_to_drug_new WHERE class_name ~ 'topical' -AND concept_name !~* 'topical|mucosal|Drug Implant|Prefilled Applicator|Shampoo|Paste|Medicated Pad|Transdermal System|Soap|Powder Spray|Medicated Patch|Douche|vaginal|\yNail\y|Intrauterine System|Mouthwash' +AND concept_name !~* 'topical|mucosal|Drug Implant|Prefilled Applicator|Shampoo|Paste|Medicated Pad|Transdermal System|Soap|Powder Spray|Medicated Patch|Douche|vaginal|\yNail\y|Intrauterine System|Mouthwash|Oil|Intraperitoneal Solution|Urethral Gel' AND concept_name !~* '\yStick|Rectal Foam|Medicated Tape|Medicated Guaze|Paint|Rectal|Nasal|Otic|Ophthalmic Solution|Dry Powder|Urethral Suppository|Intrauterine|irrigation|Cement|Cream|ointment|spray|gum|enema|Ophthalmic' AND class_name !~* 'oral|vaginal|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; @@ -1729,7 +1767,7 @@ INSERT INTO wrong_df SELECT *, 'inhalant mismatch' FROM class_to_drug_new WHERE class_name ~* 'inhalant' -AND concept_name !~* 'inhal|nasal|Powder Spray' +AND concept_name !~* 'inhal|nasal|Powder Spray|Oral Spray| Oral Powder|Prefilled Applicator' AND class_name !~ 'oral|local oral|vaginal|parenteral|transdermal|ophthalmic|otic|rectal|systemic|topical' AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; @@ -1738,9 +1776,10 @@ INSERT INTO wrong_df SELECT *, 'parenteral mismatch' FROM class_to_drug_new WHERE class_name ~* 'parenteral' -AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant' +AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant|transdermal' AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|ophthalmic|otic|nasal|rectal|systemic|urethral' AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + -- clean up otic|ophthalmic INSERT INTO wrong_df SELECT *, 'otic|ophthalmic mismatch' @@ -1759,7 +1798,7 @@ AND concept_name ~* 'nasal' AND concept_name !~ 'metered' AND class_name !~* 'rectal|nasal|vaginal|topical' AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' -AND concept_name ~* 'caine|azoline|thrombin|sodium chloride|glycerol|acetylcysteine|chlorhexidine|amlexanox|ammonia|Chlorobutanol|phenylephrine|dexamethasone|Ephedrine|Histamine|Hydrocortisone|Peppermint oil|pyrilamine|dexpanthenol|Phentolamine|Sodium Chloride|Cellulose|glycerin|pantenol|tetrahydrozoline|triamcinolone' +AND concept_name ~* 'caine|azoline|thrombin|sodium chloride|glycerol|acetylcysteine|chlorhexidine|amlexanox|ammonia|Chlorobutanol|phenylephrine|Peppermint oil|pyrilamine|dexpanthenol|Phentolamine|Sodium Chloride|Cellulose|glycerin|pantenol|tetrahydrozoline|triamcinolone|dexamethasone|Ephedrine|Histamine|Hydrocortisone' -- ; -- clean up systemic forms 2 @@ -1798,7 +1837,8 @@ WHERE class_name ~* 'systemic' AND concept_name !~* '\...$|\...\) \} Pack$|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' AND class_name !~* 'rectal|nasal|vaginal|topical' AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' -AND concept_class_id ~ 'Pack'; +AND concept_class_id ~ 'Pack' +and class_code not in ('R03AC130'); -- formoterol Powder for Oral Suspension| -- clean up systemic packs 2 - check This next time INSERT INTO wrong_df @@ -1808,40 +1848,24 @@ WHERE class_name ~* 'systemic' AND concept_name !~* '\.\.\.$|\.\.\.\) \} Pack|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' AND class_name !~* 'rectal|nasal|vaginal|topical' AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' -AND concept_class_id ~ 'Pack'; +AND concept_class_id ~ 'Pack' +AND concept_code <> 'C02AC01'; -- clonidine Medicated Patch + +-- wrong vaginal rings +INSERT INTO wrong_df +SELECT * +FROM class_to_drug_new +WHERE class_name ~ 'vaginal ring' +AND concept_name !~* 'ring|insert|system|implant'; + +DELETE FROM wrong_df where class_code in ( 'C02AC01', 'G03GA08', 'R03AC13'); -- look at them and remove from class_to_drug_new DELETE FROM class_to_drug_new WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM wrong_df); -- 891 - --- add links from ATC Monocomponent Classes which do not have Dose Forms to Ingredients (in the future this concepts could be processed manually, some of them have mappings) --- (order = 26) -INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT * -FROM concept_manual -WHERE concept_code NOT IN (SELECT class_code FROM class_to_drug_new) -AND concept_class_Id = 'ATC 5th' -AND invalid_reason IS NULL -AND concept_code NOT IN (SELECT class_code FROM atc_inexistent)) -SELECT DISTINCT a.concept_code, - a.concept_name, - c.*, - 26 as concept_order, - 'ATC Monocomp Class WO Dose Form to Ingredient' -FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (concept_code_1,'\w+') = a.concept_code - JOIN relationship_to_concept r ON r.concept_code_1 = i.concept_code_2 - JOIN concept c - ON c.concept_id = r.concept_id_2 - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND a.concept_code NOT IN (SELECT class_code FROM dev_combo) - and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new); -- 78 --- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (do we need them?) +-- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (do we need them?) order = 24 INSERT INTO class_to_drug_new WITH t1 AS @@ -1856,7 +1880,7 @@ AND concept_code NOT IN (SELECT class_code FROM atc_inexistent) SELECT DISTINCT a.concept_code, a.concept_name, c.*, - 27 as concept_order, + 24 as concept_order, 'ATC Combo Class to Ingredient' FROM t1 a JOIN dev_combo k @@ -1867,6 +1891,22 @@ FROM t1 a AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new);--314 + +-- wrong match +DELETE +FROM class_to_drug_new +WHERE class_code = 'J01CR04' +AND concept_name ~* 'sulbactam'; + +-- remove wrong mapping of glafenine; oral (from old class_to_drug) +DELETE +FROM class_to_drug_new +WHERE class_code = 'N02BG03'; + +-- Remove Drug Components if any +DELETE +FROM class_to_drug_new +WHERE concept_class_id ~ '\yComp'; -- remove suspicious mapping of inexistent drugs (this table should be checked before) DELETE @@ -1887,15 +1927,57 @@ DELETE FROM class_to_drug_new WHERE class_code IN (SELECT concept_code FROM concept_manual - WHERE invalid_reason IS NOT NULL); -- 0 + WHERE invalid_reason IS NOT NULL); + +-- remove non-standard mappings if any +DELETE +FROM class_to_drug_new a +WHERE NOT EXISTS (SELECT 1 + FROM devv5.concept c + WHERE c.standard_concept = 'S' + AND c.concept_id = a.concept_id); -- remove duplicates if any (however they should not be there - can be solved in the future) ---SELECT * DELETE FROM class_to_drug_new WHERE CTID NOT IN (SELECT MAX(CTID) FROM class_to_drug_new GROUP BY class_code, - concept_id) - ;-- 345 + concept_id); + +-- assemble the final of class_to_drug table in the following Concept Order: +/*1. Manual +2. Mono: Ingredient A; form +3. Mono: Ingredient A +4. Combo: Ingredient A + Ingredient B +5. Combo: Ingredient A + group B +6. Combo: Ingredient A, combination; form +7. Combo: Ingredient A, combination +8. Any packs*/ + +TRUNCATE TABLE class_to_drug; +INSERT INTO class_to_drug +SELECT DISTINCT class_code, + class_name, + concept_id, + concept_name, + concept_class_id, + CASE + WHEN class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_order = 11 AND concept_name ~ ' / ' THEN 7 + WHEN concept_class_id ~ 'Pack' THEN 8-- 14, 15, 16, 17, 18, + WHEN concept_order IN (11,13) THEN 1-- ATC Class to Drug Product from concept_relationship_manual + WHEN concept_order IN (1,12) THEN 2-- ATC Monocomp Class + WHEN concept_order = 20 THEN 3-- Mono: Ingredient A + WHEN concept_order IN (5,6,7) THEN 4-- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingreds) + WHEN concept_order IN (9,10,50,60,70) THEN 5-- Combo: Ingredient A OR Group A + group B + WHEN concept_order IN (3,4,8) THEN 6-- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient + WHEN concept_order IN (2,21,22,23,24) THEN 7-- 7. Combo: Ingredient A, combination -- 24 - to Ingredients (skipped) + END AS concept_order +FROM class_to_drug_new +; -- 12 + +DELETE +FROM class_to_drug +WHERE concept_class_id ~ 'Comp'; + -- run load_stage.sql From 9d99919815764e97ed2e293070b6daad819efbc8 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 16:57:34 +0300 Subject: [PATCH 24/45] refactoring --- ATC/load_stage.sql | 135 ++++++++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 69 deletions(-) diff --git a/ATC/load_stage.sql b/ATC/load_stage.sql index 3ee2ab7ae..6e39159b5 100644 --- a/ATC/load_stage.sql +++ b/ATC/load_stage.sql @@ -14,7 +14,8 @@ * limitations under the License. * * Authors: Anna Ostropolets, Polina Talapova, Timur Vakhitov -* Date: Jul 2021 +* Date: Oct 2021 + **************************************************************************/ DO $_$ BEGIN @@ -57,7 +58,7 @@ INSERT INTO concept_relationship_stage ( concept_code_2, vocabulary_id_1, vocabulary_id_2, - relationship_id, + relationship_id, valid_start_date, valid_end_date, invalid_reason @@ -68,17 +69,17 @@ SELECT DISTINCT d.concept_code AS concept_code_1, 'SNOMED' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, 'SNOMED - ATC eq' AS relationship_id, - d.valid_start_date, + d.valid_start_date AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason FROM concept d JOIN sources.rxnconso r ON r.code = d.concept_code AND r.sab = 'SNOMEDCT_US' AND r.code != 'NOCODE' -JOIN sources.rxnconso r2 ON r.rxcui = r2.rxcui +JOIN sources.rxnconso r2 ON r2.rxcui = r.rxcui AND r2.sab = 'ATC' AND r2.code != 'NOCODE' -JOIN concept_manual e ON r2.code = e.concept_code +JOIN concept_manual e ON e.concept_code = r2.code AND e.concept_class_id != 'ATC 5th' -- Ingredients only to RxNorm AND e.vocabulary_id = 'ACT' WHERE d.vocabulary_id = 'SNOMED' @@ -96,14 +97,15 @@ SELECT uppr.concept_code AS concept_code_1, FROM concept_stage uppr, concept_stage lowr, vocabulary v -WHERE uppr.invalid_reason is null AND lowr.invalid_reason is null -- to exclude deprecated or updated from the hierarchy +WHERE uppr.invalid_reason IS null +AND lowr.invalid_reason IS null -- to exclude deprecated or updated from the hierarchy AND ( -(LENGTH(uppr.concept_code) IN (4,5) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 1)) -OR (LENGTH(uppr.concept_code) IN (3,7) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 2)) +(LENGTH(uppr.concept_code) IN (4, 5) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 1)) +OR (LENGTH(uppr.concept_code) IN (3, 7) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 2)) ) AND uppr.vocabulary_id = 'ATC' AND lowr.vocabulary_id = 'ATC' - AND v.vocabulary_id = 'ATC'; --6493 + AND v.vocabulary_id = 'ATC'; -- 6493 -- add 'ATC - RxNorm' relationships between ATC Classes and RxN/RxE Drug Products using class_to_drug table INSERT INTO concept_relationship_stage @@ -193,25 +195,27 @@ INSERT INTO concept_relationship_stage valid_end_date, invalid_reason ) -with t1 as ( -select distinct class_code, class_name, concept_id, rnk from dev_combo where rnk in (1,2,3)), -t2 as ( +with t1 AS ( +SELECT DISTINCT class_code, class_name, concept_id, rnk + FROM dev_combo + WHERE rnk IN (1, 2, 3)), +t2 AS ( SELECT a.class_code AS concept_code_1, c.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, c.vocabulary_id AS vocabulary_id_2, -case rnk when 1 then 'ATC - RxNorm pr lat' -when 2 then 'ATC - RxNorm sec lat' -when 3 then 'ATC - RxNorm pr up' ---when 4 then 'ATC - RxNorm sec up' -end AS relationship_id, + CASE rnk WHEN 1 THEN 'ATC - RxNorm pr lat' + WHEN 2 THEN 'ATC - RxNorm sec lat' + WHEN 3 THEN 'ATC - RxNorm pr up' + END AS relationship_id, CURRENT_DATE AS valid_start_date, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason FROM t1 a -JOIN concept c on c.concept_id = a.concept_id -JOIN concept_manual k ON k.concept_code = a.class_code AND k.invalid_reason IS NULL -AND c.standard_concept = 'S') +JOIN concept c ON c.concept_id = a.concept_id +JOIN concept_manual k ON k.concept_code = a.class_code + AND k.invalid_reason IS NULL + AND c.standard_concept = 'S') SELECT DISTINCT * FROM t2 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs @@ -255,7 +259,7 @@ UNION ALL SELECT class_code, class_name, concept_id FROM ing_pr_up_sec_up_excl WHERE rnk = 3 ), -t2 as ( +t2 AS ( SELECT DISTINCT c.class_code AS concept_code_1, cc.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, @@ -265,9 +269,9 @@ SELECT DISTINCT c.class_code AS concept_code_1, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason FROM class_to_drug c - JOIN concept_ancestor ON descendant_concept_id = c.concept_id + JOIN concept_ancestor ca ON ca.descendant_concept_id = c.concept_id JOIN concept cc - ON cc.concept_id = ancestor_concept_id + ON cc.concept_id = ca.ancestor_concept_id AND cc.standard_concept = 'S' AND cc.concept_class_id = 'Ingredient' AND cc.vocabulary_id IN ('RxNorm', 'RxNorm Extension') @@ -299,7 +303,7 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) - with t1 as ( + with t1 AS ( SELECT c.concept_code AS concept_code_1, cc.concept_code AS concept_code_2, c.vocabulary_id AS vocabulary_id_1, @@ -323,9 +327,9 @@ WHERE NOT EXISTS (SELECT 1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 AND crs.relationship_id = cs.relationship_id - and crs.invalid_reason = cs.invalid_reason); -- 8365 + AND crs.invalid_reason = cs.invalid_reason); -- 8365 --- Deprecate accessory links for invalid codes +-- deprecate accessory links for invalid codes INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -350,11 +354,10 @@ SELECT FROM concept_relationship cr JOIN concept c ON concept_id_1 = c.concept_id JOIN concept cc ON concept_id_2 = cc.concept_id - JOIN Concept_manual k on k.concept_code = c.concept_code - and k.invalid_reason is not null ---AND cc.standard_concept is null -- non-standard + JOIN concept_manual k ON k.concept_code = c.concept_code + AND k.invalid_reason IS NOT null WHERE c.vocabulary_id = 'ATC' -AND cr.invalid_reason IS NULL) +AND cr.invalid_reason IS NULL) SELECT * FROM t1 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs @@ -385,11 +388,11 @@ WITH t1 AS AND invalid_reason IS NULL ), t2 AS ( -SELECT DISTINCT a.concept_code_1, --k.concept_name, - a.concept_code_2,-- d.concept_name, +SELECT DISTINCT a.concept_code_1, + a.concept_code_2, a.vocabulary_id_1, a.vocabulary_id_2, - 'Maps to' as relationship_id, + 'Maps to' AS relationship_id, a.valid_start_date, a.valid_end_date, a.invalid_reason @@ -412,7 +415,7 @@ WHERE NOT EXISTS (SELECT 1 AND crs.vocabulary_id_1 = 'ATC' AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id -- and crs.invalid_reason is null + AND crs.relationship_id = cs.relationship_id ); -- 4374 -- add mirroring 'Maps to' of 'ATC - RxNorm sec lat' relationships for ATC Combo Classes (1-to-1) @@ -438,14 +441,12 @@ WITH T1 AS JOIN concept c ON c.concept_id = a.concept_id WHERE rnk = 2 ), -t2 as ( -SELECT DISTINCT class_code as concept_code_1, - -- class_name, - concept_code as concept_code_2, - -- concept_name, - 'ATC' as vocabulary_id_1, - vocabulary_id as vocabulary_id_2, - 'Maps to' as relationship_id, +t2 AS ( +SELECT DISTINCT class_code AS concept_code_1, + concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + vocabulary_id AS vocabulary_id_2, + 'Maps to' AS relationship_id, CURRENT_DATE AS valid_start_date, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason @@ -462,7 +463,7 @@ WHERE NOT EXISTS (SELECT 1 AND crs.vocabulary_id_1 = 'ATC' AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = 'Maps to' -- and crs.invalid_reason is null + AND crs.relationship_id = 'Maps to' ); -- 209 INSERT INTO concept_relationship_stage ( @@ -487,14 +488,12 @@ WITH T1 AS JOIN concept c ON c.concept_id = a.concept_id WHERE rnk = 2 ), -t2 as ( -SELECT DISTINCT class_code as concept_code_1, - -- class_name, - concept_code as concept_code_2, - -- concept_name, - 'ATC' as vocabulary_id_1, - vocabulary_id as vocabulary_id_2, - 'Maps to' as relationship_id, +t2 AS ( +SELECT DISTINCT class_code AS concept_code_1, + concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + vocabulary_id AS vocabulary_id_2, + 'Maps to' AS relationship_id, CURRENT_DATE AS valid_start_date, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason @@ -503,7 +502,9 @@ WHERE class_code IN (SELECT class_code FROM t1 GROUP BY class_code HAVING COUNT(1) <= 3) -- Combo Classes with COUNT(1) > 3 were added from concept_relationship_manual -and not exists (select 1 from atc_one_to_many_excl b where b.atc_code = a.class_code and a.concept_id = b.concept_id) +AND NOT EXISTS (SELECT 1 FROM atc_one_to_many_excl b + WHERE b.atc_code = a.class_code + AND a.concept_id = b.concept_id) ) SELECT * FROM t2 cs WHERE NOT EXISTS (SELECT 1 @@ -512,12 +513,12 @@ WHERE NOT EXISTS (SELECT 1 AND crs.vocabulary_id_1 = 'ATC' AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = 'Maps to' -- and crs.invalid_reason is null + AND crs.relationship_id = 'Maps to' ) -AND concept_code_1 not in ('P01BF05', 'J07AG52', 'J07BD51') -- artenimol and piperaquine|hemophilus influenzae B, combinations with pertussis and toxoids; systemic|measles, combinations with mumps, live attenuated; systemic +AND concept_code_1 NOT IN ('P01BF05', 'J07AG52', 'J07BD51') -- artenimol and piperaquine|hemophilus influenzae B, combinations with pertussis and toxoids; systemic|measles, combinations with mumps, live attenuated; systemic ; -- 193 --- Add synonyms to concept_synonym stage for each of the rxcui/code combinations +-- add synonyms to concept_synonym stage for each of the rxcui/code combinations INSERT INTO concept_synonym_stage ( synonym_concept_code, synonym_name, @@ -541,25 +542,25 @@ BEGIN PERFORM VOCABULARY_PACK.CheckReplacementMappings(); END $_$; --- Add mapping FROM deprecated to fresh concepts +-- add mappings from deprecated to fresh codes DO $_$ BEGIN PERFORM VOCABULARY_PACK.AddFreshMAPSTO(); END $_$; --- Deprecate 'Maps to' mappings to deprecated AND upgraded concepts +-- deprecate 'Maps to' mappings to deprecated AND updated codes DO $_$ BEGIN PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO(); END $_$; --- DELETE ambiguous 'Maps to' mappings +-- remove ambiguous 'Maps to' mappings DO $_$ BEGIN PERFORM VOCABULARY_PACK.DELETEAmbiguousMAPSTO(); END $_$; ---16. Build reverse relationship. This is necessary for next point +-- build reverse relationships in order to take the next sept INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -591,7 +592,7 @@ WHERE NOT EXISTS ( AND r.reverse_relationship_id = i.relationship_id ); ---17. Deprecate all relationships in concept_relationship that do not exist in concept_relationship_stage +-- deprecate all relationships from the concept_relationship table which do not exist in the concept_relationship_stage INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -603,8 +604,8 @@ INSERT INTO concept_relationship_stage ( invalid_reason ) SELECT DISTINCT - a.concept_code, --a.concept_name, - b.concept_code, --b.concept_name, + a.concept_code, + b.concept_code, a.vocabulary_id, b.vocabulary_id, relationship_id, @@ -640,15 +641,11 @@ WHERE concept_code_1 IN ('C10AA55','J05AE06','C10AA52','C10AA53','C10AA51','N02A AND concept_code_2 IN ('17767','85762','7393','1191','161', '11149') AND invalid_reason IS NULL; -- 6 ---delete duplicate (to do: define its origin) +-- delete duplicate (to do: define its origin) DELETE FROM concept_relationship_stage WHERE ctid NOT IN (SELECT MIN(ctid) FROM concept_relationship_stage - - GROUP BY concept_code_1, - concept_code_2, - relationship_id, - invalid_reason - ) +GROUP BY concept_code_1, concept_code_2, relationship_id, invalid_reason + ) and concept_code_1 = 'H01BA06';-- 1 From b2a33f8ac337f6e03639f62761665832239a3c1a Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:34:38 +0300 Subject: [PATCH 25/45] refactoring --- ATC/load_interim.sql | 54 ++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 780aac239..4fb9eb97a 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -1279,7 +1279,7 @@ SELECT DISTINCT b.concept_code AS class_code, c.*, 13 as concept_order, 'ATC Class from old class_to_drug' -FROM sources_class_to_drug_160921 a +FROM sources_class_to_drug_old a JOIN concept_manual b ON b.concept_code = a.class_code JOIN concept c ON a.concept_id = c.concept_id @@ -1436,8 +1436,8 @@ SELECT DISTINCT class_code, 19 as concept_order, 'Additional Pack: semi-manual contraceptive' FROM class_to_drug_new f - JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) - JOIN devv5.concept c + JOIN concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) + JOIN concept c ON c.concept_id = descendant_concept_id AND c.concept_class_id LIKE '%Pack%' WHERE f.class_code ~ 'G03FB|G03AB' @@ -1910,17 +1910,17 @@ WHERE concept_class_id ~ '\yComp'; -- remove suspicious mapping of inexistent drugs (this table should be checked before) DELETE ---select * FROM class_to_drug_new -WHERE class_code IN (SELECT class_code FROM atc_inexistent); -- 16 +WHERE class_code IN (SELECT class_code FROM atc_inexistent); -- 14 -- remove wrong mappings DELETE ---select * FROM class_to_drug_new -WHERE class_code||concept_code IN (SELECT concept_code_1||concept_code_2 - FROM concept_relationship_manual - WHERE invalid_reason IS NOT NULL);-- 3 + WHERE class_code||concept_code IN ( + SELECT concept_code_1||concept_code_2 + FROM concept_relationship_manual + WHERE invalid_reason IS NOT NULL + AND relationship_id = 'ATC - RxNorm'); -- 0 -- remove dead ATC Classes if any DELETE @@ -1945,7 +1945,7 @@ WHERE CTID NOT IN (SELECT MAX(CTID) GROUP BY class_code, concept_id); --- assemble the final of class_to_drug table in the following Concept Order: +-- usgin the concep_order field from the class_to_drgu_new table, assemble the final table of class_to_drug in the following Concept Order: /*1. Manual 2. Mono: Ingredient A; form 3. Mono: Ingredient A @@ -1963,21 +1963,27 @@ SELECT DISTINCT class_code, concept_name, concept_class_id, CASE + WHEN class_code||concept_code IN (SELECT concept_code_1||concept_code_2 + FROM concept_relationship_manual WHERE relationship_id = 'ATC - RxNorm' AND invalid_reason IS NULL) + AND class_code IN (SELECT class_code FROM combo_pull) and concept_class_id !~ 'Pack' + AND concept_order <> 11 THEN 1 -- if ATC Combo has an entry in crm, its higher concept_order values have to be converted to 1 as well + WHEN class_code||concept_code IN (SELECT concept_code_1||concept_code_2 + FROM concept_relationship_manual WHERE relationship_id = 'ATC - RxNorm' AND invalid_reason IS NULL) + AND class_code NOT IN (SELECT class_code FROM combo_pull) and concept_class_id !~ 'Pack' and concept_name !~ ' \/ ' + AND concept_order <> 11 THEN 1 -- if ATC Mono has an entry in crm, its higher concept_order values have to be converted to 1 as well WHEN class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_order = 11 AND concept_name ~ ' / ' THEN 7 - WHEN concept_class_id ~ 'Pack' THEN 8-- 14, 15, 16, 17, 18, - WHEN concept_order IN (11,13) THEN 1-- ATC Class to Drug Product from concept_relationship_manual - WHEN concept_order IN (1,12) THEN 2-- ATC Monocomp Class - WHEN concept_order = 20 THEN 3-- Mono: Ingredient A - WHEN concept_order IN (5,6,7) THEN 4-- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingreds) - WHEN concept_order IN (9,10,50,60,70) THEN 5-- Combo: Ingredient A OR Group A + group B - WHEN concept_order IN (3,4,8) THEN 6-- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient - WHEN concept_order IN (2,21,22,23,24) THEN 7-- 7. Combo: Ingredient A, combination -- 24 - to Ingredients (skipped) + WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18, + WHEN concept_order IN (11,13) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual + WHEN concept_order IN (1,12) THEN 2 -- ATC Monocomp Class + WHEN concept_order = 20 THEN 3 -- Mono: Ingredient A + WHEN concept_order IN (5,6,7) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingreds) + WHEN concept_order IN (9,10,50,60,70) THEN 5 -- Combo: Ingredient A OR Group A + group B + WHEN concept_order IN (3,4,8) THEN 6 -- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient + WHEN concept_order IN (2,21,22,23,24) THEN 7 -- 7. Combo: Ingredient A, combination -- 24 - to Ingredients (skipped) END AS concept_order FROM class_to_drug_new -; -- 12 - -DELETE -FROM class_to_drug -WHERE concept_class_id ~ 'Comp'; +; -- 116679 --- run load_stage.sql +/*-- update class_to_drug in the schema of 'sources' +SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); +-- run load_stage.sql */ From b9a977f446bf618fe273b6123743fc6ed0ecb06c Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:38:35 +0300 Subject: [PATCH 26/45] Update load_interim.sql --- ATC/load_interim.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 4fb9eb97a..8ed09487a 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -1972,14 +1972,14 @@ SELECT DISTINCT class_code, AND class_code NOT IN (SELECT class_code FROM combo_pull) and concept_class_id !~ 'Pack' and concept_name !~ ' \/ ' AND concept_order <> 11 THEN 1 -- if ATC Mono has an entry in crm, its higher concept_order values have to be converted to 1 as well WHEN class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_order = 11 AND concept_name ~ ' / ' THEN 7 - WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18, - WHEN concept_order IN (11,13) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual - WHEN concept_order IN (1,12) THEN 2 -- ATC Monocomp Class + WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18 (note, that 19 doesn't exist) + WHEN concept_order IN (11, 13) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual + WHEN concept_order IN (1, 12) THEN 2 -- ATC Monocomp Class WHEN concept_order = 20 THEN 3 -- Mono: Ingredient A - WHEN concept_order IN (5,6,7) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingreds) - WHEN concept_order IN (9,10,50,60,70) THEN 5 -- Combo: Ingredient A OR Group A + group B - WHEN concept_order IN (3,4,8) THEN 6 -- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient - WHEN concept_order IN (2,21,22,23,24) THEN 7 -- 7. Combo: Ingredient A, combination -- 24 - to Ingredients (skipped) + WHEN concept_order IN (5, 6, 7) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingreds) + WHEN concept_order IN (9, 10, 50, 60, 70) THEN 5 -- Combo: Ingredient A OR Group A + group B + WHEN concept_order IN (3, 4, 8) THEN 6 -- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient + WHEN concept_order IN (2, 21, 22, 23, 24) THEN 7 -- 7. Combo: Ingredient A, combination END AS concept_order FROM class_to_drug_new ; -- 116679 From f61c6412c19f7fad62e450af6de4ba2c388c0417 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:45:05 +0300 Subject: [PATCH 27/45] Update load_stage.sql --- ATC/load_stage.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ATC/load_stage.sql b/ATC/load_stage.sql index 6e39159b5..b9eb29aed 100644 --- a/ATC/load_stage.sql +++ b/ATC/load_stage.sql @@ -15,7 +15,7 @@ * * Authors: Anna Ostropolets, Polina Talapova, Timur Vakhitov * Date: Oct 2021 - +* Total script execution time: 4m 58s **************************************************************************/ DO $_$ BEGIN From 2b38b53722c5a627de3a03e4bbe58b883cd2977e Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:53:06 +0300 Subject: [PATCH 28/45] Update readme.md --- ATC/readme.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ATC/readme.md b/ATC/readme.md index 21c55794d..9347ada64 100644 --- a/ATC/readme.md +++ b/ATC/readme.md @@ -7,8 +7,12 @@ * Working directory, e.g. *dev_atc*. #### I - Manual work #### -1. Create a backup of the **class_drugs_scraper** table -2. There are 3 options: +1. Create a backup of the **class_drugs_scraper** and **class_to_drug** tables. +```sql +CREATE TABLE class_drugs_scraper_mmyy AS SELECT * FROM class_drugs_scraper; +CREATE TABLE class_to_drug_old AS SELECT * FROM class_to_drug; -- note, that postfix '_old' is obligatory to add, because it is used in further steps. +``` +3. There are 3 options: **Scenario A** - addendum of new ATC codes - Insert them manually into the **class_drugs_scraper** table, populating the following fields: class_code (varchar), class_name (varchar), ddd (varchar), u (varchar), adm_r (varchar), note (varchar), valid_start_date (date), valid_end_date (date), change_type (varchar). The query example: ```sql @@ -33,12 +37,16 @@ FROM atc_addendum_MMYY; #### II - Machinery #### 1. Run *load_input.sql*, which populates the **input tables** of **drug_concept_stage, internal_relationship_stage, relationship_to_concept** and ATC-specific **dev_combo**. -2. Run *load_interim.sql*, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products -3. Run *load_stage.sql* which populates the **staging tables** of **concept_stage, concept_relationship_stage**, and **concept_synonym_stage** -4. Run *generic_update.sql* +2. Run *load_interim.sql*, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products +3. Run the procedure which overwrites the new version of class_to_drug instead old one in the schema of 'sources' ```sql -SELECT devv5.genericupdate(); +SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); +``` +5. Run *load_stage.sql* which populates the **staging tables** of **concept_stage, concept_relationship_stage**, and **concept_synonym_stage** +6. Run *generic_update.sql* ```sql +SELECT devv5.genericupdate(); +``` 6. Perform **post-processing**: ```sql DO $_$ From 06f41f5aa4a201bf583997143d407e3f3700056a Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 19:54:09 +0300 Subject: [PATCH 29/45] Update load_interim.sql --- ATC/load_interim.sql | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 8ed09487a..99cdf125f 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -1602,13 +1602,18 @@ INSERT INTO class_to_drug_new -- ATC Combo Class to Drug Product which is out of drug hierarchy (order = 23) INSERT INTO class_to_drug_new - SELECT DISTINCT -atc_code,atc_name,c.*, +SELECT DISTINCT b.concept_code, + b.concept_name, + c.*, 23 AS conept_order, 'ATC Combo Class to Drug Product which is out of drug hierarchy' - FROM MISSING a - join concept c using (concept_id) - where atc_code||concept_id not in (select class_code||concept_id from class_to_drug_new); -- 2890 +FROM missing a + JOIN concept_manual b + ON b.concept_code = a.atc_code + AND b.invalid_reason IS NULL + JOIN concept c USING (concept_id) +WHERE c.concept_id NOT IN (42731911, 40046823, 36264365, 42481935, 43730307, 42800420); +and b.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) -- remove excessive links to children among Packs WITH t1 AS @@ -1984,6 +1989,7 @@ SELECT DISTINCT class_code, FROM class_to_drug_new ; -- 116679 -/*-- update class_to_drug in the schema of 'sources' -SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); --- run load_stage.sql */ +-- update class_to_drug in the schema of 'sources' +SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); + +-- run load_stage.sql From 1341872f26172c2d38964e77708ee843373e140f Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Fri, 1 Oct 2021 22:09:44 +0300 Subject: [PATCH 30/45] Update load_interim.sql --- ATC/load_interim.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 99cdf125f..d77782624 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -1612,8 +1612,8 @@ FROM missing a ON b.concept_code = a.atc_code AND b.invalid_reason IS NULL JOIN concept c USING (concept_id) -WHERE c.concept_id NOT IN (42731911, 40046823, 36264365, 42481935, 43730307, 42800420); -and b.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) +WHERE c.concept_id NOT IN (42731911, 40046823, 36264365, 42481935, 43730307, 42800420) +AND b.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2833 -- remove excessive links to children among Packs WITH t1 AS @@ -1688,8 +1688,7 @@ ON a1.class_code = a2.class_code AND a1.concept_id <> a2.concept_id and a2.concept_class_id !~ 'Pack' AND a1.class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') - ) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic - ; + ); -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen |varicella, live attenuated; systemic |zoster, live attenuated; systemic -- clean up the same issue among exclusions - Packs WITH t0 AS @@ -1950,7 +1949,7 @@ WHERE CTID NOT IN (SELECT MAX(CTID) GROUP BY class_code, concept_id); --- usgin the concep_order field from the class_to_drgu_new table, assemble the final table of class_to_drug in the following Concept Order: +-- usgin the concep_order field from the class_to_drgu_new table, assemble the final table of class_to_drug and re-assign concept_order value in the following way: /*1. Manual 2. Mono: Ingredient A; form 3. Mono: Ingredient A From 4892653b80be01fa7f4c9579b8db61b95f66fa74 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sat, 2 Oct 2021 12:18:58 +0300 Subject: [PATCH 31/45] Update load_input.sql --- ATC/load_input.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 200bb5b9d..36bd8c1c1 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -2062,6 +2062,13 @@ AND class_code = 'J07AM51'; -- remove wrong rank for the Mono ATC of B03AD02 ferrous fumarate, combinations DELETE from dev_combo where class_code = 'B03AD02' and rnk = 2; + +-- remove duplicates (to do: prevent the entry of duplicates in previous steps) +DELETE +FROM dev_combo +WHERE ctid NOT IN (SELECT MIN(ctid) + FROM dev_combo +GROUP BY class_code, class_name, concept_id, rnk); /*************************************** ******* relationship_to_concept ******** ****************************************/ From 6ddbf949f4f61bb18f2aa08523e94dbd220ab933 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sat, 2 Oct 2021 12:53:01 +0300 Subject: [PATCH 32/45] Update load_input.sql --- ATC/load_input.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 36bd8c1c1..58ee0a299 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -15,7 +15,7 @@ * * Authors: Anna Ostropolets, Polina Talapova * Date: Jul 2021 -* Total script execution time: 21m 27s +* Total script execution time: 27m 33s **************************************************************************/ DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; @@ -1902,8 +1902,7 @@ UPDATE dev_combo WHERE class_code = 'R05FB02' AND class = 'cough suppressants' AND concept_id = 1103137 -AND rnk = 3; - +AND rnk = 3; /******************************************* **** ADD ODDMENTS TO THE INPUT TABLES ***** ********************************************/ From 87da4e05f6e4b3caa5add7cb1b7b3bc487f4f60d Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sat, 2 Oct 2021 14:12:50 +0300 Subject: [PATCH 33/45] Update load_interim.sql --- ATC/load_interim.sql | 376 ++++++++++++++++++++++--------------------- 1 file changed, 190 insertions(+), 186 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index d77782624..24fba35ce 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -482,7 +482,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 5, + 5 AS concept_order, 'ATC Combo Class: Primary lateral + Secondary lateral, 4 ingreds' AS order_desc FROM t1 a JOIN t1 b @@ -533,7 +533,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 50, + 50 AS concept_order, 'ATC Combo Class: Primary lateral + Secondary upward, 4 ingreds' AS order_desc FROM t1 a JOIN t1 b @@ -584,7 +584,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 6, + 6 AS concept_order, 'ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds' AS order_desc FROM t1 a JOIN t1 b @@ -626,7 +626,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 60, + 60 AS concept_order, 'ATC Combo Class: Primary lateral + Secondary upward, 3 ingreds' AS order_desc FROM t1 a JOIN t1 b @@ -669,7 +669,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 7, + 7 AS concept_order, 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' AS order_desc FROM t1 a JOIN t1 b @@ -677,10 +677,10 @@ FROM t1 a AND b.concept_id = a.concept_id AND a.ing_id <> b.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND a.class_code NOT IN (SELECT class_code FROM class_to_drug_new) +AND b.rnk <> 1 +AND a.class_code NOT IN (SELECT class_code FROM class_to_drug_new) AND a.class_name NOT LIKE '%,%and%' AND a.class_code !~ '^J07' -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2426 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2426 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary upward', 2 ingreds' (order = 70) INSERT INTO class_to_drug_new @@ -708,7 +708,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 70, + 70 AS concept_order, 'ATC Combo Class: Primary lateral + Secondary upward, 2 ingreds' AS order_desc FROM t1 a JOIN t1 b @@ -716,8 +716,8 @@ FROM t1 a AND b.concept_id = a.concept_id AND a.ing_id <> b.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 0 +AND b.rnk <> 1 +AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 0 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' in combination with other drugs and an excluded Ingredient (order = 8) INSERT INTO class_to_drug_new @@ -737,7 +737,7 @@ t2 AS (SELECT DISTINCT class_code, class_name, c.*, - 11, + 11 AS concept_order, b.ing_name FROM t1 a JOIN rx_all_combo b @@ -759,8 +759,8 @@ SELECT DISTINCT class_code, valid_start_date, valid_end_date, invalid_reason, - 8 as concept_order, - 'ATC Combo Class: Primary lateral in combination with excluded Ingredient' + 8 AS concept_order, + 'ATC Combo Class: Primary lateral in combination with excluded Ingredient' AS order_desc FROM t2 WHERE concept_name ~ ' / ' AND concept_id NOT IN (SELECT d_id @@ -788,7 +788,7 @@ FROM atc_all_combo a JOIN rx_all_combo b USING (ing_id,df_id) JOIN concept c ON c.concept_id = d_id WHERE a.class_code IN (SELECT class_code FROM ing_pr_up_sec_up_excl)), -t2 as ( +t2 AS ( SELECT DISTINCT a.class_code, a.class_name, a.concept_id, @@ -801,7 +801,7 @@ SELECT DISTINCT a.class_code, a.valid_start_date, a.valid_end_date, a.invalid_reason, - 0, -- no match up to now, but it can happen in the future + 0 AS concept_order, -- no match up to now, but it can happen in the future 'ATC Combo Class: Primary upward + Secondary upward with excluded ingredient' AS order_desc FROM t1 a JOIN t1 b @@ -810,7 +810,7 @@ FROM t1 a AND a.ing_id <> b.ing_id WHERE a.rnk = 3 AND b.rnk <> 3) -SELECT DISTINCT * from t2 +SELECT DISTINCT * FROM t2 WHERE concept_name ~ ' / ' AND concept_id NOT IN (SELECT d_id FROM rx_all_combo a @@ -818,7 +818,7 @@ AND concept_id NOT IN (SELECT d_id ON LOWER (b.ing_name) = LOWER (a.ing_name) AND LOWER (b.df_name) = LOWER (a.df_name) AND b.rnk = 0) -AND class_code||concept_id::varchar NOT IN (SELECT class_code||concept_id::varchar FROM class_to_drug_new); --0 +AND class_code||concept_id::varchar NOT IN (SELECT class_code||concept_id::varchar FROM class_to_drug_new); --0 -- add mappings of Multicomponent ACT Classes of 'Primary upward' + 'Secondary upward' (order = 9) INSERT INTO class_to_drug_new @@ -847,8 +847,7 @@ FROM t1 a ON c.concept_id = b.d_id AND c.standard_concept = 'S' AND rnk = 3), -t3 -AS +t3 AS (SELECT DISTINCT a.class_code, a.class_name, a.ing_name, @@ -863,7 +862,7 @@ FROM t1 a AND rnk = 4) SELECT DISTINCT a.*, 9 AS concept_order, - 'ATC Combo Class: Primary upward + Secondary upward' + 'ATC Combo Class: Primary upward + Secondary upward' AS order_desc FROM t2 a JOIN t3 b ON b.class_code = a.class_code @@ -906,7 +905,7 @@ WHERE rnk = 4; -- Secondary upward -- add Primary lateral AND (Secondary lateral 1 AND/OR Secondary lateral 2) AND/OR Primary upward AND/OR Secondary upward Multicomponent ATC Classes DROP TABLE if exists full_combo; CREATE UNLOGGED TABLE full_combo -AS( +AS ( SELECT DISTINCT a.class_code, a.class_name, a.concept_id||COALESCE('-' || b.concept_id,'') ||COALESCE('-' || c.concept_id,'') ||COALESCE('-' || c1.concept_id,'') ||COALESCE('-' || d.concept_id,'') AS i_combo @@ -953,11 +952,11 @@ WHERE i_combo NOT IN (SELECT i_combo FROM full_combo); DROP TABLE IF EXISTS permutations; CREATE UNLOGGED TABLE permutations AS -SELECT distinct a.class_code, a.class_name, a.concept_id||COALESCE('-' || b.concept_id, '')||COALESCE('-' || c.concept_id, '') AS i_combo +SELECT distinct a.class_code, a.class_name, a.concept_id||COALESCE('-'||b.concept_id, '')||COALESCE('-'||c.concept_id, '') AS i_combo FROM t1 a -- from Primary lateral LEFT JOIN t2 b ON b.class_code = a.class_code -- to the 1st Secondary lateral LEFT JOIN t2 c ON c.class_code = a.class_code -- and the 2nd Secondary lateral -WHERE b.concept_id<>c.concept_id AND b.concept_id<>a.concept_id; +WHERE b.concept_id <> c.concept_id AND b.concept_id <> a.concept_id; -- add newly created permutations to the full_combo table in order to enrich the set of aggregated ATC Ingredients INSERT INTO full_combo @@ -966,9 +965,8 @@ SELECT * FROM permutations; -- create temporary table with all possible i_combos for Primary lateral in combinations (with unspecified drugs) DROP TABLE if exists ing_pr_lat_combo_to_drug; CREATE TABLE ing_pr_lat_combo_to_drug -AS -(WITH z1 -AS +AS ( +WITH z1 AS (SELECT drug_concept_id, REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo FROM rx_combo @@ -983,9 +981,8 @@ FROM ing_pr_lat_combo a -- create temporary table with all possible i_combos for Primary lateral + Secondary upward DROP TABLE if exists ing_pr_lat_sec_up_combo_to_drug; CREATE TABLE ing_pr_lat_sec_up_combo_to_drug -AS -(WITH z1 -AS +AS ( +WITH z1 AS (SELECT drug_concept_id, REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo FROM rx_combo @@ -1016,11 +1013,20 @@ AS (SELECT drug_concept_id, REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo FROM rx_combo -WHERE i_combo LIKE '%-%') SELECT DISTINCT a.class_code,a.class_name,d.i_combo FROM ing_pr_lat_combo_excl a JOIN t1 b ON b.i_combo = a.concept_id -JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code AND a1.rnk <> a.rnk AND a1.rnk = 0 JOIN t1 f ON f.i_combo = a1.concept_id --- excluded -JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id JOIN rx_combo d1 ON d1.drug_concept_id = f.drug_concept_id --- excluded +WHERE i_combo LIKE '%-%') + SELECT DISTINCT + a.class_code, + a.class_name, + d.i_combo + FROM ing_pr_lat_combo_excl a + JOIN t1 b ON b.i_combo = a.concept_id +JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code + AND a1.rnk <> a.rnk + AND a1.rnk = 0 + JOIN t1 f ON f.i_combo = a1.concept_id -- excluded +JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id + JOIN rx_combo d1 + ON d1.drug_concept_id = f.drug_concept_id -- excluded AND d1.drug_concept_id <> d.drug_concept_id); -- create a temporary table with all possible i_combos for Primary upward + Secondary upward: ing_pr_sec_up_combo_to_drug @@ -1087,19 +1093,15 @@ FROM ing_pr_sec_up_combo_excl a -- add prepared list of aggregated Ingredients of ATC Combo Classes to full_combo INSERT INTO full_combo -SELECT * -FROM ing_pr_lat_combo_to_drug +SELECT * FROM ing_pr_lat_combo_to_drug UNION -SELECT * -FROM ing_pr_lat_combo_excl_to_drug +SELECT * FROM ing_pr_lat_combo_excl_to_drug UNION -SELECT * -FROM ing_pr_sec_up_combo_to_drug +SELECT * FROM ing_pr_sec_up_combo_to_drug UNION -SELECT * -FROM ing_pr_sec_up_combo_excl_to_drug +SELECT * FROM ing_pr_sec_up_combo_excl_to_drug UNION -SELECT * from ing_pr_lat_sec_up_combo_to_drug; +SELECT * FROM ing_pr_lat_sec_up_combo_to_drug; -- create a table to order aggregated ATC ingredients by an Ingredient DROP TABLE IF EXISTS full_combo_reodered; @@ -1119,10 +1121,11 @@ CREATE INDEX i_full_combo_reodered ON full_combo_reodered (class_code, i_combo); DROP TABLE full_combo_with_form; CREATE UNLOGGED TABLE full_combo_with_form AS -SELECT DISTINCT a.class_code, - a.class_name, +SELECT DISTINCT + a.class_code, + a.class_name, a.i_combo, - r.concept_id_2::int as df_id + r.concept_id_2::int AS df_id FROM full_combo_reodered a JOIN internal_relationship_stage i ON class_code = substring (concept_code_1, '\w+') -- cut ATC code before space character JOIN drug_concept_stage b @@ -1159,11 +1162,11 @@ CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo INSERT INTO class_to_drug_new WITH t1 AS -(SELECT c.concept_id, -- Standard Drug Product +(SELECT c.concept_id, -- Standard Drug Product c.concept_name, c.concept_class_id, c.vocabulary_id, - r.concept_id_2 AS df_id, -- Dose Form + r.concept_id_2 AS df_id, -- Dose Form a.i_combo -- combination of Standard Ingredient IDs as a key for join FROM rx_combo a JOIN concept c ON c.concept_id = a.drug_concept_id @@ -1175,18 +1178,20 @@ AND r.relationship_id = 'RxNorm has dose form' AND r.invalid_reason IS NULL ) SELECT DISTINCT f.class_code, -- ATC - c.concept_name as class_name, + c.concept_name AS class_name, d.*, 10 AS conept_order, - 'ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations' + 'ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations' AS order_desc FROM full_combo_with_form f - JOIN concept_manual c on c.concept_code = f.class_code and c.invalid_reason is null and c.concept_class_id = 'ATC 5th' + JOIN concept_manual c ON c.concept_code = f.class_code + AND c.invalid_reason IS NULL + AND c.concept_class_id = 'ATC 5th' JOIN t1 r ON r.i_combo = f.i_combo -- combination of Standard Ingredient IDs AND r.df_id = f.df_id - JOIN concept d on d.concept_id = r.concept_id - and f.class_code||r.concept_id not in (select class_code||concept_id from class_to_drug_new) - and f.class_code in ( 'J07CA10', 'G03FB09', 'G03FB06', 'G03FB05', 'G03FA12', 'G03FA11','G03FA01', 'G03EA03','A11GB01'); -- 63 + JOIN concept d ON d.concept_id = r.concept_id + AND f.class_code||r.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) + AND f.class_code IN ( 'J07CA10', 'G03FB09', 'G03FB06', 'G03FB05', 'G03FA12', 'G03FA11','G03FA01', 'G03EA03','A11GB01'); -- 63 -- add manual mappings from concept_relationship_manual (order = 11) INSERT INTO class_to_drug_new @@ -1194,7 +1199,7 @@ SELECT DISTINCT a.class_code, f.concept_name AS class_name, c.*, 11 AS concept_order, - 'ATC Class to Drug Product from concept_relationship_manual' + 'ATC Class to Drug Product from concept_relationship_manual' AS order_desc FROM class_drugs_scraper a JOIN concept_relationship_manual b ON b.concept_code_1 = a.class_code JOIN concept_manual f @@ -1206,7 +1211,7 @@ FROM class_drugs_scraper a AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND c.standard_concept = 'S' AND (a.class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) -AND f.invalid_reason IS NULL AND b.invalid_reason is null +AND f.invalid_reason IS NULL AND b.invalid_reason IS null AND f.concept_class_id = 'ATC 5th'; -- 7558 -- manual clean up for Precise Ingredients and other particular cases (according to the information on the ATC WHO Website) @@ -1217,44 +1222,54 @@ OR class_code = 'N05AF02' -- clopentixol OR class_code IN ('D07AB02','D07BB04') -- hydrocortisone butyrate + combo that so far doesn't exist OR class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral OR (class_code = 'B02BD14' AND concept_name LIKE '%Tretten%') -- 2 --catridecacog -OR (class_code IN ('B02BD14','B02BD11') and concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog +OR (class_code IN ('B02BD14','B02BD11') AND concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog ; -- 310 -- add additional semi-manual mappings based on pattern-matching (order = 12) INSERT INTO class_to_drug_new -with t1 as ( -SELECT 'B02BD11' as class_code,'catridecacog' as class_name, concept_id +with t1 AS ( +SELECT 'B02BD11' AS class_code,'catridecacog' as class_name, concept_id FROM concept -WHERE (vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' - AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') +WHERE (vocabulary_id LIKE 'RxNorm%' + AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' + AND standard_concept = 'S' + AND concept_class_id = 'Clinical Drug') OR concept_id = 35603348 -- the whole hierarchy (factor XIII Injection [Tretten] Branded Drug Form) UNION ALL SELECT 'B02BD14','susoctocog alfa', concept_id FROM concept WHERE (vocabulary_id LIKE 'RxNorm%' - AND concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' AND standard_concept = 'S' AND concept_class_id = 'Clinical Drug') + AND concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' + AND standard_concept = 'S' + AND concept_class_id = 'Clinical Drug') OR concept_id IN (35603348, 44109089) -- the whole hierarchy UNION ALL SELECT 'A02BA07','ranitidine bismuth citrate',concept_id FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Tritec%' - AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' +WHERE vocabulary_id LIKE 'RxNorm%' + AND concept_name LIKE '%Tritec%' + AND standard_concept = 'S' + AND concept_class_id = 'Branded Drug Form' UNION ALL SELECT 'N05AF02','clopenthixol', concept_id FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Sordinol|Ciatyl' - AND standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' +WHERE vocabulary_id LIKE 'RxNorm%' + AND concept_name ~* 'Sordinol|Ciatyl' + AND standard_concept = 'S' + AND concept_class_id = 'Branded Drug Form' UNION ALL SELECT 'D07AB02','hydrocortisone butyrate',concept_id FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Hydrocortisone butyrate' AND concept_class_id = 'Clinical Drug' - AND standard_concept = 'S' +WHERE vocabulary_id LIKE 'RxNorm%' + AND concept_name ~* 'Hydrocortisone butyrate' + AND concept_class_id = 'Clinical Drug' + AND standard_concept = 'S' ) SELECT DISTINCT a.class_code, d.concept_name, c.*, - 12 as concept_order, - 'ATC Class with semi-manual point fix' + 12 AS concept_order, + 'ATC Class with semi-manual point fix' AS order_desc FROM t1 a JOIN concept c ON c.concept_id = a.concept_id JOIN concept_manual d @@ -1265,7 +1280,6 @@ AND a.class_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_ -- clean up erroneous amount of ingredients DELETE ---SELECT * FROM class_to_drug_new WHERE class_name LIKE '%,%and%' AND class_name NOT LIKE '%,%,%and%' @@ -1277,32 +1291,33 @@ INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, c.*, - 13 as concept_order, - 'ATC Class from old class_to_drug' + 13 AS concept_order, + 'ATC Class from old class_to_drug' AS order_desc FROM sources_class_to_drug_old a JOIN concept_manual b ON b.concept_code = a.class_code - JOIN concept c - ON a.concept_id = c.concept_id - AND c.standard_concept = 'S' and c.concept_class_id !~ 'Pack|Ingredient' + JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id !~ 'Pack|Ingredient' AND b.invalid_reason IS NULL WHERE (class_code) NOT IN (SELECT class_code FROM class_to_drug_new) -AND class_code NOT IN ('S02CA01','V03AB05','P03AC54','S02CA03','S02CA03') -AND (class_code,c.concept_id) NOT IN ( +AND class_code NOT IN ('S02CA01', 'V03AB05', 'P03AC54', 'S02CA03', 'S02CA03') +AND (class_code, c.concept_id) NOT IN ( SELECT 'A06AA02', 40031558 -- oral - otic - UNION ALL + UNION ALL SELECT 'A06AA02', 40031561 -- oral - rectal - UNION ALL + UNION ALL SELECT 'A06AA02', 40031561 -- oral - enema - UNION ALL + UNION ALL SELECT 'A06AA02', 40723180 -- oral - enema UNION ALL - SELECT 'A06AA02',41080219 + SELECT 'A06AA02', 41080219 -- oral - enema UNION ALL - SELECT 'A06AA02',41205788 + SELECT 'A06AA02', 41205788 -- oral - enema UNION ALL - SELECT 'A06AA02',43158334 - UNION ALL - SELECT 'A06AA02',40036796); -- 235 + SELECT 'A06AA02', 43158334 -- oral - enema + UNION ALL + SELECT 'A06AA02', 40036796) -- oral - enema + ; -- 235 /********************** ****** ADD PACKS ****** ***********************/ @@ -1311,8 +1326,8 @@ INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 14 as concept_order, - 'Pack: Primary lateral in combo' as order_desc + 14 AS concept_order, + 'Pack: Primary lateral in combo' AS order_desc FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id JOIN concept d @@ -1327,12 +1342,6 @@ AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') AND j.standard_concept = 'S' AND a.class_code IN (SELECT class_code FROM ing_pr_lat_combo) - /*dev_combo - WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0))*/ AND j.concept_name ~ ' / ' -- combos only AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 1174 @@ -1341,8 +1350,8 @@ INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 15 as concept_order, - 'Pack: Primary lateral in combo additional' + 15 AS concept_order, + 'Pack: Primary lateral in combo additional' AS order_desc FROM class_to_drug_new a JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id JOIN concept j ON j.concept_id = r2.concept_id_2 @@ -1351,12 +1360,6 @@ AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack', AND j.standard_concept = 'S' AND a.class_code IN (SELECT class_code FROM ing_pr_lat_combo) - /*dev_combo - WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) - AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0)) */ AND j.concept_name ~ ' / ' -- combos only AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 262 @@ -1365,8 +1368,8 @@ INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 16 as concept_order, - 'Pack: Primary lateral + Secondary lateral' + 16 AS concept_order, + 'Pack: Primary lateral + Secondary lateral' AS order_desc FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id JOIN concept d @@ -1389,8 +1392,8 @@ INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, a.class_name, j.*, - 17 as concept_order, - 'Pack: Primary lateral + Secondary upward' + 17 AS concept_order, + 'Pack: Primary lateral + Secondary upward' AS order_desc FROM class_to_drug_new a JOIN concept_relationship r ON r.concept_id_1 = a.concept_id JOIN concept d @@ -1413,8 +1416,8 @@ INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, b.concept_name AS class_name, c.*, - 18 as concept_order, - 'Additional Pack: from old c_t_d' + 18 AS concept_order, + 'Additional Pack: from old c_t_d' AS order_desc FROM sources_class_to_drug_160921 a JOIN concept_manual b ON b.concept_code = a.class_code JOIN concept c @@ -1427,14 +1430,14 @@ AND a.class_code NOT IN ('G01AF55','S03CA04','S01CA03','S02CA03')-- gives pack AND a.class_code NOT IN ('B03AE01','C07BB52','D01AC52','C10AD52') -- wrong ing combo ; -- 289 --- еnrich pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' +-- еnrich the pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' -- they are always used as packs (order = 19) INSERT INTO class_to_drug_new SELECT DISTINCT class_code, class_name, c.*, - 19 as concept_order, - 'Additional Pack: semi-manual contraceptive' + 19 AS concept_order, + 'Additional Pack: semi-manual contraceptive' AS order_desc FROM class_to_drug_new f JOIN concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) JOIN concept c @@ -1452,19 +1455,25 @@ AND concept_class_id !~ 'Pack'; -- 81 -- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 20) INSERT INTO class_to_drug_new WITH ing AS ( -SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt +SELECT concept_id_2, + concept_code_1, + COUNT(concept_id_2) AS cnt FROM ( - SELECT DISTINCT substring(i.concept_code_1,'\w+'), r.concept_code_1, concept_id_2 + SELECT DISTINCT substring(i.concept_code_1,'\w+'), + r.concept_code_1, + concept_id_2 FROM relationship_to_concept r -JOIN internal_relationship_stage i on i.concept_code_2 = r.concept_code_1 -JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' - ) a GROUP BY concept_id_2,concept_code_1 ), +JOIN internal_relationship_stage i ON i.concept_code_2 = r.concept_code_1 +JOIN drug_concept_stage d ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient') a + GROUP BY concept_id_2, concept_code_1), drug AS ( - SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 + SELECT DISTINCT SUBSTRING(concept_code_1,'\w+') AS code, + concept_code_2 FROM internal_relationship_stage i -WHERE not exists (SELECT 1 FROM internal_relationship_stage i2 - WHERE i.concept_code_2=i2.concept_code_2 - AND substring(i.concept_code_1,'\w+')!=substring(i2.concept_code_1,'\w+')) +WHERE NOT EXISTS (SELECT 1 FROM internal_relationship_stage i2 + WHERE i.concept_code_2 = i2.concept_code_2 + AND substring(i.concept_code_1,'\w+') != substring(i2.concept_code_1,'\w+')) ), drug_name AS ( SELECT DISTINCT class_code,class_name, concept_id_2 @@ -1477,19 +1486,18 @@ AND code NOT IN (SELECT class_code FROM class_to_drug_new) all_drug AS ( SELECT class_code, class_name, concept_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt FROM drug_name a -JOIN concept c on c.concept_id = a.concept_id_2 +JOIN concept c ON c.concept_id = a.concept_id_2 ) SELECT DISTINCT a.class_code, b.concept_name, c.*, - 20 as concept_order, + 20 AS concept_order, 'ATC Mono WO Dose Form to Ingredient' FROM all_drug a -join concept_manual b on b.concept_code = a.class_code and b.invalid_reason is null -join concept c on c.concept_id = a.concept_id +JOIN concept_manual b ON b.concept_code = a.class_code AND b.invalid_reason IS null +JOIN concept c ON c.concept_id = a.concept_id WHERE cnt = 1 -AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); -- 5 */ -- to check - +AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); -- 5 -- add those which are absent in the drug hierarchy -- step 1 @@ -1508,7 +1516,7 @@ AS FROM concept c WHERE domain_id = 'Drug' AND concept_id NOT IN (SELECT descendant_concept_id FROM no_atc_1) -AND c.vocabulary_id IN ('RxNorm','RxNorm Extension') +AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') AND c.concept_class_id != 'Ingredient' AND c.standard_concept = 'S'); @@ -1589,16 +1597,16 @@ CREATE INDEX i_no_atc_full_combo ON no_atc_full_combo (concept_id, i_combo,df_id -- add mappings of ATC Combo Class to Drug Product which is out of drug hierarchy (order = 22) INSERT INTO class_to_drug_new - SELECT DISTINCT k.concept_code as class_code, -- ATC - k.concept_name as class_name, + SELECT DISTINCT k.concept_code AS class_code, -- ATC + k.concept_name AS class_name, d.*, 22 AS conept_order, - 'ATC Combo Class to Drug Product which is out of drug hierarchy' + 'ATC Combo Class to Drug Product which is out of drug hierarchy' AS order_desc FROM no_atc_full_combo f - JOIN full_combo_with_form c on c.i_combo = f.i_combo and c.df_id = f.df_id is null - join concept_manual k on k.concept_code = c.class_code and k.concept_class_id = 'ATC 5th' and k.invalid_reason is null - JOIN concept d on d.concept_id = f.concept_id - and c.class_code||d.concept_id not in (select class_code||concept_id from class_to_drug_new); -- 6188 + JOIN full_combo_with_form c ON c.i_combo = f.i_combo AND c.df_id = f.df_id IS null + JOIN concept_manual k ON k.concept_code = c.class_code AND k.concept_class_id = 'ATC 5th' AND k.invalid_reason IS null + JOIN concept d ON d.concept_id = f.concept_id + AND c.class_code||d.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 6188 -- ATC Combo Class to Drug Product which is out of drug hierarchy (order = 23) INSERT INTO class_to_drug_new @@ -1606,7 +1614,7 @@ SELECT DISTINCT b.concept_code, b.concept_name, c.*, 23 AS conept_order, - 'ATC Combo Class to Drug Product which is out of drug hierarchy' + 'ATC Combo Class to Drug Product which is out of drug hierarchy' AS order_desc FROM missing a JOIN concept_manual b ON b.concept_code = a.atc_code @@ -1620,10 +1628,9 @@ WITH t1 AS ( SELECT * FROM class_to_drug_new - where concept_class_id ~ 'Pack') -delete from class_to_drug_new where class_code||concept_id in ( -SELECT a1.class_code|| - a2.concept_id + WHERE concept_class_id ~ 'Pack') +DELETE FROM class_to_drug_new WHERE class_code||concept_id IN ( +SELECT a1.class_code||a2.concept_id FROM t1 a1-- papa JOIN t1 a2 -- child ON a1.class_code = a2.class_code @@ -1631,9 +1638,9 @@ ON a1.class_code = a2.class_code ON a1.concept_id = ca.ancestor_concept_id AND a2.concept_id = ca.descendant_concept_id AND a1.concept_id <> a2.concept_id - and a2.concept_class_id ~ 'Pack' - AND a1.class_code not in ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02' ))-- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic); -- 2558 - ; -- 3920 + AND a2.concept_class_id ~ 'Pack' + AND a1.class_code NOT IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') -- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic); -- 2558 + ); -- 3920 -- remove excessive links to children among unpacked Drug Products WITH t0 AS @@ -1642,15 +1649,14 @@ WITH t0 AS FROM class_to_drug_new WHERE concept_class_id !~ 'Pack' ), -t1 as ( -select * from t0 where class_code IN (SELECT class_code +t1 AS ( +SELECT * FROM t0 WHERE class_code IN (SELECT class_code FROM class_to_drug_new GROUP BY class_code HAVING COUNT(1) >= 2)) DELETE FROM class_to_drug_new where class_code||concept_id IN ( -SELECT a1.class_code|| - a2.concept_id -- child mapping +SELECT a1.class_code||a2.concept_id -- child mapping FROM t1 a1-- papa JOIN t1 a2 -- child ON a1.class_code = a2.class_code @@ -1658,9 +1664,9 @@ ON a1.class_code = a2.class_code ON a1.concept_id = ca.ancestor_concept_id AND a2.concept_id = ca.descendant_concept_id AND a1.concept_id <> a2.concept_id - and a2.concept_class_id !~ 'Pack' - AND a1.class_code NOT IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02')) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic - ;-- 10432 + AND a2.concept_class_id !~ 'Pack' + AND a1.class_code NOT IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') -- catridecacog|susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic + );-- 10432 -- clean up the same issue among exclusions WITH t0 AS @@ -1668,17 +1674,16 @@ WITH t0 AS SELECT * FROM class_to_drug_new WHERE concept_class_id !~ 'Pack' - AND class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') + AND class_code IN ('A02BA07', 'B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') ), -t1 as ( -select * from t0 where class_code IN (SELECT class_code +t1 AS ( +SELECT * FROM t0 WHERE class_code IN (SELECT class_code FROM class_to_drug_new GROUP BY class_code HAVING COUNT(1) >= 2)) DELETE FROM class_to_drug_new where class_code||concept_id IN ( -SELECT a1.class_code|| - a1.concept_id -- papa mapping +SELECT a1.class_code||a1.concept_id -- papa mapping FROM t1 a1-- papa JOIN t1 a2 -- child ON a1.class_code = a2.class_code @@ -1686,9 +1691,9 @@ ON a1.class_code = a2.class_code ON a1.concept_id = ca.ancestor_concept_id AND a2.concept_id = ca.descendant_concept_id AND a1.concept_id <> a2.concept_id - and a2.concept_class_id !~ 'Pack' - AND a1.class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') - ); -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen |varicella, live attenuated; systemic |zoster, live attenuated; systemic + AND a2.concept_class_id !~ 'Pack' + AND a1.class_code IN ('A02BA07', 'B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02', 'J07BB01', 'N05AF02') + ); -- catridecacog|susoctocog alfa; parenteral, topical, urethral|norgestrel and estrogen|varicella, live attenuated; systemic|zoster, live attenuated; systemic -- clean up the same issue among exclusions - Packs WITH t0 AS @@ -1696,18 +1701,17 @@ WITH t0 AS SELECT * FROM class_to_drug_new WHERE concept_class_id ~ 'Pack' - AND class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') + AND class_code IN ('A02BA07', 'B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02', 'J07BB01', 'N05AF02') ), -t1 as ( -select * from t0 where class_code IN (SELECT class_code +t1 AS ( +SELECT * FROM t0 WHERE class_code IN (SELECT class_code FROM class_to_drug_new GROUP BY class_code HAVING COUNT(1) >= 2)) DELETE FROM class_to_drug_new where class_code||concept_id IN ( -SELECT a1.class_code|| - a1.concept_id -- papa mapping -FROM t1 a1-- papa +SELECT a1.class_code||a1.concept_id -- papa mapping +FROM t1 a1 -- papa JOIN t1 a2 -- child ON a1.class_code = a2.class_code JOIN concept_ancestor ca @@ -1716,28 +1720,27 @@ ON a1.class_code = a2.class_code AND a1.concept_id <> a2.concept_id and a2.concept_class_id ~ 'Pack' AND a1.class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') - ) -- catridecacog | susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic + ) -- catridecacog|susoctocog alfa; parenteral, topical, urethral|norgestrel and estrogen|varicella, live attenuated; systemic|zoster, live attenuated; systemic ;-- 19 /*************************** ******** DF CLEAN UP ******* ****************************/ -- clean up oral forms DROP TABLE IF EXISTS wrong_df; -CREATE TABLE wrong_df as ( -SELECT *, 'oral mismatch' as issue_desc +CREATE TABLE wrong_df AS ( +SELECT *, 'oral mismatch' AS issue_desc FROM class_to_drug_new WHERE SPLIT_PART(class_name,';',2) ~ 'oral' AND concept_name !~* 'oral|chew|tooth|mouth|elixir|Extended Release Suspension|buccal|Sublingual|Paste|Mucosal|Prefilled Syringe|\...|Oral Ointment|Oral Cream' AND class_name !~ 'rectal|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|nasal' AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' -AND class_code NOT IN ('A01AA04','A01AA02','G04BD08') -); +AND class_code NOT IN ('A01AA04', 'A01AA02', 'G04BD08')); INSERT INTO wrong_df -select *, 'vaginal mismatch' from class_to_drug_new where class_name ~ 'vaginal' -and concept_name !~* 'vaginal|topical|mucosal|Drug Implant|Douche|Irrigation Solution' -and class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' -and concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; -- 1 +SELECT *, 'vaginal mismatch' FROM class_to_drug_new WHERE class_name ~ 'vaginal' +AND concept_name !~* 'vaginal|topical|mucosal|Drug Implant|Douche|Irrigation Solution' +AND class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' +AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; -- 1 -- clean up rectal forms INSERT INTO wrong_df @@ -1762,7 +1765,7 @@ AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; INSERT INTO wrong_df SELECT *, 'local oral mismatch' FROM class_to_drug_new -WHERE class_name ~ 'local oral' and class_name !~ 'oral, local oral' +WHERE class_name ~ 'local oral' AND class_name !~ 'oral, local oral' AND concept_name !~* 'mouth|topical|paste|irrig|Lozenge|gum|buccal|Suspension|solution|subl|Gel|spray|Disintegrating Oral Tablet|Effervescent Oral Tablet|Oral Powder|Chewable Tablet|Oral Film|Oral Granules' AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; @@ -1842,7 +1845,7 @@ AND concept_name !~* '\...$|\...\) \} Pack$|oral|rectal|inject|chew|Cartridge| AND class_name !~* 'rectal|nasal|vaginal|topical' AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' AND concept_class_id ~ 'Pack' -and class_code not in ('R03AC130'); -- formoterol Powder for Oral Suspension| +AND class_code NOT IN ('R03AC130'); -- formoterol Powder for Oral Suspension| -- clean up systemic packs 2 - check This next time INSERT INTO wrong_df @@ -1862,14 +1865,14 @@ FROM class_to_drug_new WHERE class_name ~ 'vaginal ring' AND concept_name !~* 'ring|insert|system|implant'; -DELETE FROM wrong_df where class_code in ( 'C02AC01', 'G03GA08', 'R03AC13'); +DELETE FROM wrong_df WHERE class_code IN ( 'C02AC01', 'G03GA08', 'R03AC13'); -- look at them and remove from class_to_drug_new DELETE FROM class_to_drug_new WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM wrong_df); -- 891 --- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (do we need them?) order = 24 +-- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (order = 24) INSERT INTO class_to_drug_new WITH t1 AS @@ -1884,8 +1887,8 @@ AND concept_code NOT IN (SELECT class_code FROM atc_inexistent) SELECT DISTINCT a.concept_code, a.concept_name, c.*, - 24 as concept_order, - 'ATC Combo Class to Ingredient' + 24 AS concept_order, + 'ATC Combo Class to Ingredient' AS order_desc FROM t1 a JOIN dev_combo k ON k.class_code = a.concept_code @@ -1894,7 +1897,7 @@ FROM t1 a ON c.concept_id = k.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - and a.concept_code||c.concept_id not in (select class_code||concept_id from class_to_drug_new);--314 + AND a.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);--314 -- wrong match DELETE @@ -1912,7 +1915,7 @@ DELETE FROM class_to_drug_new WHERE concept_class_id ~ '\yComp'; --- remove suspicious mapping of inexistent drugs (this table should be checked before) +-- remove suspicious mapping of inexistent drugs (atc_inexistent should be checked before) DELETE FROM class_to_drug_new WHERE class_code IN (SELECT class_code FROM atc_inexistent); -- 14 @@ -1959,6 +1962,7 @@ WHERE CTID NOT IN (SELECT MAX(CTID) 7. Combo: Ingredient A, combination 8. Any packs*/ +-- be sure, that you have a backup of class_to_drug TRUNCATE TABLE class_to_drug; INSERT INTO class_to_drug SELECT DISTINCT class_code, @@ -1969,21 +1973,21 @@ SELECT DISTINCT class_code, CASE WHEN class_code||concept_code IN (SELECT concept_code_1||concept_code_2 FROM concept_relationship_manual WHERE relationship_id = 'ATC - RxNorm' AND invalid_reason IS NULL) - AND class_code IN (SELECT class_code FROM combo_pull) and concept_class_id !~ 'Pack' + AND class_code IN (SELECT class_code FROM combo_pull) AND concept_class_id !~ 'Pack' AND concept_order <> 11 THEN 1 -- if ATC Combo has an entry in crm, its higher concept_order values have to be converted to 1 as well WHEN class_code||concept_code IN (SELECT concept_code_1||concept_code_2 FROM concept_relationship_manual WHERE relationship_id = 'ATC - RxNorm' AND invalid_reason IS NULL) - AND class_code NOT IN (SELECT class_code FROM combo_pull) and concept_class_id !~ 'Pack' and concept_name !~ ' \/ ' + AND class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_class_id !~ 'Pack' AND concept_name !~ ' \/ ' AND concept_order <> 11 THEN 1 -- if ATC Mono has an entry in crm, its higher concept_order values have to be converted to 1 as well WHEN class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_order = 11 AND concept_name ~ ' / ' THEN 7 - WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18 (note, that 19 doesn't exist) + WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18 (note, that 19 does not exist) WHEN concept_order IN (11, 13) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual WHEN concept_order IN (1, 12) THEN 2 -- ATC Monocomp Class WHEN concept_order = 20 THEN 3 -- Mono: Ingredient A - WHEN concept_order IN (5, 6, 7) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingreds) + WHEN concept_order IN (5, 6, 7) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingredients) WHEN concept_order IN (9, 10, 50, 60, 70) THEN 5 -- Combo: Ingredient A OR Group A + group B WHEN concept_order IN (3, 4, 8) THEN 6 -- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient - WHEN concept_order IN (2, 21, 22, 23, 24) THEN 7 -- 7. Combo: Ingredient A, combination + WHEN concept_order IN (2, 21, 22, 23, 24) THEN 7 -- Combo: Ingredient A, combination END AS concept_order FROM class_to_drug_new ; -- 116679 From 6216dfc3b76218ab3f0c60c0f99fdb89dc47db51 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sat, 2 Oct 2021 18:50:31 +0300 Subject: [PATCH 34/45] Update load_interim.sql --- ATC/load_interim.sql | 148 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 5 deletions(-) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 24fba35ce..804ecdfa3 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -1563,6 +1563,42 @@ AND k.concept_class_id = 'ATC 5th' AND k.concept_code NOT IN (SELECT class_code FROM dev_combo) AND k.concept_code||p.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);--6107 +-- add Clinical or Branded Drug Forms as descendants of non-to-Form mappings if they present in the concept_ancestor table +-- simultaneous child-parent maps will be deleted in further steps +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + c.*, + 21 AS concept_order, + 'ATC Monocomp Class to Drug Product which is out of hierarchy' +FROM class_to_drug_new a + JOIN concept_ancestor ca ON ca.descendant_concept_id = concept_id + JOIN concept c + ON c.concept_id = ca.ancestor_concept_id + AND c.concept_class_id ~ 'Form' + AND c.standard_concept = 'S' +WHERE concept_order = 21 +AND NOT EXISTS (SELECT 1 FROM class_to_drug_new x WHERE x.class_code||x.concept_id = a.class_code||c.concept_id) +AND c.domain_id = 'Drug'; -- 271 + +-- add descendants of other non-to-Form mappings (in class_to_drug_new) if they present in concept_ancestor (abundant child-parent mappings will be deleted in further steps) +INSERT INTO class_to_drug_new +SELECT DISTINCT a.class_code, + a.class_name, + c.*, + a.concept_order, + a.order_desc +FROM class_to_drug_new a + JOIN concept_ancestor ca ON ca.descendant_concept_id = concept_id + JOIN concept c + ON c.concept_id = ca.ancestor_concept_id + AND c.concept_class_id ~ 'Form' and a.concept_class_id !~ 'Form' + AND c.standard_concept = 'S' +WHERE + c.domain_id = 'Drug' AND a.class_code NOT IN (SELECT class_code + FROM combo_pull) AND NOT EXISTS (SELECT 1 FROM class_to_drug_new x WHERE x.class_code||x.concept_id = a.class_code||c.concept_id) + AND a.class_code NOT IN ('B02BD11', 'B02BD14', 'D07AB02'); -- intentionally excluded catridecacog|susoctocog alfa|hydrocortisone butyrate -- no such Precise Ingredients connected to Dose Forms + -- obtain more ATC Combo classes DROP TABLE no_atc_full_combo; CREATE TABLE no_atc_full_combo @@ -1928,6 +1964,48 @@ FROM class_to_drug_new FROM concept_relationship_manual WHERE invalid_reason IS NOT NULL AND relationship_id = 'ATC - RxNorm'); -- 0 + + -- clean up wrong vaccine mappings (they should be processed manually and be stored in CRM) +-- firstly, drop wrong mono-vaccines +DELETE +FROM class_to_drug_new +WHERE class_code ~ '^J07' +AND class_code||concept_code NOT IN (SELECT concept_code_1||concept_code_2 -- not in crm + FROM concept_relationship_manual + WHERE relationship_id = 'ATC - RxNorm' + AND invalid_reason IS NULL) + AND concept_name !~ ' / ' +AND concept_name !~ 'bivalent|trivalent|pentavalent' +AND class_code NOT IN ('J07AC01', 'J07BX03', 'J07BA01', 'J07BC02', 'J07AL01','J07AP02', 'J07CA10'); + +-- secondly, drop wrong combo-vaccines +DELETE +FROM class_to_drug_new +WHERE class_code ~ '^J07' +AND class_code||concept_code NOT IN (SELECT concept_code_1||concept_code_2 + FROM concept_relationship_manual + WHERE relationship_id = 'ATC - RxNorm' + AND invalid_reason IS NULL) +AND concept_name ~ ' / ' +AND class_code NOT IN ('J07AC01', 'J07BX03', 'J07BA01', 'J07BC02', 'J07AL01','J07AP02', 'J07CA10', 'J07AM51', 'J07BJ51', 'J07AJ52', 'J07AJ51', 'J07AH08', 'J07BD52', 'J07CA02', 'J07BD54'); -- 1220 + +-- wrong mono hepatitis A +DELETE +FROM class_to_drug_new +WHERE class_code = 'J07BC02' +AND concept_name ~ ' / '; + +-- wrong mono typhoid +DELETE +FROM class_to_drug_new +WHERE class_code = 'J07AP02' +AND concept_name ~ ' / '; + +-- wrong mono pneumococcus +DELETE +FROM class_to_drug_new +WHERE class_code = 'J07AL01' +AND concept_name ~ ' / '; -- 8 -- remove dead ATC Classes if any DELETE @@ -1953,7 +2031,36 @@ WHERE CTID NOT IN (SELECT MAX(CTID) concept_id); -- usgin the concep_order field from the class_to_drgu_new table, assemble the final table of class_to_drug and re-assign concept_order value in the following way: -/*1. Manual +/* +--==== class_to_drug_new ====-- +1 ATC Monocomp Class +2 Greedy ATC Monocomp Class +3 ATC Combo Class: Primary lateral in combination +4 ATC Combo Class: Primary upward in combination +5 ATC Combo Class: Primary lateral + Secondary lateral, 4 ingreds +50 ATC Combo Class: Primary lateral + Secondary upward, 4 ingreds +6 ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds +60 ATC Combo Class: Primary lateral + Secondary upward, 3 ingreds +7 ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds +70 ATC Combo Class: Primary lateral + Secondary upward, 2 ingreds +8 ATC Combo Class: Primary lateral in combination with excluded Ingredient +9 ATC Combo Class: Primary upward + Secondary upward +10 ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations +11 ATC Class to Drug Product from concept_relationship_manual +12 ATC Class with semi-manual point fix +13 ATC Class from old class_to_drug +14 Pack: Primary lateral in combo +16 Pack: Primary lateral + Secondary lateral +17 Pack: Primary lateral + Secondary upward +18 Additional Pack: from old c_t_d +20 ATC Mono WO Dose Form to Ingredient +21 ATC Monocomp Class to Drug Product which is out of hierarchy +22 ATC Combo Class to Drug Product which is out of drug hierarchy +23 ATC Combo Class to Drug Product which is out of drug hierarchy +24 ATC Combo Class to Ingredient + +--==== class_to_drug ====-- +1. Manual 2. Mono: Ingredient A; form 3. Mono: Ingredient A 4. Combo: Ingredient A + Ingredient B @@ -1962,7 +2069,6 @@ WHERE CTID NOT IN (SELECT MAX(CTID) 7. Combo: Ingredient A, combination 8. Any packs*/ --- be sure, that you have a backup of class_to_drug TRUNCATE TABLE class_to_drug; INSERT INTO class_to_drug SELECT DISTINCT class_code, @@ -1981,16 +2087,48 @@ SELECT DISTINCT class_code, AND concept_order <> 11 THEN 1 -- if ATC Mono has an entry in crm, its higher concept_order values have to be converted to 1 as well WHEN class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_order = 11 AND concept_name ~ ' / ' THEN 7 WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18 (note, that 19 does not exist) + WHEN concept_order = 21 AND concept_class_id !~ 'Box|Product' THEN 1 -- ATC Mono out of hierarchy (as manual) WHEN concept_order IN (11, 13) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual WHEN concept_order IN (1, 12) THEN 2 -- ATC Monocomp Class WHEN concept_order = 20 THEN 3 -- Mono: Ingredient A WHEN concept_order IN (5, 6, 7) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingredients) WHEN concept_order IN (9, 10, 50, 60, 70) THEN 5 -- Combo: Ingredient A OR Group A + group B WHEN concept_order IN (3, 4, 8) THEN 6 -- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient - WHEN concept_order IN (2, 21, 22, 23, 24) THEN 7 -- Combo: Ingredient A, combination + WHEN concept_order IN (2, 22, 23, 24) THEN 7 -- Combo: Ingredient A, combination END AS concept_order -FROM class_to_drug_new -; -- 116679 +FROM class_to_drug_new; + +-- fix wrong mono +UPDATE class_to_drug + SET "order" = 7 +WHERE class_code IN ('A01AB14','B02BD30') +AND concept_name ~ ' / ' +AND "order" <> 7; + +-- update wrong order for some manual links +WITH t1 AS +( + SELECT a.*, + c.concept_code + FROM class_to_drug a + JOIN concept c ON c.concept_id = a.concept_id +), +t2 as ( +SELECT * +FROM t1 +WHERE class_code||concept_code IN (SELECT concept_code_1||concept_code_2 + FROM concept_relationship_manual + WHERE relationship_id = 'ATC - RxNorm' + AND invalid_reason IS NULL) +AND concept_class_id !~ 'Pack' +AND "order" <> 1 +AND concept_name ~ ' / ' +AND class_code IN ('A10AC04','A10AD01','A10AD02','A10AD03','A10AD04','A10AD05','G03GA02','J07AG01','J07AH03','J07AH04','J07AH08', +'J07AL01','J07AL02','J07BB02','J07BB03','J07BH02','J07BM01','J07BM02','J07BM03','J07BF02', 'J07BF03')) + +UPDATE class_to_drug + SET "order" = 1 +WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM t2); -- 105 -- update class_to_drug in the schema of 'sources' SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); From 37a5148db1ce29edd0126e745a3aced2811fe17e Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sat, 2 Oct 2021 19:09:25 +0300 Subject: [PATCH 35/45] Update load_interim.sql --- ATC/load_interim.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 804ecdfa3..60e0aa908 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -2088,6 +2088,7 @@ SELECT DISTINCT class_code, WHEN class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_order = 11 AND concept_name ~ ' / ' THEN 7 WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18 (note, that 19 does not exist) WHEN concept_order = 21 AND concept_class_id !~ 'Box|Product' THEN 1 -- ATC Mono out of hierarchy (as manual) + WHEN concept_order = 21 AND concept_class_id ~ 'Box|Product' THEN 7 WHEN concept_order IN (11, 13) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual WHEN concept_order IN (1, 12) THEN 2 -- ATC Monocomp Class WHEN concept_order = 20 THEN 3 -- Mono: Ingredient A From 40d3511b31e07087d23373bed3a201b290fba54d Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sun, 3 Oct 2021 15:28:01 +0300 Subject: [PATCH 36/45] Update load_stage.sql --- ATC/load_stage.sql | 125 ++++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 65 deletions(-) diff --git a/ATC/load_stage.sql b/ATC/load_stage.sql index b9eb29aed..6f7eefb15 100644 --- a/ATC/load_stage.sql +++ b/ATC/load_stage.sql @@ -95,17 +95,17 @@ SELECT uppr.concept_code AS concept_code_1, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason FROM concept_stage uppr, - concept_stage lowr, - vocabulary v -WHERE uppr.invalid_reason IS null -AND lowr.invalid_reason IS null -- to exclude deprecated or updated from the hierarchy + concept_stage lowr, + vocabulary v +WHERE uppr.invalid_reason IS NULL +AND lowr.invalid_reason IS NULL -- to exclude deprecated or updated codes from the hierarchy AND ( (LENGTH(uppr.concept_code) IN (4, 5) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 1)) OR (LENGTH(uppr.concept_code) IN (3, 7) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 2)) ) AND uppr.vocabulary_id = 'ATC' AND lowr.vocabulary_id = 'ATC' - AND v.vocabulary_id = 'ATC'; -- 6493 + AND v.vocabulary_id = 'ATC'; -- 6495 -- add 'ATC - RxNorm' relationships between ATC Classes and RxN/RxE Drug Products using class_to_drug table INSERT INTO concept_relationship_stage @@ -119,26 +119,32 @@ INSERT INTO concept_relationship_stage valid_end_date, invalid_reason ) -SELECT DISTINCT cs.class_code AS concept_code_1, +WITH t1 AS +( + SELECT DISTINCT + cs.class_code AS concept_code_1, c.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, c.vocabulary_id AS vocabulary_id_2, 'ATC - RxNorm' AS relationship_id, CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','yyyymmdd') AS valid_end_date, + TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason -FROM class_to_drug cs -- manually curated table with ATC Class to Rx Drug Product links -JOIN concept_manual k ON k.concept_code = cs.class_code AND k.invalid_reason IS NULL +FROM sources.class_to_drug cs -- manually curated table with ATC Class to Rx Drug Product links +JOIN concept_manual k ON k.concept_code = cs.class_code +AND k.invalid_reason IS NULL JOIN concept c ON c.concept_id = cs.concept_id +WHERE c.concept_class_id != 'Ingredient' +) + SELECT * FROM t1 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.class_code - AND crs.vocabulary_id_1 = 'ATC' - AND crs.concept_code_2 = c.concept_code - AND crs.vocabulary_id_2 = c.vocabulary_id - AND crs.relationship_id = 'ATC - RxNorm' -- and crs.invalid_reason is null - ) -AND c.concept_class_id != 'Ingredient'; -- 107533 + WHERE crs.concept_code_1 = cs.concept_code_1 + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 + AND crs.concept_code_2 = cs.concept_code_2 + AND crs.vocabulary_id_2 = cs.vocabulary_id_2 + AND crs.relationship_id = cs.relationship_id + ); -- 115047 -- add 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between ATC Classes and RxN/RxE Drug Products (using input tables and dev_combo populated during previous Steps) INSERT INTO concept_relationship_stage @@ -152,7 +158,7 @@ INSERT INTO concept_relationship_stage valid_end_date, invalid_reason ) -with t1 as ( +WITH t1 AS ( SELECT DISTINCT SUBSTRING(irs.concept_code_1,'\w+') AS concept_code_1, c.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, @@ -177,11 +183,11 @@ SELECT * FROM t1 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = 'ATC - RxNorm pr lat' -- and crs.invalid_reason is null - ); -- 3292 + AND crs.relationship_id = cs.relationship_id + ); -- 3292 -- add 'ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up' for ATC Combo Classes using dev_combo INSERT INTO concept_relationship_stage @@ -220,11 +226,11 @@ SELECT DISTINCT * FROM t2 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id -- and crs.invalid_reason is null - ); -- 3670 + AND crs.relationship_id = cs.relationship_id + ); -- 3670 -- add 'ATC - RxNorm sec up' relationships for Primary lateral in combination INSERT INTO concept_relationship_stage @@ -260,7 +266,7 @@ SELECT class_code, class_name, concept_id FROM ing_pr_up_sec_up_excl WHERE rnk = 3 ), t2 AS ( -SELECT DISTINCT c.class_code AS concept_code_1, +SELECT DISTINCT k.concept_code AS concept_code_1, cc.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, cc.vocabulary_id AS vocabulary_id_2, @@ -268,7 +274,7 @@ SELECT DISTINCT c.class_code AS concept_code_1, CURRENT_DATE AS valid_start_date, TO_DATE('20991231','YYYYMMDD') AS valid_end_date, NULL AS invalid_reason -FROM class_to_drug c +FROM sources.class_to_drug c JOIN concept_ancestor ca ON ca.descendant_concept_id = c.concept_id JOIN concept cc ON cc.concept_id = ca.ancestor_concept_id @@ -285,13 +291,12 @@ SELECT * FROM t2 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = 'ATC - RxNorm sec up' - AND crs.invalid_reason = cs.invalid_reason - ); -- 23477 - + AND crs.relationship_id = cs.relationship_id + ); -- 23313 + -- deprecate links between ATC classes and dead RxN/RxE INSERT INTO concept_relationship_stage ( concept_code_1, @@ -303,7 +308,7 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) - with t1 AS ( +WITH t1 AS ( SELECT c.concept_code AS concept_code_1, cc.concept_code AS concept_code_2, c.vocabulary_id AS vocabulary_id_1, @@ -315,7 +320,7 @@ SELECT c.concept_code AS concept_code_1, FROM concept_relationship cr JOIN concept c ON concept_id_1 = c.concept_id JOIN concept cc ON concept_id_2 = cc.concept_id -AND cc.standard_concept is null -- non-standard +AND cc.standard_concept IS NULL -- non-standard WHERE c.vocabulary_id = 'ATC' AND cc.vocabulary_id LIKE 'RxNorm%' AND cr.invalid_reason IS NULL) @@ -323,11 +328,10 @@ SELECT * FROM t1 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - AND crs.invalid_reason = cs.invalid_reason); -- 8365 + AND crs.relationship_id = cs.relationship_id); -- 8365 -- deprecate accessory links for invalid codes INSERT INTO concept_relationship_stage ( @@ -355,19 +359,19 @@ FROM concept_relationship cr JOIN concept c ON concept_id_1 = c.concept_id JOIN concept cc ON concept_id_2 = cc.concept_id JOIN concept_manual k ON k.concept_code = c.concept_code - AND k.invalid_reason IS NOT null + AND k.invalid_reason IS NOT NULL WHERE c.vocabulary_id = 'ATC' -AND cr.invalid_reason IS NULL) +AND cr.invalid_reason IS NULL +AND c.concept_code <> 'H01BA06') -- deprecated argipressin SELECT * FROM t1 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 AND crs.relationship_id = cs.relationship_id - AND crs.invalid_reason = cs.invalid_reason - ); -- 1215 + ) ; -- 1213 -- add mirroring 'Maps to' for 'ATC - RxNorm pr lat' for monocomponent ATC Classes, which do not have doubling Standard ingredients (1-to-many mappings are permissive only for ATC Combo Classes) INSERT INTO concept_relationship_stage ( @@ -388,8 +392,8 @@ WITH t1 AS AND invalid_reason IS NULL ), t2 AS ( -SELECT DISTINCT a.concept_code_1, - a.concept_code_2, +SELECT DISTINCT a.concept_code_1, --k.concept_name, + a.concept_code_2, --d.concept_name, a.vocabulary_id_1, a.vocabulary_id_2, 'Maps to' AS relationship_id, @@ -412,7 +416,7 @@ SELECT * FROM t2 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 AND crs.relationship_id = cs.relationship_id @@ -442,8 +446,8 @@ WITH T1 AS WHERE rnk = 2 ), t2 AS ( -SELECT DISTINCT class_code AS concept_code_1, - concept_code AS concept_code_2, +SELECT DISTINCT class_code AS concept_code_1,--class_name, + concept_code AS concept_code_2,-- concept_name, 'ATC' AS vocabulary_id_1, vocabulary_id AS vocabulary_id_2, 'Maps to' AS relationship_id, @@ -460,12 +464,12 @@ SELECT * FROM t2 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = 'Maps to' + AND crs.relationship_id = cs.relationship_id ); -- 209 - + INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -476,7 +480,7 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -WITH T1 AS +WITH t1 AS ( SELECT class_code, class_name, @@ -489,8 +493,8 @@ WITH T1 AS WHERE rnk = 2 ), t2 AS ( -SELECT DISTINCT class_code AS concept_code_1, - concept_code AS concept_code_2, +SELECT DISTINCT class_code AS concept_code_1, --class_name, + concept_code AS concept_code_2,-- concept_name, 'ATC' AS vocabulary_id_1, vocabulary_id AS vocabulary_id_2, 'Maps to' AS relationship_id, @@ -510,10 +514,10 @@ SELECT * FROM t2 cs WHERE NOT EXISTS (SELECT 1 FROM concept_relationship_stage crs WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = 'ATC' + AND crs.vocabulary_id_1 = cs.vocabulary_id_1 AND crs.concept_code_2 = cs.concept_code_2 AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = 'Maps to' + AND crs.relationship_id = cs.relationship_id ) AND concept_code_1 NOT IN ('P01BF05', 'J07AG52', 'J07BD51') -- artenimol and piperaquine|hemophilus influenzae B, combinations with pertussis and toxoids; systemic|measles, combinations with mumps, live attenuated; systemic ; -- 193 @@ -534,7 +538,7 @@ JOIN sources.rxnconso r ON dv.concept_code = r.code AND r.code != 'NOCODE' AND r.lat = 'ENG' AND r.sab = 'ATC' - AND r.tty IN ('PT','IN'); + AND r.tty IN ('PT','IN'); -- 6440 -- perform mapping replacement using function below DO $_$ @@ -637,15 +641,6 @@ WHERE 'ATC' IN ( -- remove suspicious replacement mapping for OLD codes DELETE FROM concept_relationship_stage -WHERE concept_code_1 IN ('C10AA55','J05AE06','C10AA52','C10AA53','C10AA51','N02AX52', 'H01BA06') +WHERE concept_code_1 IN ('C10AA55', 'J05AE06', 'C10AA52', 'C10AA53', 'C10AA51', 'N02AX52', 'H01BA06') AND concept_code_2 IN ('17767','85762','7393','1191','161', '11149') -AND invalid_reason IS NULL; -- 6 - --- delete duplicate (to do: define its origin) -DELETE -FROM concept_relationship_stage -WHERE ctid NOT IN (SELECT MIN(ctid) - FROM concept_relationship_stage -GROUP BY concept_code_1, concept_code_2, relationship_id, invalid_reason - ) - and concept_code_1 = 'H01BA06';-- 1 +AND invalid_reason IS NULL; -- 7 From 3ebbf8bfb20932b7bad9ea4bf8468401da7e60dd Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sun, 3 Oct 2021 17:20:49 +0300 Subject: [PATCH 37/45] Update load_interim.sql --- ATC/load_interim.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 60e0aa908..91b2dc0f8 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -2007,6 +2007,12 @@ FROM class_to_drug_new WHERE class_code = 'J07AL01' AND concept_name ~ ' / '; -- 8 +-- wrong old mapping +DELETE +FROM class_to_drug_new +WHERE class_code = 'N01BB52' +AND concept_name ~* 'pack'; + -- remove dead ATC Classes if any DELETE FROM class_to_drug_new From 729b03bb4f39a2c8ff93dd9381585a0f739114dc Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sun, 3 Oct 2021 20:58:58 +0300 Subject: [PATCH 38/45] Update load_interim.sql --- ATC/load_interim.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 91b2dc0f8..630b1382b 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -1903,6 +1903,11 @@ AND concept_name !~* 'ring|insert|system|implant'; DELETE FROM wrong_df WHERE class_code IN ( 'C02AC01', 'G03GA08', 'R03AC13'); +DELETE +FROM wrong_df +WHERE class_name ~ 'local oral' +AND concept_name ~* 'paste|gel|oint'; + -- look at them and remove from class_to_drug_new DELETE FROM class_to_drug_new From 33d798ff60310c7f4729255b04e8aef7e494b33f Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sun, 3 Oct 2021 21:27:11 +0300 Subject: [PATCH 39/45] Update load_interim.sql --- ATC/load_interim.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index 630b1382b..c68df6430 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -2142,6 +2142,22 @@ UPDATE class_to_drug SET "order" = 1 WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM t2); -- 105 +UPDATE class_to_drug + SET "order" = 1 +WHERE class_code IN (SELECT class_code FROM class_to_drug WHERE "order" = 1) +AND class_code IN (SELECT class_code FROM class_to_drug WHERE "order" <> 1) +AND class_code IN (SELECT class_code FROM atc_all_mono) +AND concept_name !~ ' / ' +AND "order" <> 1; + +UPDATE class_to_drug + SET "order" = 1 +WHERE class_code IN (SELECT class_code FROM class_to_drug WHERE "order" = 1) +AND class_code IN (SELECT class_code FROM class_to_drug WHERE "order" <> 1) +AND class_code NOT IN (SELECT class_code FROM atc_all_mono) +AND concept_name ~ ' / ' +AND "order" <> 1; + -- update class_to_drug in the schema of 'sources' SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); From 2575201d78752019853b744dda8a056a467cce12 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Sun, 3 Oct 2021 22:50:04 +0300 Subject: [PATCH 40/45] Update load_input.sql --- ATC/load_input.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 58ee0a299..81ac1b1b8 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -15,7 +15,7 @@ * * Authors: Anna Ostropolets, Polina Talapova * Date: Jul 2021 -* Total script execution time: 27m 33s +* Total script execution time: 17m 11s **************************************************************************/ DROP TABLE IF EXISTS drug_concept_stage CASCADE; DROP TABLE IF EXISTS internal_relationship_stage; @@ -2083,6 +2083,6 @@ FROM internal_relationship_stage JOIN concept c ON lower(concept_code_2) = lower(c.concept_name) AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - AND c.invalid_reason IS NULL; -- 5701 + AND c.invalid_reason IS NULL; -- 5699 -- run load_interim.sql From 64d38ea56064d622000a386a4d40cb3e90988272 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Mon, 4 Oct 2021 12:26:46 +0300 Subject: [PATCH 41/45] Update load_input.sql datatype FLOAT has changed to NUMERIC --- ATC/load_input.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ATC/load_input.sql b/ATC/load_input.sql index 81ac1b1b8..df3ac79d7 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -44,7 +44,7 @@ CREATE TABLE relationship_to_concept vocabulary_id_1 VARCHAR(20), concept_id_2 INT, precedence INT, - conversion_factor FLOAT); + conversion_factor NUMERIC); -- create indexes AND constraints DROP INDEX if exists irs_concept_code_1; From 7eeec8ebfb0d335e7d123d9f696913cfc437ed28 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Mon, 4 Oct 2021 12:38:08 +0300 Subject: [PATCH 42/45] Update readme.md --- ATC/readme.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/ATC/readme.md b/ATC/readme.md index 9347ada64..f0432315c 100644 --- a/ATC/readme.md +++ b/ATC/readme.md @@ -3,14 +3,15 @@ #### Prerequisites: #### * Basic knowledge of the [ATC vocabulary as a part of OMOP CDM](https://www.ohdsi.org/web/wiki/doku.php?id=documentation:vocabulary:atc) * Schema DEVV5 with copies of tables concept, concept_relationship and concept_synonym from PRODV5, fully indexed -* SNOMED, RxNorm and RxNorm Extension must be loaded first with the full release cycle. -* Working directory, e.g. *dev_atc*. +* SNOMED, RxNorm and RxNorm Extension must be loaded first with the full release cycle +* Working directory, e.g. *dev_atc* #### I - Manual work #### -1. Create a backup of the **class_drugs_scraper** and **class_to_drug** tables. +1. Prepare the working environment using the content of *manual_work* folder +2. Create a backup of the **class_drugs_scraper** and **class_to_drug** tables ```sql CREATE TABLE class_drugs_scraper_mmyy AS SELECT * FROM class_drugs_scraper; -CREATE TABLE class_to_drug_old AS SELECT * FROM class_to_drug; -- note, that postfix '_old' is obligatory to add, because it is used in further steps. +CREATE TABLE sources_class_to_drug_old AS SELECT * FROM sources.class_to_drug; -- note, that postfix '_old' is obligatory to add, because it is used in further steps ``` 3. There are 3 options: **Scenario A** - addendum of new ATC codes @@ -30,28 +31,28 @@ SELECT id, FROM atc_addendum_MMYY; ``` **Scenario B** - changes of the existing ATC codes -- Insert them as new entities into the **class_drugs_scraper** table assigning them unique *ids* and preserving old versions of ATC codes. -**Scenario C** - use Scenario A together with Scenario B if both the addendum and changes are required. +- Insert them as new entities into the **class_drugs_scraper** table assigning them unique *ids* and preserving old versions of ATC codes +**Scenario C** - use Scenario A together with Scenario B if both the addendum and changes are required -3. Using the **class_drugs_scraper** table, add new codes with _OMOPized names_ and their *original versions* to the **concept_manual** and **concept_synonym_manual** tables respectively. +4. Using the **class_drugs_scraper** table, add new codes with _OMOPized names_ and their *original versions* to the **concept_manual** and **concept_synonym_manual** tables respectively #### II - Machinery #### -1. Run *load_input.sql*, which populates the **input tables** of **drug_concept_stage, internal_relationship_stage, relationship_to_concept** and ATC-specific **dev_combo**. -2. Run *load_interim.sql*, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products -3. Run the procedure which overwrites the new version of class_to_drug instead old one in the schema of 'sources' +5. Run *load_input.sql*, which populates the **input tables** of **drug_concept_stage, internal_relationship_stage, relationship_to_concept** and ATC-specific **dev_combo** +6. Run *load_interim.sql*, which prepares the **class_to_drug** table containing links bwetween ATC Drug Classes and RxN/RxE Drug Products +7. Run the procedure which overwrites the new version of class_to_drug instead old one in the schema of 'sources' ```sql SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); ``` -5. Run *load_stage.sql* which populates the **staging tables** of **concept_stage, concept_relationship_stage**, and **concept_synonym_stage** -6. Run *generic_update.sql* +8. Run *load_stage.sql* which populates the **staging tables** of **concept_stage, concept_relationship_stage**, and **concept_synonym_stage** +9. Run *generic_update.sql* ```sql SELECT devv5.genericupdate(); ``` -6. Perform **post-processing**: +10. Perform **post-processing**: ```sql DO $_$ BEGIN PERFORM VOCABULARY_PACK.pConceptAncestor(IS_SMALL=>TRUE); END $_$; ``` -7. enjoy! <3 +11. enjoy! From 67dc1ef1111046aab1aa1d51889ee75d9a9faf23 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Mon, 4 Oct 2021 12:53:07 +0300 Subject: [PATCH 43/45] Create create_manual_tables.sql manual tables added --- ATC/manual_work/create_manual_tables.sql | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 ATC/manual_work/create_manual_tables.sql diff --git a/ATC/manual_work/create_manual_tables.sql b/ATC/manual_work/create_manual_tables.sql new file mode 100644 index 000000000..f89490025 --- /dev/null +++ b/ATC/manual_work/create_manual_tables.sql @@ -0,0 +1,71 @@ +/************************************************************************** +* Copyright 2016 Observational Health Data Sciences and Informatics (OHDSI) +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* Authors: Vocabulary Team +* Date: 2021 +**************************************************************************/ +CREATE TABLE class_drugs_scraper +( + id INT, + class_code VARCHAR(100), + class_name VARCHAR, + ddd VARCHAR(100), + u VARCHAR(100), + adm_r VARCHAR(100), + note VARCHAR(100), + valid_start_date DATE, + valid_end_date DATE, + cgange_type VARCHAR(1) +); + +CREATE TABLE atc_inexistent +( + class_code VARCHAR, + class_name VARCHAR, + ing VARCHAR, + source_standard_concept VARCHAR, + "comment" VARCHAR +); + +CREATE TABLE atc_one_to_many_excl +( + atc_id INT, + atc_code VARCHAR, + atc_name VARCHAR, + relationship_id VARCHAR, + flag VARCHAR, + concept_id INT, + concept_code VARCHAR, + concept_name VARCHAR +); + + +CREATE TABLE missing +( + atc_id INT, + atc_code VARCHAR(50), + atc_name VARCHAR(255), + atc_class VARCHAR(20), + relationship_id VARCHAR(20), + concept_id INT, + concept_code VARCHAR(50), + concept_name VARCHAR(255), + concept_class_id VARCHAR(20), + domain_id VARCHAR(20), + vocabulary_id VARCHAR(20), + valid_start_date DATE, + valid_end_date DATE, + invalid_reason VARCHAR(1) +); From 413cc57ccf1d9aa83e7ce98ccbeda89f6a807615 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Mon, 4 Oct 2021 13:19:34 +0300 Subject: [PATCH 44/45] Create readme.md manual work --- ATC/manual_work/readme.md | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ATC/manual_work/readme.md diff --git a/ATC/manual_work/readme.md b/ATC/manual_work/readme.md new file mode 100644 index 000000000..71bc03d17 --- /dev/null +++ b/ATC/manual_work/readme.md @@ -0,0 +1,48 @@ +### STEP 1 of the ATC refresh/deployment: work with manual tables +* run *create_manual_tables.sql* +* extract the [respective csv files](https://drive.google.com/drive/folders/1TUfnmGCWj6d9KmJ5rZtGevQd0XsDgNQw?usp=sharing) into newly created tables. +* extract the [respective csv file](https://drive.google.com/file/d/1z1SdeGG80NX_QkziS7Rs8mjnMncDIpwn/view?usp=sharing) into the *concept_manual* table. The file was generated using the query: +```sql +SELECT concept_name, + domain_id, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + valid_start_date, + valid_end_date, + invalid_reason +FROM concept_manual +ORDER BY vocabulary_id, concept_code, invalid_reason, valid_start_date, valid_end_date, concept_name; +``` +* extract the [respective csv file](https://drive.google.com/file/d/1RYLXRCUU4OLYk6XERZjlntYe3k1s3EjJ/view?usp=sharing) into the *concept_relationship_manual* table. The file was generated using the query: +```sql +SELECT concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason +FROM concept_relationship_manual +ORDER BY vocabulary_id_1, vocabulary_id_2, relationship_id, concept_code_1, concept_code_2, invalid_reason, valid_start_date, valid_end_date; +``` +* extract the [respective csv file](https://drive.google.com/file/d/1Cb_WdgGMuqUcBW4URFdsrWV1zR-s8JZg/view?usp=sharing) into the *concept_synonym_manual* table. The file was generated using the query: +```sql +SELECT synonym_name, + synonym_concept_code, + synonym_vocabulary_id, + language_concept_id +FROM concept_synonym_manual +ORDER BY synonym_concept_code, + synonym_name; +``` +#### csv format: +- delimiter: ',' +- encoding: 'UTF8' +- header: ON +- decimal symbol: '.' +- quote escape: with backslash \ +- quote always: FALSE +- NULL string: empty From 8a089438d17b95dc3c137f6259101bf994bc49e1 Mon Sep 17 00:00:00 2001 From: Timur Date: Thu, 23 Jun 2022 13:35:51 +0300 Subject: [PATCH 45/45] refactoring, bugfixes, performance boost --- ATC/load_input.sql | 3985 ++++++++++------- ATC/load_interim.sql | 3860 +++++++++------- ATC/load_stage.sql | 906 ++-- .../vocabulary_pack/CreateTablesCopiesATC.sql | 14 + 4 files changed, 4986 insertions(+), 3779 deletions(-) create mode 100644 working/packages/vocabulary_pack/CreateTablesCopiesATC.sql diff --git a/ATC/load_input.sql b/ATC/load_input.sql index df3ac79d7..af9937876 100644 --- a/ATC/load_input.sql +++ b/ATC/load_input.sql @@ -14,888 +14,1266 @@ * limitations under the License. * * Authors: Anna Ostropolets, Polina Talapova -* Date: Jul 2021 -* Total script execution time: 17m 11s +* Date: 2022 **************************************************************************/ -DROP TABLE IF EXISTS drug_concept_stage CASCADE; +DROP TABLE IF EXISTS drug_concept_stage; DROP TABLE IF EXISTS internal_relationship_stage; -DROP TABLE IF EXISTS relationship_to_concept CASCADE; +DROP TABLE IF EXISTS relationship_to_concept; +DROP TABLE IF EXISTS concept_rx; -- ds_stage AND pc_stage are not used in the ATC deployment -CREATE TABLE drug_concept_stage -( concept_name VARCHAR(255), - vocabulary_id VARCHAR(20), - concept_class_id VARCHAR(20), - standard_concept VARCHAR(1), - concept_code VARCHAR(50), - possible_excipient VARCHAR(1), - domain_id VARCHAR(20), - valid_start_date DATE, - valid_end_date DATE, - invalid_reason VARCHAR(1), - source_concept_class_id VARCHAR(20)); - -CREATE TABLE internal_relationship_stage -( concept_code_1 VARCHAR(50), - concept_code_2 VARCHAR(50)); - -CREATE TABLE relationship_to_concept -( concept_code_1 VARCHAR(50), - vocabulary_id_1 VARCHAR(20), - concept_id_2 INT, - precedence INT, - conversion_factor NUMERIC); - --- create indexes AND constraints -DROP INDEX if exists irs_concept_code_1; -DROP INDEX if exists irs_concept_code_2; -DROP INDEX if exists dcs_concept_code; -DROP INDEX if exists ds_drug_concept_code; -DROP INDEX if exists ds_ingredient_concept_code; -DROP INDEX if exists dcs_unique_concept_code; -DROP INDEX if exists irs_unique_concept_code; - -CREATE INDEX irs_concept_code_1 - ON internal_relationship_stage (concept_code_1); -CREATE INDEX irs_concept_code_2 - ON internal_relationship_stage (concept_code_2); -CREATE INDEX dcs_concept_code - ON drug_concept_stage (concept_code); -CREATE UNIQUE INDEX dcs_unique_concept_code - ON drug_concept_stage (concept_code); -CREATE INDEX irs_unique_concept_code - ON internal_relationship_stage (concept_code_1, concept_code_2); +CREATE TABLE drug_concept_stage ( + concept_name VARCHAR(255), + vocabulary_id VARCHAR(20), + concept_class_id VARCHAR(20), + standard_concept VARCHAR(1), + concept_code VARCHAR, --increase the length for concept_code field to infinity + possible_excipient VARCHAR(1), + domain_id VARCHAR(20), + valid_start_date DATE, + valid_end_date DATE, + invalid_reason VARCHAR(1), + source_concept_class_id VARCHAR(20) + ); + +CREATE TABLE internal_relationship_stage + --increase the length for concept_code_1 and concept_code_2 fields to infinity + ( + concept_code_1 VARCHAR, + concept_code_2 VARCHAR + ); + +CREATE TABLE relationship_to_concept ( + concept_code_1 VARCHAR, --increase the length for concept_code_1 field to infinity + vocabulary_id_1 VARCHAR(20), + concept_id_2 INT, + precedence INT2, + conversion_factor NUMERIC + ); + +--create a small table to speed up some queries +CREATE UNLOGGED TABLE concept_rx AS +SELECT * +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%'; + +CREATE INDEX idx_rx_name ON concept_rx (UPPER(concept_name)) WITH (FILLFACTOR=100); +CREATE INDEX idx_rx_concept_id ON concept_rx (concept_id) WITH (FILLFACTOR=100); +CREATE INDEX idx_rx_concept_class_id ON concept_rx (concept_class_id) WITH (FILLFACTOR=100) WHERE concept_class_id IN ('Ingredient', 'Dose Form'); + +ANALYZE concept_rx; + + /************************************************* ***** Mono ATC to internal_relationship_stage **** **************************************************/ + ---------------- -- Dose Forms -- ---------------- --- increase the LENGTH for concept_code_1 AND concept_code_2 fields to infinity -ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_1 TYPE VARCHAR; -ALTER TABLE internal_relationship_stage ALTER COLUMN concept_code_2 TYPE VARCHAR; --- create a temporary table WITH all ATC-related RxN/RxE Dose Forms (using Dose Form Groups) -DROP TABLE if exists dev_form; -CREATE TABLE dev_form -AS ( -with dev_oral -- 1 - Oral forms -as -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 IN (36217214, 36244020, 36217223, 36244035, 36217215, 36244032, 36244021, 36244031, -- Oral Product |Buccal Product |Paste product |Chewable Product|Dental Product|Disintegrating Oral Product|Lozenge Product|Wafer Product - 36244030 , 36244029, 36217220, 36244027, 36244036, 36244033, 36217216) -- Oral Powder Product|Oral Paste Product|Oral Liquid Product|Granule Product|Flake Product|Pellet Product|Pill -AND relationship_id = 'RxNorm inverse is a' -),-- sublingual route is included as well despite the fact it is processed separately -dev_sub -- 2 - Sublingual forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 in (36217214, 36244020) -- Sublingual Product|Buccal Product -AND relationship_id = 'RxNorm inverse is a' -AND d.concept_name ~* 'sublingual' -), -- should be separated FROM oral forms in the ATC vocabulary. - dev_parenteral -- 3 - Parenteral forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 in ( 36217210, 36248213, 36217221) -- Injectable Product|Intratracheal Product|Intraperitoneal Product -AND relationship_id = 'RxNorm inverse is a'), -- returns all children of Injectable Product - dev_nasal -- 4 - Nasal forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217213 -- Nasal Product -AND relationship_id = 'RxNorm inverse is a' -), -- returns all children of Nasal Product -dev_topic -- 5 - Topical forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 IN ( - 36217206,36244040,36244034,36217219, 36217222, 36217208, -- Topical Product|Soap Product|Shampoo Product|Drug Implant Product|Irrigation Product |Medicated Pad or Tape - 36217223,36217212,36217224,36217225,1146249, 36217221) -- Paste Product|Mucosal Product|Prefilled Applicator Product|Urethral Product|Pyelocalyceal Product|Intraperitoneal Product -AND relationship_id = 'RxNorm inverse is a'), - dev_mouth -- 6 - Local oral forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 in (36244022, 36217223, 36217214, 36244020, 36217215, 36244021, 36244026,-- Mouthwash Product|Paste Product|Sublingual Product|Buccal Product|Dental Product|Lozenge Product|Toothpaste Product - 36244037, 36244023, 36244041, 37498345, 36244028, 36244024) -- Oral Spray Product|Oral Ointment Product|Oral Gel Product |Oral Film Product| Oral Foam Product| Oral Cream Product -AND relationship_id = 'RxNorm inverse is a' -AND d.concept_name ~* 'mouthwash'), - dev_rectal -- 7 - Rectal forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217211 -- Rectal Product -AND relationship_id = 'RxNorm inverse is a' -), -dev_vaginal -- 8 - Vaginal forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' - AND d.invalid_reason IS NULL -WHERE concept_id_1 = 36217209 -AND relationship_id = 'RxNorm inverse is a'), -- Vaginal Product - dev_urethral AS -- 9 - Urethral forms -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217225 -- Urethral Product -AND relationship_id = 'RxNorm inverse is a'), - dev_opht -- 10 - Ophthalmic forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 in (36217218, 36217224) -- Ophthalmic Product | Prefilled Applicator (Dose Form Group) -AND relationship_id = 'RxNorm inverse is a' -AND d.concept_name ~* 'ophthalmic'), -dev_otic -- 11 - Otic forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = 36217217 -- Otic Product (Dose Form Group) -AND relationship_id = 'RxNorm inverse is a' -), -dev_inhal -- 12 - Inhalation forms -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 IN (36217207, 36244037) -- Inhalant Product| Oral Spray Product -AND relationship_id = 'RxNorm inverse is a' -), -dev_irrig -AS -(SELECT DISTINCT d.* -FROM concept_relationship r - JOIN concept c ON c.concept_Id = r.concept_id_1 - JOIN concept d - ON d.concept_id = r.concept_id_2 - AND d.invalid_reason IS NULL - AND d.concept_class_id = 'Dose Form' -WHERE concept_id_1 = '36217222' -- Irrigation Product -AND relationship_id = 'RxNorm inverse is a' -) -SELECT * FROM ( -SELECT *, 'dev_oral' as df FROM dev_oral -- 1 +-- create a temporary table with all ATC-related RxN/RxE Dose Forms (using Dose Form Groups) +DROP TABLE IF EXISTS dev_form; +CREATE UNLOGGED TABLE dev_form AS + WITH dev_oral -- 1 - Oral forms + AS ( + SELECT DISTINCT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 IN ( + 36217214, + 36244020, + 36217223, + 36244035, + 36217215, + 36244032, + 36244021, + 36244031, -- Oral Product |Buccal Product |Paste product |Chewable Product|Dental Product|Disintegrating Oral Product|Lozenge Product|Wafer Product + 36244030, + 36244029, + 36217220, + 36244027, + 36244036, + 36244033, + 36217216 + ) -- Oral Powder Product|Oral Paste Product|Oral Liquid Product|Granule Product|Flake Product|Pellet Product|Pill + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), -- sublingual route is included as well despite the fact it is processed separately + dev_sub -- 2 - Sublingual forms + AS ( + SELECT DISTINCT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 IN ( + 36217214, + 36244020 + ) -- Sublingual Product|Buccal Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + AND d.concept_name ILIKE '%sublingual%' + ), -- should be separated FROM oral forms in the ATC vocabulary. + dev_parenteral -- 3 - Parenteral forms + AS ( + SELECT DISTINCT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 IN ( + 36217210, + 36248213, + 36217221 + ) -- Injectable Product|Intratracheal Product|Intraperitoneal Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), -- returns all children of Injectable Product + dev_nasal -- 4 - Nasal forms + AS ( + SELECT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 = 36217213 -- Nasal Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), -- returns all children of Nasal Product + dev_topic -- 5 - Topical forms + AS ( + SELECT DISTINCT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 IN ( + 36217206, + 36244040, + 36244034, + 36217219, + 36217222, + 36217208, -- Topical Product|Soap Product|Shampoo Product|Drug Implant Product|Irrigation Product |Medicated Pad or Tape + 36217223, + 36217212, + 36217224, + 36217225, + 1146249, + 36217221 + ) -- Paste Product|Mucosal Product|Prefilled Applicator Product|Urethral Product|Pyelocalyceal Product|Intraperitoneal Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), + dev_mouth -- 6 - Local oral forms + AS ( + SELECT DISTINCT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 IN ( + 36244022, + 36217223, + 36217214, + 36244020, + 36217215, + 36244021, + 36244026, -- Mouthwash Product|Paste Product|Sublingual Product|Buccal Product|Dental Product|Lozenge Product|Toothpaste Product + 36244037, + 36244023, + 36244041, + 37498345, + 36244028, + 36244024 + ) -- Oral Spray Product|Oral Ointment Product|Oral Gel Product |Oral Film Product| Oral Foam Product| Oral Cream Product + AND r.relationship_id = 'RxNorm inverse is a' + AND d.concept_name ILIKE '%mouthwash%' + AND r.invalid_reason IS NULL + ), + dev_rectal -- 7 - Rectal forms + AS ( + SELECT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 = 36217211 -- Rectal Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), + dev_vaginal -- 8 - Vaginal forms + AS ( + SELECT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 = 36217209 + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), -- Vaginal Product + dev_urethral AS -- 9 - Urethral forms + ( + SELECT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 = 36217225 -- Urethral Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), + dev_opht -- 10 - Ophthalmic forms + AS ( + SELECT DISTINCT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 IN ( + 36217218, + 36217224 + ) -- Ophthalmic Product | Prefilled Applicator (Dose Form Group) + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + AND d.concept_name ILIKE '%ophthalmic%' + ), + dev_otic -- 11 - Otic forms + AS ( + SELECT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 = 36217217 -- Otic Product (Dose Form Group) + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), + dev_inhal -- 12 - Inhalation forms + AS ( + SELECT DISTINCT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 IN ( + 36217207, + 36244037 + ) -- Inhalant Product| Oral Spray Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ), + dev_irrig AS ( + SELECT d.* + FROM concept_relationship r + JOIN concept c ON c.concept_id = r.concept_id_1 + JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.invalid_reason IS NULL + AND d.concept_class_id = 'Dose Form' + WHERE r.concept_id_1 = '36217222' -- Irrigation Product + AND r.relationship_id = 'RxNorm inverse is a' + AND r.invalid_reason IS NULL + ) + +SELECT *, + 'dev_oral' AS df +FROM dev_oral -- 1 + UNION ALL -SELECT *, 'dev_sub' FROM dev_sub -- 2 + +SELECT *, + 'dev_sub' +FROM dev_sub -- 2 + UNION ALL -SELECT *, 'dev_parenteral' FROM dev_parenteral -- 3 + +SELECT *, + 'dev_parenteral' +FROM dev_parenteral -- 3 + UNION ALL -SELECT *, 'dev_nasal' FROM dev_nasal -- 4 + +SELECT *, + 'dev_nasal' +FROM dev_nasal -- 4 + UNION ALL -SELECT *, 'dev_topic' FROM dev_topic -- 5 + +SELECT *, + 'dev_topic' +FROM dev_topic -- 5 + UNION ALL -SELECT *, 'dev_mouth' FROM dev_mouth -- 6 + +SELECT *, + 'dev_mouth' +FROM dev_mouth -- 6 + UNION ALL -SELECT *,'dev_rectal' FROM dev_rectal -- 7 + +SELECT *, + 'dev_rectal' +FROM dev_rectal -- 7 + UNION ALL -SELECT *, 'dev_vaginal' FROM dev_vaginal -- 8 + +SELECT *, + 'dev_vaginal' +FROM dev_vaginal -- 8 + UNION ALL -SELECT *, 'dev_urethral' FROM dev_urethral -- 9 + +SELECT *, + 'dev_urethral' +FROM dev_urethral -- 9 + UNION ALL -SELECT *, 'dev_opht' FROM dev_opht -- 10 + +SELECT *, + 'dev_opht' +FROM dev_opht -- 10 + UNION ALL -SELECT *, 'dev_otic' FROM dev_otic -- 11 + +SELECT *, + 'dev_otic' +FROM dev_otic -- 11 + UNION ALL -SELECT *, 'dev_inhal' FROM dev_inhal -- 12 + +SELECT *, + 'dev_inhal' +FROM dev_inhal -- 12 + UNION ALL -SELECT *, 'dev_irrig' FROM dev_irrig -- 13 -)l); + +SELECT *, + 'dev_irrig' +FROM dev_irrig; -- 13 -- connect all existing RxN/RxE forms of interest from dev_form to the ATC -DROP TABLE if exists atc_to_form; -CREATE TABLE atc_to_form AS -SELECT DISTINCT -a.concept_name, a.concept_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code -FROM concept_manual a, - dev_form b -WHERE a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th' -AND -(( a.concept_name ~* 'oral|systemic|chewing gum' and b.df = 'dev_oral') -- 1 -OR (a.concept_name ~* 'sublingual' and b.df = 'dev_sub') -- 2 -OR (a.concept_name ~* 'parenteral|systemic|instillat' and b.df = 'dev_parenteral') -- 3 -OR (a.concept_name ~* 'nasal'and b.df = 'dev_nasal') -- 4 -OR (a.concept_name ~* 'topical' AND b.df = 'dev_topic') -- 5 -OR (a.concept_name ~* 'transdermal|implant|systemic' AND b.df = 'dev_topical' and b.concept_name ~* 'transdermal|Drug Implant') -OR (a.concept_name ~* 'local oral' AND b.df = 'dev_mouth') -- 7 -OR (a.concept_name ~* 'rectal' AND b.df = 'dev_rectal') -OR (a.concept_name ~* 'vaginal' AND b.df = 'dev_vaginal') -OR (a.concept_name ~* 'urethral' AND b.df = 'dev_urethral') -OR (a.concept_name ~* 'ophthalmic' AND b.df = 'dev_opht') -OR (a.concept_name ~* '\yotic' AND b.df = 'dev_otic') -OR (a.concept_name ~* 'inhalant|systemic' AND b.df = 'dev_inhal') -OR (a.concept_name ~* '\yirrigat' AND b.df = 'dev_irrig') -); +DROP TABLE IF EXISTS atc_to_form; +CREATE UNLOGGED TABLE atc_to_form AS +SELECT DISTINCT a.concept_name, + a.concept_code || ' ' || b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + b.concept_name AS concept_code_2 -- OMOP Dose Form name treated AS a code +FROM dev_form b +JOIN concept_manual a ON a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' +WHERE ( + ( + a.concept_name ~* 'oral|systemic|chewing gum' + AND b.df = 'dev_oral' + ) -- 1 + OR ( + a.concept_name ILIKE '%sublingual%' + AND b.df = 'dev_sub' + ) -- 2 + OR ( + a.concept_name ~* 'parenteral|systemic|instillat' + AND b.df = 'dev_parenteral' + ) -- 3 + OR ( + a.concept_name ILIKE '%nasal%' + AND b.df = 'dev_nasal' + ) -- 4 + OR ( + a.concept_name ILIKE '%topical%' + AND b.df = 'dev_topic' + ) -- 5 + OR ( + a.concept_name ~* 'transdermal|implant|systemic' + AND b.df = 'dev_topical' + AND b.concept_name ~* 'transdermal|Drug Implant' + ) + OR ( + a.concept_name ILIKE '%local oral%' + AND b.df = 'dev_mouth' + ) -- 7 + OR ( + a.concept_name ILIKE '%rectal%' + AND b.df = 'dev_rectal' + ) + OR ( + a.concept_name ILIKE '%vaginal%' + AND b.df = 'dev_vaginal' + ) + OR ( + a.concept_name ILIKE '%urethral%' + AND b.df = 'dev_urethral' + ) + OR ( + a.concept_name ILIKE '%ophthalmic%' + AND b.df = 'dev_opht' + ) + OR ( + a.concept_name ~* '\yotic' + AND b.df = 'dev_otic' + ) + OR ( + a.concept_name ~* 'inhalant|systemic' + AND b.df = 'dev_inhal' + ) + OR ( + a.concept_name ~* '\yirrigat' + AND b.df = 'dev_irrig' + ) + ); -- add links connecting ATC Drug Classes with the specified administration route AND RxN/RxE Dose Forms using atc_to_form -TRUNCATE internal_relationship_stage; -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -SELECT DISTINCT concept_code_1, concept_code_2 FROM atc_to_form -WHERE (concept_code_1, concept_code_2) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) +SELECT DISTINCT concept_code_1, + concept_code_2 +FROM atc_to_form; + +CREATE INDEX idx_irs_complex ON internal_relationship_stage ( + concept_code_1, + concept_code_2 + ); + +CREATE INDEX idx_irs_up_cc2 ON internal_relationship_stage (UPPER(concept_code_2)); + +ANALYZE internal_relationship_stage; -- add links connecting ATC Drug Classes with the specified administration route AND Standard Ingredients using the concept_synonym table INSERT INTO internal_relationship_stage SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code - FROM atc_to_form a - JOIN concept c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name,';.*$',''))) -- remove all unnecessary information after the semicolon - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code +FROM atc_to_form a +JOIN concept_rx c ON UPPER(c.concept_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name, ';.*$', ''))) -- remove all unnecessary information after the semicolon + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +LEFT JOIN internal_relationship_stage irs ON irs.concept_code_1 = a.concept_code_1 + AND irs.concept_code_2 = c.concept_name +WHERE irs.concept_code_1 IS NULL; -- add links between ATC Drug Classes AND Standard Ingredients using the concept_synonym table INSERT INTO internal_relationship_stage - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code - FROM atc_to_form a - JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = upper(TRIM(REGEXP_REPLACE(a.concept_name, ';.*$', ''))) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL -WHERE (concept_code_1, c.concept_name) not in (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code +FROM atc_to_form a +JOIN concept_synonym b ON UPPER(b.concept_synonym_name) = UPPER(TRIM(REGEXP_REPLACE(a.concept_name, ';.*$', ''))) +JOIN concept_rx c ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +LEFT JOIN internal_relationship_stage irs ON irs.concept_code_1 = a.concept_code_1 + AND irs.concept_code_2 = c.concept_name +WHERE irs.concept_code_1 IS NULL; + ----------------------------- -- Mono ATC W/O Dose Forms -- ----------------------------- + -- add IRS links connecting ATC Drug Classes W/O Dose Forms and Standard Ingredients using the concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) SELECT DISTINCT a.concept_code AS concept_code_1, -- for such Drug Classes use ATC code only - c.concept_name AS concept_code_2 + c.concept_name AS concept_code_2 FROM concept_manual a - JOIN concept c - ON TRIM(UPPER (REGEXP_REPLACE (c.concept_name,'\s+|\W+','','g'))) = TRIM( UPPER (REGEXP_REPLACE (a.concept_name,'\s+|\W+| \(.*\)|, combinations.*|;.*$','','g'))) -- to neglect spaces, non-word characters, additional information and dose forms - AND a.concept_class_id = 'ATC 5th' - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (a.concept_code, c.concept_name) not in (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); +JOIN concept_rx c ON TRIM(UPPER(REGEXP_REPLACE(c.concept_name, '\s+|\W+', '', 'g'))) = TRIM(UPPER(REGEXP_REPLACE(a.concept_name, '\s+|\W+| \(.*\)|, combinations.*|;.*$', '', 'g'))) -- to neglect spaces, non-word characters, additional information and dose forms + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE a.concept_class_id = 'ATC 5th' + AND ( + a.concept_code, + c.concept_name + ) NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 + FROM internal_relationship_stage + ); -- add links connecting ATC Drug Classes W/O Dose Forms and Standard Ingredients using the concept_synonym table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) SELECT DISTINCT a.concept_code AS concept_code_1, -- for such Drug Classes use ATC code only - c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name AS a code FROM concept_manual a - JOIN concept_synonym b ON TRIM(UPPER (REGEXP_REPLACE (b.concept_synonym_name,'\s+|\W+','','g'))) = TRIM(UPPER (REGEXP_REPLACE (a.concept_name,'\s+|\W+|, combinations.*|;.*$','','g'))) - JOIN concept c - ON c.concept_id = b.concept_id - AND a.concept_class_id = 'ATC 5th' - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' - AND c.invalid_reason IS NULL - WHERE (a.concept_code, c.concept_name) not IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); - --- note, name matching with Non-standard OMOP drugs and cross-walk to Standard via concept_relationship gives a lot of errors (clean up is required). That is why this step is ignored here. +JOIN concept_synonym b ON TRIM(UPPER(REGEXP_REPLACE(b.concept_synonym_name, '\s+|\W+', '', 'g'))) = TRIM(UPPER(REGEXP_REPLACE(a.concept_name, '\s+|\W+|, combinations.*|;.*$', '', 'g'))) +JOIN concept_rx c ON c.concept_id = b.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +WHERE a.concept_class_id = 'ATC 5th' + AND ( + a.concept_code, + c.concept_name + ) NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 + FROM internal_relationship_stage + ); + +ANALYZE internal_relationship_stage; + +-- note, name matching with Non-standard OMOP drugs and cross-walk to Standard via concept_relationship gives a lot of errors (clean up is required). That is why this step is ignored here. /************************** **** ATC Combo Classes **** ***************************/ + -- assemble all Multicomponent ATC Classes into one table -DROP TABLE if exists combo_pull; -CREATE TABLE combo_pull -AS -(WITH t1 -AS -(SELECT DISTINCT concept_code AS class_code, - concept_name AS class_name, - SPLIT_PART(concept_name,';','1') AS nm +DROP TABLE IF EXISTS combo_pull; +CREATE UNLOGGED TABLE combo_pull AS +SELECT DISTINCT concept_code AS class_code, + concept_name AS class_name, + SPLIT_PART(concept_name, ';', 1) AS nm FROM concept_manual WHERE ( -SPLIT_PART(concept_name,';','1') ~* ' and |\ywith|\ycomb|preparations|acids|animals|antiinfectives|compounds|lytes\y|flowers|grass pollen|\yresins|tree pollen|dust mites|multienzymes' -OR SPLIT_PART(concept_name,';','1') ~* 'organisms|antiseptics|feather|diastase|emulsions|immunoglobulins|substitutes|glycosides|cannabinoids|typhoid-|\ydrugs|\/|antiserum' -OR SPLIT_PART(concept_name,';','1') ~* 'diphtheria-|carbohydrates|diphtheria-hepatitis B|edetates|insects|medicated shampoos|phospholipids|various|bacillus|^oil|alkaloids' - ) -AND invalid_reason IS NULL -AND concept_class_id = 'ATC 5th' -AND concept_name !~* 'varicella/zoster|tositumomab/iodine|\yIUD\y|ferric oxide polymaltose' -AND concept_code <> 'G02BB02' -- vaginal ring with progestogen; vaginal -) -SELECT*FROM t1); + SPLIT_PART(concept_name, ';', 1) ~* ' and |\ywith|\ycomb|preparations|acids|animals|antiinfectives|compounds|lytes\y|flowers|grass pollen|\yresins|tree pollen|dust mites|multienzymes' + OR SPLIT_PART(concept_name, ';', 1) ~* 'organisms|antiseptics|feather|diastase|emulsions|immunoglobulins|substitutes|glycosides|cannabinoids|typhoid-|\ydrugs|\/|antiserum' + OR SPLIT_PART(concept_name, ';', 1) ~* 'diphtheria-|carbohydrates|diphtheria-hepatitis B|edetates|insects|medicated shampoos|phospholipids|various|bacillus|^oil|alkaloids' + ) + AND invalid_reason IS NULL + AND concept_class_id = 'ATC 5th' + AND concept_name !~* 'varicella/zoster|tositumomab/iodine|\yIUD\y|ferric oxide polymaltose' + AND concept_code <> 'G02BB02';-- vaginal ring with progestogen; vaginal -- create a table for Multicomponent ATC Class mappings to Standard Ingredient, start with the 1st ATC Combo Ingredient using the concept table and full name match -DROP TABLE if exists dev_combo; -CREATE TABLE dev_combo -AS -(SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ',1) AS class, - c.concept_id, - c.concept_name, - 1 AS rnk -- stands for the Primary lateral relationship - FROM combo_pull a - JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (nm,' and ',1))) +DROP TABLE IF EXISTS dev_combo; +CREATE UNLOGGED TABLE dev_combo AS +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 1) AS class, + c.concept_id, + c.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship +FROM combo_pull a +JOIN concept c ON UPPER(c.concept_name) = TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 1))) WHERE c.standard_concept = 'S' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx'); + AND c.concept_class_id = 'Ingredient' + AND c.vocabulary_id LIKE 'RxNorm%'; -- add the 1st ATC Combo Ingredient using the concept table and full name match INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ',1) AS class, - c.concept_id, - c.concept_name, - 1 AS rnk -- stands for the Primary lateral relationship +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 1) AS class, + c.concept_id, + c.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship FROM combo_pull a - JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 1))),'\w*\s*-?\s*\w+') -WHERE c.standard_concept = 'S' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx' -AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); +JOIN concept_rx c ON UPPER(c.concept_name) = SUBSTRING(TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 1))), '\w*\s*-?\s*\w+') + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +LEFT JOIN dev_combo d USING (class_code, concept_id) +WHERE d.class_code IS NULL; -- add the 1st ATC Combo Ingredient using the concept_synonym table and full name match INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ', 1) AS class, - d.concept_id, - d.concept_name, - 1 AS rnk -- stands for the Primary lateral relationship +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 1) AS class, + d.concept_id, + d.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship FROM combo_pull a -JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE(TRIM(lower(SPLIT_PART (nm,' and ', 1))), SUBSTRING(TRIM(lower(SPLIT_PART (nm,' and ', 1))), '\w*\s*-?\s*\w+')) -JOIN concept d ON d.concept_id = cs.concept_id -WHERE d.standard_concept = 'S' -AND d.concept_class_id = 'Ingredient' -AND d.vocabulary_id ~ 'Rx' -AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); +JOIN concept_synonym cs ON UPPER(cs.concept_synonym_name) IN ( + TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 1))), + SUBSTRING(TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 1))), '\w*\s*-?\s*\w+') + ) +JOIN concept_rx d ON d.concept_id = cs.concept_id + AND d.standard_concept = 'S' + AND d.concept_class_id = 'Ingredient' +LEFT JOIN dev_combo dc ON dc.class_code = a.class_code + AND dc.concept_id = d.concept_id +WHERE dc.class_code IS NULL; -- add the the 1st ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ',1) AS class, - c.concept_id, - c.concept_name, - 1 AS rnk -- stands for the Primary lateral relationship +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 1) AS class, + c.concept_id, + c.concept_name, + 1 AS rnk -- stands for the Primary lateral relationship FROM combo_pull a - JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 1))),'^\w+') -WHERE c.standard_concept = 'S' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx' -AND (class_code) NOT IN (select class_code FROM dev_combo); +JOIN concept_rx c ON UPPER(c.concept_name) = SUBSTRING(TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 1))), '^\w+') + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +LEFT JOIN dev_combo d USING (class_code) +WHERE d.class_code IS NULL; -- add the 2nd ATC Combo Ingredient using the concept table and full name match INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ', 2) AS class, - c.concept_id, - c.concept_name, - 2 AS rnk -- stands for the Secondary lateral relationship +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 2) AS class, + c.concept_id, + c.concept_name, + 2 AS rnk -- stands for the Secondary lateral relationship FROM combo_pull a - JOIN concept c ON lower (c.concept_name) = TRIM (lower (SPLIT_PART (a.nm,' and ', 2))) +JOIN concept_rx c ON UPPER(c.concept_name) = TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 2))) WHERE c.standard_concept = 'S' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx'; + AND c.concept_class_id = 'Ingredient'; -- add the 2nd ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ', 2) AS class, - c.concept_id, - c.concept_name, - 2 AS rnk +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 2) AS class, + c.concept_id, + c.concept_name, + 2 AS rnk FROM combo_pull a - JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 2))),'\w*\s*-?\s*\w+') -WHERE c.standard_concept = 'S' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx' -AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND c.concept_id NOT IN (19049024, 19136048); +JOIN concept_rx c ON UPPER(c.concept_name) = SUBSTRING(TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 2))), '\w*\s*-?\s*\w+') + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 19049024, + 19136048 + ) +LEFT JOIN dev_combo d USING (class_code, concept_id) +WHERE d.class_code IS NULL; -- add the 2nd ATC Combo Ingredient using the concept_synonym table and partial name match INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ',2) AS class, - d.concept_id, - d.concept_name, - 2 AS rnk +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 2) AS class, + d.concept_id, + d.concept_name, + 2 AS rnk FROM combo_pull a -JOIN concept_synonym cs ON lower(cs.concept_synonym_name) = COALESCE ( TRIM(lower(SPLIT_PART (nm,' and ', 2))), SUBSTRING (TRIM(lower(SPLIT_PART (nm,' and ', 2))), '\w*\s*-?\s*\w+')) -JOIN concept d ON d.concept_id = cs.concept_id -WHERE d.standard_concept = 'S' -AND d.concept_class_id = 'Ingredient' -AND d.vocabulary_id ~ '^Rx' -AND (class_code, d.concept_id) NOT IN (select class_code, concept_id FROM dev_combo); +JOIN concept_synonym cs ON UPPER(cs.concept_synonym_name) IN ( + TRIM(LOWER(SPLIT_PART(a.nm, ' and ', 2))), + SUBSTRING(TRIM(LOWER(SPLIT_PART(a.nm, ' and ', 2))), '\w*\s*-?\s*\w+') + ) +JOIN concept_rx d ON d.concept_id = cs.concept_id + AND d.standard_concept = 'S' + AND d.concept_class_id = 'Ingredient' +LEFT JOIN dev_combo dc ON dc.class_code = a.class_code + AND dc.concept_id = d.concept_id +WHERE dc.class_code IS NULL; -- add to dev_combo the 2nd ATC Combo Ingredient using the concept table and partial name match INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - SPLIT_PART(nm,' and ', 2) AS class, - c.concept_id, - c.concept_name, - 2 AS rnk +SELECT DISTINCT a.class_code, + a.class_name, + SPLIT_PART(a.nm, ' and ', 2) AS class, + c.concept_id, + c.concept_name, + 2 AS rnk FROM combo_pull a - JOIN concept c ON lower (c.concept_name) = SUBSTRING (TRIM (lower (SPLIT_PART (nm,' and ', 2))),'^\w+') -WHERE c.standard_concept = 'S' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id ~ 'Rx' -AND (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND c.concept_id NOT IN (19049024, 19136048) -and class_code in ('R03AK10', 'R03AL01', 'R03AL03', 'R03AL04', 'R03AL05', 'R03AL06', 'R03AL07', 'R03AL10'); +JOIN concept_rx c ON UPPER(c.concept_name) = SUBSTRING(TRIM(UPPER(SPLIT_PART(a.nm, ' and ', 2))), '^\w+') + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 19049024, + 19136048 + ) +LEFT JOIN dev_combo d USING (class_code, concept_id) +WHERE d.class_code IS NULL + AND a.class_code IN ( + 'R03AK10', + 'R03AL01', + 'R03AL03', + 'R03AL04', + 'R03AL05', + 'R03AL06', + 'R03AL07', + 'R03AL10' + ); -- add manual mappings for ATC Combos using concept_relationship_manual INSERT INTO dev_combo -SELECT DISTINCT -class_code, - class_name, - c.concept_name AS class, -- leave it empty - c.concept_id, - c.concept_name, - CASE WHEN relationship_id = 'ATC - RxNorm pr lat' THEN 1 - WHEN relationship_id = 'ATC - RxNorm sec lat' THEN 2 - WHEN relationship_id = 'ATC - RxNorm pr up' THEN 3 - ELSE 4 -- stands for 'ATC - RxNorm sec up' - END AS rnk +SELECT DISTINCT class_code, + a.class_name, + c.concept_name AS class, -- leave it empty + c.concept_id, + c.concept_name, + CASE + WHEN relationship_id = 'ATC - RxNorm pr lat' + THEN 1 + WHEN relationship_id = 'ATC - RxNorm sec lat' + THEN 2 + WHEN relationship_id = 'ATC - RxNorm pr up' + THEN 3 + ELSE 4 -- stands for 'ATC - RxNorm sec up' + END AS rnk FROM combo_pull a -JOIN concept_relationship_manual r ON r.concept_code_1= a.class_code -JOIN concept c ON c.concept_code = r.concept_code_2 AND c.vocabulary_id ~ 'Rx' -AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' -WHERE (class_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND r.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up'); +JOIN concept_relationship_manual r ON r.concept_code_1 = a.class_code + AND r.relationship_id IN ( + 'ATC - RxNorm pr lat', + 'ATC - RxNorm sec lat', + 'ATC - RxNorm pr up', + 'ATC - RxNorm sec up' + ) + AND r.invalid_reason IS NULL +JOIN concept_rx c ON c.concept_code = r.concept_code_2 + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' +LEFT JOIN dev_combo d USING (class_code, concept_id) +WHERE d.class_code IS NULL; -- add mappings to those Ingredients, which have problems with name matching using the hardcoding -- add Acetylsalicylic acid INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - 'acetylsalicylic acid', - 1112807, - 'aspirin', - CASE WHEN class_name ~ '^acetylsalicylic' THEN 1 ELSE 2 END -FROM dev_combo -WHERE (class_code, concept_id) NOT IN ( select class_code, 1112807 FROM dev_combo WHERE class_name ~* 'acetylsalicylic') -AND class_name ~* 'acetylsalicylic'; +SELECT DISTINCT d.class_code, + d.class_name, + 'acetylsalicylic acid', + 1112807, + 'aspirin', + CASE WHEN d.class_name LIKE 'acetylsalicylic%' THEN 1 ELSE 2 END AS rnk +FROM dev_combo d +WHERE ( + d.class_code, + d.concept_id + ) NOT IN ( + SELECT class_code, + 1112807 + FROM dev_combo + WHERE class_name ILIKE '%acetylsalicylic%' + ) + AND d.class_name ILIKE '%acetylsalicylic%'; -- add Ethinylestradiol INSERT INTO dev_combo -SELECT DISTINCT class_code, - class_name, - 'ethinylestradiol', - 1549786, - 'ethinyl estradiol', - CASE WHEN class_name ~* '^ethinylestradiol' THEN 1 ELSE 2 END -FROM dev_combo -WHERE (class_code, concept_id) NOT IN ( select class_code, 1549786 FROM dev_combo WHERE class_name ~* 'ethinylestradiol') -AND class_name ~* 'ethinylestradiol'; +SELECT DISTINCT d.class_code, + d.class_name, + 'ethinylestradiol', + 1549786, + 'ethinyl estradiol', + CASE WHEN d.class_name ILIKE 'ethinylestradiol%' THEN 1 ELSE 2 END +FROM dev_combo d +WHERE ( + d.class_code, + d.concept_id + ) NOT IN ( + SELECT class_code, + 1549786 + FROM dev_combo + WHERE class_name ILIKE '%ethinylestradiol%' + ) + AND d.class_name ILIKE '%ethinylestradiol%'; -- add Estrogen INSERT INTO dev_combo -SELECT class_code, - class_name, - 'estrogens', - 19049228, - 'estrogens', - CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END -FROM dev_combo -WHERE (class_code, concept_id) NOT IN ( select class_code, 19049228 FROM dev_combo WHERE class_name ~* 'estrogen') -AND split_part(class_name, ';', 1) ~ 'estrogen' - UNION -SELECT class_code, - class_name, - 'estrogens', - 1549080, - 'estrogens, conjugated (USP)', - CASE WHEN SPLIT_PART(class_name,';',1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END -FROM dev_combo -WHERE (class_code, concept_id) NOT IN ( select class_code, 1549080 FROM dev_combo WHERE class_name ~* 'estrogen') -AND split_part(class_name, ';', 1) ~ 'estrogen' - UNION -SELECT class_code, - class_name, - 'estrogens', - 1551673, - 'estrogens, esterified (USP)', - CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END -FROM dev_combo -WHERE (class_code, concept_id) NOT IN ( select class_code, 1551673 FROM dev_combo WHERE class_name ~* 'estrogen') -AND split_part(class_name, ';', 1) ~ 'estrogen' - UNION -SELECT class_code, - class_name, - 'estrogens', - 1596779, - 'synthetic conjugated estrogens, A', - CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END -FROM dev_combo -WHERE (class_code, concept_id) NOT IN ( select class_code, 1596779 FROM dev_combo WHERE class_name ~* 'estrogen') -AND split_part(class_name, ';', 1) ~* 'estrogen' - UNION -SELECT class_code, - class_name, - 'estrogens' AS class, - 1586808, - 'synthetic conjugated estrogens, B', - CASE WHEN split_part(class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END -FROM dev_combo -WHERE (class_code, concept_id) NOT IN ( select class_code, 1586808 FROM dev_combo WHERE class_name ~* 'estrogen') -AND split_part(class_name, ';', 1) ~* 'estrogen'; +SELECT d.class_code, + d.class_name, + 'estrogens', + 19049228, + 'estrogens', + CASE WHEN SPLIT_PART(d.class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo d +WHERE ( + d.class_code, + d.concept_id + ) NOT IN ( + SELECT class_code, + 19049228 + FROM dev_combo + WHERE class_name ILIKE '%estrogen%' + ) + AND SPLIT_PART(d.class_name, ';', 1) LIKE '%estrogen%' + +UNION + +SELECT d.class_code, + d.class_name, + 'estrogens', + 1549080, + 'estrogens, conjugated (USP)', + CASE WHEN SPLIT_PART(d.class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo d +WHERE ( + d.class_code, + d.concept_id + ) NOT IN ( + SELECT class_code, + 1549080 + FROM dev_combo + WHERE class_name ILIKE '%estrogen%' + ) + AND SPLIT_PART(d.class_name, ';', 1) LIKE '%estrogen%' + +UNION + +SELECT d.class_code, + d.class_name, + 'estrogens', + 1551673, + 'estrogens, esterified (USP)', + CASE WHEN SPLIT_PART(d.class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo d +WHERE ( + d.class_code, + d.concept_id + ) NOT IN ( + SELECT class_code, + 1551673 + FROM dev_combo + WHERE class_name ILIKE '%estrogen%' + ) + AND SPLIT_PART(d.class_name, ';', 1) LIKE '%estrogen%' + +UNION + +SELECT d.class_code, + d.class_name, + 'estrogens', + 1596779, + 'synthetic conjugated estrogens, A', + CASE WHEN SPLIT_PART(d.class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo d +WHERE ( + d.class_code, + d.concept_id + ) NOT IN ( + SELECT class_code, + 1596779 + FROM dev_combo + WHERE class_name ILIKE '%estrogen%' + ) + AND SPLIT_PART(d.class_name, ';', 1) ILIKE '%estrogen%' + +UNION + +SELECT d.class_code, + d.class_name, + 'estrogens' AS class, + 1586808, + 'synthetic conjugated estrogens, B', + CASE WHEN SPLIT_PART(d.class_name, ';', 1) ~* '^estrogens|^conjugated estrogens' THEN 1 ELSE 2 END +FROM dev_combo d +WHERE ( + d.class_code, + d.concept_id + ) NOT IN ( + SELECT class_code, + 1586808 + FROM dev_combo + WHERE class_name ILIKE '%estrogen%' + ) + AND SPLIT_PART(d.class_name, ';', 1) ILIKE '%estrogen%'; -- remove erroneous mapping of strontium ranelate to strontium -DELETE FROM dev_combo WHERE class_code = 'M05BX53' AND concept_id = 19000815; -- strontium +DELETE +FROM dev_combo +WHERE class_code = 'M05BX53' + AND concept_id = 19000815;-- strontium + /********************************************** **** Combo to internal_realtionship_stage **** ***********************************************/ + -- add links between Multicomponent Oral ATC Drug Classes AND Standard Ingredients using the tables of dev_combo and concept table. -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.class_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code -a.concept_id -FROM dev_combo a, - dev_form b, - concept_manual c -WHERE c.concept_name ~* 'oral|systemic|chewing gum' AND b.df = 'dev_oral' -AND c.concept_code = a.class_code -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code - FROM t1 a - JOIN concept c ON c.concept_id = a.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) +WITH t1 AS ( + SELECT DISTINCT a.class_code || ' ' || b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_id + FROM dev_combo a + JOIN dev_form b ON b.df = 'dev_oral' + JOIN concept_manual c ON c.concept_name ~* 'oral|systemic|chewing gum' + AND c.concept_code = a.class_code + ) +SELECT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code +FROM t1 a +JOIN concept_rx c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +LEFT JOIN internal_relationship_stage irs ON irs.concept_code_1 = a.concept_code_1 + AND irs.concept_code_2 = c.concept_name +WHERE irs.concept_code_1 IS NULL; -- add links between Multicomponent Parenteral ATC Drug Classes AND Standard Ingredients using the tables of dev_combo and concept table. -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.class_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.class_name, -a.concept_id -FROM dev_combo a, - dev_form b, - concept_manual c -WHERE c.concept_name ~ 'parenteral|systemic' AND b.df = 'dev_parenteral' -AND c.concept_code = a.class_code -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code - FROM t1 a - JOIN concept c ON c.concept_id = a.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) +WITH t1 AS ( + SELECT DISTINCT a.class_code || ' ' || b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_id + FROM dev_combo a + JOIN dev_form b ON b.df = 'dev_parenteral' + JOIN concept_manual c ON c.concept_name ~ 'parenteral|systemic' + AND c.concept_code = a.class_code + ) +SELECT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code +FROM t1 a +JOIN concept_rx c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +LEFT JOIN internal_relationship_stage irs ON irs.concept_code_1 = a.concept_code_1 + AND irs.concept_code_2 = c.concept_name +WHERE irs.concept_code_1 IS NULL; -- add links between Multicomponent Vaginal ATC Drug Classes AND Standard Ingredients using the tables of dev_combo and concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT a.class_code|| ' ' ||b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code - a.class_name, -a.concept_id -FROM dev_combo a, - dev_form b, - concept_manual c -WHERE c.concept_name ~* 'vaginal' and b.df = 'dev_vaginal' -AND c.concept_code = a.class_code -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code - FROM t1 a - JOIN concept c ON c.concept_id = a.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT concept_code_1, concept_code_2 FROM internal_relationship_stage); +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) +WITH t1 AS ( + SELECT DISTINCT a.class_code || ' ' || b.concept_name AS concept_code_1, -- ATC code + Dose Form name AS a code + a.concept_id + FROM dev_combo a + JOIN dev_form b ON b.df = 'dev_vaginal' + JOIN concept_manual c ON c.concept_name ILIKE '%vaginal%' + AND c.concept_code = a.class_code + ) +SELECT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code +FROM t1 a +JOIN concept_rx c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +LEFT JOIN internal_relationship_stage irs ON irs.concept_code_1 = a.concept_code_1 + AND irs.concept_code_2 = c.concept_name +WHERE irs.concept_code_1 IS NULL; + ------------------------------ -- Combo ATC W/O Dose Forms --- ------------------------------- + -- add links between Multicomponent ATC Drug Classes W/O Dose Forms AND Standard Ingredients using the tables of dev_combo and concept table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) -WITH t1 -AS -(SELECT DISTINCT -class_code AS concept_code_1, -- ATC code + Dose Form name AS a code -class_name, -- ATC name to be used AS a key for JOIN -concept_id -FROM dev_combo -WHERE class_name !~';' -) - SELECT DISTINCT a.concept_code_1, -- ATC code + Dose Form name AS a code - c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code - FROM t1 a - JOIN concept c ON c.concept_id = a.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient' -WHERE (concept_code_1, c.concept_name) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1), concept_code_2 FROM internal_relationship_stage); +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) +WITH t1 AS ( + SELECT DISTINCT class_code AS concept_code_1, -- ATC code + Dose Form name AS a code + concept_id + FROM dev_combo + WHERE class_name NOT LIKE '%;%' + ) +SELECT a.concept_code_1, -- ATC code + Dose Form name AS a code + c.concept_name AS concept_code_2 -- Standard Ingredient name treated AS a code +FROM t1 a +JOIN concept_rx c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' +WHERE ( + a.concept_code_1, + c.concept_name + ) NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 + FROM internal_relationship_stage + ); + /****************************** ******* manual mapping ******** *******************************/ + -- add manually created maps between ATC Classes and equivalent Standard Ingredients using crm -INSERT INTO internal_relationship_stage -( concept_code_1, concept_code_2) -WITH t1 AS (select distinct a.concept_code_1, a.relationship_id, c.concept_name as concept_code_2 - FROM concept_relationship_manual a - JOIN concept_manual b - ON b.concept_code = a.concept_code_1 AND a.invalid_reason is null - JOIN concept c - ON c.concept_code = a.concept_code_2 - AND c.vocabulary_id = a.vocabulary_id_2 - AND c.standard_concept = 'S' AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - and a.relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up')) -- use ATC-specific relationships only -SELECT DISTINCT concept_code_1, - concept_code_2 -- OMOP Ingredient name AS an ATC Drug Attribute code, -FROM t1 WHERE (concept_code_1, concept_code_2) NOT IN (SELECT SPLIT_PART(concept_code_1, ' ', 1),concept_code_2 - FROM internal_relationship_stage); +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) +WITH t1 AS ( + SELECT DISTINCT a.concept_code_1, + c.concept_name AS concept_code_2 + FROM concept_relationship_manual a + JOIN concept_manual b ON b.concept_code = a.concept_code_1 + JOIN concept_rx c ON c.concept_code = a.concept_code_2 + AND c.vocabulary_id = a.vocabulary_id_2 + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + WHERE a.invalid_reason IS NULL + AND a.relationship_id IN ( + 'ATC - RxNorm pr lat', + 'ATC - RxNorm sec lat', + 'ATC - RxNorm pr up', + 'ATC - RxNorm sec up' + ) + ) -- use ATC-specific relationships only +SELECT concept_code_1, + concept_code_2 -- OMOP Ingredient name AS an ATC Drug Attribute code, +FROM t1 +WHERE ( + concept_code_1, + concept_code_2 + ) NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 + FROM internal_relationship_stage + ); + /********************************** ******* drug_concept_stage ******** ***********************************/ -TRUNCATE drug_concept_stage; --- change length of concept_code field -ALTER TABLE drug_concept_stage ALTER COLUMN concept_code TYPE VARCHAR; -- add to DCS ATC Drug Classes as Drug Products using the internal_relationship_stage table -INSERT INTO drug_concept_stage -( - concept_name, - vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - possible_excipient, - domain_id, - valid_start_date, - valid_end_date -) -SELECT DISTINCT concept_code_1, -- ATC code + name - 'ATC', - 'Drug Product', - NULL, - concept_code_1, - NULL, - 'Drug', - TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date +INSERT INTO drug_concept_stage ( + concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date + ) +SELECT DISTINCT concept_code_1 AS concept_name, -- ATC code + name + 'ATC' AS vocabulary_id, + 'Drug Product' AS concept_class_id, + NULL AS standard_concept, + concept_code_1 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, + TO_DATE('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date FROM internal_relationship_stage; +ANALYZE internal_relationship_stage; + -- add ATC Drug Attributes in the form of Rx Dose Form names using the internal_relationship_stage table -INSERT INTO drug_concept_stage -( - concept_name, - vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - possible_excipient, - domain_id, - valid_start_date, - valid_end_date -) -SELECT DISTINCT concept_code_2 AS concept_name, -- ATC pseudo-attribute IN the form of OMOP Dose Form name - 'ATC' AS vocabulary_id, - 'Dose Form' AS concept_class_id, - NULL AS standard_concept, - concept_code_2 AS concept_code, - NULL AS possible_excipient, - 'Drug' AS domain_id, - TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date -FROM internal_relationship_stage, - concept -WHERE concept_code_2 = concept_name -AND concept_class_id = 'Dose Form' -AND vocabulary_id ~ 'RxNorm' -AND invalid_reason IS NULL; +INSERT INTO drug_concept_stage ( + concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date + ) +SELECT DISTINCT irs.concept_code_2 AS concept_name, -- ATC pseudo-attribute IN the form of OMOP Dose Form name + 'ATC' AS vocabulary_id, + 'Dose Form' AS concept_class_id, + NULL AS standard_concept, + irs.concept_code_2 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, + TO_DATE('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage irs +JOIN concept_rx c ON c.concept_name = irs.concept_code_2 + AND c.concept_class_id = 'Dose Form' + AND c.invalid_reason IS NULL; + +CREATE UNIQUE INDEX dcs_unique_cc_idx ON drug_concept_stage (concept_code); +ANALYZE drug_concept_stage; -- add ATC Drug Attributes in the form of Standard Ingredient names using the internal_relationship_stage table -INSERT INTO drug_concept_stage -( concept_name, - vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - possible_excipient, - domain_id, - valid_start_date, - valid_end_date) -SELECT DISTINCT concept_code_2, -- ATC pseudo-attribute IN the form of OMOP Ingredient name - 'ATC', - 'Ingredient', - NULL, - concept_code_2, - NULL, - 'Drug', - TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date -FROM internal_relationship_stage, - concept -WHERE upper (concept_code_2) = upper(concept_name) -AND concept_class_id = 'Ingredient' -AND vocabulary_id ~ 'RxNorm' -AND standard_concept = 'S' -AND invalid_reason IS NULL -AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); +INSERT INTO drug_concept_stage ( + concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date + ) +SELECT DISTINCT concept_code_2 AS concept_name, -- ATC pseudo-attribute IN the form of OMOP Ingredient name + 'ATC' AS vocabulary_id, + 'Ingredient' AS concept_class_id, + NULL AS standard_concept, + concept_code_2 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, + TO_DATE('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage irs +JOIN concept_rx c ON UPPER(c.concept_name) = UPPER(irs.concept_code_2) + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.invalid_reason IS NULL +LEFT JOIN drug_concept_stage dcs ON dcs.concept_code = irs.concept_code_2 +WHERE dcs.concept_code IS NULL; -- obtain additional Ingredients for those ATC Classes which are still unmapped using fuzzy match INSERT INTO internal_relationship_stage -WITH t1 AS -( - -- define concepts to map - SELECT DISTINCT class_code, - class_name - FROM class_drugs_scraper - WHERE ( - -- totally lost - class_code NOT IN (SELECT SPLIT_PART(concept_code,' ',1) FROM drug_concept_stage) - AND LENGTH(class_code) = 7 - AND class_code NOT IN (SELECT concept_code_1 - FROM concept_relationship_manual - WHERE relationship_id IN ('ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up', 'ATC - RxNorm sec up')) - AND class_code NOT IN ('B03AD04', 'V09GX01', 'V09XX03') -- ferric oxide polymaltose complexes | thallium (201Tl) chloride | selenium (75Se) norcholesterol - AND class_name !~* '^indium|^iodine|^yttrium|^RIFAMPICIN|coagulation factor' - AND change_type IN ('', 'A')) - OR ( - -- absent IN the internal_relationship_stage - class_code IN (SELECT SPLIT_PART(concept_code,' ',1) FROM drug_concept_stage) -AND class_code NOT IN (SELECT SPLIT_PART(concept_code_1,' ',1) - FROM internal_relationship_stage a - JOIN concept c - ON c.concept_name = a.concept_code_2 - AND c.standard_concept = 'S' - AND c.concept_class_id = 'Ingredient') -AND LENGTH(class_code) = 7) -OR - -- with absent Ingredient in drug_relationship_stage -(class_code IN (SELECT SPLIT_PART(concept_code_1,' ',1) -FROM internal_relationship_stage -GROUP BY concept_code_1 -HAVING COUNT(1) = 1) AND class_code NOT IN ( - SELECT SPLIT_PART(concept_code_1,' ',1) - FROM internal_relationship_stage a - JOIN concept c - ON LOWER (c.concept_name) = LOWER (a.concept_code_2) - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S')) -), --- fuzzy macth using name similarity -t2 AS -(SELECT a.*, - c.* -FROM t1 a - JOIN concept_synonym b ON lower (b.concept_synonym_name) LIKE lower (concat ('%',class_name,'%')) - JOIN concept c - ON c.concept_id = b.concept_id - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S'), --- fuzzy match WITH levenshtein -t3 AS -(SELECT * -FROM t1 a - JOIN concept c - ON devv5.levenshtein (lower (class_name),lower (concept_name)) = 1 - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND class_code NOT IN (SELECT class_code FROM t2) -), --- match with non-standard and crosswalk to Standard -t4 AS (SELECT a.*,d.* -FROM t1 a - JOIN concept c ON lower(REGEXP_REPLACE (c.concept_name, '\s+|\W+', '', 'g')) = lower(TRIM(REGEXP_REPLACE(a.class_name,';.*$|, combinations?|IN combinations?', '', 'g'))) - AND c.domain_id = 'Drug' - JOIN concept_relationship r ON r.concept_id_1 = c.concept_id - JOIN concept d ON d.concept_id = r.concept_id_2 AND d.standard_concept = 'S' AND d.concept_class_id = 'Ingredient'), -t5 AS -(SELECT * -FROM t1 a - JOIN concept c - ON lower (c.concept_name) = lower (SUBSTRING (class_name,'\w+')) - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND a.class_code NOT IN (SELECT concept_code_1 FROM concept_relationship_manual) - AND concept_id NOT IN (19018544, 19071128, 1195334, 19124906, 19049024, 40799093, 19136048) --calcium|copper|choline|magnesium|potassium|Serum|sodium - AND class_code NOT IN (SELECT class_code FROM t2) -AND class_code NOT IN (SELECT class_code FROM t3) -), -t6 AS ( -SELECT class_code,class_name,concept_id,concept_name FROM t2 -UNION ALL -SELECT class_code,class_name,concept_id,concept_name FROM t3 -UNION ALL -SELECT class_code,class_name,concept_id,concept_name FROM t4 -UNION ALL -SELECT class_code,class_name,concept_id,concept_name FROM t5) -SELECT DISTINCT class_code, --class_name, - concept_name -FROM t6 -WHERE (class_code,concept_name) NOT IN (SELECT concept_code_1, - concept_code_2 - FROM internal_relationship_stage) - AND concept_id <> 43013482; -- butyl ester of methyl vinyl ether-maleic anhydride copolymer (125 kD) +WITH t1 AS ( + -- define concepts to map + SELECT DISTINCT class_code, + class_name + FROM class_drugs_scraper cds + WHERE ( + -- totally lost + cds.class_code NOT IN ( + SELECT SPLIT_PART(concept_code, ' ', 1) + FROM drug_concept_stage + ) + AND LENGTH(cds.class_code) = 7 + AND class_code NOT IN ( + SELECT concept_code_1 + FROM concept_relationship_manual + WHERE relationship_id IN ( + 'ATC - RxNorm pr lat', + 'ATC - RxNorm sec lat', + 'ATC - RxNorm pr up', + 'ATC - RxNorm sec up' + ) + AND invalid_reason IS NULL + ) + AND cds.class_code NOT IN ( + 'B03AD04', + 'V09GX01', + 'V09XX03' + ) -- ferric oxide polymaltose complexes | thallium (201Tl) chloride | selenium (75Se) norcholesterol + AND cds.class_name !~* '^indium|^iodine|^yttrium|^RIFAMPICIN|coagulation factor' + AND cds.change_type IN ( + '', + 'A' + ) + ) + OR ( + -- absent IN the internal_relationship_stage + cds.class_code IN ( + SELECT SPLIT_PART(concept_code, ' ', 1) + FROM drug_concept_stage + ) + AND cds.class_code NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1) + FROM internal_relationship_stage a + JOIN concept_rx c ON c.concept_name = a.concept_code_2 + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + ) + AND LENGTH(cds.class_code) = 7 + ) + OR + -- with absent Ingredient in drug_relationship_stage + ( + cds.class_code IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1) + FROM internal_relationship_stage + GROUP BY concept_code_1 + HAVING COUNT(*) = 1 + ) + AND cds.class_code NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1) + FROM internal_relationship_stage a + JOIN concept_rx c ON UPPER(c.concept_name) = UPPER(a.concept_code_2) + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + ) + ) + ), + -- fuzzy macth using name similarity + t2 AS ( + SELECT a.*, + c.* + FROM t1 a + JOIN concept_synonym b ON UPPER(b.concept_synonym_name) LIKE '%' || UPPER(a.class_name) || '%' + JOIN concept_rx c ON c.concept_id = b.concept_id + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + ), + -- fuzzy match WITH levenshtein + t3 AS ( + SELECT * + FROM t1 a + JOIN concept_rx c ON devv5.LEVENSHTEIN(UPPER(a.class_name), UPPER(c.concept_name)) = 1 + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND a.class_code NOT IN ( + SELECT class_code + FROM t2 + ) + ), + -- match with non-standard and crosswalk to Standard + t4 AS ( + SELECT a.*, + d.* + FROM t1 a + JOIN concept c ON UPPER(REGEXP_REPLACE(c.concept_name, '\s+|\W+', '', 'g')) = UPPER(TRIM(REGEXP_REPLACE(a.class_name, ';.*$|, combinations?|IN combinations?', '', 'g'))) + AND c.domain_id = 'Drug' + JOIN concept_relationship r ON r.concept_id_1 = c.concept_id + AND r.invalid_reason IS NULL + JOIN concept_rx d ON d.concept_id = r.concept_id_2 + AND d.standard_concept = 'S' + AND d.concept_class_id = 'Ingredient' + ), + t5 AS ( + SELECT * + FROM t1 a + JOIN concept_rx c ON UPPER(c.concept_name) = UPPER(SUBSTRING(a.class_name, '\w+')) + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + WHERE a.class_code NOT IN ( + SELECT concept_code_1 + FROM concept_relationship_manual + ) + AND c.concept_id NOT IN ( + 19018544, + 19071128, + 1195334, + 19124906, + 19049024, + 40799093, + 19136048 + ) --calcium|copper|choline|magnesium|potassium|Serum|sodium + AND a.class_code NOT IN ( + SELECT class_code + FROM t2 + ) + AND a.class_code NOT IN ( + SELECT class_code + FROM t3 + ) + ), + t6 AS ( + SELECT class_code, + concept_id, + concept_name + FROM t2 + + UNION ALL + + SELECT class_code, + concept_id, + concept_name + FROM t3 + + UNION ALL + + SELECT class_code, + concept_id, + concept_name + FROM t4 + + UNION ALL + + SELECT class_code, + concept_id, + concept_name + FROM t5 + ) +SELECT DISTINCT class_code, + concept_name +FROM t6 t +LEFT JOIN internal_relationship_stage irs ON irs.concept_code_1 = t.class_code + AND irs.concept_code_2 = t.concept_name +WHERE irs.concept_code_1 IS NULL + AND t.concept_id <> 43013482;-- butyl ester of methyl vinyl ether-maleic anhydride copolymer (125 kD) + +ANALYZE internal_relationship_stage; + /********************************** *** FUTHER WORK WITH ATC COMBOS *** ***********************************/ @@ -903,1186 +1281,1479 @@ WHERE (class_code,concept_name) NOT IN (SELECT concept_code_1, -- add descendants of Acid preparations INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'acid preparations' AS class, - c.concept_id, - c.concept_name, - CASE WHEN a.concept_name ~* '^acid' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, -- ATC - concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id = 21600704-- ATC code of Acid preparations - AND vocabulary_id LIKE 'RxNorm%' - AND concept_class_id = 'Ingredient' -WHERE SPLIT_PART(a.concept_name,';',1) ~* 'acid preparations' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'acid preparations' AS class, + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ILIKE 'acid%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a -- ATC +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21600704 -- ATC code of Acid preparations +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%acid preparations%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Sulfonamides INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'sulfonamides' AS class, - concept_id, - c.concept_name, - CASE WHEN a.concept_name ~* '^sulfonamides|^combinations of sulfonamides' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, -- ATC - concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id = 21603038-- ATC code of sulfonamides - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - JOIN concept_relationship b ON b.concept_id_1 = ancestor_concept_id - AND b.invalid_reason is null AND b.relationship_id = 'ATC - RxNorm pr lat' -WHERE SPLIT_PART(a.concept_name,';',1) ~* 'sulfonamides' AND SPLIT_PART(a.concept_name,';',1) !~* '^short-acting sulfonamides|^intermediate-acting sulfonamides|^long-acting sulfonamides' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'sulfonamides' AS class, + c.concept_id, + c.concept_name, + CASE WHEN a.concept_name ~* '^sulfonamides|^combinations of sulfonamides' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a -- ATC +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21603038 -- ATC code of sulfonamides +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%sulfonamides%' + AND SPLIT_PART(a.concept_name, ';', 1) !~* '^short-acting sulfonamides|^intermediate-acting sulfonamides|^long-acting sulfonamides' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' + AND EXISTS ( + SELECT 1 + FROM concept_relationship cr_int + WHERE cr_int.concept_id_1 = ca.ancestor_concept_id + AND cr_int.relationship_id = 'ATC - RxNorm pr lat' + AND cr_int.invalid_reason IS NULL + ); -- add descendants of Amino acids INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'amino acids', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1)~* '^amino acids' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, -- ATC - concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21601215, 21601034) -- 21601215 B05XB Amino acids| 21601034 B02AA Amino acids - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'amino\s*acid' - AND a.concept_code <> 'B03AD01' - AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; -- ferrous amino acid complex + a.concept_name, + 'amino acids', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'amino acids%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a -- ATC +JOIN concept_ancestor ca ON ca.ancestor_concept_id IN ( + 21601215, + 21601034 + ) -- 21601215 B05XB Amino acids| 21601034 B02AA Amino acids +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'amino\s*acid' + AND a.concept_code <> 'B03AD01' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th';--ferrous amino acid complex -- add descendants of Analgesics INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'analgesics', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1)~* '^analgesics' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21604253) -- 21604253 N02 ANALGESICS ATC 2nd - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - AND c.concept_id NOT IN (939506, 950435, 964407) -- sodium bicarbonate|citric acid|salicylic acid - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'analgesics?' AND SPLIT_PART(a.concept_name,';',1) !~* '\yexcl' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'analgesics', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'analgesics%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id IN (21604253) -- 21604253 N02 ANALGESICS ATC 2nd +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 939506, + 950435, + 964407 + ) -- sodium bicarbonate|citric acid|salicylic acid +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'analgesics?' + AND SPLIT_PART(a.concept_name, ';', 1) !~* '\yexcl' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Animals INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'animals', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^animals' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, - concept c -WHERE SPLIT_PART(a.concept_name,';',1) ~* 'Animals' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' -AND (c.concept_id IN (19091701,19056189,40170543,40170448,40170341,40170416,40175840,40175865,40170916,40175984,40161698,40170420, -19095690,40170741,40170848,40161809,40161813,45892235,40171114,45892234,37496548,40170660,40172147,40175843,40175898,40175933,40171110, -40175911,40171275,40172704,40171317,40175983,40171135,35201802,40238446,40175899,40227400,40175938,19061053,19112547,43013524,40170475, -40170818,40161805,40167658,1340875,42903998,963757,40171594,37496553,40172160,35201545,40175931,35201783,789889,35201778,40175951,35201548, -40161124,42709317,40161676,40161750,40170521,40161754,40170973,40170979,40170876,40175917) -OR ( -c.concept_name ~* 'rabbit|\ycow\y|\ydog\y|\ycat\y|goose|\yhog\y|\ygland\y|hamster|\yduck|oyster|\yhorse\y|\ylamb|pancreas|brain|kidney|\ybone\y|heart|spleen|lungs|^Pacific|\yfish|\yegg\y|\ypork|shrimp|\yveal|\ytuna|chicken' -AND c.concept_name ~* 'extract' AND c.vocabulary_id LIKE 'RxNorm%' -AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' -AND c.concept_id NOT IN (46276144,40170814,40226703,43560374,40227355,42903998,40227484,19086386)) -); + a.concept_name, + 'animals', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'animals%' THEN 3 ELSE 4 END AS rnk +FROM concept c +JOIN concept_manual a ON SPLIT_PART(a.concept_name, ';', 1) ILIKE '%Animals%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' +WHERE ( + c.concept_id IN ( + 19091701, + 19056189, + 40170543, + 40170448, + 40170341, + 40170416, + 40175840, + 40175865, + 40170916, + 40175984, + 40161698, + 40170420, + 19095690, + 40170741, + 40170848, + 40161809, + 40161813, + 45892235, + 40171114, + 45892234, + 37496548, + 40170660, + 40172147, + 40175843, + 40175898, + 40175933, + 40171110, + 40175911, + 40171275, + 40172704, + 40171317, + 40175983, + 40171135, + 35201802, + 40238446, + 40175899, + 40227400, + 40175938, + 19061053, + 19112547, + 43013524, + 40170475, + 40170818, + 40161805, + 40167658, + 1340875, + 42903998, + 963757, + 40171594, + 37496553, + 40172160, + 35201545, + 40175931, + 35201783, + 789889, + 35201778, + 40175951, + 35201548, + 40161124, + 42709317, + 40161676, + 40161750, + 40170521, + 40161754, + 40170973, + 40170979, + 40170876, + 40175917 + ) + OR ( + c.concept_name ~* 'rabbit|\ycow\y|\ydog\y|\ycat\y|goose|\yhog\y|\ygland\y|hamster|\yduck|oyster|\yhorse\y|\ylamb|pancreas|brain|kidney|\ybone\y|heart|spleen|lungs|^Pacific|\yfish|\yegg\y|\ypork|shrimp|\yveal|\ytuna|chicken' + AND c.concept_name ILIKE '%extract%' + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 46276144, + 40170814, + 40226703, + 43560374, + 40227355, + 42903998, + 40227484, + 19086386 + ) + ) + ); -- add descendants of Antiinfectives INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'antiinfectives', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^antiinfectives' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, - concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21605189, 21603552, 21605145, 21601168, 21605188, 21605146) -- Antiinfectives| ANTIINFECTIVES| ANTIINFECTIVES | Antiinfectives | Antiinfectives - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - AND c.concept_id NOT IN (19044522)-- zinc sulfate - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'antiinfectives?' --AND class_name ~* '\yexcl' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'antiinfectives', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'antiinfectives%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id IN ( + 21605189, + 21603552, + 21605145, + 21601168, + 21605188, + 21605146 + ) -- Antiinfectives| ANTIINFECTIVES| ANTIINFECTIVES | Antiinfectives | Antiinfectives +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.concept_id <> 19044522 -- zinc sulfate +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'antiinfectives?' --AND class_name ~* '\yexcl' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Cadmium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'cadmium compounds', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^cadmium compounds' THEN 3 ELSE 4 END ::INT AS rnk -- groups don't have primary lateral ings -FROM concept_manual a, -concept c -WHERE lower(c.concept_name) LIKE '%cadmium %' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id LIKE 'RxNorm%' -AND c.concept_id <> 45775350 -AND SPLIT_PART(a.concept_name,';',1) ~* 'cadmium compounds?' --AND class_name ~* '\yexcl' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'cadmium compounds', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'cadmium compounds%' THEN 3 ELSE 4 END AS rnk -- groups don't have primary lateral ings +FROM concept_manual a +JOIN concept_rx c ON c.concept_name ILIKE '%cadmium %' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id <> 45775350 +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'cadmium compounds?' --AND class_name ~* '\yexcl' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Calcium (different salts) INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'calcium (different salts IN combination)', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^calcium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c -WHERE c.concept_name ~* '\ycalcium\y' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id LIKE 'RxNorm%' -AND c.concept_id NOT IN (42903945,43533002,1337191,19007595,43532262,19051475) -- calcium ion|calcium hydride|calcium hydroxide|calcium oxide|calcium peroxide|anhydrous calcium iodide -AND SPLIT_PART(a.concept_name,';',1) ~* 'calcium' AND SPLIT_PART(a.concept_name,';',1) ~* '\ysalt' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'calcium (different salts IN combination)', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'calcium (different salts IN combination)%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_name ~* '\ycalcium\y' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 42903945, + 43533002, + 1337191, + 19007595, + 43532262, + 19051475 + ) -- calcium ion|calcium hydride|calcium hydroxide|calcium oxide|calcium peroxide|anhydrous calcium iodide +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%calcium%' + AND SPLIT_PART(a.concept_name, ';', 1) ~* '\ysalt' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Calcium compounds INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'calcium compounds', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^calcium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c -WHERE c.concept_name ~* '\ycalcium\y' -AND c.concept_class_id = 'Ingredient' -AND c.vocabulary_id LIKE 'RxNorm%' -AND c.concept_id NOT IN (19014944,42903945) -AND SPLIT_PART(a.concept_name,';',1) ~* 'calcium' AND SPLIT_PART(a.concept_name,';',1) ~* '\ycompound' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'calcium compounds', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'calcium compounds%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_name ~* '\ycalcium\y' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 19014944, + 42903945 + ) +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%calcium%' + AND SPLIT_PART(a.concept_name, ';', 1) ~* '\ycompound' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Laxatives INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'contact laxatives', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^contact laxatives' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, -concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21600537) - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' -WHERE SPLIT_PART(a.concept_name,';',1) ~* 'contact' AND SPLIT_PART(a.concept_name,';',1) ~* 'laxatives?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'contact laxatives', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'contact laxatives%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21600537 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%contact%' + AND SPLIT_PART(a.concept_name, ';', 1) ~* 'laxatives?' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Corticosteroids INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'corticosteroids', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^corticosteroids?|^combinations of corticosteroids?' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, -concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21605042, 21605164, 21605200, 21605165, 21605199, 21601607, 975125) - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' -WHERE SPLIT_PART(a.concept_name,';',1) ~* 'corticosteroids?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'corticosteroids', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ~* '^corticosteroids?|^combinations of corticosteroids?' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id IN ( + 21605042, + 21605164, + 21605200, + 21605165, + 21605199, + 21601607, + 975125 + ) +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'corticosteroids?' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Cough suppressants INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'cough suppressants', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^cough suppressants|^other cough suppressants' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, -concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21603440, 21603366, 21603409, 21603395, 21603436) - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - AND c.concept_id NOT IN (943191,1139042,1189220,1781321,19008366,19039512,19041843,19050346,19058933,19071861,19088167,19095266,42904041) - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'cough' AND SPLIT_PART(a.concept_name,';',1) ~* 'suppressants?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'cough suppressants', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ~* '^cough suppressants|^other cough suppressants' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id IN ( + 21603440, + 21603366, + 21603409, + 21603395, + 21603436 + ) +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 943191, + 1139042, + 1189220, + 1781321, + 19008366, + 19039512, + 19041843, + 19050346, + 19058933, + 19071861, + 19088167, + 19095266, + 42904041 + ) +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%cough%' + AND SPLIT_PART(a.concept_name, ';', 1) ~* 'suppressants?' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add descendants of Diuretics INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'diuretics', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^diuretics' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id = 21601461 - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'diuretics?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'diuretics', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'diuretics%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21601461 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'diuretics?' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Magnesium (different salts IN combination) INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'magnesium (different salts IN combination)', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, -- ATC -concept_ancestor ca - JOIN concept c -- Rx - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21600892) - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'magnesium' AND SPLIT_PART(a.concept_name,';',1) ~* 'salt' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'magnesium (different salts IN combination)', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'magnesium (different salts IN combination)%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a -- ATC +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21600892 +JOIN concept_rx c + ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%magnesium%' + AND SPLIT_PART(a.concept_name, ';', 1) ILIKE '%salt%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Magnesium (different salts IN combination) INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'magnesium (different salts IN combination)', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^magnesium \(different salts IN combination\)' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'magnesium' AND SPLIT_PART(a.concept_name,';',1) ~* 'salt' - AND c.concept_name ~ 'magnesium' AND c.standard_concept = 'S' AND c.concept_class_id = 'Ingredient' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' -AND (a.concept_code, c.concept_id) NOT IN (select class_code, concept_id FROM dev_combo) -AND concept_id NOT IN (43532017, 37498676); -- magnesium cation | magnesium Mg-28 +SELECT DISTINCT a.concept_code, + a.concept_name, + 'magnesium (different salts IN combination)', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'magnesium (different salts IN combination)%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_name LIKE '%magnesium%' + AND c.standard_concept = 'S' + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 43532017, + 37498676 + ) -- magnesium cation | magnesium Mg-28 +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%magnesium%' + AND SPLIT_PART(a.concept_name, ';', 1) ILIKE '%salt%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' + AND ( + a.concept_code, + c.concept_id + ) NOT IN ( + SELECT class_code, + concept_id + FROM dev_combo + ); -- add ingredients indicating Multivitamins INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'multivitamins', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^multivitamins' THEN 1 ELSE 2 END ::INT AS rnk -FROM concept_manual a, - concept c -WHERE c.concept_id = 36878782 - AND SPLIT_PART(a.concept_name,';',1) ~* 'multivitamins?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; + a.concept_name, + 'multivitamins', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'multivitamins%' THEN 1 ELSE 2 END AS rnk +FROM concept_manual a +JOIN concept c ON c.concept_id = 36878782 +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'multivitamins?' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Opium alkaloids WITH morphine INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'opium alkaloids WITH morphine', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^opium alkaloids WITH morphine' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21604255) -- Natural opium alkaloids - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - AND c.concept_id NOT IN (19112635) - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'opium alkaloids WITH morphine' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'opium alkaloids WITH morphine', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'opium alkaloids WITH morphine%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21604255 -- Natural opium alkaloids +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.concept_id <> 19112635 +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%opium alkaloids WITH morphine%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Opium derivatives INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'opium derivatives', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^opium derivatives' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id = 21603396 - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - AND c.concept_id NOT IN (19021930, 1201620) - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'opium derivatives' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'opium derivatives', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'opium derivatives%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21603396 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.concept_id NOT IN ( + 19021930, + 1201620 + ) +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%opium derivatives%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Organic nitrates INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, -'organic nitrates', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^organic nitrates' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21600316) - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'organic nitrates' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'organic nitrates', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'organic nitrates%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21600316 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%organic nitrates%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add descendants of Psycholeptics INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'psycholeptics', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^psycholeptics' THEN 3 - WHEN SPLIT_PART(a.concept_name,';',1) ~ 'excl\. psycholeptics' THEN 0 - ELSE 4 END ::INT AS rnk -- 0 stands for excluded drugs -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND (ca.ancestor_concept_id = 21604489 -- -- Crotarbital|butabarbital|Phenobarbital|butalbital -OR c.concept_name ~* ('Butalbital|Lorazepam|Ethchlorvynol|Ziprasidone|Talbutal|Pentobarbital|Olanzapine|Clobazam|'|| -'Clozapine|Meprobamate|Sulpiride|Eszopiclone|Alprazolam|Loxapine|Remoxipride|Secobarbital|' || -'Promazine|Zolpidem|Prochlorperazine|Droperidol|Methohexital|Chlordiazepoxide|' || -'Chlorpromazine|Buspirone|Haloperidol|Triflupromazine|Adinazolam|Hydroxyzine|' || -'Thiopental|Fluphenazine|Dexmedetomidine|Thioridazine|Midazolam|Flurazepam|' || -'Risperidone|Propiomazine|Primidone|Halazepam|Diazepam|Trifluoperazine|Oxazepam|' || -'Methylphenobarbital|Perphenazine|Flupentixol|Triazolam|Mesoridazine|Zaleplon|' || -'Ramelteon|Acetophenazine|Melatonin|Pimozide|Methyprylon|Thiamylal|' || -'Phenobarbital|Zopiclone|Estazolam|Quetiapine|Aripiprazole|Chlorprothixene|'|| -'Paliperidone|Amobarbital|Aprobarbital|Butobarbital|Heptabarbital|Hexobarbital|'|| -'Methotrimeprazine|Glutethimide|Barbital|Camazepam|Dichloralphenazone|Flunitrazepam|'|| -'Ethyl loflazepate|Cloxazolam|Bromazepam|Clotiazepam|Chloral hydrate|Fludiazepam|'|| -'Ketazolam|Prazepam|Quazepam|Cinolazepam|Nitrazepam|Periciazine|Acepromazine|Molindone|Pipotiazine|'|| -'Thioproperazine|Thiothixene|Zuclopenthixol|Methaqualone|Fluspirilene|Iloperidone|'|| -'Cariprazine|Sertindole|Asenapine|Amisulpride|Clomethiazole|Triclofos|Mebutamate|Tofisopam')::text -) -- obtained from https://go.drugbank.com/categories/DBCAT002185 - AND c.concept_id NOT IN (742594) - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'psycholeptics?' --AND class_name !~* 'excl\. psycholeptics' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th' AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' and c.standard_concept = 'S'; +SELECT a.concept_code, + a.concept_name, + 'psycholeptics', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'psycholeptics%' THEN 3 WHEN SPLIT_PART(a.concept_name, ';', 1) LIKE '%excl. psycholeptics%' THEN 0 ELSE 4 END AS rnk -- 0 stands for excluded drugs +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21604489 -- Crotarbital|butabarbital|Phenobarbital|butalbital +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_id <> 742594 +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'psycholeptics?' --AND class_name NOT ILIKE '%excl. psycholeptics%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' + +UNION + +SELECT a.concept_code, + a.concept_name, + 'psycholeptics', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'psycholeptics%' THEN 3 WHEN SPLIT_PART(a.concept_name, ';', 1) LIKE '%excl. psycholeptics%' THEN 0 ELSE 4 END AS rnk -- 0 stands for excluded drugs +FROM concept_manual a +JOIN concept_rx c ON c.concept_name ILIKE ANY (ARRAY + ['%Butalbital%','%Lorazepam%','%Ethchlorvynol%','%Ziprasidone%','%Talbutal%','%Pentobarbital%','%Olanzapine%','%Clobazam%','%Clozapine%','%Meprobamate%', + '%Sulpiride%','%Eszopiclone%','%Alprazolam%','%Loxapine%','%Remoxipride%','%Secobarbital%','%Promazine%','%Zolpidem%','%Prochlorperazine%','%Droperidol%', + '%Methohexital%','%Chlordiazepoxide%','%Chlorpromazine%','%Buspirone%','%Haloperidol%','%Triflupromazine%','%Adinazolam%','%Hydroxyzine%','%Thiopental%', + '%Fluphenazine%','%Dexmedetomidine%','%Thioridazine%','%Midazolam%','%Flurazepam%','%Risperidone%','%Propiomazine%','%Primidone%','%Halazepam%','%Diazepam%', + '%Trifluoperazine%','%Oxazepam%','%Methylphenobarbital%','%Perphenazine%','%Flupentixol%','%Triazolam%','%Mesoridazine%','%Zaleplon%','%Ramelteon%', + '%Acetophenazine%','%Melatonin%','%Pimozide%','%Methyprylon%','%Thiamylal%','%Phenobarbital%','%Zopiclone%','%Estazolam%','%Quetiapine%','%Aripiprazole%', + '%Chlorprothixene%','%Paliperidone%','%Amobarbital%','%Aprobarbital%','%Butobarbital%','%Heptabarbital%','%Hexobarbital%','%Methotrimeprazine%','%Glutethimide%', + '%Barbital%','%Camazepam%','%Dichloralphenazone%','%Flunitrazepam%','%Ethyl loflazepate%','%Cloxazolam%','%Bromazepam%','%Clotiazepam%','%Chloral hydrate%', + '%Fludiazepam%','%Ketazolam%','%Prazepam%','%Quazepam%','%Cinolazepam%','%Nitrazepam%','%Periciazine%','%Acepromazine%','%Molindone%','%Pipotiazine%','%Thioproperazine%', + '%Thiothixene%','%Zuclopenthixol%','%Methaqualone%','%Fluspirilene%','%Iloperidone%','%Cariprazine%','%Sertindole%','%Asenapine%','%Amisulpride%','%Clomethiazole%', + '%Triclofos%','%Mebutamate%','%Tofisopam%'] + ) -- obtained from https://go.drugbank.com/categories/DBCAT002185 + AND c.concept_id <> 742594 + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'psycholeptics?' --AND class_name NOT ILIKE '%excl. psycholeptics%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' + AND EXISTS ( + SELECT 1 + FROM concept_ancestor ca + WHERE ca.descendant_concept_id = c.concept_id + ); -- add descendants of Selenium compounds INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, -'selenium compounds', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^selenium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21600908) - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'selenium compounds' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'selenium compounds', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'selenium compounds%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21600908 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%selenium compounds%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add descendants of Silver compounds INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'silver compounds', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id IN (21602248) - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'silver compounds' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'silver compounds', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'silver compounds%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21602248 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%silver compounds%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Silver INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'silver compounds', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^silver compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.concept_name ~* 'silver\y' - AND ('silver compounds', c.concept_id) NOT IN (select class, concept_id FROM dev_combo) -AND SPLIT_PART(a.concept_name,';',1) ~* 'silver compounds' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'silver compounds', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'silver compounds%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.concept_name ~* 'silver\y' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%silver compounds%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' + AND ( + 'silver compounds', + c.concept_id + ) NOT IN ( + SELECT class, + concept_id + FROM dev_combo + ); -- add descendants of Sulfonylureas INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'sulfonylureas', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^sulfonylureas?' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id - AND ca.ancestor_concept_id = 21600749 - AND c.vocabulary_id LIKE 'RxNorm%' - AND c.concept_class_id = 'Ingredient' -WHERE SPLIT_PART(a.concept_name,';',1) ~* 'sulfonylureas?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'sulfonylureas', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ~* '^sulfonylureas?' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21600749 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'sulfonylureas?' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Snake venom antiserum INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'snake venom antiserum', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^snake venom antiserum' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'antiserum' AND c.concept_name ~* 'snake' -AND SPLIT_PART(a.concept_name,';',1) ~* 'snake venom antiserum' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'snake venom antiserum', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'snake venom antiserum%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ILIKE '%antiserum%' + AND c.concept_name ILIKE '%snake%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%snake venom antiserum%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Aluminium preparations INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'aluminium preparations', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^aluminium preparations' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'aluminium|aluminum' - AND SPLIT_PART(a.concept_name,';',1) ~* 'aluminium preparations' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'aluminium preparations', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'aluminium preparations%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'aluminium|aluminum' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%aluminium preparations%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Aluminium compounds INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'aluminium compounds', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^aluminium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'aluminium|aluminum' - AND SPLIT_PART(a.concept_name,';',1) ~* 'aluminium compounds' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'aluminium compounds', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'aluminium compounds%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'aluminium|aluminum' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%aluminium compounds%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Lactic acid producing organisms INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'lactic acid producing organisms', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^lactic acid producing organisms' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'lactobacil' - AND SPLIT_PART(a.concept_name,';',1) ~* 'lactic acid producing organisms' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'lactic acid producing organisms', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'lactic acid producing organisms%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ILIKE '%lactobacil%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%lactic acid producing organisms%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating Lactobacillus INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'lactobacillus', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^lactobacillus' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'lactobacil' - AND SPLIT_PART(a.concept_name,';',1) ~* 'lactobacillus' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'lactobacillus', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'lactobacillus%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ILIKE '%lactobacil%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%lactobacillus%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating Magnesium compounds INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'magnesium compounds', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^magnesium compounds' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'magnesium' - AND SPLIT_PART(a.concept_name,';',1) ~* 'magnesium compounds' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'magnesium compounds', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'magnesium compounds%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ILIKE '%magnesium%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%magnesium compounds%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating Grass pollen INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'grass pollen', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^grass pollen' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'grass' AND c.concept_name ~* 'pollen' - AND SPLIT_PART(a.concept_name,';',1) ~* 'grass pollen' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'grass pollen', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'grass pollen%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ILIKE '%grass%' + AND c.concept_name ILIKE '%pollen%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%grass pollen%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating Oil INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'oil', - c.concept_id, - c.concept_name, - 3 -- hardcoded - FROM concept_manual a, - concept c -WHERE c.vocabulary_id IN ('RxNorm','RxNorm Extension') -AND c.concept_class_id = 'Ingredient' -AND c.standard_concept = 'S' -AND c.concept_name ~* '\yoil\y|\yoleum\y' -AND SPLIT_PART(a.concept_name,';',1) ~* '^oil' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'oil', + c.concept_id, + c.concept_name, + 3 -- hardcoded +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* '\yoil\y|\yoleum\y' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE 'oil%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Flowers INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'flowers', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^flowers' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* '\yflower\y' AND c.concept_name ~* 'extract' - AND SPLIT_PART(a.concept_name,';',1) ~* '^flowers' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'flowers', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'flowers%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* '\yflower\y' + AND c.concept_name ILIKE '%extract%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE 'flowers%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating Fumaric acid derivatives INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'fumaric acid derivatives', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^fumaric acid derivatives' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, - concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'fumarate\y' - AND SPLIT_PART(a.concept_name,';',1) ~* 'fumaric acid derivatives' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'fumaric acid derivatives', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'fumaric acid derivatives%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* 'fumarate\y' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%fumaric acid derivatives%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add descendants of Proton pump inhibitors INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'proton pump inhibitors', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^proton pump inhibitors' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor s - JOIN concept c - ON descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - AND ancestor_concept_id IN (21600095) - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'proton pump inhibitors?' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'proton pump inhibitors', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'proton pump inhibitors%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21600095 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' +WHERE SPLIT_PART(a.concept_name, ';', 1) ~* 'proton pump inhibitors?' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add descendants of Thiazides INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'thiazides', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^thiazides' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept_ancestor ca - JOIN concept c - ON ca.descendant_concept_id = c.concept_id AND c.concept_class_id = 'Ingredient' AND c.standard_concept = 'S' - AND ca.ancestor_concept_id IN (21601463) - WHERE SPLIT_PART(a.concept_name,';',1) ~* 'thiazides' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'thiazides', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'thiazides%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_ancestor ca ON ca.ancestor_concept_id = 21601463 +JOIN concept_rx c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%thiazides%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Electrolytes - INSERT INTO dev_combo +INSERT INTO dev_combo SELECT DISTINCT a.concept_code, - a.concept_name, - 'electrolytes', - c.concept_id, - c.concept_name, - 3 -- hardcoded rank for electrolytes (no 4) - FROM concept_manual a, - concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* ('^magnesium sulfate|^ammonium chloride|^sodium chloride|^sodium acetate|^magnesium chloride^|potassium lactate|^sodium glycerophosphate|^magnesium phosphate|^potassium chloride|^calcium chloride' - || '^sodium bicarbonate|^hydrochloric acid|^potassium acetate|^zinc chloride|^sodium phosphate|^potassium bicarbonate|^succinic acid|^sodium lactate|^sodium gluconate|^sodium fumarate') - AND SPLIT_PART(a.concept_name,';',1) ~* 'electrolytes' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - + a.concept_name, + 'electrolytes', + c.concept_id, + c.concept_name, + 3 -- hardcoded rank for electrolytes (no 4) +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* ('^magnesium sulfate|^ammonium chloride|^sodium chloride|^sodium acetate|^magnesium chloride^|potassium lactate|^sodium glycerophosphate|^magnesium phosphate|^potassium chloride|^calcium chloride' + || '^sodium bicarbonate|^hydrochloric acid|^potassium acetate|^zinc chloride|^sodium phosphate|^potassium bicarbonate|^succinic acid|^sodium lactate|^sodium gluconate|^sodium fumarate') +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%electrolytes%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating bismuth preparations INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'bismuth preparations', - c.concept_id, - c.concept_name, - 3 -- hardcoded rank for bismuth preparations (no 4) - FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* ('\ybismuth') - AND SPLIT_PART(a.concept_name,';',1) ~* 'bismuth preparations' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'bismuth preparations', + c.concept_id, + c.concept_name, + 3 -- hardcoded rank for bismuth preparations (no 4) +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* '\ybismuth' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%bismuth preparations%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating Artificial Tears INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'artificial tears', - c.concept_id, - c.concept_name, - 3 -- hardcoded rank - FROM concept_manual a,concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.standard_concept = 'S' - AND c.concept_name ~* 'carboxymethylcellulose$|carboxypolymethylene|polyvinyl alcohol$|hydroxypropyl methylcellulose$|^hypromellose$|hydroxypropyl cellulose|^hyaluronate' - AND c.concept_class_id = 'Ingredient' - AND SPLIT_PART(a.concept_name,';',1) ~* 'artificial tears' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; - +SELECT DISTINCT a.concept_code, + a.concept_name, + 'artificial tears', + c.concept_id, + c.concept_name, + 3 -- hardcoded rank +FROM concept_manual a +JOIN concept_rx c ON c.standard_concept = 'S' + AND c.concept_name ~* 'carboxymethylcellulose$|carboxypolymethylene|polyvinyl alcohol$|hydroxypropyl methylcellulose$|^hypromellose$|hydroxypropyl cellulose|^hyaluronate' + AND c.concept_class_id = 'Ingredient' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%artificial tears%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; + -- add ingredients indicating Potassium-sparing agents INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'potassium-sparing agents', - c.concept_id, - c.concept_name, - CASE WHEN SPLIT_PART(a.concept_name,';',1) ~* '^potassium-sparing agents' THEN 3 ELSE 4 END ::INT AS rnk -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* '\yamiloride|triamterene|spironolactone|eplerenone|finerenone|canrenone|canrenoic acid' - AND SPLIT_PART(a.concept_name,';',1) ~* 'potassium-sparing agents' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'potassium-sparing agents', + c.concept_id, + c.concept_name, + CASE WHEN SPLIT_PART(a.concept_name, ';', 1) ILIKE 'potassium-sparing agents%' THEN 3 ELSE 4 END AS rnk +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ~* '\yamiloride|triamterene|spironolactone|eplerenone|finerenone|canrenone|canrenoic acid' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%potassium-sparing agents%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Ethiodized oil INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'ethyl esters of iodised fatty acids', - c.concept_id, - c.concept_name, - 1 -- hradcoded -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'ethiodized oil' - AND SPLIT_PART(a.concept_name,';',1) ~* '^ethyl esters of iodised fatty acids' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'ethyl esters of iodised fatty acids', + c.concept_id, + c.concept_name, + 1 -- hardcoded +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ILIKE '%ethiodized oil%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE 'ethyl esters of iodised fatty acids%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Ophthalmic Antibiotics INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'antibiotics ophthalmic', - c.concept_id, - c.concept_name, - 3 -- hradcoded -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND SPLIT_PART(a.concept_name,';',1) ~* 'antibiotics' and SPLIT_PART(a.concept_name,';',1) ~* 'combination' and SPLIT_PART(a.concept_name,';',1) ~* 'ophthalmic' -AND lower (substring(c.concept_name, '\w+')) in ( -'azithromycin','bacitracin','besifloxacin','ciprofloxacin','erythromycin','gatifloxacin','gentamicin','levofloxacin', -'moxifloxacin','ofloxacin','sulfacetamide','tobramycin','polymyxin B','trimethoprim','sulfacetamide','neomycin','gramicidin' - ) -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'antibiotics ophthalmic', + c.concept_id, + c.concept_name, + 3 -- hradcoded +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND LOWER(SUBSTRING(c.concept_name, '\w+')) IN ( + 'azithromycin', + 'bacitracin', + 'besifloxacin', + 'ciprofloxacin', + 'erythromycin', + 'gatifloxacin', + 'gentamicin', + 'levofloxacin', + 'moxifloxacin', + 'ofloxacin', + 'sulfacetamide', + 'tobramycin', + 'polymyxin B', + 'trimethoprim', + 'sulfacetamide', + 'neomycin', + 'gramicidin' + ) +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE ALL (ARRAY['%antibiotics%','%combination%','%ophthalmic%']) + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add ingredients indicating Topical Antibiotics INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'antibiotics topical', - c.concept_id, - c.concept_name, - 4 -- hradcoded -FROM concept_manual a, concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND SPLIT_PART(a.concept_name,';',1) ~* 'antibiotics' and SPLIT_PART(a.concept_name,';',1) ~* 'combination' and SPLIT_PART(a.concept_name,';',1) ~* 'topical' -AND lower(c.concept_name)in ( -'mupirocin','sulfacetamide','retapamulin','silver sulfadiazine','polymyxin b','bacitracin', -'neomycin','ozenoxacin','erythromycin','mafenide','gentamicin','demeclocycline','retapamulin', -'chlortetracycline','virginiamycin','chloramphenicol','oxytetracycline','tetracycline') -AND a.invalid_reason IS NULL -AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'antibiotics topical', + c.concept_id, + c.concept_name, + 4 -- hradcoded +FROM concept_manual a +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE ALL (ARRAY ['%antibiotics%','%combination%','%topical%']) + AND UPPER(c.concept_name) IN ( + 'MUPIROCIN', + 'SULFACETAMIDE', + 'RETAPAMULIN', + 'SILVER SULFADIAZINE', + 'POLYMYXIN B', + 'BACITRACIN', + 'NEOMYCIN', + 'OZENOXACIN', + 'ERYTHROMYCIN', + 'MAFENIDE', + 'GENTAMICIN', + 'DEMECLOCYCLINE', + 'RETAPAMULIN', + 'CHLORTETRACYCLINE', + 'VIRGINIAMYCIN', + 'CHLORAMPHENICOL', + 'OXYTETRACYCLINE', + 'TETRACYCLINE' + ) + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- add excluded Trimethoprim INSERT INTO dev_combo -SELECT DISTINCT a.concept_code, - a.concept_name, - 'excl. trimethoprim', - c.concept_id, - c.concept_name, - 0 -- hardcoded rank - FROM concept_manual a, -- ATC - concept c -WHERE c.vocabulary_id IN ( 'RxNorm', 'RxNorm Extension') - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND c.concept_name ~* 'trimethoprim' - AND SPLIT_PART(a.concept_name,';',1) ~* 'excl. trimethoprim' -AND a.invalid_reason IS NULL AND a.concept_class_id = 'ATC 5th'; +SELECT DISTINCT a.concept_code, + a.concept_name, + 'excl. trimethoprim', + c.concept_id, + c.concept_name, + 0 -- hardcoded rank +FROM concept_manual a -- ATC +JOIN concept_rx c ON c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' + AND c.concept_name ILIKE '%trimethoprim%' +WHERE SPLIT_PART(a.concept_name, ';', 1) ILIKE '%excl. trimethoprim%' + AND a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th'; -- perform dev_combo cleanup -- fix Vitamin D AND analogues IN combination UPDATE dev_combo - SET rnk = 3 +SET rnk = 3 WHERE rnk = 1 -AND class_code = 'A11CC20'; + AND class_code = 'A11CC20'; -- fix erroneous rnk of 1 for Ingredient groups UPDATE dev_combo - SET rnk = 3 -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND rnk = 1; +SET rnk = 3 +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 3 + ) + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 2, + 4 + ) + ) + AND rnk = 1; -- add missing codeine INSERT INTO dev_combo SELECT DISTINCT class_code, - class_name, - 'codeine', - 1189596, - 'dihydrocodeine', - 1 + class_name, + 'codeine', + 1189596, + 'dihydrocodeine', + 1 FROM dev_combo -WHERE class_code = 'N02AA59'; +WHERE class_code = 'N02AA59'; -- remove doubling ingredients with different rank, remaining those which are Primary lateral DELETE FROM dev_combo -WHERE (class_code,concept_id,rnk) IN (SELECT a.class_code, - a.concept_id, - a.rnk - FROM dev_combo a - JOIN dev_combo b - ON a.class_code = b.class_code - AND a.concept_id = b.concept_id - WHERE a.rnk > 1 - AND b.rnk = 1); +WHERE ( + class_code, + concept_id, + rnk + ) IN ( + SELECT a.class_code, + a.concept_id, + a.rnk + FROM dev_combo a + JOIN dev_combo b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND b.rnk = 1 + WHERE a.rnk > 1 + ); DELETE FROM dev_combo -WHERE class_name ~ 'antiinfectives' -AND rnk = 4 -AND concept_id IN (19010309,19136048,1036884,19049024,989878,961145,19018544,917006,914335); +WHERE class_name LIKE '%antiinfectives%' + AND rnk = 4 + AND concept_id IN ( + 19010309, + 19136048, + 1036884, + 19049024, + 989878, + 961145, + 19018544, + 917006, + 914335 + ); UPDATE dev_combo - SET rnk = 3 -WHERE class_name ~ 'lactic acid producing organisms' -AND rnk = 4 -AND concept_name ~* 'Saccharomyces|Bacillus|Bifidobacterium|Enterococcus|Escherichia|Streptococcus'; +SET rnk = 3 +WHERE class_name LIKE '%lactic acid producing organisms%' + AND rnk = 4 + AND concept_name ~* 'Saccharomyces|Bacillus|Bifidobacterium|Enterococcus|Escherichia|Streptococcus'; DELETE FROM dev_combo -WHERE class_name ~ 'lactic acid producing organisms' -AND rnk = 4; +WHERE class_name LIKE '%lactic acid producing organisms%' + AND rnk = 4; UPDATE dev_combo - SET rnk = 3 -WHERE class_name ~ 'opium derivatives' -AND rnk = 1; +SET rnk = 3 +WHERE class_name LIKE '%opium derivatives%' + AND rnk = 1; DELETE FROM dev_combo WHERE class_code = 'R05FB01' -AND class_name = 'cough suppressants and mucolytics' -AND class = 'cough suppressants' -AND concept_id = 19057932 -AND rnk = 3; + AND class_name = 'cough suppressants and mucolytics' + AND class = 'cough suppressants' + AND concept_id = 19057932 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'R05FB01' -AND class_name = 'cough suppressants and mucolytics' -AND class = 'cough suppressants' -AND concept_id = 19071999 -AND rnk = 3; + AND class_name = 'cough suppressants and mucolytics' + AND class = 'cough suppressants' + AND concept_id = 19071999 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1790868 -AND rnk = 3; -- 1 + AND class = 'sulfonamides' + AND concept_id = 1790868 + AND rnk = 3;-- 1 UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1734104 -AND rnk = 3; -- 1 + AND class = 'sulfonamides' + AND concept_id = 1734104 + AND rnk = 3;-- 1 UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1748975 -AND rnk = 3; -- 1 + AND class = 'sulfonamides' + AND concept_id = 1748975 + AND rnk = 3;-- 1 UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1778162 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1778162 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1797513 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1797513 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1754994 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1754994 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1742253 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1742253 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1707164 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1707164 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1721543 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1721543 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 923081 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 923081 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 19023254 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 19023254 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 19024197 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 19024197 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 19037983 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 19037983 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 19070251 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 19070251 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1836948 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1836948 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'J01RA02' -AND class = 'sulfonamides' -AND concept_id = 1702559 -AND rnk = 3; + AND class = 'sulfonamides' + AND concept_id = 1702559 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'R05FB02' -AND class = 'cough suppressants' -AND concept_id = 43012226 -AND rnk = 3; + AND class = 'cough suppressants' + AND concept_id = 43012226 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'R05FB02' -AND class = 'cough suppressants' -AND concept_id = 912362 -AND rnk = 3; + AND class = 'cough suppressants' + AND concept_id = 912362 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'R05FB02' -AND class = 'cough suppressants' -AND concept_id = 19060831 -AND rnk = 3; + AND class = 'cough suppressants' + AND concept_id = 19060831 + AND rnk = 3; UPDATE dev_combo - SET rnk = 3 +SET rnk = 3 WHERE class_code = 'R05FB02' -AND class = '' -AND concept_id = 1140088 -AND rnk = 4; + AND class = '' + AND concept_id = 1140088 + AND rnk = 4; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'R05FB02' -AND class = 'cough suppressants' -AND concept_id = 19063951 -AND rnk = 3; + AND class = 'cough suppressants' + AND concept_id = 19063951 + AND rnk = 3; UPDATE dev_combo - SET rnk = 4 +SET rnk = 4 WHERE class_code = 'R05FB02' -AND class = 'cough suppressants' -AND concept_id = 1103137 -AND rnk = 3; + AND class = 'cough suppressants' + AND concept_id = 1103137 + AND rnk = 3; + /******************************************* **** ADD ODDMENTS TO THE INPUT TABLES ***** ********************************************/ + -- add to links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of RxN/RxE Standard Ingredient names using the dev_combo table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) SELECT DISTINCT class_code, -- ATC - c.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code + c.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code FROM dev_combo a - JOIN concept c ON c.concept_id = a.concept_id AND c.standard_concept = 'S' -AND (a.class_code, c.concept_name) NOT IN ( -SELECT SPLIT_PART(concept_code_1, ' ', 1), - concept_code_2 -FROM internal_relationship_stage) -WHERE LENGTH (class_code) = 7 -AND rnk <> 0; +JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND ( + a.class_code, + c.concept_name + ) NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 + FROM internal_relationship_stage + ) +WHERE LENGTH(class_code) = 7 + AND rnk <> 0; -- add more links between ATC Classes indicating Ingredient Groups AND ATC Drug Attributes in the form of RxN/RxE Standard Ingredient names using the dev_combo table -INSERT INTO internal_relationship_stage -(concept_code_1, concept_code_2) +INSERT INTO internal_relationship_stage ( + concept_code_1, + concept_code_2 + ) SELECT DISTINCT class_code, -- ATC - c.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code + c.concept_name -- OMOP Ingredient name AS an ATC Drug Attribute code FROM dev_combo a - JOIN concept c ON lower(c.concept_name) = lower(a.concept_name) - AND c.standard_concept = 'S' -AND (a.class_code, c.concept_name) NOT IN ( -SELECT SPLIT_PART(concept_code_1, ' ', 1), - concept_code_2 -FROM internal_relationship_stage) -WHERE LENGTH (class_code) = 7 -AND rnk <> 0; +JOIN concept c ON UPPER(c.concept_name) = UPPER(a.concept_name) + AND c.standard_concept = 'S' + AND ( + a.class_code, + c.concept_name + ) NOT IN ( + SELECT SPLIT_PART(concept_code_1, ' ', 1), + concept_code_2 + FROM internal_relationship_stage + ) +WHERE LENGTH(class_code) = 7 + AND rnk <> 0; -- add ATC Classes of Ingredient Groups AS Drug Products using the internal_relationship_stage and class_drugs_scraper tables -INSERT INTO drug_concept_stage -( concept_name, - vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - possible_excipient, - domain_id, - valid_start_date, - valid_end_date -) +INSERT INTO drug_concept_stage ( + concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date + ) SELECT DISTINCT b.class_name AS concept_name, - 'ATC' AS vocabulary_id, - 'Drug Product' AS concept_class_id, - NULL AS standard_concept, - concept_code_1 AS concept_code, - NULL AS possible_excipient, - 'Drug' AS domain_id, - TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date + 'ATC' AS vocabulary_id, + 'Drug Product' AS concept_class_id, + NULL AS standard_concept, + concept_code_1 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, + TO_DATE('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date FROM internal_relationship_stage a JOIN class_drugs_scraper b ON b.class_code = SPLIT_PART(a.concept_code_1, ' ', 1) -WHERE concept_code_1 NOT IN (select concept_code FROM drug_concept_stage); +LEFT JOIN drug_concept_stage dcs ON dcs.concept_code = a.concept_code_1 +WHERE dcs.concept_code IS NULL; -- add ATC Drug Attributes IN the form of Standard Ingredient names using the internal_relationship_stage and concept tables -INSERT INTO drug_concept_stage -( concept_name, - vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - possible_excipient, - domain_id, - valid_start_date, - valid_end_date) +INSERT INTO drug_concept_stage ( + concept_name, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + possible_excipient, + domain_id, + valid_start_date, + valid_end_date + ) SELECT DISTINCT concept_code_2 AS concept_name, -- ATC pseudo-attribute IN the form of OMOP Dose Form name - 'ATC' AS vocabulary_id, - c.concept_class_id AS concept_class_id, - NULL AS standard_concept, -- check all standard_concept values - concept_code_2 AS concept_code, - NULL AS possible_excipient, - 'Drug' AS domain_id, - TO_DATE ('19700101', 'YYYYMMDD') AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date -FROM internal_relationship_stage, - concept c WHERE lower(c.concept_name) = lower(concept_code_2) -AND concept_class_id = 'Ingredient' -AND vocabulary_id IN ('RxNorm', 'RxNorm Extension') -AND invalid_reason IS NULL -AND concept_code_2 NOT IN (select concept_code FROM drug_concept_stage); + 'ATC' AS vocabulary_id, + c.concept_class_id AS concept_class_id, + NULL AS standard_concept, -- check all standard_concept values + irs.concept_code_2 AS concept_code, + NULL AS possible_excipient, + 'Drug' AS domain_id, + TO_DATE('19700101', 'YYYYMMDD') AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date +FROM internal_relationship_stage irs +JOIN concept_rx c ON UPPER(c.concept_name) = UPPER(irs.concept_code_2) + AND c.concept_class_id = 'Ingredient' + AND c.invalid_reason IS NULL +LEFT JOIN drug_concept_stage dcs ON dcs.concept_code = irs.concept_code_2 +WHERE dcs.concept_code IS NULL; + +ANALYZE drug_concept_stage; -- remove dead deprecated or updated ATC codes DELETE FROM internal_relationship_stage -WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT concept_code - FROM concept_manual - WHERE invalid_reason IS NOT NULL); +WHERE SUBSTRING(concept_code_1, '\w+') IN ( + SELECT concept_code + FROM concept_manual + WHERE invalid_reason IS NOT NULL + ); DELETE FROM drug_concept_stage -WHERE concept_code IN (SELECT concept_code - FROM concept_manual - WHERE invalid_reason IS NOT NULL); +WHERE concept_code IN ( + SELECT concept_code + FROM concept_manual + WHERE invalid_reason IS NOT NULL + ); -- remove mappings of ATC Classes which are nonexistent in OMOP (old and wrong) DELETE FROM drug_concept_stage -WHERE concept_code IN (SELECT class_code FROM atc_inexistent) -AND concept_code NOT IN (SELECT class_code FROM dev_combo); +WHERE concept_code IN ( + SELECT class_code + FROM atc_inexistent + ) + AND concept_code NOT IN ( + SELECT class_code + FROM dev_combo + ); DELETE FROM internal_relationship_stage -WHERE SUBSTRING(concept_code_1,'\w+') IN (SELECT class_code FROM atc_inexistent) -AND SUBSTRING(concept_code_1,'\w+') NOT IN (SELECT class_code FROM dev_combo) -AND concept_code_1 !~ '\s+'; +WHERE SUBSTRING(concept_code_1, '\w+') IN ( + SELECT class_code + FROM atc_inexistent + ) + AND SUBSTRING(concept_code_1, '\w+') NOT IN ( + SELECT class_code + FROM dev_combo + ) + AND concept_code_1 !~ '\s+'; -- remove mappings which have been deprecated in concept_relationship_manual -WITH t1 AS -( - SELECT * - FROM concept_relationship_manual - WHERE relationship_id IN ('ATC - RxNorm pr lat','ATC - RxNorm sec up','ATC - RxNorm pr up','ATC - RxNorm sec lat') -) DELETE +DELETE FROM dev_combo -WHERE class_code||concept_id IN (SELECT DISTINCT a.class_code||c.concept_id - FROM dev_combo a - JOIN concept c ON c.concept_id = a.concept_id - JOIN t1 r - ON r.concept_code_1 = a.class_code - AND r.concept_code_2 = c.concept_code - AND r.invalid_reason IS NOT NULL); +WHERE class_code || concept_id IN ( + SELECT a.class_code || c.concept_id + FROM dev_combo a + JOIN concept c ON c.concept_id = a.concept_id + JOIN concept_relationship_manual crm ON crm.concept_code_1 = a.class_code + AND crm.concept_code_2 = c.concept_code + AND crm.relationship_id IN ( + 'ATC - RxNorm pr lat', + 'ATC - RxNorm sec up', + 'ATC - RxNorm pr up', + 'ATC - RxNorm sec lat' + ) + AND crm.invalid_reason IS NOT NULL + ); -- clean up sulfonamides DELETE FROM dev_combo -WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id - FROM dev_combo - WHERE class_name ~ 'sulfonamides' - AND (concept_id IN (SELECT concept_id FROM dev_combo WHERE class_code = 'J01EB20') OR concept_name ~* '^sulfa') - AND rnk = 3) -AND class_name ~ 'sulfonamides' -AND rnk = 3; -- 16 +WHERE class_code || concept_id NOT IN ( + SELECT class_code || concept_id + FROM dev_combo + WHERE class_name LIKE '%sulfonamides%' + AND ( + concept_id IN ( + SELECT concept_id + FROM dev_combo + WHERE class_code = 'J01EB20' + ) + OR concept_name ILIKE 'sulfa%' + ) + AND rnk = 3 + ) + AND class_name LIKE '%sulfonamides%' + AND rnk = 3;-- 16 -- add missing sulfonamides INSERT INTO dev_combo WITH t1 -AS -(SELECT concept_id, - concept_name, - rnk -FROM dev_combo -WHERE class_name ~ 'sulfonamides' -AND (concept_id IN (SELECT concept_id FROM dev_combo WHERE class_code = 'J01EB20') OR concept_name ~* '^sulfa') -AND rnk = 3) SELECT DISTINCT class_code,class_name,'sulfonamides',b.*FROM dev_combo a,t1 b WHERE class_name ~ 'sulfonamides' -AND class_code||b.concept_id NOT IN (SELECT class_code||concept_id FROM dev_combo); --40 +AS ( + SELECT concept_id, + concept_name, + rnk + FROM dev_combo + WHERE class_name LIKE '%sulfonamides%' + AND ( + concept_id IN ( + SELECT concept_id + FROM dev_combo + WHERE class_code = 'J01EB20' + ) + OR concept_name ILIKE 'sulfa%' + ) + AND rnk = 3 + ) +SELECT DISTINCT class_code, + class_name, + 'sulfonamides', + b.* +FROM dev_combo a +CROSS JOIN t1 b +WHERE a.class_name LIKE '%sulfonamides%' + AND class_code || b.concept_id NOT IN ( + SELECT class_code || concept_id + FROM dev_combo + );--40 -- assign correct rnk 529303 798304 diphtheria toxoid vaccine, inactivated UPDATE dev_combo - SET rnk = 2 +SET rnk = 2 WHERE concept_id = 529303 -AND class_code = 'J07AM51'; + AND class_code = 'J07AM51'; -- remove wrong rank for the Mono ATC of B03AD02 ferrous fumarate, combinations -DELETE from dev_combo where class_code = 'B03AD02' -and rnk = 2; +DELETE +FROM dev_combo +WHERE class_code = 'B03AD02' + AND rnk = 2; -- remove duplicates (to do: prevent the entry of duplicates in previous steps) -DELETE -FROM dev_combo -WHERE ctid NOT IN (SELECT MIN(ctid) - FROM dev_combo -GROUP BY class_code, class_name, concept_id, rnk); +DELETE +FROM dev_combo d +WHERE EXISTS ( + SELECT 1 + FROM dev_combo d_int + WHERE d_int.class_code = d.class_code + AND d_int.class_name = d.class_name + AND d_int.concept_id = d.concept_id + AND d_int.rnk = d.rnk + AND d_int.ctid > d.ctid + ); + /*************************************** ******* relationship_to_concept ******** ****************************************/ + -- add mappings of ATC Drug Attributes to OMOP Equivalents (Standard for the Ingredients and valid for the Dose Forms) -TRUNCATE relationship_to_concept; -ALTER TABLE relationship_to_concept ALTER COLUMN concept_code_1 TYPE VARCHAR; -INSERT INTO relationship_to_concept -( concept_code_1, vocabulary_id_1, concept_id_2) -SELECT DISTINCT concept_code_2 AS concept_code_1, -- ATC attribute IN the form of OMOP Dose Form OR Ingredient name - 'ATC' AS vocabulary_id_1, - c.concept_id AS concept_id_2 -- OMOP concept_id -FROM internal_relationship_stage - JOIN concept c - ON lower(concept_code_2) = lower(c.concept_name) - AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - AND c.invalid_reason IS NULL; -- 5699 - - -- run load_interim.sql +INSERT INTO relationship_to_concept ( + concept_code_1, + vocabulary_id_1, + concept_id_2 + ) +SELECT DISTINCT irs.concept_code_2 AS concept_code_1, -- ATC attribute IN the form of OMOP Dose Form OR Ingredient name + 'ATC' AS vocabulary_id_1, + c.concept_id AS concept_id_2 -- OMOP concept_id +FROM internal_relationship_stage irs +JOIN concept_rx c ON UPPER(c.concept_name) = UPPER(irs.concept_code_2) + AND c.invalid_reason IS NULL;-- 5699 + +--clean up +DROP TABLE concept_rx, + dev_form, + atc_to_form; + +-- run load_interim.sql \ No newline at end of file diff --git a/ATC/load_interim.sql b/ATC/load_interim.sql index c68df6430..328b2c5bd 100644 --- a/ATC/load_interim.sql +++ b/ATC/load_interim.sql @@ -14,1937 +14,2283 @@ * limitations under the License. * * Authors: Anna Ostropolets, Polina Talapova, Timur Vakhitov -* Date: Sep 2021 -* Total script execution time: 33m 56s +* Date: 2022 **************************************************************************/ + /******************** ***** RX COMBO ***** *********************/ + -- create an interim table of rx_combo with aggregated concept_ids of RxN/RxN Standard Ingredients DROP TABLE IF EXISTS rx_combo; -CREATE TABLE rx_combo AS -SELECT drug_concept_id, - string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo -FROM devv5.drug_strength - JOIN devv5.concept ON concept_id = drug_concept_id AND - concept_class_id IN ('Clinical Drug Form', 'Ingredient') -- 'Clinical Drug Comp' doesn't exist in ATCs +CREATE UNLOGGED TABLE rx_combo AS +SELECT ds.drug_concept_id, + ARRAY_AGG(ds.ingredient_concept_id) AS i_combo +FROM drug_strength ds +JOIN concept c ON c.concept_id = ds.drug_concept_id + AND c.concept_class_id IN ( + 'Clinical Drug Form', + 'Ingredient' + ) -- 'Clinical Drug Comp' doesn't exist in ATCs GROUP BY drug_concept_id; +CREATE INDEX idx_rx_combo_dcid ON rx_combo (drug_concept_id) WITH (FILLFACTOR=100); +ANALYZE rx_combo; + -- create an interim table of rx_all_combo as the assemblage of Multicomponent RxN/RxE Drug Products, RxN/RxE Standard Ingredients and RxN/RxE Dose Forms DROP TABLE rx_all_combo; -CREATE TABLE rx_all_combo -AS -(SELECT DISTINCT c.concept_id AS d_id, - c.concept_name AS d_name, - d.concept_id AS ing_id, - d.concept_name AS ing_name, - k.concept_id AS df_id, - k.concept_name AS df_name +CREATE UNLOGGED TABLE rx_all_combo AS +SELECT c.concept_id AS d_id, + c.concept_name AS d_name, + d.concept_id AS ing_id, + d.concept_name AS ing_name, + k.concept_id AS df_id, + k.concept_name AS df_name FROM concept c - JOIN concept_relationship r ON r.concept_id_1 = c.concept_id - JOIN concept d ON d.concept_id = r.concept_id_2--df - JOIN concept_relationship r2 ON r2.concept_id_1 = c.concept_id - JOIN concept k ON k.concept_id = r2.concept_id_2 +JOIN concept_relationship r ON r.concept_id_1 = c.concept_id + AND r.relationship_id = 'RxNorm has ing' + AND r.invalid_reason IS NULL +JOIN concept d ON d.concept_id = r.concept_id_2 --df + AND d.standard_concept = 'S' + AND d.concept_class_id = 'Ingredient' + AND d.vocabulary_id LIKE 'RxNorm%' +JOIN concept_relationship r2 ON r2.concept_id_1 = c.concept_id + AND r2.relationship_id = 'RxNorm has dose form' + AND r2.invalid_reason IS NULL +JOIN concept k ON k.concept_id = r2.concept_id_2 + AND k.concept_class_id = 'Dose Form' + AND k.invalid_reason IS NULL WHERE c.concept_class_id = 'Clinical Drug Form' -AND c.standard_concept = 'S' -AND d.standard_concept = 'S' -AND d.concept_class_id = 'Ingredient' -AND r.relationship_id = 'RxNorm has ing' -AND r2.relationship_id = 'RxNorm has dose form' -AND k.concept_class_id = 'Dose Form' -AND k.invalid_reason IS NULL -AND r.invalid_reason IS NULL -AND r2.invalid_reason IS NULL); + AND c.standard_concept = 'S'; + +CREATE INDEX idx_rx_all_combo1 ON rx_all_combo (ing_id, df_id, d_id) WITH (FILLFACTOR=100); +CREATE INDEX idx_rx_all_combo2 ON rx_all_combo (LOWER(ing_name), LOWER(df_name), d_id) WITH (FILLFACTOR=100); +ANALYZE rx_all_combo; -- create an interim table of atc_all_combo as the assemblage of Multicomponent ATC Classes, RxN/RxE Standard Ingredients and RxN/RxE Dose Forms DROP TABLE IF EXISTS atc_all_combo; -CREATE TABLE atc_all_combo -AS -(SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id AS ing_id, - a.concept_name AS ing_name, - c.concept_id AS df_id, - c.concept_name AS df_name, - rnk +CREATE UNLOGGED TABLE atc_all_combo AS +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id AS ing_id, + a.concept_name AS ing_name, + c.concept_id AS df_id, + c.concept_name AS df_name, + a.rnk FROM dev_combo a - JOIN internal_relationship_stage b ON SPLIT_PART (b.concept_code_1,' ',1) = a.class_code - JOIN drug_concept_stage d - ON d.concept_code = b.concept_code_2 - AND d.concept_class_id = 'Dose Form' - JOIN concept c - ON c.concept_name = d.concept_name - AND c.invalid_reason IS NULL); +JOIN internal_relationship_stage b ON SPLIT_PART(b.concept_code_1, ' ', 1) = a.class_code +JOIN drug_concept_stage d ON d.concept_code = b.concept_code_2 + AND d.concept_class_id = 'Dose Form' +JOIN concept c ON c.concept_name = d.concept_name + AND c.invalid_reason IS NULL; + +CREATE INDEX idx_atc_all_combo ON atc_all_combo (ing_id, df_id, class_code) WITH (FILLFACTOR=100); +ANALYZE atc_all_combo; + +CREATE INDEX idx_irs_cc1 ON internal_relationship_stage (SUBSTRING (concept_code_1,'\w+')) WITH (FILLFACTOR=100); +ANALYZE internal_relationship_stage; -- create an interim table of atc_all_combo as the assemblage of Monocomponent ATC Classes, RxN/RxE Standard Ingredients and RxN/RxE Dose Forms -drop table if exists atc_all_mono; -CREATE TABLE atc_all_mono -AS -( -SELECT DISTINCT a1.concept_code as class_code, - a1.concept_name as class_name, - c.concept_id AS ing_id, - c.concept_name AS ing_name, - d.concept_id AS df_id, - d.concept_name AS df_name, - 1 as rnk +DROP TABLE IF EXISTS tmp_irs_dcs; +CREATE UNLOGGED TABLE tmp_irs_dcs AS +SELECT i.concept_code_1, + i.concept_code_2, + d.concept_class_id +FROM ( + SELECT DISTINCT SUBSTRING(concept_code_1, '\w+') AS concept_code_1, + concept_code_2 + FROM internal_relationship_stage + ) i +JOIN drug_concept_stage d ON d.concept_code = i.concept_code_2 + AND d.concept_class_id IN ( + 'Ingredient', + 'Dose Form' + ); + +DROP TABLE IF EXISTS atc_all_mono; +CREATE UNLOGGED TABLE atc_all_mono AS +SELECT DISTINCT a1.concept_code AS class_code, + a1.concept_name AS class_name, + c.concept_id AS ing_id, + c.concept_name AS ing_name, + d.concept_id AS df_id, + d.concept_name AS df_name, + 1 AS rnk FROM concept_manual a1 - JOIN internal_relationship_stage b1 ON substring (b1.concept_code_1, '\w+') = a1.concept_code - JOIN drug_concept_stage d1 - ON d1.concept_code = b1.concept_code_2 - AND d1.concept_class_id = 'Ingredient' - JOIN concept c - ON lower(c.concept_name) = lower( d1.concept_code) and c.standard_concept = 'S' and c.domain_id = 'Drug' - JOIN concept_manual a2 on a2.concept_code = a1.concept_code and a2.invalid_reason is null - JOIN internal_relationship_stage b2 ON substring (b2.concept_code_1, '\w+') = a2.concept_code - JOIN drug_concept_stage d2 ON d2.concept_code = b2.concept_code_2 - AND d2.concept_class_id = 'Dose Form' - JOIN concept d on lower(d.concept_name) = lower(d2.concept_code) - AND d2.invalid_reason IS NULL and d.domain_id = 'Drug' - AND a1.concept_code not in (select class_code from dev_combo) - ); +JOIN tmp_irs_dcs b1 ON b1.concept_code_1 = a1.concept_code + AND b1.concept_class_id = 'Ingredient' +JOIN concept c ON UPPER(c.concept_name) = UPPER(b1.concept_code_2) + AND c.standard_concept = 'S' + AND c.domain_id = 'Drug' +JOIN tmp_irs_dcs b2 ON b2.concept_code_1 = a1.concept_code + AND b2.concept_class_id = 'Dose Form' +JOIN concept d ON UPPER(d.concept_name) = UPPER(b2.concept_code_2) + AND d.domain_id = 'Drug' +LEFT JOIN dev_combo dc ON dc.class_code = a1.concept_code +WHERE dc.class_code IS NULL + AND a1.invalid_reason IS NULL; + + /************************************ ***** PREPARE ATC COMBO CLASSES ***** *************************************/ + -- create an interim table which contains pure Multicomponent ATC Classes with semantic links of 'Primary lateral' + 'Secondary lateral' (ranks of 1 and 2 in dev_combo) DROP TABLE IF EXISTS ing_pr_lat_sec_lat; -CREATE TABLE ing_pr_lat_sec_lat -AS -SELECT DISTINCT class_code, - class_name, - concept_id, - concept_name, - rnk +CREATE UNLOGGED TABLE ing_pr_lat_sec_lat AS +SELECT class_code, + class_name, + concept_id, + concept_name, + rnk FROM dev_combo -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1)-- Primary lateral -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 2)-- Secondary lateral -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3)-- exclude Priamry upward -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0); -- exclude Secondary upward +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 1 + ) -- Primary lateral + AND class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 2 + ) -- Secondary lateral + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 3, + 4, + 0 + ) + );-- exclude Priamry/Secondary upward -- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary lateral' + 'Secondary upward' (ranks of 1 and 4 in dev_combo) -DROP TABLE if exists ing_pr_lat_sec_up; -CREATE TABLE ing_pr_lat_sec_up -AS -(SELECT * +DROP TABLE IF EXISTS ing_pr_lat_sec_up; +CREATE UNLOGGED TABLE ing_pr_lat_sec_up AS +SELECT * FROM dev_combo a -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) -); +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 1 + ) + AND class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 4 + ) + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 2, + 3, + 0 + ) + ); -- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary lateral' in combination with other drugs (rank of 1 in dev_combo) -DROP TABLE if exists ing_pr_lat_combo; -CREATE TABLE ing_pr_lat_combo -AS -(SELECT * +DROP TABLE IF EXISTS ing_pr_lat_combo; +CREATE UNLOGGED TABLE ing_pr_lat_combo AS +SELECT * FROM dev_combo -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0) -); +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 1 + ) + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 2, + 3, + 4, + 0 + ) + ); -- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary lateral' in combination with other drugs and excluded Ingredient -- (ranks of 1 and 0 in dev_combo) -DROP TABLE if exists ing_pr_lat_combo_excl; -CREATE TABLE ing_pr_lat_combo_excl -AS -(SELECT * +DROP TABLE IF EXISTS ing_pr_lat_combo_excl; +CREATE UNLOGGED TABLE ing_pr_lat_combo_excl AS +SELECT * FROM dev_combo -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 1) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 3) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 0) -); - +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 1 + ) + AND class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 0 + ) + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 2, + 3, + 4 + ) + ); + -- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary upward' in combination with other drugs (rank of 3 in dev_combo) DROP TABLE IF EXISTS ing_pr_up_combo; -CREATE TABLE ing_pr_up_combo -AS -(SELECT DISTINCT class_code, - class_name, - concept_id, - concept_name, - rnk +CREATE UNLOGGED TABLE ing_pr_up_combo AS +SELECT class_code, + class_name, + concept_id, + concept_name, + rnk FROM dev_combo -WHERE class_code IN (SELECT class_code FROM dev_combo WHERE rnk = 3)-- include Priamry upward -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 1)-- exclude Primary lateral -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 2)-- exclude Secondary lateral -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 4) -AND class_code NOT IN (SELECT class_code FROM dev_combo WHERE rnk = 0));-- exclude Secondary upward +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 3 + ) -- include Priamry upward + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 1, + 2, + 4, + 0 + ) + );-- exclude Primary/Secondary lateral -- add the same Ingredients marked with rank of 1 to create permutation INSERT INTO ing_pr_up_combo -- Primary upward -SELECT class_code, class_name, concept_id, concept_name, 1 - FROM ing_pr_up_combo; - +SELECT class_code, + class_name, + concept_id, + concept_name, + 1 +FROM ing_pr_up_combo; + -- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary upward' + 'Secondary upward' (ranks of 3 and 4 in dev_combo) DROP TABLE IF EXISTS ing_pr_up_sec_up; -CREATE TABLE ing_pr_up_sec_up AS ( -SELECT DISTINCT class_code, - class_name, - concept_id, - concept_name, - rnk +CREATE UNLOGGED TABLE ing_pr_up_sec_up AS +SELECT class_code, + class_name, + concept_id, + concept_name, + rnk FROM dev_combo -WHERE class_code IN (SELECT class_code - FROM dev_combo - WHERE rnk = 3) -- include Priamry upward -AND class_code NOT IN (SELECT class_code - FROM dev_combo - WHERE rnk = 1) -- exclude Primary lateral -AND class_code NOT IN (SELECT class_code - FROM dev_combo - WHERE rnk = 2) -- exclude Secondary lateral -AND class_code IN (SELECT class_code - FROM dev_combo - WHERE rnk = 4) -and class_code not in (select class_code from dev_combo where rnk = 0) -- exclude Excluded -); +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 3 + ) -- include Priamry upward + AND class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 4 + ) + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 1, + 2, + 0 + ) + ); -- exclude Primary/Secondary lateral -- create an interim table which contains Multicomponent ATC Classes with semantic links of 'Primary upward' + 'Secondary upward' with excluded Ingredient -- ranks of 3, 4 and 0 in dev_combo +-- note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product DROP TABLE IF EXISTS ing_pr_up_sec_up_excl; -CREATE TABLE ing_pr_up_sec_up_excl AS ( -SELECT distinct class_code, - class_name, - concept_id, - concept_name, - rnk +CREATE UNLOGGED TABLE ing_pr_up_sec_up_excl AS +SELECT class_code, + class_name, + concept_id, + concept_name, + rnk FROM dev_combo -WHERE class_code IN (SELECT class_code - FROM dev_combo - WHERE rnk = 3) -- include Priamry upward -AND class_code NOT IN (SELECT class_code - FROM dev_combo - WHERE rnk = 1) -- exclude Primary lateral -AND class_code NOT IN (SELECT class_code - FROM dev_combo - WHERE rnk = 2) -- exclude Secondary lateral -AND class_code IN (SELECT class_code - FROM dev_combo - WHERE rnk = 4) -AND class_code IN (SELECT class_code - FROM dev_combo - WHERE rnk = 0) -);-- Note, Secondary upward links cannot stand alone in combinations with unmentioned RxN/RxE Drug Product - --- create indices -DROP INDEX IF EXISTS idx_atc; -CREATE INDEX idx_atc ON atc_all_combo (class_code, df_id, ing_id); -ANALYZE atc_all_combo; -DROP INDEX IF EXISTS idx_rx; -CREATE INDEX idx_rx ON rx_all_combo (ing_id, df_id); -ANALYZE rx_all_combo; -DROP INDEX IF EXISTS idx_rx_0; -CREATE INDEX idx_rx_0 ON rx_combo (i_combo); -ANALYZE rx_combo; -DROP INDEX IF EXISTS idx_rx_1; -CREATE INDEX idx_rx_1 ON rx_combo (drug_concept_id, i_combo); -ANALYZE rx_combo; -DROP INDEX IF EXISTS idx_rx_3; -CREATE INDEX idx_rx_3 ON rx_all_combo (lower(ing_name), lower(df_name)); -ANALYZE rx_all_combo; -DROP INDEX IF EXISTS idx_rx_4; -CREATE INDEX idx_rx_4 ON atc_all_combo (lower(ing_name), lower(df_name)); -ANALYZE atc_all_combo; -DROP INDEX IF EXISTS irs_x; -CREATE INDEX irs_x ON internal_relationship_stage (SUBSTRING (concept_code_1,'\w+')); -ANALYZE internal_relationship_stage; +WHERE class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 3 + ) -- include Priamry upward + AND class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 4 + ) + AND class_code IN ( + SELECT class_code + FROM dev_combo + WHERE rnk = 0 + ) + AND class_code NOT IN ( + SELECT class_code + FROM dev_combo + WHERE rnk IN ( + 1, + 2 + ) + );-- exclude Primary/Secondary lateral + /************************* ***** CLASS TO DRUG ****** **************************/ + -- assemble a table containing ATC Drug Classes which are as ancestors hierarchically connected to RxN/RxE Drug Products (the first line target - Clinical Drug Form) -- add mappings from Monocomponent ATC Classes to respective Monocomponent RxN/RxE Drug Products (order = 1) -DROP TABLE if exists class_to_drug_new; -CREATE TABLE class_to_drug_new -AS -(WITH atc -AS -(SELECT DISTINCT a.class_code, - a.class_name, - df_id, - ARRAY_AGG(a.ing_id) OVER (PARTITION BY a.class_code) ings -FROM atc_all_mono a),rx AS (SELECT DISTINCT r.d_id, r.d_name, r.df_id, ARRAY_AGG(r.ing_id) OVER (PARTITION BY r.d_id) ings - FROM rx_all_combo r) - SELECT DISTINCT atc.class_code, - atc.class_name, - c.*, - 1 AS concept_order, - 'ATC Monocomp Class' AS order_desc +DROP TABLE IF EXISTS class_to_drug_new; +CREATE UNLOGGED TABLE class_to_drug_new AS + WITH atc AS ( + SELECT DISTINCT a.class_code, + a.class_name, + df_id, + ARRAY_AGG(a.ing_id) OVER (PARTITION BY a.class_code) ings + FROM atc_all_mono a + ), + rx AS ( + SELECT DISTINCT r.d_id, + r.d_name, + r.df_id, + ARRAY_AGG(r.ing_id) OVER (PARTITION BY r.d_id) ings + FROM rx_all_combo r + ) +SELECT DISTINCT atc.class_code, + atc.class_name, + c.*, + 1 AS concept_order, + 'ATC Monocomp Class' AS order_desc FROM atc - JOIN rx USING (df_id) - JOIN concept c ON c.concept_id = rx.d_id -WHERE atc.ings @> rx.ings -AND atc.ings <@ rx.ings); +JOIN rx USING (df_id) +JOIN concept c ON c.concept_id = rx.d_id +WHERE rx.ings @> atc.ings + AND rx.ings <@ atc.ings; -- add more mappings from Monocomponent ATC Classes to respective Monocomponent RxN/RxE Drug Products (order = 1) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT * -FROM concept_manual -WHERE invalid_reason IS NULL -AND concept_class_id = 'ATC 5th' -AND concept_code NOT IN (SELECT class_code FROM dev_combo)), -- get attributes -t2 -AS -(SELECT DISTINCT a.concept_code AS class_code, - a.concept_name AS class_name, - d.concept_code AS ing_name, - d2.concept_code AS df_name +WITH t1 +AS ( + SELECT a.concept_code AS class_code, + a.concept_name AS class_name, + i.concept_code_2 AS ing_name, + i2.concept_code_2 AS df_name + FROM concept_manual a + JOIN tmp_irs_dcs i ON i.concept_code_1 = a.concept_code + AND i.concept_class_id = 'Ingredient' + JOIN tmp_irs_dcs i2 ON i2.concept_code_1 = a.concept_code + AND i2.concept_class_id = 'Dose Form' + LEFT JOIN dev_combo dc ON dc.class_code = a.concept_code + WHERE a.invalid_reason IS NULL + AND a.concept_class_id = 'ATC 5th' + AND dc.class_code IS NULL + ) +SELECT DISTINCT a.class_code, + a.class_name, + c.*, + 1 AS concept_order, + 'ATC Monocomp Class' AS order_desc FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d - ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient' - LEFT JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - LEFT JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form') -SELECT DISTINCT class_code, - class_name, - c.*, - 1 AS concept_order, - 'ATC Monocomp Class' as order_desc -FROM t2 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - AND d_name !~ ' / ' -- exclude Multicomponent - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' -WHERE class_code NOT IN (SELECT class_code FROM dev_combo) - AND class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);-- 1085 +JOIN rx_all_combo b ON LOWER(b.ing_name) = LOWER(a.ing_name) + AND LOWER(b.df_name) = LOWER(a.df_name) + AND b.d_name NOT LIKE '% / %' -- exclude Multicomponent +JOIN concept c ON c.concept_id = b.d_id + AND c.standard_concept = 'S' +WHERE a.class_code || c.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 1085 -- add mappings of "greedy" ATC Monocomponent Classes, which are matched with Multicomponent RxN/RxE Drug Products (order = 2) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT i.class_code,i.class_name,i.ing_id,i.ing_name,i.df_id,i.df_name,i.rnk,i.d_id,i.d_name -FROM (SELECT a.class_code, - a.class_name, - a.ing_id, - a.ing_name, - a.df_id, - a.df_name, - a.rnk, - r.d_id, - r.d_name, - ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, - ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks - FROM atc_all_mono a - LEFT JOIN rx_all_combo r USING (ing_id,df_id) - ) i -WHERE i.d_id IS NOT NULL) - SELECT DISTINCT - class_code, - class_name, - c.*, - 2 AS concept_order, - 'Greedy ATC Monocomp Class' AS order_desc - FROM t1 a - JOIN concept c ON c.concept_id = a.d_id - WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 60223 +WITH t1 AS ( + SELECT i.class_code, + i.class_name, + i.ing_id, + i.ing_name, + i.df_id, + i.df_name, + i.rnk, + i.d_id, + i.d_name + FROM ( + SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + a.rnk, + r.d_id, + r.d_name, + ARRAY_AGG(a.rnk) FILTER(WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, + ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks + FROM atc_all_mono a + LEFT JOIN rx_all_combo r USING (ing_id, df_id) + ) i + WHERE i.d_id IS NOT NULL + ) +SELECT DISTINCT a.class_code, + class_name, + c.*, + 2 AS concept_order, + 'Greedy ATC Monocomp Class' AS order_desc +FROM t1 a +JOIN concept c ON c.concept_id = a.d_id +WHERE a.class_code || concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 60223 -- add more mappings of "greedy" ATC Monocomponent Classes, which are matched with Multicomponent RxN/RxE Drug Products (order = 2) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT * -FROM concept_manual -WHERE invalid_reason IS NULL -AND concept_class_id = 'ATC 5th'), -- get attributes -t2 -AS -(SELECT DISTINCT a.concept_code AS class_code, - a.concept_name AS class_name, - d.concept_code AS ing_name, - d2.concept_code AS df_name -FROM t1 a - JOIN internal_relationship_stage i ON SUBSTRING (i.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d - ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient' - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.concept_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form') +WITH t1 AS ( + SELECT DISTINCT cm.concept_code AS class_code, + cm.concept_name AS class_name, + i.concept_code_2 AS ing_name, + i2.concept_code_2 AS df_name + FROM concept_manual cm + JOIN tmp_irs_dcs i ON i.concept_code_1 = cm.concept_code + AND i.concept_class_id = 'Ingredient' + JOIN tmp_irs_dcs i2 ON i2.concept_code_1 = cm.concept_code + AND i2.concept_class_id = 'Dose Form' + WHERE cm.invalid_reason IS NULL + AND cm.concept_class_id = 'ATC 5th' + ) SELECT DISTINCT class_code, - class_name, - c.*, - 2 AS concept_order, - 'Greedy ATC Monocomp Class' as order_desc -FROM t2 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND (class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) - AND class_code NOT IN (SELECT class_code FROM dev_combo); + class_name, + c.*, + 2 AS concept_order, + 'Greedy ATC Monocomp Class' AS order_desc +FROM t1 a +JOIN rx_all_combo b ON LOWER(b.ing_name) = LOWER(a.ing_name) + AND LOWER(b.df_name) = LOWER(a.df_name) +JOIN concept c ON c.concept_id = b.d_id + AND c.standard_concept = 'S' +WHERE NOT EXISTS ( + SELECT 1 + FROM class_to_drug_new cdn + WHERE cdn.class_code = a.class_code + AND cdn.concept_id = c.concept_id + ) + AND NOT EXISTS ( + SELECT 1 + FROM dev_combo d + WHERE d.class_code = a.class_code + ); -- add mappings of Multicomponent ACT Classes of 'Primary lateral' in combination (order = 3) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT i.class_code,i.class_name,i.ing_id,i.ing_name,i.df_id,i.df_name,i.rnk,i.d_id,i.d_name -FROM (SELECT a.class_code, - a.class_name, - a.ing_id, - a.ing_name, - a.df_id, - a.df_name, - a.rnk, - r.d_id, - r.d_name, - ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, - ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks - FROM atc_all_combo a - LEFT JOIN rx_all_combo r USING (ing_id,df_id) - WHERE class_code IN (SELECT class_code FROM ing_pr_lat_combo)) i -WHERE i.d_id IS NOT NULL) -SELECT DISTINCT - class_code, - class_name, - c.*, - 3 AS concept_order, - 'ATC Combo Class: Primary lateral in combination' AS order_desc +WITH t1 AS ( + SELECT i.class_code, + i.class_name, + i.ing_id, + i.ing_name, + i.df_id, + i.df_name, + i.rnk, + i.d_id, + i.d_name + FROM ( + SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + a.rnk, + r.d_id, + r.d_name, + ARRAY_AGG(a.rnk) FILTER(WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, + ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks + FROM atc_all_combo a + LEFT JOIN rx_all_combo r USING (ing_id, df_id) + WHERE class_code IN ( + SELECT class_code + FROM ing_pr_lat_combo + ) + ) i + WHERE i.d_id IS NOT NULL + ) +SELECT DISTINCT class_code, + class_name, + c.*, + 3 AS concept_order, + 'ATC Combo Class: Primary lateral in combination' AS order_desc FROM t1 a - JOIN concept c - ON c.concept_id = a.d_id - AND c.concept_name ~ ' / ' -WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 9201 +JOIN concept c ON c.concept_id = a.d_id + AND c.concept_name LIKE '% / %' +WHERE a.class_code || concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 9201 -- add mappings of Multicomponent ACT Classes of 'Primary upward' in combination with other drugs (order = 4) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT i.class_code,i.class_name,i.ing_id,i.ing_name,i.df_id,i.df_name,i.rnk,i.d_id,i.d_name -FROM (SELECT a.class_code, - a.class_name, - a.ing_id, - a.ing_name, - a.df_id, - a.df_name, - a.rnk, - r.d_id, - r.d_name, - ARRAY_AGG(a.rnk) FILTER (WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, - ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks - FROM atc_all_combo a - LEFT JOIN rx_all_combo r USING (ing_id,df_id) - WHERE class_code IN (SELECT class_code FROM ing_pr_up_combo)) i -WHERE i.d_id IS NOT NULL) -SELECT DISTINCT - class_code, - class_name, - c.*, - 4 AS concept_order, - 'ATC Combo Class: Primary upward in combination' AS order_desc +WITH t1 AS ( + SELECT i.class_code, + i.class_name, + i.ing_id, + i.ing_name, + i.df_id, + i.df_name, + i.rnk, + i.d_id, + i.d_name + FROM ( + SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + a.rnk, + r.d_id, + r.d_name, + ARRAY_AGG(a.rnk) FILTER(WHERE r.ing_id IS NOT NULL) OVER (PARTITION BY a.class_code) matched_ranks, + ARRAY_AGG(a.rnk) OVER (PARTITION BY a.class_code) all_ranks + FROM atc_all_combo a + LEFT JOIN rx_all_combo r USING (ing_id, df_id) + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_up_combo + ) + ) i + WHERE i.d_id IS NOT NULL + ) +SELECT DISTINCT class_code, + class_name, + c.*, + 4 AS concept_order, + 'ATC Combo Class: Primary upward in combination' AS order_desc FROM t1 a - JOIN concept c - ON c.concept_id = a.d_id - AND c.concept_name ~ ' / ' -WHERE class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 14307 +JOIN concept c ON c.concept_id = a.d_id + AND c.concept_name LIKE '% / %' +WHERE a.class_code || concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 14307 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary lateral, 4 ingreds' (order = 5) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk -FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_lat)) +WITH t1 AS ( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + a.rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id, df_id) + JOIN concept c ON c.concept_id = b.d_id + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_lat + ) + ) SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.domain_id, - a.vocabulary_id, - a.concept_class_id, - a.standard_concept, - a.concept_code, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason, - 5 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary lateral, 4 ingreds' AS order_desc + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 5 AS concept_order, + 'ATC Combo Class: Primary lateral + Secondary lateral, 4 ingreds' AS order_desc FROM t1 a - JOIN t1 b - ON b.class_code = a.class_code - AND b.concept_id = a.concept_id - AND a.ing_id <> b.ing_id - JOIN t1 c - ON c.class_code = a.class_code - AND c.concept_id = b.concept_id - AND a.ing_id <> c.ing_id - AND b.ing_id <> c.ing_id - JOIN t1 d - ON d.class_code = a.class_code - AND d.concept_id = b.concept_id - AND a.ing_id <> d.ing_id - AND d.ing_id <> c.ing_id - AND d.ing_id <> b.ing_id +JOIN t1 b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id +JOIN t1 c ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id +JOIN t1 d ON d.class_code = a.class_code + AND d.concept_id = b.concept_id + AND a.ing_id <> d.ing_id + AND d.ing_id <> c.ing_id + AND d.ing_id <> b.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND c.rnk <> 1 -AND d.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id -FROM class_to_drug_new) -and a.class_code !~ '^J07'; -- 269 + AND b.rnk <> 1 + AND c.rnk <> 1 + AND d.rnk <> 1 + AND a.class_code || a.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + ) + AND a.class_code NOT LIKE 'J07%';-- 269 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary upward', 4 ingreds (order = 50) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk -FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_up)) +WITH t1 AS ( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + a.rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id, df_id) + JOIN concept c ON c.concept_id = b.d_id + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_up + ) + ) SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.domain_id, - a.vocabulary_id, - a.concept_class_id, - a.standard_concept, - a.concept_code, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason, - 50 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary upward, 4 ingreds' AS order_desc + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 50 AS concept_order, + 'ATC Combo Class: Primary lateral + Secondary upward, 4 ingreds' AS order_desc FROM t1 a - JOIN t1 b - ON b.class_code = a.class_code - AND b.concept_id = a.concept_id - AND a.ing_id <> b.ing_id - JOIN t1 c - ON c.class_code = a.class_code - AND c.concept_id = b.concept_id - AND a.ing_id <> c.ing_id - AND b.ing_id <> c.ing_id - JOIN t1 d - ON d.class_code = a.class_code - AND d.concept_id = b.concept_id - AND a.ing_id <> d.ing_id - AND d.ing_id <> c.ing_id - AND d.ing_id <> b.ing_id +JOIN t1 b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id +JOIN t1 c ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id +JOIN t1 d ON d.class_code = a.class_code + AND d.concept_id = b.concept_id + AND a.ing_id <> d.ing_id + AND d.ing_id <> c.ing_id + AND d.ing_id <> b.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND c.rnk <> 1 -AND d.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) -; -- 141 - + AND b.rnk <> 1 + AND c.rnk <> 1 + AND d.rnk <> 1 + AND a.class_code || a.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 141 + -- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary lateral', 3 ingreds (order = 6) INSERT INTO class_to_drug_new -WITH t1 AS -( - SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk - FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id - WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_lat) -) +WITH t1 AS ( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id, df_id) + JOIN concept c ON c.concept_id = b.d_id + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_lat + ) + ) SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.domain_id, - a.vocabulary_id, - a.concept_class_id, - a.standard_concept, - a.concept_code, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason, - 6 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds' AS order_desc + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 6 AS concept_order, + 'ATC Combo Class: Primary lateral + Secondary lateral, 3 ingreds' AS order_desc FROM t1 a - JOIN t1 b - ON b.class_code = a.class_code - AND b.concept_id = a.concept_id - AND a.ing_id <> b.ing_id - JOIN t1 c - ON c.class_code = a.class_code - AND c.concept_id = b.concept_id - AND a.ing_id <> c.ing_id - AND b.ing_id <> c.ing_id +JOIN t1 b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id +JOIN t1 c ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND c.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 261 + AND b.rnk <> 1 + AND c.rnk <> 1 + AND a.class_code || a.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 261 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary upward', 3 ingreds (order = 60) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk -FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_up)) +WITH t1 AS ( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id, df_id) + JOIN concept c ON c.concept_id = b.d_id + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_up + ) + ) SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.domain_id, - a.vocabulary_id, - a.concept_class_id, - a.standard_concept, - a.concept_code, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason, - 60 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary upward, 3 ingreds' AS order_desc + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 60 AS concept_order, + 'ATC Combo Class: Primary lateral + Secondary upward, 3 ingreds' AS order_desc FROM t1 a - JOIN t1 b - ON b.class_code = a.class_code - AND b.concept_id = a.concept_id - AND a.ing_id <> b.ing_id - JOIN t1 c - ON c.class_code = a.class_code - AND c.concept_id = b.concept_id - AND a.ing_id <> c.ing_id - AND b.ing_id <> c.ing_id +JOIN t1 b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id +JOIN t1 c ON c.class_code = a.class_code + AND c.concept_id = b.concept_id + AND a.ing_id <> c.ing_id + AND b.ing_id <> c.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND c.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2 + AND b.rnk <> 1 + AND c.rnk <> 1 + AND a.class_code || a.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 2 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary lateral', 2 ingreds' (order = 7) INSERT INTO class_to_drug_new -WITH t1 AS -( - SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk - FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id - WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_lat) -) -SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.domain_id, - a.vocabulary_id, - a.concept_class_id, - a.standard_concept, - a.concept_code, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason, - 7 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' AS order_desc +WITH t1 AS ( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id, df_id) + JOIN concept c ON c.concept_id = b.d_id + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_lat + ) + ) +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 7 AS concept_order, + 'ATC Combo Class: Primary lateral + Secondary lateral, 2 ingreds' AS order_desc FROM t1 a - JOIN t1 b - ON b.class_code = a.class_code - AND b.concept_id = a.concept_id - AND a.ing_id <> b.ing_id +JOIN t1 b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND a.class_code NOT IN (SELECT class_code FROM class_to_drug_new) -AND a.class_name NOT LIKE '%,%and%' AND a.class_code !~ '^J07' -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2426 + AND b.rnk <> 1 + AND a.class_code NOT IN ( + SELECT class_code + FROM class_to_drug_new + ) + AND a.class_name NOT LIKE '%,%and%' + AND a.class_code NOT LIKE 'J07%' + AND a.class_code || a.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 2426 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' + 'Secondary upward', 2 ingreds' (order = 70) INSERT INTO class_to_drug_new -WITH t1 AS -( - SELECT a.class_code, - a.class_name, - a.ing_id, - c.*, - rnk - FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id - WHERE a.class_code IN (SELECT class_code FROM ing_pr_lat_sec_up) -) -SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.domain_id, - a.vocabulary_id, - a.concept_class_id, - a.standard_concept, - a.concept_code, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason, - 70 AS concept_order, - 'ATC Combo Class: Primary lateral + Secondary upward, 2 ingreds' AS order_desc +WITH t1 AS ( + SELECT a.class_code, + a.class_name, + a.ing_id, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id, df_id) + JOIN concept c ON c.concept_id = b.d_id + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_up + ) + ) +SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 70 AS concept_order, + 'ATC Combo Class: Primary lateral + Secondary upward, 2 ingreds' AS order_desc FROM t1 a - JOIN t1 b - ON b.class_code = a.class_code - AND b.concept_id = a.concept_id - AND a.ing_id <> b.ing_id +JOIN t1 b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id WHERE a.rnk = 1 -AND b.rnk <> 1 -AND a.class_code||a.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 0 + AND b.rnk <> 1 + AND a.class_code || a.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 0 -- add mappings of Multicomponent ACT Classes of 'Primary lateral' in combination with other drugs and an excluded Ingredient (order = 8) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT DISTINCT a.class_code AS class_code, - a.class_name AS class_name, - a.concept_name AS ing_name, - d2.concept_code AS df_name, - rnk -FROM ing_pr_lat_combo_excl a - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form'), -t2 AS -(SELECT DISTINCT class_code, - class_name, - c.*, - 11 AS concept_order, - b.ing_name -FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND rnk = 1) +WITH t1 AS ( + SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk + FROM ing_pr_lat_combo_excl a + JOIN internal_relationship_stage i2 ON SUBSTRING(i2.concept_code_1, '\w+') = a.class_code + JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form' + ), + t2 AS ( + SELECT DISTINCT class_code, + class_name, + c.*, + 11 AS concept_order, + b.ing_name + FROM t1 a + JOIN rx_all_combo b ON LOWER(b.ing_name) = LOWER(a.ing_name) + AND LOWER(b.df_name) = LOWER(a.df_name) + JOIN concept c ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 1 + ) SELECT DISTINCT class_code, - class_name, - concept_id, - concept_name, - domain_id, - vocabulary_id, - concept_class_id, - standard_concept, - concept_code, - valid_start_date, - valid_end_date, - invalid_reason, - 8 AS concept_order, - 'ATC Combo Class: Primary lateral in combination with excluded Ingredient' AS order_desc + class_name, + concept_id, + concept_name, + domain_id, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + valid_start_date, + valid_end_date, + invalid_reason, + 8 AS concept_order, + 'ATC Combo Class: Primary lateral in combination with excluded Ingredient' AS order_desc FROM t2 -WHERE concept_name ~ ' / ' -AND concept_id NOT IN (SELECT d_id - FROM rx_all_combo a - JOIN t1 b - ON LOWER (b.ing_name) = LOWER (a.ing_name) - AND LOWER (b.df_name) = LOWER (a.df_name) - AND b.rnk = 0) -AND class_code||concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 0 +WHERE concept_name LIKE '% / %' + AND concept_id NOT IN ( + SELECT d_id + FROM rx_all_combo a + JOIN t1 b ON LOWER(b.ing_name) = LOWER(a.ing_name) + AND LOWER(b.df_name) = LOWER(a.df_name) + AND b.rnk = 0 + ) + AND class_code || concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 0 -- add mappings of Multicomponent ACT Classes of 'Primary upward' + 'Secondary upward' with excluded ingredient' -- currently, no such (order = 0) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT a.class_code, - a.class_name, - a.ing_id, - a.ing_name, - a.df_id, - a.df_name, - c.*, - rnk -FROM atc_all_combo a - JOIN rx_all_combo b USING (ing_id,df_id) - JOIN concept c ON c.concept_id = d_id -WHERE a.class_code IN (SELECT class_code FROM ing_pr_up_sec_up_excl)), -t2 AS ( -SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.domain_id, - a.vocabulary_id, - a.concept_class_id, - a.standard_concept, - a.concept_code, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason, - 0 AS concept_order, -- no match up to now, but it can happen in the future - 'ATC Combo Class: Primary upward + Secondary upward with excluded ingredient' AS order_desc -FROM t1 a - JOIN t1 b - ON b.class_code = a.class_code - AND b.concept_id = a.concept_id - AND a.ing_id <> b.ing_id -WHERE a.rnk = 3 -AND b.rnk <> 3) -SELECT DISTINCT * FROM t2 -WHERE concept_name ~ ' / ' -AND concept_id NOT IN (SELECT d_id - FROM rx_all_combo a - JOIN t1 b - ON LOWER (b.ing_name) = LOWER (a.ing_name) - AND LOWER (b.df_name) = LOWER (a.df_name) - AND b.rnk = 0) -AND class_code||concept_id::varchar NOT IN (SELECT class_code||concept_id::varchar FROM class_to_drug_new); --0 +WITH t1 AS ( + SELECT a.class_code, + a.class_name, + a.ing_id, + a.ing_name, + a.df_id, + a.df_name, + c.*, + rnk + FROM atc_all_combo a + JOIN rx_all_combo b USING (ing_id, df_id) + JOIN concept c ON c.concept_id = d_id + WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_up_sec_up_excl + ) + ), + t2 AS ( + SELECT DISTINCT a.class_code, + a.class_name, + a.concept_id, + a.concept_name, + a.domain_id, + a.vocabulary_id, + a.concept_class_id, + a.standard_concept, + a.concept_code, + a.valid_start_date, + a.valid_end_date, + a.invalid_reason, + 0 AS concept_order, -- no match up to now, but it can happen in the future + 'ATC Combo Class: Primary upward + Secondary upward with excluded ingredient' AS order_desc + FROM t1 a + JOIN t1 b ON b.class_code = a.class_code + AND b.concept_id = a.concept_id + AND a.ing_id <> b.ing_id + WHERE a.rnk = 3 + AND b.rnk <> 3 + ) +SELECT DISTINCT * +FROM t2 +WHERE concept_name LIKE '% / %' + AND concept_id NOT IN ( + SELECT d_id + FROM rx_all_combo a + JOIN t1 b ON LOWER(b.ing_name) = LOWER(a.ing_name) + AND LOWER(b.df_name) = LOWER(a.df_name) + AND b.rnk = 0 + ) + AND class_code || concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );--0 -- add mappings of Multicomponent ACT Classes of 'Primary upward' + 'Secondary upward' (order = 9) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT DISTINCT a.class_code AS class_code, - a.class_name AS class_name, - a.concept_name AS ing_name, - d2.concept_code AS df_name, - rnk -FROM ing_pr_up_sec_up a - JOIN internal_relationship_stage i2 ON SUBSTRING (i2.concept_code_1,'\w+') = a.class_code - JOIN drug_concept_stage d2 - ON d2.concept_code = i2.concept_code_2 - AND d2.concept_class_id = 'Dose Form'), -t2 -AS -(SELECT DISTINCT class_code, - class_name, - c.* -FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND rnk = 3), -t3 AS -(SELECT DISTINCT a.class_code, - a.class_name, - a.ing_name, - c.* -FROM t1 a - JOIN rx_all_combo b - ON lower (b.ing_name) = lower (a.ing_name) - AND lower (b.df_name) = lower (a.df_name) - JOIN concept c - ON c.concept_id = b.d_id - AND c.standard_concept = 'S' - AND rnk = 4) +WITH t1 AS ( + SELECT DISTINCT a.class_code AS class_code, + a.class_name AS class_name, + a.concept_name AS ing_name, + d2.concept_code AS df_name, + rnk + FROM ing_pr_up_sec_up a + JOIN internal_relationship_stage i2 ON SUBSTRING(i2.concept_code_1, '\w+') = a.class_code + JOIN drug_concept_stage d2 ON d2.concept_code = i2.concept_code_2 + AND d2.concept_class_id = 'Dose Form' + ), + t2 AS ( + SELECT DISTINCT class_code, + class_name, + c.* + FROM t1 a + JOIN rx_all_combo b ON LOWER(b.ing_name) = LOWER(a.ing_name) + AND LOWER(b.df_name) = LOWER(a.df_name) + JOIN concept c ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 3 + ), + t3 AS ( + SELECT DISTINCT a.class_code, + a.class_name, + a.ing_name, + c.* + FROM t1 a + JOIN rx_all_combo b ON LOWER(b.ing_name) = LOWER(a.ing_name) + AND LOWER(b.df_name) = LOWER(a.df_name) + JOIN concept c ON c.concept_id = b.d_id + AND c.standard_concept = 'S' + AND rnk = 4 + ) SELECT DISTINCT a.*, - 9 AS concept_order, - 'ATC Combo Class: Primary upward + Secondary upward' AS order_desc + 9 AS concept_order, + 'ATC Combo Class: Primary upward + Secondary upward' AS order_desc FROM t2 a - JOIN t3 b - ON b.class_code = a.class_code - AND a.concept_id = b.concept_id -WHERE a.concept_name ~ ' / '; -- 325 +JOIN t3 b ON b.class_code = a.class_code + AND a.concept_id = b.concept_id +WHERE a.concept_name LIKE '% / %';-- 325 + ------------------------- ---- GET MORE LINKS ----- ------------------------- -- create an interim table with all Primary lateral Multicomponent ATC Classes (rnk = 1 in dev_combo) DROP TABLE IF EXISTS t1; -CREATE UNLOGGED TABLE t1 -AS -SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk - FROM dev_combo WHERE rnk = 1; -- Primary lateral +CREATE UNLOGGED TABLE t1 AS +SELECT DISTINCT class_code, + class_name, + concept_id, + concept_name, + rnk +FROM dev_combo +WHERE rnk = 1;-- Primary lateral -- create an interim table with all Secondary lateral Multicomponent ATC Classes (rnk = 2 in dev_combo) DROP TABLE IF EXISTS t2; -CREATE UNLOGGED TABLE t2 -AS -SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk - FROM dev_combo WHERE rnk = 2; -- Secondary lateral +CREATE UNLOGGED TABLE t2 AS +SELECT DISTINCT class_code, + class_name, + concept_id, + concept_name, + rnk +FROM dev_combo +WHERE rnk = 2;-- Secondary lateral -- create an interim table with all Primary upward Multicomponent ATC Classes DROP TABLE IF EXISTS t3; -CREATE UNLOGGED TABLE t3 -AS -SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk - FROM dev_combo -WHERE rnk = 3; -- Primary upward +CREATE UNLOGGED TABLE t3 AS +SELECT DISTINCT class_code, + class_name, + concept_id, + concept_name, + rnk +FROM dev_combo +WHERE rnk = 3;-- Primary upward -- create an interim table with all Secondary upward Multicomponent ATC DROP TABLE IF EXISTS t4; -CREATE UNLOGGED TABLE t4 -AS -SELECT DISTINCT class_code, class_name, concept_id, concept_name, rnk - FROM dev_combo -WHERE rnk = 4; -- Secondary upward +CREATE UNLOGGED TABLE t4 AS +SELECT DISTINCT class_code, + class_name, + concept_id, + concept_name, + rnk +FROM dev_combo +WHERE rnk = 4;-- Secondary upward -- create an interim table with aggregated ATC Ingredients per one Multicomponent ATC Class (no more than 3 ingredients per Class is recommended) -- add Primary lateral AND (Secondary lateral 1 AND/OR Secondary lateral 2) AND/OR Primary upward AND/OR Secondary upward Multicomponent ATC Classes -DROP TABLE if exists full_combo; -CREATE UNLOGGED TABLE full_combo -AS ( +DROP TABLE IF EXISTS full_combo; +CREATE UNLOGGED TABLE full_combo AS SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id||COALESCE('-' || b.concept_id,'') ||COALESCE('-' || c.concept_id,'') ||COALESCE('-' || c1.concept_id,'') ||COALESCE('-' || d.concept_id,'') AS i_combo + a.class_name, + ARRAY [a.concept_id] || b.concept_id || c.concept_id || c1.concept_id || d.concept_id AS i_combo FROM t1 a - JOIN t2 b ON b.class_code = a.class_code-- rank 2 - LEFT JOIN t2 c ON c.class_code = b.class_code-- rank 2 - LEFT JOIN t3 c1 ON c1.class_code = c.class_code-- rank 3 -- is it possible? usually there is no combinations as rnk=1 + rnk=3 - LEFT JOIN t4 d ON d.class_code = c1.class_code-- rank 4 - AND a.concept_id <> b.concept_id - AND a.concept_id <> c.concept_id - AND a.concept_id <> d.concept_id - AND a.concept_id <> c1.concept_id - AND b.concept_id <> c.concept_id - AND b.concept_id <> c1.concept_id - AND b.concept_id <> d.concept_id - AND c.concept_id <> c1.concept_id - AND c1.concept_id <> d.concept_id -ORDER BY class_code); +JOIN t2 b ON b.class_code = a.class_code -- rank 2 +LEFT JOIN t2 c ON c.class_code = b.class_code -- rank 2 +LEFT JOIN t3 c1 ON c1.class_code = c.class_code -- rank 3 -- is it possible? usually there is no combinations as rnk=1 + rnk=3 +LEFT JOIN t4 d ON d.class_code = c1.class_code -- rank 4 +WHERE a.concept_id <> b.concept_id + AND a.concept_id <> c.concept_id + AND a.concept_id <> d.concept_id + AND a.concept_id <> c1.concept_id + AND b.concept_id <> c.concept_id + AND b.concept_id <> c1.concept_id + AND b.concept_id <> d.concept_id + AND c.concept_id <> c1.concept_id + AND c1.concept_id <> d.concept_id; -- add Primary lateral AND/OR Secondary lateral AND/OR Primary upward AND/OR Secondary upward INSERT INTO full_combo -WITH z1 -AS -(SELECT DISTINCT a.class_code, - a.class_name, - a.concept_id||COALESCE('-' || b.concept_id,'') ||COALESCE('-' || c.concept_id,'') ||COALESCE('-' || d.concept_id,'') AS i_combo -FROM t1 a - JOIN t2 b ON b.class_code = a.class_code-- rank 2 - LEFT JOIN t3 c ON c.class_code = b.class_code-- rank 3 - LEFT JOIN t4 d - ON d.class_code = c.class_code-- rank 4 - AND a.concept_id <> b.concept_id - AND a.concept_id <> c.concept_id - AND a.concept_id <> d.concept_id - AND b.concept_id <> c.concept_id - AND b.concept_id <> d.concept_id -ORDER BY class_code -) -SELECT * +WITH z1 AS ( + SELECT DISTINCT a.class_code, + a.class_name, + ARRAY [a.concept_id] || b.concept_id || c.concept_id || d.concept_id AS i_combo + FROM t1 a + JOIN t2 b ON b.class_code = a.class_code -- rank 2 + LEFT JOIN t3 c ON c.class_code = b.class_code -- rank 3 + LEFT JOIN t4 d ON d.class_code = c.class_code -- rank 4 + WHERE a.concept_id <> b.concept_id + AND a.concept_id <> c.concept_id + AND a.concept_id <> d.concept_id + AND b.concept_id <> c.concept_id + AND b.concept_id <> d.concept_id + ) +SELECT z1.* FROM z1 -WHERE i_combo NOT IN (SELECT i_combo FROM full_combo); - --- create a temporary table with Ingredient permutations - additional layer of mappings: Primary lateral AND/OR Secondary lateral 1 AND/OR Secondary lateral 2 -DROP TABLE IF EXISTS permutations; -CREATE UNLOGGED TABLE permutations -AS -SELECT distinct a.class_code, a.class_name, a.concept_id||COALESCE('-'||b.concept_id, '')||COALESCE('-'||c.concept_id, '') AS i_combo - FROM t1 a -- from Primary lateral -LEFT JOIN t2 b ON b.class_code = a.class_code -- to the 1st Secondary lateral -LEFT JOIN t2 c ON c.class_code = a.class_code -- and the 2nd Secondary lateral -WHERE b.concept_id <> c.concept_id AND b.concept_id <> a.concept_id; +LEFT JOIN full_combo fc ON fc.i_combo @> z1.i_combo + AND fc.i_combo <@ z1.i_combo --equivalent of 'NOT IN' +WHERE fc.i_combo IS NULL; --- add newly created permutations to the full_combo table in order to enrich the set of aggregated ATC Ingredients +-- add additional layer of mappings (Primary lateral AND/OR Secondary lateral 1 AND/OR Secondary lateral 2) to the full_combo table in order to enrich the set of aggregated ATC Ingredients INSERT INTO full_combo -SELECT * FROM permutations; +SELECT DISTINCT a.class_code, + a.class_name, + ARRAY [a.concept_id] || b.concept_id || c.concept_id AS i_combo +FROM t1 a -- from Primary lateral +LEFT JOIN t2 b ON b.class_code = a.class_code -- to the 1st Secondary lateral +LEFT JOIN t2 c ON c.class_code = a.class_code -- and the 2nd Secondary lateral +WHERE b.concept_id <> c.concept_id + AND b.concept_id <> a.concept_id; -- create temporary table with all possible i_combos for Primary lateral in combinations (with unspecified drugs) -DROP TABLE if exists ing_pr_lat_combo_to_drug; -CREATE TABLE ing_pr_lat_combo_to_drug -AS ( -WITH z1 AS -(SELECT drug_concept_id, - REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo -FROM rx_combo -WHERE i_combo LIKE '%-%') - SELECT DISTINCT a.class_code, - a.class_name, - d.i_combo +DROP TABLE IF EXISTS ing_pr_lat_combo_to_drug; +CREATE UNLOGGED TABLE ing_pr_lat_combo_to_drug AS +SELECT DISTINCT a.class_code, + a.class_name, + d.i_combo FROM ing_pr_lat_combo a - JOIN z1 b ON b.i_combo = a.concept_id - JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id); +JOIN ( + SELECT drug_concept_id, + UNNEST(i_combo) AS i_combo + FROM rx_combo + ) b ON b.i_combo = a.concept_id +JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id; -- create temporary table with all possible i_combos for Primary lateral + Secondary upward -DROP TABLE if exists ing_pr_lat_sec_up_combo_to_drug; -CREATE TABLE ing_pr_lat_sec_up_combo_to_drug -AS ( -WITH z1 AS -(SELECT drug_concept_id, - REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo -FROM rx_combo -WHERE i_combo LIKE '%-%') +DROP TABLE IF EXISTS ing_pr_lat_sec_up_combo_to_drug; +CREATE UNLOGGED TABLE ing_pr_lat_sec_up_combo_to_drug AS + WITH z1 AS ( + SELECT drug_concept_id, + UNNEST(i_combo) AS i_combo + FROM rx_combo + ) SELECT DISTINCT a.class_code, - a.class_name, - c.i_combo + a.class_name, + c.i_combo FROM ing_pr_lat_sec_up a - JOIN z1 b - ON b.i_combo = a.concept_id - AND a.rnk = 1 - JOIN ing_pr_lat_sec_up a1 - ON a1.class_code = a.class_code - AND a1.concept_id <> a.concept_id - JOIN z1 j - ON j.i_combo = a1.concept_id - AND a1.rnk = 4 - JOIN rx_combo c - ON c.drug_concept_id = b.drug_concept_id - AND c.drug_concept_id = j.drug_concept_id); +JOIN z1 b ON b.i_combo = a.concept_id + AND a.rnk = 1 +JOIN ing_pr_lat_sec_up a1 ON a1.class_code = a.class_code + AND a1.concept_id <> a.concept_id +JOIN z1 j ON j.i_combo = a1.concept_id + AND a1.rnk = 4 +JOIN rx_combo c ON c.drug_concept_id = b.drug_concept_id + AND c.drug_concept_id = j.drug_concept_id; -- create temporary table with all possible i_combos for Primary lateral in combination with excluded Ingredient -DROP TABLE if exists ing_pr_lat_combo_excl_to_drug; -CREATE TABLE ing_pr_lat_combo_excl_to_drug -AS -(WITH t1 -AS -(SELECT drug_concept_id, - REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo -FROM rx_combo -WHERE i_combo LIKE '%-%') - SELECT DISTINCT - a.class_code, - a.class_name, - d.i_combo - FROM ing_pr_lat_combo_excl a - JOIN t1 b ON b.i_combo = a.concept_id -JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code - AND a1.rnk <> a.rnk - AND a1.rnk = 0 - JOIN t1 f ON f.i_combo = a1.concept_id -- excluded -JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id - JOIN rx_combo d1 - ON d1.drug_concept_id = f.drug_concept_id -- excluded -AND d1.drug_concept_id <> d.drug_concept_id); +DROP TABLE IF EXISTS ing_pr_lat_combo_excl_to_drug; +CREATE UNLOGGED TABLE ing_pr_lat_combo_excl_to_drug AS + WITH t1 AS ( + SELECT drug_concept_id, + UNNEST(i_combo) AS i_combo + FROM rx_combo + ) +SELECT DISTINCT a.class_code, + a.class_name, + d.i_combo +FROM ing_pr_lat_combo_excl a +JOIN t1 b ON b.i_combo = a.concept_id +JOIN ing_pr_lat_combo_excl a1 ON a1.class_code = a.class_code + AND a1.rnk <> a.rnk + AND a1.rnk = 0 +JOIN t1 f ON f.i_combo = a1.concept_id -- excluded +JOIN rx_combo d ON d.drug_concept_id = b.drug_concept_id +JOIN rx_combo d1 ON d1.drug_concept_id = f.drug_concept_id -- excluded + AND d1.drug_concept_id <> d.drug_concept_id; -- create a temporary table with all possible i_combos for Primary upward + Secondary upward: ing_pr_sec_up_combo_to_drug -DROP TABLE if exists ing_pr_sec_up_combo_to_drug; -CREATE TABLE ing_pr_sec_up_combo_to_drug -AS -(WITH t1 -AS -(SELECT drug_concept_id, - REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo -FROM rx_combo -WHERE i_combo LIKE '%-%') +DROP TABLE IF EXISTS ing_pr_sec_up_combo_to_drug; +CREATE UNLOGGED TABLE ing_pr_sec_up_combo_to_drug AS + WITH t1 AS ( + SELECT drug_concept_id, + UNNEST(i_combo) AS i_combo + FROM rx_combo + ) + SELECT DISTINCT a.class_code, - a.class_name, - c.i_combo + a.class_name, + c.i_combo FROM ing_pr_up_sec_up a - JOIN t1 b - ON b.i_combo = a.concept_id - AND a.rnk = 3 - JOIN ing_pr_up_sec_up a1 - ON a1.class_code = a.class_code - AND a1.concept_id <> a.concept_id - JOIN t1 j - ON j.i_combo = a1.concept_id - AND a1.rnk = 4 - JOIN rx_combo c - ON c.drug_concept_id = b.drug_concept_id - AND c.drug_concept_id = j.drug_concept_id); - +JOIN t1 b ON b.i_combo = a.concept_id + AND a.rnk = 3 +JOIN ing_pr_up_sec_up a1 ON a1.class_code = a.class_code + AND a1.concept_id <> a.concept_id +JOIN t1 j ON j.i_combo = a1.concept_id + AND a1.rnk = 4 +JOIN rx_combo c ON c.drug_concept_id = b.drug_concept_id + AND c.drug_concept_id = j.drug_concept_id; + -- create temporary table with all possible i_combos for Primary upward + Secondary upward with excluded Ingredients -- currently no match, but query works ing_pr_sec_up_combo_excl_to_drug -DROP TABLE if exists ing_pr_sec_up_combo_excl_to_drug; -CREATE TABLE ing_pr_sec_up_combo_excl_to_drug -AS -(WITH t1 -AS -(SELECT drug_concept_id, - REGEXP_SPLIT_TO_TABLE(i_combo,'-')::INT AS i_combo -FROM rx_combo -WHERE i_combo LIKE '%-%') +DROP TABLE IF EXISTS ing_pr_sec_up_combo_excl_to_drug; +CREATE UNLOGGED TABLE ing_pr_sec_up_combo_excl_to_drug AS + WITH t1 AS ( + SELECT drug_concept_id, + UNNEST(i_combo) AS i_combo + FROM rx_combo + ) + SELECT DISTINCT a.class_code, - a.class_name, - c.i_combo + a.class_name, + c.i_combo FROM ing_pr_sec_up_combo_excl a - JOIN t1 b - ON b.i_combo = a.concept_id - AND a.rnk = 3 - JOIN ing_pr_sec_up_combo_excl a1 - ON a1.class_code = a.class_code - AND a1.concept_id <> a.concept_id - JOIN t1 j - ON j.i_combo = a1.concept_id - AND a1.rnk = 4 - JOIN ing_pr_sec_up_combo_excl a2 - ON a2.class_code = a.class_code - AND a2.rnk = 0 - JOIN rx_combo c - ON c.drug_concept_id = b.drug_concept_id - AND c.drug_concept_id = j.drug_concept_id - JOIN rx_combo c1 - ON c1.drug_concept_id = j.drug_concept_id - AND c1.drug_concept_id = c.drug_concept_id - AND c.i_combo !~ a2.concept_id::VARCHAR); +JOIN t1 b ON b.i_combo = a.concept_id + AND a.rnk = 3 +JOIN ing_pr_sec_up_combo_excl a1 ON a1.class_code = a.class_code + AND a1.concept_id <> a.concept_id +JOIN t1 j ON j.i_combo = a1.concept_id + AND a1.rnk = 4 +JOIN ing_pr_sec_up_combo_excl a2 ON a2.class_code = a.class_code + AND a2.rnk = 0 +JOIN rx_combo c ON c.drug_concept_id = b.drug_concept_id + AND c.drug_concept_id = j.drug_concept_id +JOIN rx_combo c1 ON c1.drug_concept_id = j.drug_concept_id + AND c1.drug_concept_id = c.drug_concept_id + AND NOT c.i_combo && ARRAY [a2.concept_id]; -- add prepared list of aggregated Ingredients of ATC Combo Classes to full_combo INSERT INTO full_combo SELECT * FROM ing_pr_lat_combo_to_drug -UNION +UNION ALL SELECT * FROM ing_pr_lat_combo_excl_to_drug -UNION +UNION ALL SELECT * FROM ing_pr_sec_up_combo_to_drug -UNION +UNION ALL SELECT * FROM ing_pr_sec_up_combo_excl_to_drug -UNION +UNION ALL SELECT * FROM ing_pr_lat_sec_up_combo_to_drug; --- create a table to order aggregated ATC ingredients by an Ingredient -DROP TABLE IF EXISTS full_combo_reodered; -CREATE UNLOGGED TABLE full_combo_reodered AS -SELECT DISTINCT fc.class_code, - fc.class_name, - l.i_combo -FROM full_combo fc -CROSS JOIN LATERAL(SELECT STRING_AGG(s0.ing, '-' ORDER BY s0.ing::INT) AS i_combo FROM ( - SELECT UNNEST(STRING_TO_ARRAY(fc.i_combo, '-')) AS ing - ) AS s0) l; - --- create index to make the script faster -CREATE INDEX i_full_combo_reodered ON full_combo_reodered (class_code, i_combo); +CREATE INDEX idx_full_combo_cc ON full_combo (class_code) WITH (FILLFACTOR=100); +ANALYZE full_combo; --create a table containing aggregated Ingredients + Dose Forms for ATC Combo Classes DROP TABLE full_combo_with_form; -CREATE UNLOGGED TABLE full_combo_with_form -AS -SELECT DISTINCT - a.class_code, - a.class_name, - a.i_combo, - r.concept_id_2::int AS df_id -FROM full_combo_reodered a - JOIN internal_relationship_stage i ON class_code = substring (concept_code_1, '\w+') -- cut ATC code before space character - JOIN drug_concept_stage b - ON lower(b.concept_code) = lower(i.concept_code_2) - AND b.concept_class_id = 'Dose Form' - JOIN relationship_to_concept r ON r.concept_code_1 = i.concept_code_2; -/******************** -***** REFERENCE ***** -*********************/ --- create a temporary table of reference containing links between ATC source codes and combinations of ATC_codes AND Dose Forms from drug_concept_stage -DROP TABLE if exists reference; -CREATE TABLE reference -AS -SELECT DISTINCT class_code, - concept_code -FROM drug_concept_stage - LEFT JOIN class_drugs_scraper ON SPLIT_PART (concept_name,' ',1) = class_code; - --- add ATC Combo Classes WO Dose Forms using the 'reference' table -INSERT INTO full_combo_with_form -(class_code, i_combo) +CREATE UNLOGGED TABLE full_combo_with_form AS +SELECT DISTINCT a.class_code, + a.class_name, + a.i_combo, + r.concept_id_2 AS df_id +FROM full_combo a +JOIN internal_relationship_stage i ON SUBSTRING(i.concept_code_1, '\w+') = a.class_code -- cut ATC code before space character +JOIN drug_concept_stage b ON LOWER(b.concept_code) = LOWER(i.concept_code_2) + AND b.concept_class_id = 'Dose Form' +JOIN relationship_to_concept r ON r.concept_code_1 = i.concept_code_2; + +-- add ATC Combo Classes WO Dose Forms using links between ATC source codes and combinations of ATC_codes AND Dose Forms from drug_concept_stage +INSERT INTO full_combo_with_form ( + class_code, + i_combo + ) SELECT DISTINCT f.class_code, - i_combo - FROM full_combo_reodered f - JOIN reference r ON r.class_code = f.class_code -WHERE r.concept_code = r.class_code; + f.i_combo +FROM full_combo f +JOIN ( + SELECT DISTINCT cds.class_code + FROM drug_concept_stage dcs + JOIN class_drugs_scraper cds ON cds.class_code = SPLIT_PART(dcs.concept_name, ' ', 1) + WHERE dcs.concept_code = cds.class_code + ) r ON r.class_code = f.class_code; + +CREATE INDEX idx_full_combo_with_form ON full_combo_with_form (df_id, class_code) WITH (FILLFACTOR=100); +ANALYZE full_combo_with_form; -CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo,df_id); /******************************* ******** CLASS TO DRUG ********* ********************************/ + -- add the 2nd portion of mappings of Multicomponent ATC Classes -- ATC Combo Classes with Dose Forms using full_combo_with_form and rx_combo (order = 10) INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT c.concept_id, -- Standard Drug Product - c.concept_name, - c.concept_class_id, - c.vocabulary_id, - r.concept_id_2 AS df_id, -- Dose Form - a.i_combo -- combination of Standard Ingredient IDs as a key for join - FROM rx_combo a - JOIN concept c ON c.concept_id = a.drug_concept_id - JOIN concept_relationship r ON r.concept_id_1 = c.concept_id -WHERE c.concept_class_id = 'Clinical Drug Form' -AND c.vocabulary_id LIKE 'RxNorm%' -AND c.invalid_reason IS NULL -AND r.relationship_id = 'RxNorm has dose form' -AND r.invalid_reason IS NULL - ) - SELECT DISTINCT f.class_code, -- ATC - c.concept_name AS class_name, - d.*, - 10 AS conept_order, - 'ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations' AS order_desc - FROM full_combo_with_form f - JOIN concept_manual c ON c.concept_code = f.class_code - AND c.invalid_reason IS NULL +WITH t1 AS ( + SELECT c.concept_id, -- Standard Drug Product + c.concept_name, + c.concept_class_id, + c.vocabulary_id, + r.concept_id_2 AS df_id, -- Dose Form + a.i_combo -- combination of Standard Ingredient IDs as a key for join + FROM rx_combo a + JOIN concept c ON c.concept_id = a.drug_concept_id + JOIN concept_relationship r ON r.concept_id_1 = c.concept_id + WHERE c.concept_class_id = 'Clinical Drug Form' + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.invalid_reason IS NULL + AND r.relationship_id = 'RxNorm has dose form' + AND r.invalid_reason IS NULL + ) +SELECT DISTINCT f.class_code, -- ATC + c.concept_name AS class_name, + d.*, + 10 AS conept_order, + 'ATC Combo Class with Dose Form to Clinical Drug Form by additional permutations' AS order_desc +FROM full_combo_with_form f +JOIN concept_manual c ON c.concept_code = f.class_code + AND c.invalid_reason IS NULL AND c.concept_class_id = 'ATC 5th' - JOIN t1 r - ON r.i_combo = f.i_combo -- combination of Standard Ingredient IDs - AND r.df_id = f.df_id - JOIN concept d ON d.concept_id = r.concept_id - AND f.class_code||r.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new) - AND f.class_code IN ( 'J07CA10', 'G03FB09', 'G03FB06', 'G03FB05', 'G03FA12', 'G03FA11','G03FA01', 'G03EA03','A11GB01'); -- 63 +JOIN t1 r ON r.i_combo @> f.i_combo + AND r.i_combo <@ f.i_combo -- combination of Standard Ingredient IDs + AND r.df_id = f.df_id +JOIN concept d ON d.concept_id = r.concept_id +WHERE f.class_code || r.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + ) + AND f.class_code IN ( + 'J07CA10', + 'G03FB09', + 'G03FB06', + 'G03FB05', + 'G03FA12', + 'G03FA11', + 'G03FA01', + 'G03EA03', + 'A11GB01' + );-- 63 -- add manual mappings from concept_relationship_manual (order = 11) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, - f.concept_name AS class_name, - c.*, - 11 AS concept_order, - 'ATC Class to Drug Product from concept_relationship_manual' AS order_desc + f.concept_name AS class_name, + c.*, + 11 AS concept_order, + 'ATC Class to Drug Product from concept_relationship_manual' AS order_desc FROM class_drugs_scraper a - JOIN concept_relationship_manual b ON b.concept_code_1 = a.class_code - JOIN concept_manual f - ON f.concept_code = a.class_code - AND b.relationship_Id IN ('ATC - RxNorm') - JOIN concept c - ON c.concept_code = b.concept_code_2 - AND c.vocabulary_id = b.vocabulary_id_2 - AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - AND c.standard_concept = 'S' - AND (a.class_code,c.concept_id) NOT IN (SELECT class_code,concept_id FROM class_to_drug_new) -AND f.invalid_reason IS NULL AND b.invalid_reason IS null -AND f.concept_class_id = 'ATC 5th'; -- 7558 +JOIN concept_relationship_manual b ON b.concept_code_1 = a.class_code +JOIN concept_manual f ON f.concept_code = a.class_code + AND f.invalid_reason IS NULL + AND f.concept_class_id = 'ATC 5th' +JOIN concept c ON c.concept_code = b.concept_code_2 + AND c.vocabulary_id = b.vocabulary_id_2 + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.standard_concept = 'S' + AND a.class_code || c.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + ) +WHERE b.relationship_Id = 'ATC - RxNorm' + AND b.invalid_reason IS NULL;-- 7558 -- manual clean up for Precise Ingredients and other particular cases (according to the information on the ATC WHO Website) DELETE FROM class_to_drug_new -WHERE (class_code = 'A02BA07' AND concept_class_id = 'Clinical Drug Form') -- Branded Drug 'Tritec' -OR class_code = 'N05AF02' -- clopentixol -OR class_code IN ('D07AB02','D07BB04') -- hydrocortisone butyrate + combo that so far doesn't exist -OR class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral -OR (class_code = 'B02BD14' AND concept_name LIKE '%Tretten%') -- 2 --catridecacog -OR (class_code IN ('B02BD14','B02BD11') AND concept_class_id = 'Ingredient')-- susoctocog alfa | catridecacog -; -- 310 +WHERE ( + class_code = 'A02BA07' + AND concept_class_id = 'Clinical Drug Form' + ) -- Branded Drug 'Tritec' + OR class_code = 'N05AF02' -- clopentixol + OR class_code IN ( + 'D07AB02', + 'D07BB04' + ) -- hydrocortisone butyrate + combo that so far doesn't exist + OR class_code = 'C01DA05' -- pentaerithrityl tetranitrate; oral + OR ( + class_code = 'B02BD14' + AND concept_name LIKE '%Tretten%' + ) -- 2 --catridecacog + OR ( + class_code IN ( + 'B02BD14', + 'B02BD11' + ) + AND concept_class_id = 'Ingredient' + );-- susoctocog alfa | catridecacog + -- 310 -- add additional semi-manual mappings based on pattern-matching (order = 12) INSERT INTO class_to_drug_new -with t1 AS ( -SELECT 'B02BD11' AS class_code,'catridecacog' as class_name, concept_id -FROM concept -WHERE (vocabulary_id LIKE 'RxNorm%' - AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' - AND standard_concept = 'S' - AND concept_class_id = 'Clinical Drug') - OR concept_id = 35603348 -- the whole hierarchy (factor XIII Injection [Tretten] Branded Drug Form) -UNION ALL -SELECT 'B02BD14','susoctocog alfa', concept_id -FROM concept -WHERE (vocabulary_id LIKE 'RxNorm%' - AND concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' - AND standard_concept = 'S' - AND concept_class_id = 'Clinical Drug') - OR concept_id IN (35603348, 44109089) -- the whole hierarchy -UNION ALL -SELECT 'A02BA07','ranitidine bismuth citrate',concept_id -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' - AND concept_name LIKE '%Tritec%' - AND standard_concept = 'S' - AND concept_class_id = 'Branded Drug Form' -UNION ALL -SELECT 'N05AF02','clopenthixol', concept_id -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' - AND concept_name ~* 'Sordinol|Ciatyl' - AND standard_concept = 'S' - AND concept_class_id = 'Branded Drug Form' -UNION ALL -SELECT 'D07AB02','hydrocortisone butyrate',concept_id -FROM concept -WHERE vocabulary_id LIKE 'RxNorm%' - AND concept_name ~* 'Hydrocortisone butyrate' - AND concept_class_id = 'Clinical Drug' - AND standard_concept = 'S' - ) +WITH t1 AS ( + SELECT 'B02BD11' AS class_code, + 'catridecacog' AS class_name, + concept_id + FROM concept + WHERE ( + vocabulary_id LIKE 'RxNorm%' + AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' + AND standard_concept = 'S' + AND concept_class_id = 'Clinical Drug' + ) + OR concept_id = 35603348 -- the whole hierarchy (factor XIII Injection [Tretten] Branded Drug Form) + + UNION ALL + + SELECT 'B02BD14', + 'susoctocog alfa', + concept_id + FROM concept + WHERE ( + vocabulary_id LIKE 'RxNorm%' + AND concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' + AND standard_concept = 'S' + AND concept_class_id = 'Clinical Drug' + ) + OR concept_id IN ( + 35603348, + 44109089 + ) -- the whole hierarchy + + UNION ALL + + SELECT 'A02BA07', + 'ranitidine bismuth citrate', + concept_id + FROM concept + WHERE vocabulary_id LIKE 'RxNorm%' + AND concept_name LIKE '%Tritec%' + AND standard_concept = 'S' + AND concept_class_id = 'Branded Drug Form' + + UNION ALL + + SELECT 'N05AF02', + 'clopenthixol', + concept_id + FROM concept + WHERE vocabulary_id LIKE 'RxNorm%' + AND concept_name ~* 'Sordinol|Ciatyl' + AND standard_concept = 'S' + AND concept_class_id = 'Branded Drug Form' + + UNION ALL + + SELECT 'D07AB02', + 'hydrocortisone butyrate', + concept_id + FROM concept + WHERE vocabulary_id LIKE 'RxNorm%' + AND concept_name ILIKE '%Hydrocortisone butyrate%' + AND concept_class_id = 'Clinical Drug' + AND standard_concept = 'S' + ) SELECT DISTINCT a.class_code, - d.concept_name, - c.*, - 12 AS concept_order, - 'ATC Class with semi-manual point fix' AS order_desc + d.concept_name, + c.*, + 12 AS concept_order, + 'ATC Class with semi-manual point fix' AS order_desc FROM t1 a - JOIN concept c ON c.concept_id = a.concept_id - JOIN concept_manual d - ON d.concept_code = a.class_code - AND d.concept_class_id = 'ATC 5th' - AND d.invalid_reason IS NULL -AND a.class_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 13 - +JOIN concept c ON c.concept_id = a.concept_id +JOIN concept_manual d ON d.concept_code = a.class_code + AND d.concept_class_id = 'ATC 5th' + AND d.invalid_reason IS NULL +WHERE a.class_code || c.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 13 + -- clean up erroneous amount of ingredients DELETE FROM class_to_drug_new WHERE class_name LIKE '%,%and%' - AND class_name NOT LIKE '%,%,%and%' - AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT LIKE '% / % / %'; -- 14 + AND class_name NOT LIKE '%,%,%and%' + AND NOT class_name ~* 'comb|other|whole root|selective' + AND concept_name NOT LIKE '% / % / %';-- 14 ---- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug (order = 13) +-- add missing Clinical Drug Forms and Clinical Drugs using previous version of class_to_drug (order = 13) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, - b.concept_name AS class_name, - c.*, - 13 AS concept_order, - 'ATC Class from old class_to_drug' AS order_desc + b.concept_name AS class_name, + c.*, + 13 AS concept_order, + 'ATC Class from old class_to_drug' AS order_desc FROM sources_class_to_drug_old a - JOIN concept_manual b ON b.concept_code = a.class_code - JOIN concept c ON c.concept_id = a.concept_id - AND c.standard_concept = 'S' - AND c.concept_class_id !~ 'Pack|Ingredient' - AND b.invalid_reason IS NULL -WHERE (class_code) NOT IN (SELECT class_code FROM class_to_drug_new) -AND class_code NOT IN ('S02CA01', 'V03AB05', 'P03AC54', 'S02CA03', 'S02CA03') -AND (class_code, c.concept_id) NOT IN ( - SELECT 'A06AA02', 40031558 -- oral - otic - UNION ALL - SELECT 'A06AA02', 40031561 -- oral - rectal - UNION ALL - SELECT 'A06AA02', 40031561 -- oral - enema - UNION ALL - SELECT 'A06AA02', 40723180 -- oral - enema - UNION ALL - SELECT 'A06AA02', 41080219 -- oral - enema - UNION ALL - SELECT 'A06AA02', 41205788 -- oral - enema - UNION ALL - SELECT 'A06AA02', 43158334 -- oral - enema - UNION ALL - SELECT 'A06AA02', 40036796) -- oral - enema - ; -- 235 +JOIN concept_manual b ON b.concept_code = a.class_code + AND b.invalid_reason IS NULL +JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' + AND c.concept_class_id !~ 'Pack|Ingredient' +WHERE a.class_code NOT IN ( + SELECT class_code + FROM class_to_drug_new + ) + AND a.class_code NOT IN ( + 'S02CA01', + 'V03AB05', + 'P03AC54', + 'S02CA03', + 'S02CA03' + ) +AND (a.class_code, c.concept_id) NOT IN ( + SELECT 'A06AA02', 40031558 -- oral - otic + UNION ALL + SELECT 'A06AA02', 40031561 -- oral - rectal + UNION ALL + SELECT 'A06AA02', 40031561 -- oral - enema + UNION ALL + SELECT 'A06AA02', 40723180 -- oral - enema + UNION ALL + SELECT 'A06AA02', 41080219 -- oral - enema + UNION ALL + SELECT 'A06AA02', 41205788 -- oral - enema + UNION ALL + SELECT 'A06AA02', 43158334 -- oral - enema + UNION ALL + SELECT 'A06AA02', 40036796); -- oral - enema +-- 235 + /********************** ****** ADD PACKS ****** ***********************/ + -- add packs of Primary lateral only in combination (order = 14) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, - a.class_name, - j.*, - 14 AS concept_order, - 'Pack: Primary lateral in combo' AS order_desc + a.class_name, + j.*, + 14 AS concept_order, + 'Pack: Primary lateral in combo' AS order_desc FROM class_to_drug_new a - JOIN concept_relationship r ON r.concept_id_1 = a.concept_id - JOIN concept d - ON d.concept_Id = r.concept_id_2 - AND d.concept_class_id = 'Clinical Drug' - JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id - JOIN concept j ON j.concept_id = r2.concept_id_2 +JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + AND r.invalid_reason IS NULL +JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.concept_class_id = 'Clinical Drug' +JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id + AND r2.invalid_reason IS NULL +JOIN concept j ON j.concept_id = r2.concept_id_2 + AND j.concept_class_id IN ( + 'Clinical Pack', + 'Clinical Pack Box', + 'Branded Pack' + ) + AND j.concept_name LIKE '% / %' -- combos only + AND j.standard_concept = 'S' WHERE a.concept_class_id = 'Clinical Drug Form' -AND r.invalid_reason IS NULL -AND r2.invalid_reason IS NULL -AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') -AND j.standard_concept = 'S' -AND a.class_code IN (SELECT class_code - FROM ing_pr_lat_combo) -AND j.concept_name ~ ' / ' -- combos only -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 1174 + AND a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_combo + ) + AND a.class_code || j.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 1174 -- add additional packs of Primary lateral Ingredients in combination (Class A, combinations, order = 15) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, - a.class_name, - j.*, - 15 AS concept_order, - 'Pack: Primary lateral in combo additional' AS order_desc + a.class_name, + j.*, + 15 AS concept_order, + 'Pack: Primary lateral in combo additional' AS order_desc FROM class_to_drug_new a - JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id - JOIN concept j ON j.concept_id = r2.concept_id_2 -WHERE r2.invalid_reason IS NULL -AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack','Branded Pack Box') -AND j.standard_concept = 'S' -AND a.class_code IN (SELECT class_code - FROM ing_pr_lat_combo) -AND j.concept_name ~ ' / ' -- combos only -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 262 +JOIN concept_relationship r2 ON r2.concept_id_1 = a.concept_id + AND r2.invalid_reason IS NULL +JOIN concept j ON j.concept_id = r2.concept_id_2 + AND j.concept_class_id IN ( + 'Clinical Pack', + 'Clinical Pack Box', + 'Branded Pack', + 'Branded Pack Box' + ) + AND j.standard_concept = 'S' + AND j.concept_name LIKE '% / %' -- combos only +WHERE a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_combo + ) + AND a.class_code || j.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 262 -- add packs of Primary lateral + Secondary lateral (Class A AND Class B, order = 16) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, - a.class_name, - j.*, - 16 AS concept_order, - 'Pack: Primary lateral + Secondary lateral' AS order_desc + a.class_name, + j.*, + 16 AS concept_order, + 'Pack: Primary lateral + Secondary lateral' AS order_desc FROM class_to_drug_new a - JOIN concept_relationship r ON r.concept_id_1 = a.concept_id - JOIN concept d - ON d.concept_Id = r.concept_id_2 - AND d.concept_class_id = 'Clinical Drug' - JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id - JOIN concept j ON j.concept_id = r2.concept_id_2 +JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + AND r.invalid_reason IS NULL +JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.concept_class_id = 'Clinical Drug' +JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id + AND r2.invalid_reason IS NULL +JOIN concept j ON j.concept_id = r2.concept_id_2 + AND j.concept_class_id IN ( + 'Clinical Pack', + 'Clinical Pack Box', + 'Branded Pack' + ) + AND j.standard_concept = 'S' + AND j.concept_name LIKE '% / %' WHERE a.concept_class_id = 'Clinical Drug Form' -AND r.invalid_reason IS NULL -AND r2.invalid_reason IS NULL -AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') -AND j.standard_concept = 'S' -AND a.class_code IN (SELECT class_code - FROM ing_pr_lat_sec_lat) -AND j.concept_name ~ ' / ' -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 3830 + AND a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_lat + ) + AND a.class_code || j.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 3830 -- add packs of Primary lateral and Secondary upward (Class A + Class D, order = 17) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, - a.class_name, - j.*, - 17 AS concept_order, - 'Pack: Primary lateral + Secondary upward' AS order_desc + a.class_name, + j.*, + 17 AS concept_order, + 'Pack: Primary lateral + Secondary upward' AS order_desc FROM class_to_drug_new a - JOIN concept_relationship r ON r.concept_id_1 = a.concept_id - JOIN concept d - ON d.concept_Id = r.concept_id_2 - AND d.concept_class_id = 'Clinical Drug' - JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id - JOIN concept j ON j.concept_id = r2.concept_id_2 +JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + AND r.invalid_reason IS NULL +JOIN concept d ON d.concept_id = r.concept_id_2 + AND d.concept_class_id = 'Clinical Drug' +JOIN concept_relationship r2 ON r2.concept_id_1 = d.concept_id + AND r2.invalid_reason IS NULL +JOIN concept j ON j.concept_id = r2.concept_id_2 + AND j.concept_class_id IN ( + 'Clinical Pack', + 'Clinical Pack Box', + 'Branded Pack' + ) + AND j.standard_concept = 'S' + AND j.concept_name LIKE '% / %' WHERE a.concept_class_id = 'Clinical Drug Form' -AND r.invalid_reason IS NULL -AND r2.invalid_reason IS NULL -AND j.concept_class_id IN ('Clinical Pack','Clinical Pack Box','Branded Pack') -AND j.standard_concept = 'S' -AND a.class_code IN (SELECT class_code - FROM ing_pr_lat_sec_up) -AND j.concept_name ~ ' / ' -AND (a.class_code,j.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 218 + AND a.class_code IN ( + SELECT class_code + FROM ing_pr_lat_sec_up + ) + AND a.class_code || j.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 218 --- add missing Packs using previous version of class_to_drug (order = 18) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code AS class_code, - b.concept_name AS class_name, - c.*, - 18 AS concept_order, - 'Additional Pack: from old c_t_d' AS order_desc -FROM sources_class_to_drug_160921 a - JOIN concept_manual b ON b.concept_code = a.class_code - JOIN concept c - ON a.concept_id = c.concept_id - AND c.standard_concept = 'S' - AND b.invalid_reason IS NULL -WHERE (class_code,a.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new) -AND a.concept_class_id ~ 'Pack' -AND a.class_code NOT IN ('G01AF55','S03CA04','S01CA03','S02CA03')-- gives packs with erroneous forms -AND a.class_code NOT IN ('B03AE01','C07BB52','D01AC52','C10AD52') -- wrong ing combo -; -- 289 + b.concept_name AS class_name, + c.*, + 18 AS concept_order, + 'Additional Pack: from old c_t_d' AS order_desc +FROM sources_class_to_drug_old a +JOIN concept_manual b ON b.concept_code = a.class_code + AND b.invalid_reason IS NULL +JOIN concept c ON c.concept_id = a.concept_id + AND c.standard_concept = 'S' +WHERE a.class_code || a.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + ) + AND a.concept_class_id LIKE '%Pack%' + AND a.class_code NOT IN ( + 'G01AF55', + 'S03CA04', + 'S01CA03', + 'S02CA03' + ) -- gives packs with erroneous forms + AND a.class_code NOT IN ( + 'B03AE01', + 'C07BB52', + 'D01AC52', + 'C10AD52' + );-- wrong ing combo + -- 289 -- еnrich the pool of links to Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' -- they are always used as packs (order = 19) INSERT INTO class_to_drug_new SELECT DISTINCT class_code, - class_name, - c.*, - 19 AS concept_order, - 'Additional Pack: semi-manual contraceptive' AS order_desc + class_name, + c.*, + 19 AS concept_order, + 'Additional Pack: semi-manual contraceptive' AS order_desc FROM class_to_drug_new f - JOIN concept_ancestor ca ON ca.ancestor_concept_id = CAST (f.concept_id AS INT) - JOIN concept c - ON c.concept_id = descendant_concept_id - AND c.concept_class_id LIKE '%Pack%' +JOIN concept_ancestor ca ON ca.ancestor_concept_id = f.concept_id +JOIN concept c ON c.concept_id = ca.descendant_concept_id + AND c.concept_class_id LIKE '%Pack%' WHERE f.class_code ~ 'G03FB|G03AB' -AND (f.class_code, c.concept_id) NOT IN (SELECT class_code, concept_id FROM class_to_drug_new); -- 264 + AND f.class_code || c.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 264 -- get rid of all other concept_class_ids except Packs for 'G03FB Progestogens and estrogens, sequential preparations' AND 'G03AB Progestogens and estrogens, sequential preparations' DELETE FROM class_to_drug_new WHERE class_code ~ 'G03FB|G03AB' -- Progestogens and estrogens -AND concept_class_id !~ 'Pack'; -- 81 + AND concept_class_id NOT LIKE '%Pack%';-- 81 --- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 20) +-- add links from ATC Classes WO Dose Forms specified, however their possible ancestors are unique (order = 20) INSERT INTO class_to_drug_new - WITH ing AS ( -SELECT concept_id_2, - concept_code_1, - COUNT(concept_id_2) AS cnt -FROM ( - SELECT DISTINCT substring(i.concept_code_1,'\w+'), - r.concept_code_1, - concept_id_2 -FROM relationship_to_concept r -JOIN internal_relationship_stage i ON i.concept_code_2 = r.concept_code_1 -JOIN drug_concept_stage d ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient') a - GROUP BY concept_id_2, concept_code_1), -drug AS ( - SELECT DISTINCT SUBSTRING(concept_code_1,'\w+') AS code, - concept_code_2 -FROM internal_relationship_stage i -WHERE NOT EXISTS (SELECT 1 FROM internal_relationship_stage i2 - WHERE i.concept_code_2 = i2.concept_code_2 - AND substring(i.concept_code_1,'\w+') != substring(i2.concept_code_1,'\w+')) -), -drug_name AS ( -SELECT DISTINCT class_code,class_name, concept_id_2 -FROM ing -JOIN drug ON concept_code_2 = concept_code_1 -JOIN class_drugs_scraper ON code = class_code -WHERE cnt=1 AND class_name = concept_code_1 -AND code NOT IN (SELECT class_code FROM class_to_drug_new) -), -all_drug AS ( -SELECT class_code, class_name, concept_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt -FROM drug_name a -JOIN concept c ON c.concept_id = a.concept_id_2 -) +WITH ing AS ( + SELECT a.concept_id_2, + a.concept_code_1, + COUNT(a.concept_id_2) AS cnt + FROM ( + SELECT DISTINCT SUBSTRING(i.concept_code_1, '\w+'), + r.concept_code_1, + r.concept_id_2 + FROM relationship_to_concept r + JOIN internal_relationship_stage i ON i.concept_code_2 = r.concept_code_1 + JOIN drug_concept_stage d ON d.concept_code = i.concept_code_2 + AND d.concept_class_id = 'Ingredient' + ) a + GROUP BY a.concept_id_2, + a.concept_code_1 + HAVING COUNT(a.concept_id_2) = 1 + ), + drug AS ( + SELECT DISTINCT SUBSTRING(concept_code_1, '\w+') AS code, + concept_code_2 + FROM internal_relationship_stage i + WHERE NOT EXISTS ( + SELECT 1 + FROM internal_relationship_stage i2 + WHERE i2.concept_code_2 = i.concept_code_2 + AND SUBSTRING(i2.concept_code_1, '\w+') <> SUBSTRING(i.concept_code_1, '\w+') + ) + ), + drug_name AS ( + SELECT DISTINCT cds.class_code, + cds.class_name, + i.concept_id_2 + FROM ing i + JOIN drug d ON d.concept_code_2 = i.concept_code_1 + JOIN class_drugs_scraper cds ON cds.class_code = d.code + AND cds.class_name = i.concept_code_1 + WHERE d.code NOT IN ( + SELECT class_code + FROM class_to_drug_new + ) + ), + all_drug AS ( + SELECT * + FROM ( + SELECT a.class_code, + a.class_name, + a.concept_id_2, + COUNT(a.concept_id_2) OVER (PARTITION BY a.class_code) AS cnt + FROM drug_name a + ) s0 + WHERE s0.cnt = 1 + ) SELECT DISTINCT a.class_code, - b.concept_name, - c.*, - 20 AS concept_order, - 'ATC Mono WO Dose Form to Ingredient' + b.concept_name, + c.*, + 20 AS concept_order, + 'ATC Mono WO Dose Form to Ingredient' FROM all_drug a -JOIN concept_manual b ON b.concept_code = a.class_code AND b.invalid_reason IS null -JOIN concept c ON c.concept_id = a.concept_id -WHERE cnt = 1 -AND (class_code) NOT IN (SELECT class_code FROM class_to_drug_new); -- 5 +JOIN concept_manual b ON b.concept_code = a.class_code + AND b.invalid_reason IS NULL +JOIN concept c ON c.concept_id = a.concept_id_2 +WHERE a.class_code NOT IN ( + SELECT class_code + FROM class_to_drug_new + );-- 5 -- add those which are absent in the drug hierarchy -- step 1 -DROP TABLE if exists no_atc_1; -CREATE TABLE no_atc_1 -AS -(SELECT descendant_concept_id -FROM concept_ancestor - JOIN dev_atc.class_to_drug_new n ON ancestor_concept_id = n.concept_id); - --- step 2 -DROP TABLE if exists no_atc_2; -CREATE TABLE no_atc_2 -AS -(SELECT DISTINCT c.* +DROP TABLE IF EXISTS no_atc_2; +CREATE UNLOGGED TABLE no_atc_1 AS +SELECT c.* FROM concept c -WHERE domain_id = 'Drug' -AND concept_id NOT IN (SELECT descendant_concept_id FROM no_atc_1) -AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension') -AND c.concept_class_id != 'Ingredient' -AND c.standard_concept = 'S'); - --- step 3 -DROP TABLE if exists no_atc_1_with_form; -CREATE TABLE no_atc_1_with_form -AS -(SELECT DISTINCT a.concept_id, - a.concept_name, - a.concept_class_id, - a.vocabulary_id, - b.ingredient_concept_id AS ing_id, - d.concept_name AS ing_nm, - d.standard_concept, - g.concept_id AS df_id, - g.concept_name AS df_nm -FROM no_atc_2 a - JOIN drug_strength b ON b.drug_concept_id = a.concept_id - JOIN concept d ON d.concept_id = b.ingredient_concept_id - JOIN concept_relationship r ON r.concept_id_1 = a.concept_id - JOIN concept g - ON g.concept_id = r.concept_id_2 - AND g.concept_class_id = 'Dose Form'); +LEFT JOIN ( + SELECT ca.descendant_concept_id + FROM concept_ancestor ca + WHERE ca.ancestor_concept_id IN ( + SELECT concept_id + FROM class_to_drug_new + ) + ) d ON d.descendant_concept_id = c.concept_id +WHERE d.descendant_concept_id IS NULL + AND c.domain_id = 'Drug' + AND c.vocabulary_id LIKE 'RxNorm%' + AND c.concept_class_id <> 'Ingredient' + AND c.standard_concept = 'S'; + +-- step 2 +DROP TABLE IF EXISTS no_atc_1_with_form; +CREATE UNLOGGED TABLE no_atc_1_with_form AS +SELECT DISTINCT a.concept_id, + a.concept_name, + a.concept_class_id, + a.vocabulary_id, + b.ingredient_concept_id AS ing_id, + d.concept_name AS ing_nm, + d.standard_concept, + g.concept_id AS df_id, + g.concept_name AS df_nm +FROM no_atc_1 a +JOIN drug_strength b ON b.drug_concept_id = a.concept_id +JOIN concept d ON d.concept_id = b.ingredient_concept_id +JOIN concept_relationship r ON r.concept_id_1 = a.concept_id + AND r.invalid_reason IS NULL +JOIN concept g ON g.concept_id = r.concept_id_2 + AND g.concept_class_id = 'Dose Form'; + -- add additional mappings for hierarchical absentees (order = 21) INSERT INTO class_to_drug_new SELECT DISTINCT k.concept_code AS class_code, - k.concept_name AS class_name, - p.*, - 21 as concept_order, - 'ATC Monocomp Class to Drug Product which is out of hierarchy' + k.concept_name AS class_name, + p.*, + 21 AS concept_order, + 'ATC Monocomp Class to Drug Product which is out of hierarchy' FROM no_atc_1_with_form a - JOIN internal_relationship_stage i ON lower (i.concept_code_2) = lower (a.ing_nm) - JOIN internal_relationship_stage i2 - ON lower (i2.concept_code_2) = lower (a.df_nm) - AND i.concept_code_1 = i2.concept_code_1 - JOIN concept_manual k ON k.concept_code = SUBSTRING (i.concept_code_1,'\w+') - JOIN concept p - ON p.concept_id = a.concept_id - AND p.standard_concept = 'S' -WHERE a.concept_name !~ ' / ' -AND k.invalid_reason IS NULL -AND k.concept_class_id = 'ATC 5th' -AND k.concept_code NOT IN (SELECT class_code FROM dev_combo) -AND k.concept_code||p.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);--6107 +JOIN internal_relationship_stage i ON LOWER(i.concept_code_2) = LOWER(a.ing_nm) +JOIN internal_relationship_stage i2 ON LOWER(i2.concept_code_2) = LOWER(a.df_nm) + AND i2.concept_code_1 = i.concept_code_1 +JOIN concept_manual k ON k.concept_code = SUBSTRING(i.concept_code_1, '\w+') + AND k.invalid_reason IS NULL + AND k.concept_class_id = 'ATC 5th' +JOIN concept p ON p.concept_id = a.concept_id + AND p.standard_concept = 'S' +WHERE a.concept_name NOT LIKE '% / %' + AND k.concept_code NOT IN ( + SELECT class_code + FROM dev_combo + ) + AND k.concept_code || p.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );--6107 -- add Clinical or Branded Drug Forms as descendants of non-to-Form mappings if they present in the concept_ancestor table -- simultaneous child-parent maps will be deleted in further steps INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, - a.class_name, - c.*, - 21 AS concept_order, - 'ATC Monocomp Class to Drug Product which is out of hierarchy' + a.class_name, + c.*, + 21 AS concept_order, + 'ATC Monocomp Class to Drug Product which is out of hierarchy' FROM class_to_drug_new a - JOIN concept_ancestor ca ON ca.descendant_concept_id = concept_id - JOIN concept c - ON c.concept_id = ca.ancestor_concept_id - AND c.concept_class_id ~ 'Form' - AND c.standard_concept = 'S' -WHERE concept_order = 21 -AND NOT EXISTS (SELECT 1 FROM class_to_drug_new x WHERE x.class_code||x.concept_id = a.class_code||c.concept_id) -AND c.domain_id = 'Drug'; -- 271 +JOIN concept_ancestor ca ON ca.descendant_concept_id = a.concept_id +JOIN concept c ON c.concept_id = ca.ancestor_concept_id + AND c.concept_class_id LIKE '%Form%' + AND c.standard_concept = 'S' + AND c.domain_id = 'Drug' +WHERE a.concept_order = 21 + AND NOT EXISTS ( + SELECT 1 + FROM class_to_drug_new x + WHERE x.class_code || x.concept_id = a.class_code || c.concept_id + );-- 271 -- add descendants of other non-to-Form mappings (in class_to_drug_new) if they present in concept_ancestor (abundant child-parent mappings will be deleted in further steps) INSERT INTO class_to_drug_new SELECT DISTINCT a.class_code, - a.class_name, - c.*, - a.concept_order, - a.order_desc + a.class_name, + c.*, + a.concept_order, + a.order_desc FROM class_to_drug_new a - JOIN concept_ancestor ca ON ca.descendant_concept_id = concept_id - JOIN concept c - ON c.concept_id = ca.ancestor_concept_id - AND c.concept_class_id ~ 'Form' and a.concept_class_id !~ 'Form' - AND c.standard_concept = 'S' -WHERE - c.domain_id = 'Drug' AND a.class_code NOT IN (SELECT class_code - FROM combo_pull) AND NOT EXISTS (SELECT 1 FROM class_to_drug_new x WHERE x.class_code||x.concept_id = a.class_code||c.concept_id) - AND a.class_code NOT IN ('B02BD11', 'B02BD14', 'D07AB02'); -- intentionally excluded catridecacog|susoctocog alfa|hydrocortisone butyrate -- no such Precise Ingredients connected to Dose Forms - +JOIN concept_ancestor ca ON ca.descendant_concept_id = a.concept_id +JOIN concept c ON c.concept_id = ca.ancestor_concept_id + AND c.concept_class_id LIKE '%Form%' + AND c.standard_concept = 'S' + AND c.domain_id = 'Drug' +WHERE a.concept_class_id NOT LIKE '%Form' + AND a.class_code NOT IN ( + SELECT class_code + FROM combo_pull + ) + AND NOT EXISTS ( + SELECT 1 + FROM class_to_drug_new x + WHERE x.class_code || x.concept_id = a.class_code || c.concept_id + ) + AND a.class_code NOT IN ( + 'B02BD11', + 'B02BD14', + 'D07AB02' + );-- intentionally excluded catridecacog|susoctocog alfa|hydrocortisone butyrate -- no such Precise Ingredients connected to Dose Forms + -- obtain more ATC Combo classes DROP TABLE no_atc_full_combo; -CREATE TABLE no_atc_full_combo -AS -SELECT DISTINCT concept_id, - concept_name, - df_id, - STRING_AGG(ing_id::VARCHAR,'-' ORDER BY ing_id::INT) AS i_combo +CREATE UNLOGGED TABLE no_atc_full_combo AS +SELECT concept_id, + df_id, + ARRAY_AGG(ing_id) AS i_combo FROM no_atc_1_with_form GROUP BY concept_id, - concept_name, - df_id; + concept_name, + df_id; -DROP TABLE IF EXISTS no_atc_reodered; -CREATE UNLOGGED TABLE no_atc_reodered AS -SELECT DISTINCT fc.concept_id, -- missing Rx Drug - fc.concept_name, - df_id, - l.i_combo -FROM no_atc_full_combo fc -CROSS JOIN LATERAL(SELECT STRING_AGG(s0.ing, '-' ORDER BY s0.ing::INT) AS i_combo FROM ( - SELECT UNNEST(STRING_TO_ARRAY(fc.i_combo, '-')) AS ing - ) AS s0) l; - -INSERT INTO no_atc_full_combo -(concept_id, i_combo, df_id) -SELECT DISTINCT concept_id, - i_combo, df_id - FROM no_atc_reodered ; - -CREATE INDEX i_no_atc_full_combo ON no_atc_full_combo (concept_id, i_combo,df_id); +CREATE INDEX idx_no_atc_full_combo ON no_atc_full_combo USING GIN (i_combo); +ANALYZE no_atc_full_combo; -- add mappings of ATC Combo Class to Drug Product which is out of drug hierarchy (order = 22) INSERT INTO class_to_drug_new - SELECT DISTINCT k.concept_code AS class_code, -- ATC - k.concept_name AS class_name, - d.*, - 22 AS conept_order, - 'ATC Combo Class to Drug Product which is out of drug hierarchy' AS order_desc - FROM no_atc_full_combo f - JOIN full_combo_with_form c ON c.i_combo = f.i_combo AND c.df_id = f.df_id IS null - JOIN concept_manual k ON k.concept_code = c.class_code AND k.concept_class_id = 'ATC 5th' AND k.invalid_reason IS null - JOIN concept d ON d.concept_id = f.concept_id - AND c.class_code||d.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 6188 - +SELECT DISTINCT k.concept_code AS class_code, -- ATC + k.concept_name AS class_name, + d.*, + 22 AS conept_order, + 'ATC Combo Class to Drug Product which is out of drug hierarchy' AS order_desc +FROM full_combo_with_form c +JOIN no_atc_full_combo f ON f.i_combo @> c.i_combo + AND f.i_combo <@ c.i_combo + AND f.df_id = c.df_id +JOIN concept_manual k ON k.concept_code = c.class_code + AND k.concept_class_id = 'ATC 5th' + AND k.invalid_reason IS NULL +JOIN concept d ON d.concept_id = f.concept_id +WHERE NOT EXISTS ( + SELECT 1 + FROM class_to_drug_new cdn + WHERE cdn.class_code = c.class_code + AND cdn.concept_id = f.concept_id + );-- 6188 + -- ATC Combo Class to Drug Product which is out of drug hierarchy (order = 23) INSERT INTO class_to_drug_new SELECT DISTINCT b.concept_code, - b.concept_name, - c.*, - 23 AS conept_order, - 'ATC Combo Class to Drug Product which is out of drug hierarchy' AS order_desc + b.concept_name, + c.*, + 23 AS conept_order, + 'ATC Combo Class to Drug Product which is out of drug hierarchy' AS order_desc FROM missing a - JOIN concept_manual b - ON b.concept_code = a.atc_code - AND b.invalid_reason IS NULL - JOIN concept c USING (concept_id) -WHERE c.concept_id NOT IN (42731911, 40046823, 36264365, 42481935, 43730307, 42800420) -AND b.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new); -- 2833 +JOIN concept_manual b ON b.concept_code = a.atc_code + AND b.invalid_reason IS NULL +JOIN concept c ON c.concept_id = a.concept_id +WHERE c.concept_id NOT IN ( + 42731911, + 40046823, + 36264365, + 42481935, + 43730307, + 42800420 + ) + AND b.concept_code || c.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );-- 2833 -- remove excessive links to children among Packs -WITH t1 AS -( - SELECT * - FROM class_to_drug_new - WHERE concept_class_id ~ 'Pack') -DELETE FROM class_to_drug_new WHERE class_code||concept_id IN ( -SELECT a1.class_code||a2.concept_id -FROM t1 a1-- papa - JOIN t1 a2 -- child -ON a1.class_code = a2.class_code - JOIN concept_ancestor ca - ON a1.concept_id = ca.ancestor_concept_id - AND a2.concept_id = ca.descendant_concept_id - AND a1.concept_id <> a2.concept_id - AND a2.concept_class_id ~ 'Pack' - AND a1.class_code NOT IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') -- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic); -- 2558 - ); -- 3920 - +DELETE +FROM class_to_drug_new c USING ( + SELECT a1.class_code || a2.concept_id AS class_and_cid + FROM class_to_drug_new a1 -- papa + JOIN class_to_drug_new a2 -- child + ON a1.class_code = a2.class_code + JOIN concept_ancestor ca ON ca.ancestor_concept_id = a1.concept_id + AND ca.descendant_concept_id = a2.concept_id + WHERE a1.concept_id <> a2.concept_id + AND a1.concept_class_id LIKE '%Pack%' + AND a2.concept_class_id LIKE '%Pack%' + AND a1.class_code NOT IN ( + 'A02BA07', + 'B02BD11', + 'B02BD14', + 'J07BK01', + 'J07BK02', + 'J07AE02', + 'J07BB01', + 'N05AF02' + ) -- susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic -- 2558 + ) h +WHERE c.class_code || c.concept_id = h.class_and_cid;-- 3920 + -- remove excessive links to children among unpacked Drug Products -WITH t0 AS -( - SELECT * - FROM class_to_drug_new - WHERE concept_class_id !~ 'Pack' -), -t1 AS ( -SELECT * FROM t0 WHERE class_code IN (SELECT class_code - FROM class_to_drug_new - GROUP BY class_code - HAVING COUNT(1) >= 2)) +WITH t0 +AS ( + SELECT * + FROM ( + SELECT cdn.*, + COUNT(*) OVER (PARTITION BY cdn.class_code) cc_cnt + FROM class_to_drug_new cdn + ) s0 + WHERE s0.cc_cnt >= 2 + AND s0.concept_class_id NOT LIKE '%Pack%' + ) DELETE -FROM class_to_drug_new where class_code||concept_id IN ( -SELECT a1.class_code||a2.concept_id -- child mapping -FROM t1 a1-- papa - JOIN t1 a2 -- child -ON a1.class_code = a2.class_code - JOIN concept_ancestor ca - ON a1.concept_id = ca.ancestor_concept_id - AND a2.concept_id = ca.descendant_concept_id - AND a1.concept_id <> a2.concept_id - AND a2.concept_class_id !~ 'Pack' - AND a1.class_code NOT IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') -- catridecacog|susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic - );-- 10432 - +FROM class_to_drug_new c USING ( + SELECT a1.class_code || a2.concept_id AS class_and_cid -- child mapping + FROM t0 a1 -- papa + JOIN t0 a2 -- child + ON a1.class_code = a2.class_code + JOIN concept_ancestor ca ON ca.ancestor_concept_id = a1.concept_id + AND ca.descendant_concept_id = a2.concept_id + WHERE a1.concept_id <> a2.concept_id + AND a1.class_code NOT IN ( + 'A02BA07', + 'B02BD11', + 'B02BD14', + 'J07BK01', + 'J07BK02', + 'J07AE02', + 'J07BB01', + 'N05AF02' + ) -- catridecacog|susoctocog alfa; parenteral, topical, urethral | norgestrel and estrogen | varicella, live attenuated; systemic |zoster, live attenuated; systemic + ) h -- 10432 +WHERE c.class_code || c.concept_id = h.class_and_cid; + -- clean up the same issue among exclusions -WITH t0 AS -( - SELECT * - FROM class_to_drug_new - WHERE concept_class_id !~ 'Pack' - AND class_code IN ('A02BA07', 'B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') -), -t1 AS ( -SELECT * FROM t0 WHERE class_code IN (SELECT class_code - FROM class_to_drug_new - GROUP BY class_code - HAVING COUNT(1) >= 2)) +WITH t0 +AS ( + SELECT * + FROM ( + SELECT cdn.*, + COUNT(*) OVER (PARTITION BY cdn.class_code) cc_cnt + FROM class_to_drug_new cdn + ) s0 + WHERE s0.cc_cnt >= 2 + AND s0.concept_class_id NOT LIKE '%Pack%' + AND s0.class_code IN ( + 'A02BA07', + 'B02BD11', + 'B02BD14', + 'J07BK01', + 'J07BK02', + 'J07AE02', + 'J07BB01', + 'N05AF02' + ) -- catridecacog|susoctocog alfa; parenteral, topical, urethral|norgestrel and estrogen|varicella, live attenuated; systemic|zoster, live attenuated; systemic + ) DELETE -FROM class_to_drug_new where class_code||concept_id IN ( -SELECT a1.class_code||a1.concept_id -- papa mapping -FROM t1 a1-- papa - JOIN t1 a2 -- child -ON a1.class_code = a2.class_code - JOIN concept_ancestor ca - ON a1.concept_id = ca.ancestor_concept_id - AND a2.concept_id = ca.descendant_concept_id - AND a1.concept_id <> a2.concept_id - AND a2.concept_class_id !~ 'Pack' - AND a1.class_code IN ('A02BA07', 'B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02', 'J07BB01', 'N05AF02') - ); -- catridecacog|susoctocog alfa; parenteral, topical, urethral|norgestrel and estrogen|varicella, live attenuated; systemic|zoster, live attenuated; systemic - +FROM class_to_drug_new c USING ( + SELECT a1.class_code || a1.concept_id AS class_and_cid -- papa mapping + FROM t0 a1 -- papa + JOIN t0 a2 -- child + ON a1.class_code = a2.class_code + JOIN concept_ancestor ca ON ca.ancestor_concept_id = a1.concept_id + AND ca.descendant_concept_id = a2.concept_id + WHERE a1.concept_id <> a2.concept_id + ) h +WHERE c.class_code || c.concept_id = h.class_and_cid; + -- clean up the same issue among exclusions - Packs -WITH t0 AS -( - SELECT * - FROM class_to_drug_new - WHERE concept_class_id ~ 'Pack' - AND class_code IN ('A02BA07', 'B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02', 'J07BB01', 'N05AF02') -), -t1 AS ( -SELECT * FROM t0 WHERE class_code IN (SELECT class_code - FROM class_to_drug_new - GROUP BY class_code - HAVING COUNT(1) >= 2)) +WITH t0 +AS ( + SELECT * + FROM ( + SELECT cdn.*, + COUNT(*) OVER (PARTITION BY cdn.class_code) cc_cnt + FROM class_to_drug_new cdn + ) s0 + WHERE s0.cc_cnt >= 2 + AND s0.concept_class_id LIKE '%Pack%' + AND s0.class_code IN ( + 'A02BA07', + 'B02BD11', + 'B02BD14', + 'J07BK01', + 'J07BK02', + 'J07AE02', + 'J07BB01', + 'N05AF02' + ) -- catridecacog|susoctocog alfa; parenteral, topical, urethral|norgestrel and estrogen|varicella, live attenuated; systemic|zoster, live attenuated; systemic + ) DELETE -FROM class_to_drug_new where class_code||concept_id IN ( -SELECT a1.class_code||a1.concept_id -- papa mapping -FROM t1 a1 -- papa - JOIN t1 a2 -- child -ON a1.class_code = a2.class_code - JOIN concept_ancestor ca - ON a1.concept_id = ca.ancestor_concept_id - AND a2.concept_id = ca.descendant_concept_id - AND a1.concept_id <> a2.concept_id - and a2.concept_class_id ~ 'Pack' - AND a1.class_code IN ('A02BA07','B02BD11', 'B02BD14', 'J07BK01', 'J07BK02', 'J07AE02','J07BB01','N05AF02') - ) -- catridecacog|susoctocog alfa; parenteral, topical, urethral|norgestrel and estrogen|varicella, live attenuated; systemic|zoster, live attenuated; systemic - ;-- 19 +FROM class_to_drug_new c USING ( + SELECT a1.class_code || a1.concept_id AS class_and_cid -- papa mapping + FROM t0 a1 -- papa + JOIN t0 a2 -- child + ON a1.class_code = a2.class_code + JOIN concept_ancestor ca ON ca.ancestor_concept_id = a1.concept_id + AND ca.descendant_concept_id = a2.concept_id + WHERE a1.concept_id <> a2.concept_id + ) h +WHERE c.class_code || c.concept_id = h.class_and_cid;-- 19 + + /*************************** ******** DF CLEAN UP ******* ****************************/ + -- clean up oral forms DROP TABLE IF EXISTS wrong_df; -CREATE TABLE wrong_df AS ( -SELECT *, 'oral mismatch' AS issue_desc +CREATE UNLOGGED TABLE wrong_df AS +SELECT *, + 'oral mismatch' AS issue_desc +FROM class_to_drug_new +WHERE SPLIT_PART(class_name, ';', 2) LIKE '%oral%' + AND concept_name !~* 'oral|chew|tooth|mouth|elixir|Extended Release Suspension|buccal|Sublingual|Paste|Mucosal|Prefilled Syringe|\...|Oral Ointment|Oral Cream' + AND class_name !~ 'rectal|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|nasal' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + AND class_code NOT IN ( + 'A01AA04', + 'A01AA02', + 'G04BD08' + ) + +UNION ALL + +SELECT *, + 'vaginal mismatch' FROM class_to_drug_new -WHERE SPLIT_PART(class_name,';',2) ~ 'oral' -AND concept_name !~* 'oral|chew|tooth|mouth|elixir|Extended Release Suspension|buccal|Sublingual|Paste|Mucosal|Prefilled Syringe|\...|Oral Ointment|Oral Cream' -AND class_name !~ 'rectal|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|nasal' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' -AND class_code NOT IN ('A01AA04', 'A01AA02', 'G04BD08')); - -INSERT INTO wrong_df -SELECT *, 'vaginal mismatch' FROM class_to_drug_new WHERE class_name ~ 'vaginal' -AND concept_name !~* 'vaginal|topical|mucosal|Drug Implant|Douche|Irrigation Solution' -AND class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; -- 1 +WHERE class_name LIKE '%vaginal%' + AND concept_name !~* 'vaginal|topical|mucosal|Drug Implant|Douche|Irrigation Solution' + AND class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'-- 1 + +UNION ALL -- clean up rectal forms -INSERT INTO wrong_df -SELECT *, 'rectal mismatch' +SELECT *, + 'rectal mismatch' FROM class_to_drug_new -WHERE class_name ~ 'rectal' -AND concept_name !~* 'rectal|topical|mucosal|enema' -AND class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|systemic' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; +WHERE class_name LIKE '%rectal%' + AND concept_name !~* 'rectal|topical|mucosal|enema' + AND class_name !~ 'oral|topical|inhalant|parenteral|transdermal|otic|vaginal|local oral|systemic' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + +UNION ALL -- clean up topical forms -INSERT INTO wrong_df -SELECT *, 'topical mismatch' +SELECT *, + 'topical mismatch' FROM class_to_drug_new -WHERE class_name ~ 'topical' -AND concept_name !~* 'topical|mucosal|Drug Implant|Prefilled Applicator|Shampoo|Paste|Medicated Pad|Transdermal System|Soap|Powder Spray|Medicated Patch|Douche|vaginal|\yNail\y|Intrauterine System|Mouthwash|Oil|Intraperitoneal Solution|Urethral Gel' -AND concept_name !~* '\yStick|Rectal Foam|Medicated Tape|Medicated Guaze|Paint|Rectal|Nasal|Otic|Ophthalmic Solution|Dry Powder|Urethral Suppository|Intrauterine|irrigation|Cement|Cream|ointment|spray|gum|enema|Ophthalmic' -AND class_name !~* 'oral|vaginal|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; +WHERE class_name LIKE '%topical%' + AND concept_name !~* 'topical|mucosal|Drug Implant|Prefilled Applicator|Shampoo|Paste|Medicated Pad|Transdermal System|Soap|Powder Spray|Medicated Patch|Douche|vaginal|\yNail\y|Intrauterine System|Mouthwash|Oil|Intraperitoneal Solution|Urethral Gel' + AND concept_name !~* '\yStick|Rectal Foam|Medicated Tape|Medicated Guaze|Paint|Rectal|Nasal|Otic|Ophthalmic Solution|Dry Powder|Urethral Suppository|Intrauterine|irrigation|Cement|Cream|ointment|spray|gum|enema|Ophthalmic' + AND class_name !~* 'oral|vaginal|inhalant|parenteral|transdermal|otic|rectal|local oral|systemic' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + +UNION ALL -- clean up local oral forms -INSERT INTO wrong_df -SELECT *, 'local oral mismatch' +SELECT *, + 'local oral mismatch' FROM class_to_drug_new -WHERE class_name ~ 'local oral' AND class_name !~ 'oral, local oral' -AND concept_name !~* 'mouth|topical|paste|irrig|Lozenge|gum|buccal|Suspension|solution|subl|Gel|spray|Disintegrating Oral Tablet|Effervescent Oral Tablet|Oral Powder|Chewable Tablet|Oral Film|Oral Granules' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; +WHERE class_name LIKE '%local oral%' + AND class_name NOT LIKE '%oral, local oral%' + AND concept_name !~* 'mouth|topical|paste|irrig|Lozenge|gum|buccal|Suspension|solution|subl|Gel|spray|Disintegrating Oral Tablet|Effervescent Oral Tablet|Oral Powder|Chewable Tablet|Oral Film|Oral Granules' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + +UNION ALL -- clean up inhalant -INSERT INTO wrong_df -SELECT *, 'inhalant mismatch' +SELECT *, + 'inhalant mismatch' FROM class_to_drug_new -WHERE class_name ~* 'inhalant' -AND concept_name !~* 'inhal|nasal|Powder Spray|Oral Spray| Oral Powder|Prefilled Applicator' -AND class_name !~ 'oral|local oral|vaginal|parenteral|transdermal|ophthalmic|otic|rectal|systemic|topical' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; +WHERE class_name ILIKE '%inhalant%' + AND concept_name !~* 'inhal|nasal|Powder Spray|Oral Spray| Oral Powder|Prefilled Applicator' + AND class_name !~ 'oral|local oral|vaginal|parenteral|transdermal|ophthalmic|otic|rectal|systemic|topical' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + +UNION ALL -- clean up parenteral -INSERT INTO wrong_df -SELECT *, 'parenteral mismatch' +SELECT *, + 'parenteral mismatch' FROM class_to_drug_new -WHERE class_name ~* 'parenteral' -AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant|transdermal' -AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|ophthalmic|otic|nasal|rectal|systemic|urethral' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; +WHERE class_name ILIKE '%parenteral%' + AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant|transdermal' + AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|ophthalmic|otic|nasal|rectal|systemic|urethral' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + +UNION ALL -- clean up otic|ophthalmic -INSERT INTO wrong_df -SELECT *, 'otic|ophthalmic mismatch' +SELECT *, + 'otic|ophthalmic mismatch' FROM class_to_drug_new WHERE class_name ~* 'otic|ophthalmic' -AND concept_name !~* 'otic|ophthalmic|topical|Prefilled Applicator' -AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|nasal|rectal|systemic|urethral|parenteral' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; + AND concept_name !~* 'otic|ophthalmic|topical|Prefilled Applicator' + AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|nasal|rectal|systemic|urethral|parenteral' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + +UNION ALL -- clean up systemic forms -INSERT INTO wrong_df -SELECT *, 'systemic mismatch' +SELECT *, + 'systemic mismatch' FROM class_to_drug_new -WHERE class_name ~* 'systemic' -AND concept_name ~* 'nasal' -AND concept_name !~ 'metered' -AND class_name !~* 'rectal|nasal|vaginal|topical' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' -AND concept_name ~* 'caine|azoline|thrombin|sodium chloride|glycerol|acetylcysteine|chlorhexidine|amlexanox|ammonia|Chlorobutanol|phenylephrine|Peppermint oil|pyrilamine|dexpanthenol|Phentolamine|Sodium Chloride|Cellulose|glycerin|pantenol|tetrahydrozoline|triamcinolone|dexamethasone|Ephedrine|Histamine|Hydrocortisone' -- -; +WHERE class_name ILIKE '%systemic%' + AND concept_name ILIKE '%nasal%' + AND concept_name NOT LIKE '%metered%' + AND class_name !~* 'rectal|nasal|vaginal|topical' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + AND concept_name ~* 'caine|azoline|thrombin|sodium chloride|glycerol|acetylcysteine|chlorhexidine|amlexanox|ammonia|Chlorobutanol|phenylephrine|Peppermint oil|pyrilamine|dexpanthenol|Phentolamine|Sodium Chloride|Cellulose|glycerin|pantenol|tetrahydrozoline|triamcinolone|dexamethasone|Ephedrine|Histamine|Hydrocortisone' + +UNION ALL -- clean up systemic forms 2 -INSERT INTO wrong_df -SELECT *, 'systemic mismatch 2' +SELECT *, + 'systemic mismatch 2' FROM class_to_drug_new -WHERE class_name ~* 'systemic' -AND concept_name !~* '\.\.\.$|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' -AND class_name !~* 'rectal|nasal|vaginal|topical' -AND concept_name !~* 'Alfentanil|scopolamine|amyl nitrite|Heroin|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp'; +WHERE class_name ILIKE '%systemic%' + AND concept_name !~* '\.\.\.$|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' + AND class_name !~* 'rectal|nasal|vaginal|topical' + AND concept_name !~* 'Alfentanil|scopolamine|amyl nitrite|Heroin|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + +UNION ALL -- clean up instillation packs -INSERT INTO wrong_df -SELECT *, 'instill pack mismatch' +SELECT *, + 'instill pack mismatch' FROM class_to_drug_new -WHERE class_name ~ 'instill' -AND concept_name !~* 'instil|irrig' -AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' -AND concept_name !~ 'Intratracheal|Phospholipids / Soybean Oil'; +WHERE class_name LIKE '%instill%' + AND concept_name !~* 'instil|irrig' + AND concept_class_id !~ 'Pack|Ingredient|Clinical Drug Comp' + AND concept_name !~ 'Intratracheal|Phospholipids / Soybean Oil' + +UNION ALL -- clean up parenteral packs -INSERT INTO wrong_df -SELECT *, 'parenteral pack mismatch' +SELECT *, + 'parenteral pack mismatch' FROM class_to_drug_new -WHERE class_name ~* 'parenteral' -AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant' -AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|ophthalmic|otic|nasal|rectal|systemic|urethral' -AND concept_class_id ~ 'Pack'; +WHERE class_name ILIKE '%parenteral%' + AND concept_name !~* 'inject|prefill|intrav|intram|cartridge|UNT|Intraperitoneal Solution|Irrigation Solution|MG\/ML|Inhal|Nasal|\...|Drug Implant' + AND class_name !~* 'oral|local oral|vaginal|inhalant|transdermal|topical|ophthalmic|otic|nasal|rectal|systemic|urethral' + AND concept_class_id LIKE '%Pack%' + +UNION ALL -- clean up systemic packs -INSERT INTO wrong_df -SELECT *, 'systemic pack mismatch 2' +SELECT *, + 'systemic pack mismatch 2' FROM class_to_drug_new -WHERE class_name ~* 'systemic' -AND concept_name !~* '\...$|\...\) \} Pack$|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' -AND class_name !~* 'rectal|nasal|vaginal|topical' -AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' -AND concept_class_id ~ 'Pack' -AND class_code NOT IN ('R03AC130'); -- formoterol Powder for Oral Suspension| +WHERE class_name ILIKE '%systemic%' + AND concept_name !~* '\...$|\...\) \} Pack$|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' + AND class_name !~* 'rectal|nasal|vaginal|topical' + AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' + AND concept_class_id LIKE '%Pack%' + AND class_code <> 'R03AC130' -- formoterol Powder for Oral Suspension| + +UNION ALL -- clean up systemic packs 2 - check This next time -INSERT INTO wrong_df -SELECT * , 'systemic pack mismatch 3' +SELECT *, + 'systemic pack mismatch 3' FROM class_to_drug_new -WHERE class_name ~* 'systemic' -AND concept_name !~* '\.\.\.$|\.\.\.\) \} Pack|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' -AND class_name !~* 'rectal|nasal|vaginal|topical' -AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' -AND concept_class_id ~ 'Pack' -AND concept_code <> 'C02AC01'; -- clonidine Medicated Patch +WHERE class_name ILIKE '%systemic%' + AND concept_name !~* '\.\.\.$|\.\.\.\) \} Pack|oral|rectal|inject|chew|Cartridge|Sublingual|MG\ML|syringe|influenza|pertussis|intravenous|Sublingual|Amyl Nitrite|implant|transdermal|Intramuscular|\yInhal|rotavirus|papillomavirus|\ymening|pneumococ|Streptococ|buccal|Extended Release Suspension|Metered Dose Nasal Spray' + AND class_name !~* 'rectal|nasal|vaginal|topical' + AND concept_name !~* 'Alfentanil|amyl nitrite|Chloroform|dihydroergotamine|estradiol|fentanyl|Furosemide|gonadorelin|Isosorbide Dinitrate|Ketamine|ketorolac|midazolam|naloxone|Thyrotropin-Releasing Hormone|succimer Kit for radiopharmaceutical preparation' + AND concept_class_id LIKE '%Pack%' + AND concept_code <> 'C02AC01' -- clonidine Medicated Patch + +UNION ALL -- wrong vaginal rings -INSERT INTO wrong_df -SELECT * +SELECT *, + 'vaginal rings mismatch' FROM class_to_drug_new -WHERE class_name ~ 'vaginal ring' -AND concept_name !~* 'ring|insert|system|implant'; +WHERE class_name LIKE '%vaginal ring%' + AND concept_name !~* 'ring|insert|system|implant'; -DELETE FROM wrong_df WHERE class_code IN ( 'C02AC01', 'G03GA08', 'R03AC13'); +DELETE +FROM wrong_df +WHERE class_code IN ( + 'C02AC01', + 'G03GA08', + 'R03AC13' + ); DELETE FROM wrong_df -WHERE class_name ~ 'local oral' -AND concept_name ~* 'paste|gel|oint'; +WHERE class_name LIKE '%local oral%' + AND concept_name ~* 'paste|gel|oint'; -- look at them and remove from class_to_drug_new DELETE FROM class_to_drug_new -WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM wrong_df); -- 891 - +WHERE class_code || concept_id IN ( + SELECT class_code || concept_id + FROM wrong_df + );-- 891 + -- add links from ATC Combo Classes which do not have Dose Forms to Ingredients (order = 24) -INSERT INTO class_to_drug_new -WITH t1 -AS -(SELECT * -FROM concept_manual -WHERE concept_code NOT IN (SELECT class_code FROM class_to_drug_new) -AND concept_class_Id = 'ATC 5th' -AND invalid_reason IS NULL -AND concept_code NOT IN (SELECT class_code FROM atc_inexistent) - -) +INSERT INTO class_to_drug_new SELECT DISTINCT a.concept_code, - a.concept_name, - c.*, - 24 AS concept_order, - 'ATC Combo Class to Ingredient' AS order_desc -FROM t1 a - JOIN dev_combo k - ON k.class_code = a.concept_code - AND rnk IN (1, 2) - JOIN concept c - ON c.concept_id = k.concept_id - AND c.concept_class_id = 'Ingredient' - AND c.standard_concept = 'S' - AND a.concept_code||c.concept_id NOT IN (SELECT class_code||concept_id FROM class_to_drug_new);--314 - + a.concept_name, + c.*, + 24 AS concept_order, + 'ATC Combo Class to Ingredient' AS order_desc +FROM concept_manual a +JOIN dev_combo k ON k.class_code = a.concept_code + AND k.rnk IN ( + 1, + 2 + ) +JOIN concept c ON c.concept_id = k.concept_id + AND c.concept_class_id = 'Ingredient' + AND c.standard_concept = 'S' +WHERE a.concept_code NOT IN ( + SELECT class_code + FROM class_to_drug_new + + UNION ALL + + SELECT class_code + FROM atc_inexistent + ) + AND a.concept_class_id = 'ATC 5th' + AND a.invalid_reason IS NULL + AND a.concept_code || c.concept_id NOT IN ( + SELECT class_code || concept_id + FROM class_to_drug_new + );--314 + -- wrong match DELETE FROM class_to_drug_new WHERE class_code = 'J01CR04' -AND concept_name ~* 'sulbactam'; + AND concept_name ILIKE '%sulbactam%'; -- remove wrong mapping of glafenine; oral (from old class_to_drug) DELETE @@ -1958,89 +2304,115 @@ WHERE concept_class_id ~ '\yComp'; -- remove suspicious mapping of inexistent drugs (atc_inexistent should be checked before) DELETE -FROM class_to_drug_new -WHERE class_code IN (SELECT class_code FROM atc_inexistent); -- 14 +FROM class_to_drug_new cdn USING atc_inexistent ai +WHERE cdn.class_code = ai.class_code;-- 14 -- remove wrong mappings DELETE -FROM class_to_drug_new - WHERE class_code||concept_code IN ( - SELECT concept_code_1||concept_code_2 - FROM concept_relationship_manual - WHERE invalid_reason IS NOT NULL - AND relationship_id = 'ATC - RxNorm'); -- 0 - - -- clean up wrong vaccine mappings (they should be processed manually and be stored in CRM) --- firstly, drop wrong mono-vaccines +FROM class_to_drug_new cdn USING concept_relationship_manual crm +WHERE cdn.class_code || cdn.concept_code = crm.concept_code_1 || crm.concept_code_2 + AND crm.invalid_reason IS NOT NULL + AND crm.relationship_id = 'ATC - RxNorm';-- 0 + +-- clean up wrong vaccine mappings (they should be processed manually and be stored in CRM) +-- firstly, drop wrong mono-vaccines DELETE FROM class_to_drug_new -WHERE class_code ~ '^J07' -AND class_code||concept_code NOT IN (SELECT concept_code_1||concept_code_2 -- not in crm - FROM concept_relationship_manual - WHERE relationship_id = 'ATC - RxNorm' - AND invalid_reason IS NULL) - AND concept_name !~ ' / ' -AND concept_name !~ 'bivalent|trivalent|pentavalent' -AND class_code NOT IN ('J07AC01', 'J07BX03', 'J07BA01', 'J07BC02', 'J07AL01','J07AP02', 'J07CA10'); +WHERE class_code LIKE 'J07%' + AND class_code || concept_code NOT IN ( + SELECT concept_code_1 || concept_code_2 -- not in crm + FROM concept_relationship_manual + WHERE relationship_id = 'ATC - RxNorm' + AND invalid_reason IS NULL + ) + AND concept_name NOT LIKE '% / %' + AND concept_name !~ 'bivalent|trivalent|pentavalent' + AND class_code NOT IN ( + 'J07AC01', + 'J07BX03', + 'J07BA01', + 'J07BC02', + 'J07AL01', + 'J07AP02', + 'J07CA10' + ); -- secondly, drop wrong combo-vaccines DELETE FROM class_to_drug_new -WHERE class_code ~ '^J07' -AND class_code||concept_code NOT IN (SELECT concept_code_1||concept_code_2 - FROM concept_relationship_manual - WHERE relationship_id = 'ATC - RxNorm' - AND invalid_reason IS NULL) -AND concept_name ~ ' / ' -AND class_code NOT IN ('J07AC01', 'J07BX03', 'J07BA01', 'J07BC02', 'J07AL01','J07AP02', 'J07CA10', 'J07AM51', 'J07BJ51', 'J07AJ52', 'J07AJ51', 'J07AH08', 'J07BD52', 'J07CA02', 'J07BD54'); -- 1220 - --- wrong mono hepatitis A +WHERE class_code LIKE 'J07%' + AND class_code || concept_code NOT IN ( + SELECT concept_code_1 || concept_code_2 + FROM concept_relationship_manual + WHERE relationship_id = 'ATC - RxNorm' + AND invalid_reason IS NULL + ) + AND concept_name LIKE '% / %' + AND class_code NOT IN ( + 'J07AC01', + 'J07BX03', + 'J07BA01', + 'J07BC02', + 'J07AL01', + 'J07AP02', + 'J07CA10', + 'J07AM51', + 'J07BJ51', + 'J07AJ52', + 'J07AJ51', + 'J07AH08', + 'J07BD52', + 'J07CA02', + 'J07BD54' + );-- 1220 + +-- wrong mono hepatitis A DELETE FROM class_to_drug_new WHERE class_code = 'J07BC02' -AND concept_name ~ ' / '; + AND concept_name LIKE '% / %'; -- wrong mono typhoid DELETE FROM class_to_drug_new WHERE class_code = 'J07AP02' -AND concept_name ~ ' / '; + AND concept_name LIKE '% / %'; -- wrong mono pneumococcus DELETE FROM class_to_drug_new WHERE class_code = 'J07AL01' -AND concept_name ~ ' / '; -- 8 + AND concept_name LIKE '% / %';-- 8 -- wrong old mapping DELETE FROM class_to_drug_new WHERE class_code = 'N01BB52' -AND concept_name ~* 'pack'; + AND concept_name ILIKE '%pack%'; -- remove dead ATC Classes if any DELETE -FROM class_to_drug_new -WHERE class_code IN (SELECT concept_code - FROM concept_manual - WHERE invalid_reason IS NOT NULL); +FROM class_to_drug_new cdn USING concept_manual cm +WHERE cdn.class_code = cm.concept_code + AND cm.invalid_reason IS NOT NULL; -- remove non-standard mappings if any DELETE FROM class_to_drug_new a -WHERE NOT EXISTS (SELECT 1 - FROM devv5.concept c - WHERE c.standard_concept = 'S' - AND c.concept_id = a.concept_id); - +WHERE NOT EXISTS ( + SELECT 1 + FROM concept c + WHERE c.standard_concept = 'S' + AND c.concept_id = a.concept_id + ); + -- remove duplicates if any (however they should not be there - can be solved in the future) DELETE -FROM class_to_drug_new -WHERE CTID NOT IN (SELECT MAX(CTID) - FROM class_to_drug_new - GROUP BY class_code, - concept_id); - +FROM class_to_drug_new cdn USING class_to_drug_new cdn_int +WHERE cdn.class_code = cdn_int.class_code + AND cdn.concept_id = cdn_int.concept_id + AND cdn.ctid > cdn_int.ctid; + -- usgin the concep_order field from the class_to_drgu_new table, assemble the final table of class_to_drug and re-assign concept_order value in the following way: /* --==== class_to_drug_new ====-- @@ -2082,83 +2454,179 @@ WHERE CTID NOT IN (SELECT MAX(CTID) TRUNCATE TABLE class_to_drug; INSERT INTO class_to_drug -SELECT DISTINCT class_code, - class_name, - concept_id, - concept_name, - concept_class_id, - CASE - WHEN class_code||concept_code IN (SELECT concept_code_1||concept_code_2 - FROM concept_relationship_manual WHERE relationship_id = 'ATC - RxNorm' AND invalid_reason IS NULL) - AND class_code IN (SELECT class_code FROM combo_pull) AND concept_class_id !~ 'Pack' - AND concept_order <> 11 THEN 1 -- if ATC Combo has an entry in crm, its higher concept_order values have to be converted to 1 as well - WHEN class_code||concept_code IN (SELECT concept_code_1||concept_code_2 - FROM concept_relationship_manual WHERE relationship_id = 'ATC - RxNorm' AND invalid_reason IS NULL) - AND class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_class_id !~ 'Pack' AND concept_name !~ ' \/ ' - AND concept_order <> 11 THEN 1 -- if ATC Mono has an entry in crm, its higher concept_order values have to be converted to 1 as well - WHEN class_code NOT IN (SELECT class_code FROM combo_pull) AND concept_order = 11 AND concept_name ~ ' / ' THEN 7 - WHEN concept_class_id ~ 'Pack' THEN 8 -- 14, 15, 16, 17, 18 (note, that 19 does not exist) - WHEN concept_order = 21 AND concept_class_id !~ 'Box|Product' THEN 1 -- ATC Mono out of hierarchy (as manual) - WHEN concept_order = 21 AND concept_class_id ~ 'Box|Product' THEN 7 - WHEN concept_order IN (11, 13) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual - WHEN concept_order IN (1, 12) THEN 2 -- ATC Monocomp Class - WHEN concept_order = 20 THEN 3 -- Mono: Ingredient A - WHEN concept_order IN (5, 6, 7) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingredients) - WHEN concept_order IN (9, 10, 50, 60, 70) THEN 5 -- Combo: Ingredient A OR Group A + group B - WHEN concept_order IN (3, 4, 8) THEN 6 -- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient - WHEN concept_order IN (2, 22, 23, 24) THEN 7 -- Combo: Ingredient A, combination - END AS concept_order +SELECT class_code, + class_name, + concept_id, + concept_name, + concept_class_id, + CASE WHEN class_code || concept_code IN ( + SELECT concept_code_1 || concept_code_2 + FROM concept_relationship_manual + WHERE relationship_id = 'ATC - RxNorm' + AND invalid_reason IS NULL + ) + AND class_code IN ( + SELECT class_code + FROM combo_pull + ) + AND concept_class_id NOT LIKE '%Pack%' + AND concept_order <> 11 THEN 1 -- if ATC Combo has an entry in crm, its higher concept_order values have to be converted to 1 as well + WHEN class_code || concept_code IN ( + SELECT concept_code_1 || concept_code_2 + FROM concept_relationship_manual + WHERE relationship_id = 'ATC - RxNorm' + AND invalid_reason IS NULL + ) + AND class_code NOT IN ( + SELECT class_code + FROM combo_pull + ) + AND concept_class_id NOT LIKE '%Pack%' + AND concept_name NOT LIKE '% / %' + AND concept_order <> 11 THEN 1 -- if ATC Mono has an entry in crm, its higher concept_order values have to be converted to 1 as well + WHEN class_code NOT IN ( + SELECT class_code + FROM combo_pull + ) + AND concept_order = 11 + AND concept_name LIKE '% / %' THEN 7 WHEN concept_class_id LIKE '%Pack%' THEN 8 -- 14, 15, 16, 17, 18 (note, that 19 does not exist) + WHEN concept_order = 21 + AND concept_class_id !~ 'Box|Product' THEN 1 -- ATC Mono out of hierarchy (as manual) + WHEN concept_order = 21 + AND concept_class_id ~ 'Box|Product' THEN 7 WHEN concept_order IN ( + 11, + 13 + ) THEN 1 -- ATC Class to Drug Product from concept_relationship_manual + WHEN concept_order IN ( + 1, + 12 + ) THEN 2 -- ATC Monocomp Class + WHEN concept_order = 20 THEN 3 -- Mono: Ingredient A + WHEN concept_order IN ( + 5, + 6, + 7 + ) THEN 4 -- ATC Combo Class: Primary lateral + Secondary lateral (2, 3 and 4 ingredients) + WHEN concept_order IN ( + 9, + 10, + 50, + 60, + 70 + ) THEN 5 -- Combo: Ingredient A OR Group A + group B + WHEN concept_order IN ( + 3, + 4, + 8 + ) THEN 6 -- ATC Combo Class: Primary lateral in combination | ATC Combo Class: Primary lateral in combination with excluded Ingredient + WHEN concept_order IN ( + 2, + 22, + 23, + 24 + ) THEN 7 -- Combo: Ingredient A, combination + END AS concept_order FROM class_to_drug_new; -- fix wrong mono UPDATE class_to_drug - SET "order" = 7 -WHERE class_code IN ('A01AB14','B02BD30') -AND concept_name ~ ' / ' -AND "order" <> 7; +SET concept_order = 7 +WHERE class_code IN ( + 'A01AB14', + 'B02BD30' + ) + AND concept_name LIKE '% / %' + AND concept_order <> 7; -- update wrong order for some manual links -WITH t1 AS -( - SELECT a.*, - c.concept_code - FROM class_to_drug a - JOIN concept c ON c.concept_id = a.concept_id -), -t2 as ( -SELECT * -FROM t1 -WHERE class_code||concept_code IN (SELECT concept_code_1||concept_code_2 - FROM concept_relationship_manual - WHERE relationship_id = 'ATC - RxNorm' - AND invalid_reason IS NULL) -AND concept_class_id !~ 'Pack' -AND "order" <> 1 -AND concept_name ~ ' / ' -AND class_code IN ('A10AC04','A10AD01','A10AD02','A10AD03','A10AD04','A10AD05','G03GA02','J07AG01','J07AH03','J07AH04','J07AH08', -'J07AL01','J07AL02','J07BB02','J07BB03','J07BH02','J07BM01','J07BM02','J07BM03','J07BF02', 'J07BF03')) - -UPDATE class_to_drug - SET "order" = 1 -WHERE class_code||concept_id IN (SELECT class_code||concept_id FROM t2); -- 105 - -UPDATE class_to_drug - SET "order" = 1 -WHERE class_code IN (SELECT class_code FROM class_to_drug WHERE "order" = 1) -AND class_code IN (SELECT class_code FROM class_to_drug WHERE "order" <> 1) -AND class_code IN (SELECT class_code FROM atc_all_mono) -AND concept_name !~ ' / ' -AND "order" <> 1; - -UPDATE class_to_drug - SET "order" = 1 -WHERE class_code IN (SELECT class_code FROM class_to_drug WHERE "order" = 1) -AND class_code IN (SELECT class_code FROM class_to_drug WHERE "order" <> 1) -AND class_code NOT IN (SELECT class_code FROM atc_all_mono) -AND concept_name ~ ' / ' -AND "order" <> 1; +UPDATE class_to_drug ctd +SET concept_order = 1 +FROM ( + SELECT ctd.class_code || c.concept_id AS class_and_cid + FROM class_to_drug ctd + JOIN concept c ON c.concept_id = ctd.concept_id + AND c.concept_class_id NOT LIKE '%Pack%' + JOIN concept_relationship_manual crm ON crm.concept_code_1 || crm.concept_code_2 = ctd.class_code || c.concept_code + AND crm.relationship_id = 'ATC - RxNorm' + AND crm.invalid_reason IS NULL + WHERE ctd.concept_order <> 1 + AND ctd.concept_name LIKE '% / %' + AND ctd.class_code IN ( + 'A10AC04', + 'A10AD01', + 'A10AD02', + 'A10AD03', + 'A10AD04', + 'A10AD05', + 'G03GA02', + 'J07AG01', + 'J07AH03', + 'J07AH04', + 'J07AH08', + 'J07AL01', + 'J07AL02', + 'J07BB02', + 'J07BB03', + 'J07BH02', + 'J07BM01', + 'J07BM02', + 'J07BM03', + 'J07BF02', + 'J07BF03' + ) + ) u +WHERE ctd.class_code || ctd.concept_id = u.class_and_cid + AND concept_order <> 1;-- 105 + +UPDATE class_to_drug cdn +SET concept_order = 1 +WHERE EXISTS ( + SELECT 1 + FROM class_to_drug cdn_int + WHERE cdn_int.class_code = cdn.class_code + AND cdn_int.concept_order = 1 + ) + AND EXISTS ( + SELECT 1 + FROM class_to_drug cdn_int + WHERE cdn_int.class_code = cdn.class_code + AND cdn_int.concept_order <> 1 + ) + AND EXISTS ( + SELECT 1 + FROM atc_all_mono a + WHERE a.class_code = cdn.class_code + ) + AND cdn.concept_name NOT LIKE '% / %' + AND cdn.concept_order <> 1; + + +UPDATE class_to_drug cdn +SET concept_order = 1 +WHERE EXISTS ( + SELECT 1 + FROM class_to_drug cdn_int + WHERE cdn_int.class_code = cdn.class_code + AND cdn_int.concept_order = 1 + ) + AND EXISTS ( + SELECT 1 + FROM class_to_drug cdn_int + WHERE cdn_int.class_code = cdn.class_code + AND cdn_int.concept_order <> 1 + ) + AND EXISTS ( + SELECT 1 + FROM atc_all_mono a + WHERE a.class_code = cdn.class_code + ) + AND cdn.concept_name LIKE '% / %' + AND cdn.concept_order <> 1; -- update class_to_drug in the schema of 'sources' -SELECT * FROM vocabulary_pack.CreateTablesCopiesATC (); + DO $_$ + BEGIN + PERFORM VOCABULARY_PACK.CreateTablesCopiesATC(); + END $_$; --- run load_stage.sql +-- run load_stage.sql \ No newline at end of file diff --git a/ATC/load_stage.sql b/ATC/load_stage.sql index 6f7eefb15..3e55031bd 100644 --- a/ATC/load_stage.sql +++ b/ATC/load_stage.sql @@ -14,54 +14,56 @@ * limitations under the License. * * Authors: Anna Ostropolets, Polina Talapova, Timur Vakhitov -* Date: Oct 2021 -* Total script execution time: 4m 58s +* Date: 2022 **************************************************************************/ +--1. Update latest_update field to new date DO $_$ BEGIN PERFORM VOCABULARY_PACK.SetLatestUpdate( - pVocabularyName => 'ATC', - pVocabularyDate => (SELECT vocabulary_date FROM sources.rxnatomarchive LIMIT 1), - pVocabularyVersion => (SELECT vocabulary_version FROM sources.rxnatomarchive LIMIT 1), + pVocabularyName => 'ATC', + pVocabularyDate => (SELECT vocabulary_date FROM sources.rxnatomarchive LIMIT 1), + pVocabularyVersion => (SELECT vocabulary_version FROM sources.rxnatomarchive LIMIT 1), pVocabularyDevSchema => 'DEV_ATC' ); END $_$; --- truncate all working tables +--2. Truncate all working tables TRUNCATE TABLE concept_stage; TRUNCATE TABLE concept_relationship_stage; TRUNCATE TABLE concept_synonym_stage; TRUNCATE TABLE pack_content_stage; TRUNCATE TABLE drug_strength_stage; --- add all ATC codes to staging tables using the function which processes the concept_manual table +--3. Add all ATC codes to staging tables using the function which processes the concept_manual table DO $_$ BEGIN PERFORM VOCABULARY_PACK.ProcessManualConcepts(); END $_$; --- add manually created relationships using the function which processes the concept_relationship_manual table +ANALYZE concept_stage; + +--4. Add manually created relationships using the function which processes the concept_relationship_manual table DO $_$ BEGIN PERFORM VOCABULARY_PACK.ProcessManualRelationships(); END $_$; --- add manually created synonyms using the function processing the concept_synonym_manual +--5. Add manually created synonyms using the function processing the concept_synonym_manual DO $_$ BEGIN PERFORM VOCABULARY_PACK.ProcessManualSynonyms(); END $_$; +--6. Fill the concept_relationship_stage -- add 1) 'SNOMED - ATC eq' relationships between SNOMED Drugs and Higher ATC Classes (excl. 5th) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, vocabulary_id_1, vocabulary_id_2, - relationship_id, + relationship_id, valid_start_date, - valid_end_date, - invalid_reason + valid_end_date ) -- crosslinks between SNOMED Drug Class AND ATC Classes (not ATC 5th) SELECT DISTINCT d.concept_code AS concept_code_1, @@ -70,234 +72,235 @@ SELECT DISTINCT d.concept_code AS concept_code_1, 'ATC' AS vocabulary_id_2, 'SNOMED - ATC eq' AS relationship_id, d.valid_start_date AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date FROM concept d JOIN sources.rxnconso r ON r.code = d.concept_code AND r.sab = 'SNOMEDCT_US' - AND r.code != 'NOCODE' + AND r.code <> 'NOCODE' JOIN sources.rxnconso r2 ON r2.rxcui = r.rxcui AND r2.sab = 'ATC' - AND r2.code != 'NOCODE' -JOIN concept_manual e ON e.concept_code = r2.code - AND e.concept_class_id != 'ATC 5th' -- Ingredients only to RxNorm - AND e.vocabulary_id = 'ACT' + AND r2.code <> 'NOCODE' +JOIN concept_stage e ON e.concept_code = r2.code + AND e.concept_class_id <> 'ATC 5th' -- Ingredients only to RxNorm + AND e.vocabulary_id = 'ATC' WHERE d.vocabulary_id = 'SNOMED' AND d.invalid_reason IS NULL + UNION ALL + -- 2) 'Is a' relationships between ATC Classes using mrconso (internal ATC hierarchy) SELECT uppr.concept_code AS concept_code_1, lowr.concept_code AS concept_code_2, 'ATC' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, 'Is a' AS relationship_id, - v.latest_UPDATE AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM concept_stage uppr, - concept_stage lowr, - vocabulary v -WHERE uppr.invalid_reason IS NULL -AND lowr.invalid_reason IS NULL -- to exclude deprecated or updated codes from the hierarchy -AND ( -(LENGTH(uppr.concept_code) IN (4, 5) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 1)) -OR (LENGTH(uppr.concept_code) IN (3, 7) AND lowr.concept_code = SUBSTR(uppr.concept_code,1,LENGTH(uppr.concept_code) - 2)) -) + v.latest_update AS valid_start_date, + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date +FROM concept_stage uppr +JOIN concept_stage lowr ON lowr.vocabulary_id = 'ATC' + AND lowr.invalid_reason IS NULL -- to exclude deprecated or updated codes from the hierarchy +JOIN vocabulary v ON v.vocabulary_id = 'ATC' +WHERE uppr.invalid_reason IS NULL AND uppr.vocabulary_id = 'ATC' - AND lowr.vocabulary_id = 'ATC' - AND v.vocabulary_id = 'ATC'; -- 6495 + AND ( + ( + LENGTH(uppr.concept_code) IN ( + 4, + 5 + ) + AND lowr.concept_code = SUBSTR(uppr.concept_code, 1, LENGTH(uppr.concept_code) - 1) + ) + OR ( + LENGTH(uppr.concept_code) IN ( + 3, + 7 + ) + AND lowr.concept_code = SUBSTR(uppr.concept_code, 1, LENGTH(uppr.concept_code) - 2) + ) + );-- 6495 --- add 'ATC - RxNorm' relationships between ATC Classes and RxN/RxE Drug Products using class_to_drug table -INSERT INTO concept_relationship_stage -( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason -) -WITH t1 AS -( - SELECT DISTINCT - cs.class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM sources.class_to_drug cs -- manually curated table with ATC Class to Rx Drug Product links -JOIN concept_manual k ON k.concept_code = cs.class_code -AND k.invalid_reason IS NULL - JOIN concept c ON c.concept_id = cs.concept_id -WHERE c.concept_class_id != 'Ingredient' -) - SELECT * FROM t1 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ); -- 115047 +--6.1 add 'ATC - RxNorm' relationships between ATC Classes and RxN/RxE Drug Products using class_to_drug table +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date + ) +SELECT ctd.class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date +FROM sources.class_to_drug ctd -- manually curated table with ATC Class to Rx Drug Product links +JOIN concept c ON c.concept_id = ctd.concept_id + AND c.concept_class_id <> 'Ingredient' +WHERE EXISTS ( + SELECT 1 + FROM concept_stage cs_int + WHERE cs_int.concept_code = ctd.class_code + AND cs_int.invalid_reason IS NULL + ) + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = ctd.class_code + AND crs_int.vocabulary_id_1 = 'ATC' + AND crs_int.concept_code_2 = c.concept_code + AND crs_int.vocabulary_id_2 = c.vocabulary_id + AND crs_int.relationship_id = 'ATC - RxNorm' + );-- 115047 --- add 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between ATC Classes and RxN/RxE Drug Products (using input tables and dev_combo populated during previous Steps) -INSERT INTO concept_relationship_stage -( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason -) -WITH t1 AS ( -SELECT DISTINCT SUBSTRING(irs.concept_code_1,'\w+') AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm pr lat' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason +--6.2 add 'ATC - RxNorm pr lat' relationships indicating Primary unambiguous links between ATC Classes and RxN/RxE Drug Products (using input tables and dev_combo populated during previous Steps) +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date + ) +SELECT DISTINCT SUBSTRING(irs.concept_code_1, '\w+') AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date FROM internal_relationship_stage irs - JOIN relationship_to_concept rtc ON lower (irs.concept_code_2) = lower (rtc.concept_code_1) - JOIN concept c - ON concept_id_2 = c.concept_id - AND c.concept_class_id = 'Ingredient' - JOIN concept_manual k - ON k.concept_code = SUBSTRING (irs.concept_code_1,'\w+') - AND k.invalid_reason IS NULL -WHERE NOT EXISTS (SELECT 1 - FROM dev_combo t - WHERE LOWER(t.concept_name) = LOWER(rtc.concept_code_1) - AND t.class_code = SUBSTRING(irs.concept_code_1,'\w+'))) -SELECT * FROM t1 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ); -- 3292 - --- add 'ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up' for ATC Combo Classes using dev_combo -INSERT INTO concept_relationship_stage -( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason -) -with t1 AS ( -SELECT DISTINCT class_code, class_name, concept_id, rnk - FROM dev_combo - WHERE rnk IN (1, 2, 3)), -t2 AS ( -SELECT a.class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - CASE rnk WHEN 1 THEN 'ATC - RxNorm pr lat' - WHEN 2 THEN 'ATC - RxNorm sec lat' - WHEN 3 THEN 'ATC - RxNorm pr up' - END AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM t1 a -JOIN concept c ON c.concept_id = a.concept_id -JOIN concept_manual k ON k.concept_code = a.class_code +JOIN relationship_to_concept rtc ON lower(irs.concept_code_2) = lower(rtc.concept_code_1) +JOIN concept c ON concept_id_2 = c.concept_id + AND c.concept_class_id = 'Ingredient' +JOIN concept_stage k ON k.concept_code = SUBSTRING(irs.concept_code_1, '\w+') AND k.invalid_reason IS NULL - AND c.standard_concept = 'S') -SELECT DISTINCT * FROM t2 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ); -- 3670 +WHERE NOT EXISTS ( + SELECT 1 + FROM dev_combo t + WHERE LOWER(t.concept_name) = LOWER(rtc.concept_code_1) + AND t.class_code = SUBSTRING(irs.concept_code_1, '\w+') + ) + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = SUBSTRING(irs.concept_code_1, '\w+') + AND crs_int.vocabulary_id_1 = 'ATC' + AND crs_int.concept_code_2 = c.concept_code + AND crs_int.vocabulary_id_2 = c.vocabulary_id + AND crs_int.relationship_id = 'ATC - RxNorm pr lat' + );-- 3292 --- add 'ATC - RxNorm sec up' relationships for Primary lateral in combination -INSERT INTO concept_relationship_stage -( - concept_code_1, - concept_code_2, - vocabulary_id_1, - vocabulary_id_2, - relationship_id, - valid_start_date, - valid_end_date, - invalid_reason -) -WITH t1 AS -- the list of the main Ingredients -(SELECT class_code, - class_name, - concept_id -FROM ing_pr_lat_sec_up WHERE rnk = 1 -UNION ALL -SELECT class_code, -class_name, concept_id FROM ing_pr_up_sec_up WHERE rnk = 3 -UNION ALL -SELECT class_code, -class_name, concept_id FROM ing_pr_up_combo WHERE rnk = 3 -UNION ALL -SELECT class_code, -class_name, concept_id FROM ing_pr_lat_combo WHERE rnk = 1 -UNION ALL -SELECT class_code, -class_name, concept_id FROM Ing_pr_lat_combo_excl WHERE rnk = 1 -UNION ALL -SELECT class_code, -class_name, concept_id FROM ing_pr_up_sec_up_excl WHERE rnk = 3 -), -t2 AS ( + +--6.3 add 'ATC - RxNorm pr lat', 'ATC - RxNorm sec lat', 'ATC - RxNorm pr up' for ATC Combo Classes using dev_combo +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date + ) +SELECT * +FROM ( + SELECT dc.class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + CASE rnk WHEN 1 THEN 'ATC - RxNorm pr lat' WHEN 2 THEN 'ATC - RxNorm sec lat' WHEN 3 THEN 'ATC - RxNorm pr up' END AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date + FROM dev_combo dc + JOIN concept c ON c.concept_id = dc.concept_id + AND c.standard_concept = 'S' + JOIN concept_stage k ON k.concept_code = dc.class_code + AND k.invalid_reason IS NULL + WHERE dc.rnk IN ( + 1, + 2, + 3 + ) + ) s0 +WHERE NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = s0.concept_code_1 + AND crs_int.vocabulary_id_1 = 'ATC' + AND crs_int.concept_code_2 = s0.concept_code_2 + AND crs_int.vocabulary_id_2 = s0.vocabulary_id_2 + AND crs_int.relationship_id = s0.relationship_id + );-- 3670 + +--6.4 add 'ATC - RxNorm sec up' relationships for Primary lateral in combination +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date + ) SELECT DISTINCT k.concept_code AS concept_code_1, - cc.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - cc.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm sec up' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason + cc.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + cc.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date FROM sources.class_to_drug c - JOIN concept_ancestor ca ON ca.descendant_concept_id = c.concept_id - JOIN concept cc - ON cc.concept_id = ca.ancestor_concept_id - AND cc.standard_concept = 'S' - AND cc.concept_class_id = 'Ingredient' - AND cc.vocabulary_id IN ('RxNorm', 'RxNorm Extension') - JOIN concept_manual k - ON k.concept_code = c.class_code - AND k.invalid_reason IS NULL - AND k.concept_code IN (SELECT class_code FROM dev_combo) -WHERE (c.class_code,cc.concept_id) NOT IN (SELECT class_code, concept_id FROM t1)-- exclude main Ingredients -) -SELECT * FROM t2 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ); -- 23313 - --- deprecate links between ATC classes and dead RxN/RxE +JOIN concept_ancestor ca ON ca.descendant_concept_id = c.concept_id +JOIN concept cc ON cc.concept_id = ca.ancestor_concept_id + AND cc.standard_concept = 'S' + AND cc.concept_class_id = 'Ingredient' + AND cc.vocabulary_id LIKE 'RxNorm%' +JOIN concept_stage k ON k.concept_code = c.class_code + AND k.invalid_reason IS NULL +-- exclude main Ingredients +LEFT JOIN ing_pr_lat_sec_up i1 ON i1.rnk = 1 + AND i1.class_code = c.class_code + AND i1.concept_id = cc.concept_id +LEFT JOIN ing_pr_up_sec_up i2 ON i2.rnk = 3 + AND i2.class_code = c.class_code + AND i2.concept_id = cc.concept_id +LEFT JOIN ing_pr_up_combo i3 ON i3.rnk = 3 + AND i3.class_code = c.class_code + AND i3.concept_id = cc.concept_id +LEFT JOIN ing_pr_lat_combo i4 ON i4.rnk = 1 + AND i4.class_code = c.class_code + AND i4.concept_id = cc.concept_id +LEFT JOIN ing_pr_lat_combo_excl i5 ON i5.rnk = 1 + AND i5.class_code = c.class_code + AND i5.concept_id = cc.concept_id +LEFT JOIN ing_pr_up_sec_up_excl i6 ON i6.rnk = 3 + AND i6.class_code = c.class_code + AND i6.concept_id = cc.concept_id +WHERE EXISTS ( + SELECT 1 + FROM dev_combo dc_int + WHERE dc_int.class_code = k.concept_code + ) + AND i1.class_code IS NULL + AND i2.class_code IS NULL + AND i3.class_code IS NULL + AND i4.class_code IS NULL + AND i5.class_code IS NULL + AND i6.class_code IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = k.concept_code + AND crs_int.vocabulary_id_1 = 'ATC' + AND crs_int.concept_code_2 = cc.concept_code + AND crs_int.vocabulary_id_2 = cc.vocabulary_id + AND crs_int.relationship_id = 'ATC - RxNorm sec up' + );-- 23313 + +--6.5 deprecate links between ATC classes and dead RxN/RxE INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -308,32 +311,32 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -WITH t1 AS ( SELECT c.concept_code AS concept_code_1, - cc.concept_code AS concept_code_2, - c.vocabulary_id AS vocabulary_id_1, - cc.vocabulary_id AS vocabulary_id_2, - relationship_id AS relationship_id, - cr.valid_start_date AS valid_start_date, - CURRENT_DATE AS valid_end_date, - 'D' AS invalid_reason + cc.concept_code AS concept_code_2, + c.vocabulary_id AS vocabulary_id_1, + cc.vocabulary_id AS vocabulary_id_2, + relationship_id AS relationship_id, + cr.valid_start_date AS valid_start_date, + CURRENT_DATE AS valid_end_date, + 'D' AS invalid_reason FROM concept_relationship cr - JOIN concept c ON concept_id_1 = c.concept_id - JOIN concept cc ON concept_id_2 = cc.concept_id -AND cc.standard_concept IS NULL -- non-standard -WHERE c.vocabulary_id = 'ATC' -AND cc.vocabulary_id LIKE 'RxNorm%' -AND cr.invalid_reason IS NULL) -SELECT * FROM t1 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id); -- 8365 +JOIN concept c ON c.concept_id = cr.concept_id_1 + AND c.vocabulary_id = 'ATC' +JOIN concept cc ON cc.concept_id = cr.concept_id_2 + AND cc.vocabulary_id LIKE 'RxNorm%' + AND cc.standard_concept IS NULL -- non-standard +WHERE cr.invalid_reason IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = c.concept_code + AND crs_int.vocabulary_id_1 = c.vocabulary_id + AND crs_int.concept_code_2 = cc.concept_code + AND crs_int.vocabulary_id_2 = cc.vocabulary_id + AND crs_int.relationship_id = cr.relationship_id + );-- 8365 --- deprecate accessory links for invalid codes +--6.6 deprecate accessory links for invalid codes INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -344,36 +347,33 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -WITH t1 AS - ( -SELECT - c.concept_code AS concept_code_1, - cc.concept_code AS concept_code_2, - c.vocabulary_id AS vocabulary_id_1, - cc.vocabulary_id AS vocabulary_id_2, - relationship_id AS relationship_id, - cr.valid_start_date AS valid_start_date, - CURRENT_DATE AS valid_end_date, - 'D' AS invalid_reason +SELECT c.concept_code AS concept_code_1, + cc.concept_code AS concept_code_2, + c.vocabulary_id AS vocabulary_id_1, + cc.vocabulary_id AS vocabulary_id_2, + relationship_id AS relationship_id, + cr.valid_start_date AS valid_start_date, + CURRENT_DATE AS valid_end_date, + 'D' AS invalid_reason FROM concept_relationship cr - JOIN concept c ON concept_id_1 = c.concept_id - JOIN concept cc ON concept_id_2 = cc.concept_id - JOIN concept_manual k ON k.concept_code = c.concept_code - AND k.invalid_reason IS NOT NULL -WHERE c.vocabulary_id = 'ATC' -AND cr.invalid_reason IS NULL -AND c.concept_code <> 'H01BA06') -- deprecated argipressin -SELECT * FROM t1 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ) ; -- 1213 +JOIN concept c ON c.concept_id = cr.concept_id_1 + AND c.vocabulary_id = 'ATC' +JOIN concept cc ON cc.concept_id = cr.concept_id_2 +JOIN concept_stage k ON k.concept_code = c.concept_code + AND k.invalid_reason IS NOT NULL +WHERE cr.invalid_reason IS NULL + AND c.concept_code <> 'H01BA06' -- deprecated argipressin + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = c.concept_code + AND crs_int.vocabulary_id_1 = c.vocabulary_id + AND crs_int.concept_code_2 = cc.concept_code + AND crs_int.vocabulary_id_2 = cc.vocabulary_id + AND crs_int.relationship_id = cr.relationship_id + );-- 1213 --- add mirroring 'Maps to' for 'ATC - RxNorm pr lat' for monocomponent ATC Classes, which do not have doubling Standard ingredients (1-to-many mappings are permissive only for ATC Combo Classes) +--6.7 add mirroring 'Maps to' for 'ATC - RxNorm pr lat' for monocomponent ATC Classes, which do not have doubling Standard ingredients (1-to-many mappings are permissive only for ATC Combo Classes) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -381,48 +381,46 @@ INSERT INTO concept_relationship_stage ( vocabulary_id_2, relationship_id, valid_start_date, - valid_end_date, - invalid_reason + valid_end_date ) -WITH t1 AS -( - SELECT * - FROM concept_relationship_stage - WHERE relationship_id = 'ATC - RxNorm pr lat' - AND invalid_reason IS NULL -), -t2 AS ( -SELECT DISTINCT a.concept_code_1, --k.concept_name, - a.concept_code_2, --d.concept_name, - a.vocabulary_id_1, - a.vocabulary_id_2, - 'Maps to' AS relationship_id, - a.valid_start_date, - a.valid_end_date, - a.invalid_reason -FROM (SELECT *, - COUNT(concept_code_1) OVER (PARTITION BY concept_code_1) AS cnt - FROM t1) a - JOIN concept_manual k - ON k.concept_code = a.concept_code_1 - AND k.invalid_reason IS NULL - JOIN concept d - ON d.concept_code = a.concept_code_2 - AND d.vocabulary_id = a.vocabulary_id_2 - AND d.standard_concept = 'S' -WHERE cnt = 1 -- can be just one -) -SELECT * FROM t2 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ); -- 4374 - --- add mirroring 'Maps to' of 'ATC - RxNorm sec lat' relationships for ATC Combo Classes (1-to-1) +SELECT s0.concept_code_1, + s0.concept_code_2, + s0.vocabulary_id_1, + s0.vocabulary_id_2, + s0.relationship_id, + s0.valid_start_date, + s0.valid_end_date +FROM ( + SELECT crs.concept_code_1, --k.concept_name, + crs.concept_code_2, --d.concept_name, + crs.vocabulary_id_1, + crs.vocabulary_id_2, + 'Maps to' AS relationship_id, + crs.valid_start_date, + crs.valid_end_date, + COUNT(crs.concept_code_2) OVER (PARTITION BY crs.concept_code_1) AS cnt + FROM concept_relationship_stage crs + JOIN concept_stage k ON k.concept_code = crs.concept_code_1 + AND k.vocabulary_id = crs.vocabulary_id_1 + AND k.invalid_reason IS NULL + JOIN concept d ON d.concept_code = crs.concept_code_2 + AND d.vocabulary_id = crs.vocabulary_id_2 + AND d.standard_concept = 'S' + WHERE crs.relationship_id = 'ATC - RxNorm pr lat' + AND crs.invalid_reason IS NULL + ) s0 +WHERE s0.cnt = 1 -- can be just one + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = s0.concept_code_1 + AND crs_int.vocabulary_id_1 = s0.vocabulary_id_1 + AND crs_int.concept_code_2 = s0.concept_code_2 + AND crs_int.vocabulary_id_2 = s0.vocabulary_id_2 + AND crs_int.relationship_id = 'Maps to' + );-- 4374 + +--6.8 add mirroring 'Maps to' of 'ATC - RxNorm sec lat' relationships for ATC Combo Classes (1-to-1) INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -430,46 +428,40 @@ INSERT INTO concept_relationship_stage ( vocabulary_id_2, relationship_id, valid_start_date, - valid_end_date, - invalid_reason + valid_end_date ) -WITH T1 AS -( - SELECT class_code, - class_name, - a.concept_id, - c.concept_code, - a.concept_name, - c.vocabulary_id - FROM dev_combo a - JOIN concept c ON c.concept_id = a.concept_id - WHERE rnk = 2 -), -t2 AS ( -SELECT DISTINCT class_code AS concept_code_1,--class_name, - concept_code AS concept_code_2,-- concept_name, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'Maps to' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM t1 -WHERE class_code IN (SELECT class_code - FROM t1 - GROUP BY class_code - HAVING COUNT(1) = 1) - ) -SELECT * FROM t2 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ); -- 209 - +SELECT s0.concept_code_1, + s0.concept_code_2, + s0.vocabulary_id_1, + s0.vocabulary_id_2, + s0.relationship_id, + s0.valid_start_date, + s0.valid_end_date +FROM ( + SELECT a.class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'Maps to' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, + COUNT(*) OVER (PARTITION BY a.class_code) AS cnt + FROM dev_combo a + JOIN concept c ON c.concept_id = a.concept_id + WHERE a.rnk = 2 + ) s0 +WHERE s0.cnt = 1 + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = s0.concept_code_1 + AND crs_int.vocabulary_id_1 = 'ATC' + AND crs_int.concept_code_2 = s0.concept_code_2 + AND crs_int.vocabulary_id_2 = s0.vocabulary_id_2 + AND crs_int.relationship_id = s0.relationship_id + );-- 209 + +--6.9 INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -477,94 +469,96 @@ INSERT INTO concept_relationship_stage ( vocabulary_id_2, relationship_id, valid_start_date, - valid_end_date, - invalid_reason + valid_end_date ) -WITH t1 AS -( - SELECT class_code, - class_name, - a.concept_id, - c.concept_code, - a.concept_name, - c.vocabulary_id - FROM dev_combo a - JOIN concept c ON c.concept_id = a.concept_id - WHERE rnk = 2 -), -t2 AS ( -SELECT DISTINCT class_code AS concept_code_1, --class_name, - concept_code AS concept_code_2,-- concept_name, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'Maps to' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231','YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM t1 a -WHERE class_code IN (SELECT class_code - FROM t1 - GROUP BY class_code - HAVING COUNT(1) <= 3) -- Combo Classes with COUNT(1) > 3 were added from concept_relationship_manual -AND NOT EXISTS (SELECT 1 FROM atc_one_to_many_excl b - WHERE b.atc_code = a.class_code - AND a.concept_id = b.concept_id) -) -SELECT * FROM t2 cs -WHERE NOT EXISTS (SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.concept_code_1 - AND crs.vocabulary_id_1 = cs.vocabulary_id_1 - AND crs.concept_code_2 = cs.concept_code_2 - AND crs.vocabulary_id_2 = cs.vocabulary_id_2 - AND crs.relationship_id = cs.relationship_id - ) -AND concept_code_1 NOT IN ('P01BF05', 'J07AG52', 'J07BD51') -- artenimol and piperaquine|hemophilus influenzae B, combinations with pertussis and toxoids; systemic|measles, combinations with mumps, live attenuated; systemic -; -- 193 - --- add synonyms to concept_synonym stage for each of the rxcui/code combinations +SELECT s0.concept_code_1, + s0.concept_code_2, + s0.vocabulary_id_1, + s0.vocabulary_id_2, + s0.relationship_id, + s0.valid_start_date, + s0.valid_end_date +FROM ( + SELECT a.class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'Maps to' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, + COUNT(*) OVER (PARTITION BY a.class_code) AS cnt, + c.concept_id + FROM dev_combo a + JOIN concept c ON c.concept_id = a.concept_id + WHERE a.rnk = 2 + ) s0 +LEFT JOIN atc_one_to_many_excl atme ON atme.atc_code = s0.concept_code_1 + AND atme.concept_id = s0.concept_id +WHERE atme.atc_code IS NULL + AND s0.cnt <= 3 -- Combo Classes with COUNT(*) > 3 were added from concept_relationship_manual + AND NOT EXISTS ( + SELECT 1 + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = s0.concept_code_1 + AND crs_int.vocabulary_id_1 = 'ATC' + AND crs_int.concept_code_2 = s0.concept_code_2 + AND crs_int.vocabulary_id_2 = s0.vocabulary_id_2 + AND crs_int.relationship_id = s0.relationship_id + ) + AND concept_code_1 NOT IN ( + 'P01BF05', + 'J07AG52', + 'J07BD51' + );-- artenimol and piperaquine|hemophilus influenzae B, combinations with pertussis and toxoids; systemic|measles, combinations with mumps, live attenuated; systemic -- 193 + +--7. Add synonyms to concept_synonym stage for each of the rxcui/code combinations INSERT INTO concept_synonym_stage ( synonym_concept_code, synonym_name, synonym_vocabulary_id, language_concept_id ) -SELECT DISTINCT dv.concept_code AS synonym_concept_code, - SUBSTR(r.str, 1, 1000) AS synonym_name, - dv.vocabulary_id AS synonym_vocabulary_id, +SELECT cs.concept_code AS synonym_concept_code, + vocabulary_pack.CutConceptSynonymName(r.str) AS synonym_name, + cs.vocabulary_id AS synonym_vocabulary_id, 4180186 AS language_concept_id -FROM concept_manual dv -JOIN sources.rxnconso r ON dv.concept_code = r.code - AND r.code != 'NOCODE' +FROM concept_stage cs +JOIN sources.rxnconso r ON r.code = cs.concept_code + AND r.code <> 'NOCODE' AND r.lat = 'ENG' - AND r.sab = 'ATC' - AND r.tty IN ('PT','IN'); -- 6440 + AND r.sab = 'ATC' + AND r.tty IN ( + 'PT', + 'IN' + );-- 6440 + +ANALYZE concept_relationship_stage; --- perform mapping replacement using function below +--8. Perform mapping replacement using function below DO $_$ BEGIN PERFORM VOCABULARY_PACK.CheckReplacementMappings(); END $_$; --- add mappings from deprecated to fresh codes +--9. Add mappings from deprecated to fresh codes DO $_$ BEGIN PERFORM VOCABULARY_PACK.AddFreshMAPSTO(); END $_$; --- deprecate 'Maps to' mappings to deprecated AND updated codes +--10. Deprecate 'Maps to' mappings to deprecated AND updated codes DO $_$ BEGIN PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO(); END $_$; --- remove ambiguous 'Maps to' mappings +--11. Remove ambiguous 'Maps to' mappings DO $_$ BEGIN PERFORM VOCABULARY_PACK.DELETEAmbiguousMAPSTO(); END $_$; --- build reverse relationships in order to take the next sept +--12. Build reverse relationships in order to take the next step INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -575,7 +569,7 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT DISTINCT crs.concept_code_2, +SELECT crs.concept_code_2, crs.concept_code_1, crs.vocabulary_id_2, crs.vocabulary_id_1, @@ -588,15 +582,17 @@ JOIN relationship r ON r.relationship_id = crs.relationship_id WHERE NOT EXISTS ( -- the inverse record SELECT 1 - FROM concept_relationship_stage i - WHERE crs.concept_code_1 = i.concept_code_2 - AND crs.concept_code_2 = i.concept_code_1 - AND crs.vocabulary_id_1 = i.vocabulary_id_2 - AND crs.vocabulary_id_2 = i.vocabulary_id_1 - AND r.reverse_relationship_id = i.relationship_id + FROM concept_relationship_stage crs_int + WHERE crs_int.concept_code_1 = crs.concept_code_2 + AND crs_int.vocabulary_id_1 = crs.vocabulary_id_2 + AND crs_int.concept_code_2 = crs.concept_code_1 + AND crs_int.vocabulary_id_2 = crs.vocabulary_id_1 + AND crs_int.relationship_id = r.reverse_relationship_id ); --- deprecate all relationships from the concept_relationship table which do not exist in the concept_relationship_stage +ANALYZE concept_relationship_stage; + +--13. Deprecate all relationships from the concept_relationship table which do not exist in the concept_relationship_stage INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -607,8 +603,7 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT DISTINCT - a.concept_code, +SELECT a.concept_code, b.concept_code, a.vocabulary_id, b.vocabulary_id, @@ -621,8 +616,17 @@ JOIN concept_relationship r ON a.concept_id = concept_id_1 AND r.invalid_reason IS NULL AND r.relationship_id NOT IN ( 'Concept replaced by', - 'Concept replaces','Drug has drug class', 'Drug class of drug', 'Subsumes', - 'Is a', 'ATC - SNOMED eq', 'SNOMED - ATC eq', 'VA Class to ATC eq', 'ATC to VA Class eq', 'ATC to NDFRT eq', 'NDFRT to ATC eq' + 'Concept replaces', + 'Drug has drug class', + 'Drug class of drug', + 'Subsumes', + 'Is a', + 'ATC - SNOMED eq', + 'SNOMED - ATC eq', + 'VA Class to ATC eq', + 'ATC to VA Class eq', + 'ATC to NDFRT eq', + 'NDFRT to ATC eq' ) JOIN concept b ON b.concept_id = concept_id_2 WHERE 'ATC' IN ( @@ -637,10 +641,60 @@ WHERE 'ATC' IN ( AND crs_int.vocabulary_id_1 = a.vocabulary_id AND crs_int.vocabulary_id_2 = b.vocabulary_id AND crs_int.relationship_id = r.relationship_id - ); -- 5418 - --- remove suspicious replacement mapping for OLD codes -DELETE FROM concept_relationship_stage -WHERE concept_code_1 IN ('C10AA55', 'J05AE06', 'C10AA52', 'C10AA53', 'C10AA51', 'N02AX52', 'H01BA06') -AND concept_code_2 IN ('17767','85762','7393','1191','161', '11149') -AND invalid_reason IS NULL; -- 7 + );-- 5418 + +--14. Remove suspicious replacement mapping for OLD codes +DELETE +FROM concept_relationship_stage +WHERE concept_code_1 IN ( + 'C10AA55', + 'J05AE06', + 'C10AA52', + 'C10AA53', + 'C10AA51', + 'N02AX52', + 'H01BA06' + ) + AND concept_code_2 IN ( + '17767', + '85762', + '7393', + '1191', + '161', + '11149' + ) + AND invalid_reason IS NULL;-- 7 + +--15. Clean up +DROP TABLE rx_combo, + rx_all_combo, + atc_all_combo, + tmp_irs_dcs, + atc_all_mono, + ing_pr_lat_sec_lat, + ing_pr_lat_sec_up, + ing_pr_lat_combo, + ing_pr_lat_combo_excl, + ing_pr_up_combo, + ing_pr_up_sec_up, + ing_pr_up_sec_up_excl, + class_to_drug_new, + t1, + t2, + t3, + t4, + full_combo, + ing_pr_lat_combo_to_drug, + ing_pr_lat_sec_up_combo_to_drug, + ing_pr_lat_combo_excl_to_drug, + ing_pr_sec_up_combo_to_drug, + ing_pr_sec_up_combo_excl_to_drug, + full_combo_with_form, + no_atc_1, + no_atc_1_with_form, + no_atc_full_combo, + wrong_df, + combo_pull, + dev_combo; + +-- At the end, the three tables concept_stage, concept_relationship_stage and concept_synonym_stage should be ready to be fed into the generic_update.sql script \ No newline at end of file diff --git a/working/packages/vocabulary_pack/CreateTablesCopiesATC.sql b/working/packages/vocabulary_pack/CreateTablesCopiesATC.sql new file mode 100644 index 000000000..54af75f81 --- /dev/null +++ b/working/packages/vocabulary_pack/CreateTablesCopiesATC.sql @@ -0,0 +1,14 @@ +CREATE OR REPLACE FUNCTION vocabulary_pack.CreateTablesCopiesATC () +RETURNS void AS +$BODY$ +/* + This procedure creates a copy of dev_atc.class_to_drug when ATC is ready for release +*/ +BEGIN + DELETE FROM sources.class_to_drug; + INSERT INTO sources.class_to_drug SELECT * FROM dev_atc.class_to_drug; + ANALYZE sources.class_to_drug; +END; +$BODY$ +LANGUAGE 'plpgsql' +SECURITY DEFINER; \ No newline at end of file