V0.10.0 release - uclh-alpha (#98)

v0.10.0 - uclh-alpha release

* Algorithm tweak to default to adverse reaction in allergy paragraphs (#93)

* 🗃️ Update lookup data (#94)

* 🐛 Fixed vtm deduplication bug (#96)

* ✨ Add numbering to problem concepts (#99)

* 🔧 Negex defaults to off in configs

* ⬆️ Python upgraded to 3.11

* 🔥 "(converted)" debug label removed

* 👷 Install pytorch cpu in CI

* 👷 Run tests on dev PRs

.github/workflows/ci.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     branches:
       - master
+      - dev
 
 jobs:
   Test:
@@ -16,11 +17,12 @@ jobs:
       - name: install python3
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8.10
+          python-version: 3.11
       - name: install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install pytest
+          pip install torch --index-url https://download.pytorch.org/whl/cpu
           pip install ./
           pip list
       - name: download model

configs/miade_config.yaml

@@ -0,0 +1,13 @@
+models:
+  problems: f25ec9423958e8d6
+  meds/allergies: a146c741501cf1f7
+annotators:
+  problems: ProblemsAnnotator
+  meds/allergies: MedsAllergiesAnnotator
+general:
+  problems:
+    negation_detection: None
+    disable: []
+  meds/allergies:
+    negation_detection: None
+    disable: []

src/miade/annotators.py

@@ -224,6 +224,11 @@ def process_paragraphs(note: Note, concepts: List[Concept]) -> List[Concept]:
                                               f"{Relevance.IRRELEVANT} for concept ({concept.id} | {concept.name}): "
                                               f"paragraph is {paragraph.type}")
                                     meta.value = Relevance.IRRELEVANT
+                                if meta.name == "substance_category":
+                                    log.debug(f"Converted {meta.value} to "
+                                              f"{SubstanceCategory.ADVERSE_REACTION} for concept ({concept.id} | {concept.name}): "
+                                              f"paragraph is {paragraph.type}")
+                                    meta.value = SubstanceCategory.ADVERSE_REACTION
                         elif paragraph.type == ParagraphType.exam or paragraph.type == ParagraphType.ddx or paragraph.type == ParagraphType.plan:
                             # problem is irrelevant and allergy is irrelevant
                             for meta in concept.meta:
@@ -243,7 +248,7 @@ def process_paragraphs(note: Note, concepts: List[Concept]) -> List[Concept]:
 
         # if more than 10 concepts in prob or imp or pmh sections, return only those and ignore all other concepts
         if len(prob_concepts) > 10:
-            log.debug(f"Ignoring concepts elsewhere in the document because total prob "
+            log.debug(f"Ignoring concepts elsewhere in the document because "
                       f"concepts in prob, imp, pmh sections exceed 10: {len(prob_concepts)}")
             return prob_concepts
         else:
@@ -254,20 +259,25 @@ def process_paragraphs(note: Note, concepts: List[Concept]) -> List[Concept]:
     def deduplicate(concepts: List[Concept], record_concepts: Optional[List[Concept]]) -> List[Concept]:
         if record_concepts is not None:
             record_ids = {record_concept.id for record_concept in record_concepts}
+            record_names = {record_concept.name for record_concept in record_concepts}
         else:
             record_ids = set()
+            record_names = set()
 
         # Use an OrderedDict to keep track of ids as it preservers original MedCAT order (the order it appears in text)
         filtered_concepts: List[Concept] = []
-        existing_ids = OrderedDict()
+        existing_concepts = OrderedDict()
 
         # Filter concepts that are in record or exist in concept list
         for concept in concepts:
-            if concept.id in record_ids or concept.id in existing_ids:
+            if concept.id is not None and (concept.id in record_ids or concept.id in existing_concepts):
                 log.debug(f"Removed concept ({concept.id} | {concept.name}): concept id exists in record")
+            # check name match for null ids - VTM deduplication
+            elif concept.id is None and (concept.name in record_names or concept.name in existing_concepts.values()):
+                log.debug(f"Removed concept ({concept.id} | {concept.name}): concept name exists in record")
             else:
                 filtered_concepts.append(concept)
-                existing_ids[concept.id] = None
+                existing_concepts[concept.id] = concept.name
 
         return filtered_concepts
 
@@ -301,6 +311,14 @@ def add_dosages_to_concepts(
 
         return concepts
 
+    @staticmethod
+    def add_numbering_to_name(concepts: List[Concept]) -> List[Concept]:
+        # Prepend numbering to problem concepts e.g. 00 asthma, 01 stroke...
+        for i, concept in enumerate(concepts):
+            concept.name = f"{i:02} {concept.name}"
+
+        return concepts
+
     def __call__(
         self,
         note: Note,
@@ -432,6 +450,9 @@ def __call__(
         if "deduplicator" not in self.config.disable:
             concepts = self.deduplicate(concepts, record_concepts)
 
+        if "add_numbering" not in self.config.disable:
+            concepts = self.add_numbering_to_name(concepts)
+
         return concepts
 
 
@@ -464,7 +485,6 @@ def _validate_and_convert_substance(self, concept) -> bool:
             log.debug(f"Converted concept ({concept.id} | {concept.name}) to "
                       f"({lookup_result['subsetId']} | {concept.name + tag}): valid Epic allergen subset")
             concept.id = str(lookup_result["subsetId"])
-            concept.name += tag
 
             # then check the allergen type from lookup result - e.g. drug, food
             try:
@@ -488,7 +508,6 @@ def _validate_and_convert_reaction(self, concept) -> bool:
                       f"({lookup_result} | {concept.name + tag}): valid Epic reaction subset")
 
             concept.id = str(lookup_result)
-            concept.name += tag
             return True
         else:
             log.warning(f"Reaction not found in Epic subset conversion for concept {concept.__str__()}")

src/miade/concept.py

@@ -46,10 +46,8 @@ def __init__(
         if linked_concepts is None:
             self.linked_concepts = []
 
-
     @classmethod
     def from_entity(cls, entity: [Dict]):
-
         meta_anns = None
         if entity["meta_anns"]:
             meta_anns = [MetaAnnotations(**value) for value in entity["meta_anns"].values()]

src/miade/core.py

@@ -11,6 +11,7 @@
 from .note import Note
 from .annotators import Annotator, ProblemsAnnotator, MedsAllergiesAnnotator
 from .dosageextractor import DosageExtractor
+from .utils.metaannotationstypes import SubstanceCategory
 from .utils.miade_cat import MiADE_CAT
 from .utils.modelfactory import ModelFactory
 from .utils.annotatorconfig import AnnotatorConfig