🚨 ruff lint

pyproject.toml

@@ -58,3 +58,6 @@ miade = ["data/*.csv"]
 
 [tool.ruff]
 line-length = 120
+
+[tool.ruff.lint]
+ignore = ["E721"]

src/miade/annotators.py

@@ -2,6 +2,8 @@
 import logging
 import pkgutil
 import re
+from enum import Enum
+
 import pandas as pd
 
 from typing import List, Optional, Tuple, Dict
@@ -15,7 +17,13 @@
 from .note import Note
 from .paragraph import ParagraphType
 from .dosageextractor import DosageExtractor
-from .utils.metaannotationstypes import *
+from .utils.metaannotationstypes import (
+    Presence,
+    Relevance,
+    ReactionPos,
+    SubstanceCategory,
+    Severity,
+)
 from .utils.annotatorconfig import AnnotatorConfig
 
 log = logging.getLogger(__name__)

src/miade/core.py

@@ -3,15 +3,13 @@
 import yaml
 import logging
 
-from negspacy.negation import Negex
 from pathlib import Path
 from typing import List, Optional, Dict
 
 from .concept import Concept, Category
 from .note import Note
-from .annotators import Annotator, ProblemsAnnotator, MedsAllergiesAnnotator
+from .annotators import Annotator
 from .dosageextractor import DosageExtractor
-from .utils.metaannotationstypes import SubstanceCategory
 from .utils.miade_cat import MiADE_CAT
 from .utils.modelfactory import ModelFactory
 from .utils.annotatorconfig import AnnotatorConfig
@@ -127,7 +125,7 @@ def _load_model_factory(self, custom_annotators: Optional[List[Annotator]] = Non
                     continue
                 mapped_models[name] = cat_model
         else:
-            log.warning(f"No model ids configured!")
+            log.warning("No model ids configured!")
 
         mapped_annotators = {}
         # {name: <class Annotator>}
@@ -145,7 +143,7 @@ def _load_model_factory(self, custom_annotators: Optional[List[Annotator]] = Non
                     except AttributeError as e:
                         log.warning(f"{annotator_string} not found: {e}")
         else:
-            log.warning(f"No annotators configured!")
+            log.warning("No annotators configured!")
 
         mapped_configs = {}
         if "general" in config_dict:

src/miade/dosage.py

@@ -97,15 +97,15 @@ def parse_dose(text: str, quantities: List[str], units: List[str], results: Dict
         else:
             try:
                 quantity_dosage.value = float(quantities[0])
-            except:
+            except ValueError:
                 quantity_dosage.value = float(re.sub(r"[^\d.]+", "", quantities[0]))
         quantity_dosage.unit = units[0]
     elif len(quantities) == 2 and len(units) == 2:
         quantities.sort()
         try:
             quantity_dosage.low = float(quantities[0])
             quantity_dosage.high = float(quantities[1])
-        except:
+        except ValueError:
             quantity_dosage.low = float(re.sub(r"[^\d.]+", "", quantities[0]))
             quantity_dosage.high = float(re.sub(r"[^\d.]+", "", quantities[1]))
         if units[0] == units[1]:
@@ -162,7 +162,7 @@ def parse_frequency(text: str, results: Dict) -> Optional[Frequency]:
     if results["freq"] is not None and results["time"] is not None:
         try:
             frequency_dosage.value = results["time"] / results["freq"]
-        except ZeroDivisionError as e:
+        except ZeroDivisionError:
             frequency_dosage.value = None
         # here i convert time to hours if not institution specified
         # (every X hrs as opposed to X times day) but it's arbitrary really...

src/miade/dosageextractor.py

@@ -5,9 +5,6 @@
 from typing import Optional
 
 from .dosage import Dosage
-from .drugdoseade.preprocessor import Preprocessor
-from .drugdoseade.pattern_matcher import PatternMatcher
-from .drugdoseade.entities_refiner import EntitiesRefiner
 
 
 log = logging.getLogger(__name__)

src/miade/metaannotations.py

@@ -1,8 +1,16 @@
 from typing import Optional
 from pydantic import BaseModel, validator
-
-from .utils.metaannotationstypes import *
-
+from enum import Enum
+
+from .utils.metaannotationstypes import (
+    Presence,
+    Relevance,
+    Laterality,
+    ReactionPos,
+    SubstanceCategory,
+    AllergyType,
+    Severity,
+)
 
 META_ANNS_DICT = {
     "presence": Presence,

src/miade/model_builders/cdbbuilder.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 from shutil import copy
 from typing import List, Optional

src/miade/model_builders/preprocess_elg.py

@@ -1,4 +1,4 @@
-from pandas import DataFrame, read_csv, isnull
+from pandas import DataFrame, read_csv
 from pathlib import Path
 
 

src/miade/model_builders/preprocess_snomeduk.py

@@ -2,7 +2,6 @@
     with a few minor changes adapted to reading snomed UK folder paths"""
 
 import os
-import json
 import re
 import hashlib
 import pandas as pd

src/miade/utils/miade_cat.py

@@ -3,9 +3,9 @@
 import pandas as pd
 from copy import deepcopy
 
-from typing import Union, List, Tuple, Optional, Dict, Iterable, Set
+from typing import List, Tuple, Optional, Dict, Set
 
-from tqdm.autonotebook import tqdm, trange
+from tqdm.autonotebook import trange
 from spacy.tokens import Span, Doc
 
 from medcat.cat import CAT

src/scripts/miade.py

@@ -240,7 +240,7 @@ def create_bbpe_tokenizer(
             data.append(tokenizer.encode(line).tokens)
             step += 1
 
-    log.info(f"Started training word2vec model with tokenized text...")
+    log.info("Started training word2vec model with tokenized text...")
     w2v = Word2Vec(data, vector_size=300, min_count=1)
 
     log.info(f"Creating embeddings matrix, vocab size {tokenizer.get_vocab_size()}")
@@ -271,7 +271,7 @@ def create_metacats(
 ):
     log.info(f"Loading tokenizer from {tokenizer_path}/...")
     tokenizer = TokenizerWrapperBPE.load(str(tokenizer_path))
-    log.info(f"Loading embeddings from embeddings.npy...")
+    log.info("Loading embeddings from embeddings.npy...")
     embeddings = np.load(str(os.path.join(tokenizer_path, "embeddings.npy")))
 
     assert len(embeddings) == tokenizer.get_size(), (

streamlit_app/app.py

@@ -1,3 +1,5 @@
+# ruff: noqa: F811
+
 import os
 import json
 from time import sleep
@@ -21,7 +23,13 @@
 
 from medcat.cat import CAT
 from miade.utils.miade_meta_cat import MiADE_MetaCAT
-from utils import *
+from utils import (
+    load_documents,
+    load_annotations,
+    get_valid_annotations,
+    get_probs_meta_classes_data,
+    get_meds_meta_classes_data,
+)
 
 load_dotenv(find_dotenv())
 

streamlit_app/utils.py

@@ -1,8 +1,5 @@
 import pandas as pd
 
-from medcat.meta_cat import MetaCAT
-from medcat.config_meta_cat import ConfigMetaCAT
-from medcat.tokenizers.meta_cat_tokenizers import TokenizerWrapperBPE
 
 from typing import Optional
 
@@ -42,7 +39,7 @@ def get_probs_meta_classes_data(
     cuis = []
     names = []
     texts = []
-    tokens = []
+    # tokens = []
     for ann in annotations:
         r_labels.append(ann.meta_relevance)
         p_labels.append(ann.meta_presence)
@@ -91,7 +88,7 @@ def get_meds_meta_classes_data(
     cuis = []
     names = []
     texts = []
-    tokens = []
+    # tokens = []
     for ann in annotations:
         substance_labels.append(ann.meta_substance_cat)
         allergy_labels.append(ann.meta_allergy_type)

tests/conftest.py

@@ -8,7 +8,15 @@
 from miade.note import Note
 from miade.concept import Concept, Category
 from miade.metaannotations import MetaAnnotations
-from miade.utils.metaannotationstypes import *
+from miade.utils.metaannotationstypes import (
+    Presence,
+    Relevance,
+    Laterality,
+    ReactionPos,
+    SubstanceCategory,
+    AllergyType,
+    Severity,
+)
 from miade.utils.miade_cat import MiADE_CAT
 
 

tests/test_cdbbuilder.py

@@ -1,5 +1,3 @@
-from pathlib import Path
-
 from miade.model_builders import CDBBuilder
 
 

tests/test_core.py

@@ -2,7 +2,11 @@
 from miade.concept import Concept, Category
 from miade.annotators import Annotator
 from miade.metaannotations import MetaAnnotations
-from miade.utils.metaannotationstypes import *
+from miade.utils.metaannotationstypes import (
+    Presence,
+    Relevance,
+    Laterality,
+)
 
 
 def test_core(model_directory_path, test_note, test_negated_note, test_duplicated_note):

tests/test_dosageextractor.py

@@ -1,4 +1,3 @@
-import pytest
 from pandas import isnull
 
 # from devtools import debug

tests/test_install.py

@@ -1,2 +1,2 @@
 def test_install():
-    import miade
+    pass

tests/test_note.py

@@ -2,7 +2,11 @@
 from miade.paragraph import Paragraph, ParagraphType
 from miade.metaannotations import MetaAnnotations
 
-from miade.utils.metaannotationstypes import *
+from miade.utils.metaannotationstypes import (
+    Presence,
+    Relevance,
+    SubstanceCategory,
+)
 
 
 def test_note(model_directory_path, test_clean_and_paragraphing_note, test_paragraph_chunking_concepts):