WIP: DWDSmor Integration and Annotation Pipeline Refactoring

.dir-locals.el

@@ -0,0 +1 @@
+((python-mode . ((eval . (pyvenv-workon "wordprofile")))))

.gitignore

@@ -32,3 +32,4 @@ tests/testdata/db
 test-data/
 
 .coverage*
+.mypy_cache/

pyproject.toml

@@ -25,23 +25,27 @@ dynamic = ["readme", "version", "dependencies", "optional-dependencies"]
 [project.urls]
 Homepage = "https://github.com/zentrum-lexikographie/wordprofile"
 
+[project.scripts]
+wp-annotate = "wordprofile.annotate:main"
+wp-ddc2conllu = "wordprofile.ddc:main"
+
 [tool.isort]
 profile = "black"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 
-[[tool.mypy.overrides]]
-module = ["pymysql", "MySQLdb"]
+[tool.mypy]
+disable_error_code = "import-untyped"
 ignore_missing_imports = true
 
 [tool.setuptools.dynamic]
 readme = {file = ["README.md"], content-type = "text/markdown"}
 version = {attr = "wordprofile.version.__version__"}
-dependencies = { file = ["requirements/base.in"] }
-optional-dependencies.dev = { file = ["requirements/dev.in"] }
-optional-dependencies.api = { file = ["requirements/api.in"] }
-optional-dependencies.build = { file = ["requirements/build.in"] }
+dependencies = { file = ["requirements/base.txt"] }
+optional-dependencies.dev = { file = ["requirements/dev.txt"] }
+optional-dependencies.api = { file = ["requirements/api.txt"] }
+optional-dependencies.build = { file = ["requirements/build.txt"] }
 
 [tool.setuptools.packages.find]
 exclude = ["tests"]

requirements/build.in

@@ -1,6 +1,8 @@
 -c base.txt
+cachetools
 conllu
+dwdsmor
+lingua-language-detector
 spacy[cuda11x]==3.7.5
 spacy-transformers
-de_hdt_lg @ https://huggingface.co/zentrum-lexikographie/de_hdt_lg/resolve/main/de_hdt_lg-any-py3-none-any.whl
-de_hdt_dist @ https://huggingface.co/zentrum-lexikographie/de_hdt_dist/resolve/main/de_hdt_dist-any-py3-none-any.whl
+tqdm

requirements/build.txt

@@ -8,6 +8,8 @@ annotated-types==0.7.0
     # via pydantic
 blis==0.7.11
     # via thinc
+cachetools==5.5.1
+    # via -r requirements/build.in
 catalogue==2.0.10
     # via
     #   spacy
@@ -34,9 +36,7 @@ cymem==2.0.10
     #   preshed
     #   spacy
     #   thinc
-de-hdt-dist @ https://huggingface.co/zentrum-lexikographie/de_hdt_dist/resolve/main/de_hdt_dist-any-py3-none-any.whl
-    # via -r requirements/build.in
-de-hdt-lg @ https://huggingface.co/zentrum-lexikographie/de_hdt_lg/resolve/main/de_hdt_lg-any-py3-none-any.whl
+dwdsmor==0.2.1
     # via -r requirements/build.in
 fastrlock==0.8.3
     # via cupy-cuda11x
@@ -50,14 +50,17 @@ fsspec==2024.10.0
     # via
     #   huggingface-hub
     #   torch
-huggingface-hub==0.27.0
+huggingface-hub[cli]==0.27.0
     # via
+    #   dwdsmor
     #   tokenizers
     #   transformers
 idna==3.10
     # via
     #   -c /home/gregor/repositories/zdl/wordprofile/requirements/base.txt
     #   requests
+inquirerpy==0.3.4
+    # via huggingface-hub
 jinja2==3.1.4
     # via
     #   spacy
@@ -66,6 +69,8 @@ langcodes==3.5.0
     # via spacy
 language-data==1.3.0
     # via langcodes
+lingua-language-detector==2.0.2
+    # via -r requirements/build.in
 marisa-trie==1.2.1
     # via language-data
 markdown-it-py==3.0.0
@@ -130,10 +135,14 @@ packaging==24.2
     #   thinc
     #   transformers
     #   weasel
+pfzy==0.3.4
+    # via inquirerpy
 preshed==3.0.9
     # via
     #   spacy
     #   thinc
+prompt-toolkit==3.0.50
+    # via inquirerpy
 pydantic==2.10.3
     # via
     #   confection
@@ -144,6 +153,8 @@ pydantic-core==2.27.1
     # via pydantic
 pygments==2.18.0
     # via rich
+python-dotenv==1.0.1
+    # via dwdsmor
 pyyaml==6.0.2
     # via
     #   huggingface-hub
@@ -160,15 +171,15 @@ rich==13.9.4
     # via typer
 safetensors==0.4.5
     # via transformers
+sfst-transduce==1.2.1
+    # via dwdsmor
 shellingham==1.5.4
     # via typer
 smart-open==7.1.0
     # via weasel
 spacy[cuda11x]==3.7.5
     # via
     #   -r requirements/build.in
-    #   de-hdt-dist
-    #   de-hdt-lg
     #   spacy-transformers
 spacy-alignments==0.9.1
     # via spacy-transformers
@@ -177,9 +188,7 @@ spacy-legacy==3.0.12
 spacy-loggers==1.0.5
     # via spacy
 spacy-transformers==1.3.5
-    # via
-    #   -r requirements/build.in
-    #   de-hdt-dist
+    # via -r requirements/build.in
 srsly==2.5.0
     # via
     #   confection
@@ -189,6 +198,8 @@ srsly==2.5.0
     #   weasel
 sympy==1.13.1
     # via torch
+tabulate==0.9.0
+    # via dwdsmor
 thinc==8.2.5
     # via spacy
 tokenizers==0.15.2
@@ -197,6 +208,7 @@ torch==2.5.1
     # via spacy-transformers
 tqdm==4.67.1
     # via
+    #   -r requirements/build.in
     #   huggingface-hub
     #   spacy
     #   transformers
@@ -223,6 +235,8 @@ wasabi==1.1.3
     #   spacy
     #   thinc
     #   weasel
+wcwidth==0.2.13
+    # via prompt-toolkit
 weasel==0.4.1
     # via spacy
 wrapt==1.17.0