Merge pull request #100 from investigativedata/develop

v2.2.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1.1
+current_version = 2.2.0
 commit = True
 tag = True
 message = 🔖 Bump version: {current_version} → {new_version}

.github/workflows/build-docker.yml

@@ -34,7 +34,7 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push release
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           # platforms: linux/amd64,linux/arm64

.github/workflows/python.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install poetry
@@ -31,15 +31,15 @@ jobs:
     - name: set PY
       run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
     - name: Set up poetry cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: .venv
         key: venv-${{ runner.os }}-${{ env.PY }}-${{ hashFiles('**/poetry.lock') }}
     - name: Ensure cache is healthy
       if: steps.cache.outputs.cache-hit == 'true'
       run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
     - name: Set up pre-commit cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pre-commit
         key: pre-commit-${{ runner.os }}-${{ env.PY }}-${{ hashFiles('.pre-commit-config.yaml') }}

.gitignore

@@ -46,6 +46,7 @@ htmlcov/
 .nox/
 .coverage
 .coverage.*
+coverage.*
 .cache
 nosetests.xml
 coverage.xml

.pre-commit-config.yaml

@@ -5,7 +5,7 @@
 # * Run "pre-commit install".
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -20,7 +20,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.16.0
     hooks:
       - id: pyupgrade
         args: [ "--py310-plus" ]
@@ -31,26 +31,26 @@ repos:
     - id: absolufy-imports
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         args: ["--profile", "black"]
 
   - repo: https://github.com/psf/black
-    rev: 23.11.0
+    rev: 24.4.2
     hooks:
       - id: black
 
   - repo: https://github.com/csachs/pyproject-flake8
-    rev: v6.1.0
+    rev: v7.0.0
     hooks:
       - id: pyproject-flake8
         additional_dependencies: [ flake8-bugbear ]
         args: [ "--extend-ignore", "E203, E501, B008" ]
         exclude: (test_[\w]+\.py)$
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         exclude: (test_[\w]+\.py|\.csv|\.json|\.lock)$
@@ -68,7 +68,7 @@ repos:
       - id: rst-inline-touching-normal
 
   - repo: https://github.com/python-poetry/poetry
-    rev: 1.7.0
+    rev: 1.8.0
     hooks:
       - id: poetry-check
       - id: poetry-lock

Makefile

@@ -11,7 +11,7 @@ followthemoney.store:
 	poetry run ftmq --store-dataset gdho -i ./tests/fixtures/gdho.ftm.json -o $(FTM_STORE_URI)
 
 test: followthemoney.store
-	poetry run pytest -s --cov=ftmstore_fastapi --cov-report term-missing -v
+	poetry run pytest -s --cov=ftmstore_fastapi --cov-report lcov -v
 
 typecheck:
 	# pip install types-python-jose

VERSION

@@ -1 +1 @@
-2.1.1
+2.2.0

ftmstore_fastapi/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.1"
+__version__ = "2.2.0"

ftmstore_fastapi/serialize.py

@@ -5,13 +5,13 @@
 
 from collections import defaultdict
 from collections.abc import Iterable
-from typing import Any, Union
+from typing import Any, Self, Union
 
 from banal import clean_dict
 from fastapi import Request
 from followthemoney.types import registry
 from ftmq.aggregations import AggregatorResult
-from ftmq.model import Catalog, Coverage, Dataset
+from ftmq.model import Catalog, Dataset, DatasetStats
 from ftmq.types import CE, CEGenerator
 from furl import furl
 from pydantic import BaseModel, Field
@@ -59,13 +59,13 @@ def from_entity(
         )
 
 
-EntityResponse.update_forward_refs()
+EntityResponse.model_rebuild()
 
 
 class EntitiesResponse(BaseModel):
     total: int
     items: int
-    coverage: Coverage
+    stats: DatasetStats
     query: ViewQueryParams
     url: str
     next_url: str | None = None
@@ -77,36 +77,36 @@ def from_view(
         cls,
         request: Request,
         entities: CEGenerator,
-        coverage: Coverage,
+        stats: DatasetStats,
         adjacents: Iterable[CE] | None = None,
         authenticated: bool | None = False,
     ) -> "EntitiesResponse":
         query = ViewQueryParams.from_request(request, authenticated)
-        url = furl(request.url)
-        query_data = clean_dict(query.dict())
+        url = furl(str(request.url))
+        query_data = clean_dict(query.model_dump())
         query_data.pop("schema_", None)
         url.args.update(query_data)
         entities = [EntityResponse.from_entity(e, adjacents) for e in entities]
         response = cls(
-            total=coverage.entities,
+            total=stats.entity_count,
             items=len(entities),
             query=query,
             entities=entities,
-            coverage=coverage,
+            stats=stats,
             url=str(url),
         )
         if query.page > 1:
             url.args["page"] = query.page - 1
             response.prev_url = str(url)
-        if query.limit * query.page < coverage.entities:
+        if query.limit * query.page < stats.entity_count:
             url.args["page"] = query.page + 1
             response.next_url = str(url)
         return response
 
 
 class AggregationResponse(BaseModel):
     total: int
-    coverage: Coverage
+    stats: DatasetStats
     query: ViewQueryParams
     url: str
     aggregations: Aggregations
@@ -115,13 +115,13 @@ class AggregationResponse(BaseModel):
     def from_view(
         cls,
         request: Request,
-        coverage: Coverage,
+        stats: DatasetStats,
         aggregations: AggregatorResult,
         authenticated: bool | None = False,
-    ) -> "AggregationResponse":
+    ) -> Self:
         query = ViewQueryParams.from_request(request, authenticated)
-        url = furl(request.url)
-        query_data = clean_dict(query.dict())
+        url = furl(str(request.url))
+        query_data = clean_dict(query.model_dump())
         query_data.pop("schema_", None)
         url.args.update(query_data)
 
@@ -132,9 +132,9 @@ def from_view(
                 agg_data[field][func] = value
 
         return cls(
-            total=coverage.entities,
+            total=stats.entity_count,
             query=query,
-            coverage=coverage,
+            stats=stats,
             aggregations=agg_data,
             url=str(url),
         )
@@ -144,9 +144,9 @@ class DatasetResponse(Dataset):
     entities_url: str | None = None
 
     @classmethod
-    def from_dataset(cls, request: Request, dataset: Dataset) -> "DatasetResponse":
+    def from_dataset(cls, request: Request, dataset: Dataset) -> Self:
         return cls(
-            **dataset.dict(),
+            **dataset.model_dump(),
             entities_url=f"{request.base_url}entities?dataset={dataset.name}",
         )
 
@@ -155,7 +155,7 @@ class CatalogResponse(Catalog):
     datasets: list[DatasetResponse]
 
     @classmethod
-    def from_catalog(cls, request: Request, catalog: Catalog) -> "CatalogResponse":
+    def from_catalog(cls, request: Request, catalog: Catalog) -> Self:
         return cls(
             datasets=[
                 DatasetResponse.from_dataset(request, d) for d in catalog.datasets

ftmstore_fastapi/store.py

@@ -23,7 +23,7 @@
 def get_catalog(uri: str | None = CATALOG) -> Catalog:
     uri = uri or CATALOG
     if uri is not None:
-        return Catalog.from_uri(uri)
+        return Catalog._from_uri(uri)
     return Catalog()
 
 
@@ -66,7 +66,7 @@ def __init__(
         self.query = self.store.query()
         self.view = self.store.default_view()
 
-        self.coverage = self.query.coverage
+        self.stats = self.query.stats
         self.aggregations = self.query.aggregations
         self.get_adjacents = self.query.get_adjacents
 

ftmstore_fastapi/views.py

@@ -60,7 +60,7 @@ def dataset_list(request: Request) -> CatalogResponse:
     datasets: list[Dataset] = []
     for dataset in catalog.datasets:
         view = get_view(dataset.name)
-        dataset.coverage = view.coverage()
+        dataset.apply_stats(view.stats())
         datasets.append(dataset)
     catalog.datasets = datasets
     return CatalogResponse.from_catalog(request, catalog)
@@ -70,7 +70,7 @@ def dataset_list(request: Request) -> CatalogResponse:
 def dataset_detail(request: Request, name: str) -> DatasetResponse:
     view = get_view(name)
     dataset = get_dataset(name)
-    dataset.coverage = view.coverage()
+    dataset.apply_stats(view.stats())
     return DatasetResponse.from_dataset(request, dataset)
 
 
@@ -91,7 +91,7 @@ def entity_list(
         request=request,
         entities=entities,
         adjacents=adjacents,
-        coverage=view.coverage(query),
+        stats=view.stats(query),
         authenticated=authenticated,
     )
 
@@ -127,5 +127,5 @@ def aggregation(request: Request) -> AggregationResponse:
     return AggregationResponse.from_view(
         request=request,
         aggregations=view.aggregations(query),
-        coverage=view.coverage(query),
+        stats=view.stats(query),
     )