refactor(db): Use scoped session for tasks (#24)

.github/workflows/quality_check.yml

@@ -9,7 +9,8 @@ permissions:
   pull-requests: write
 
 jobs:
-  validate-polylith:
+  project-quality:
+    name: Validate project quality
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

bases/ecoindex/backend/VERSION

@@ -1 +1 @@
-3.1.1
+3.1.4

bases/ecoindex/backend/main.py

@@ -1,20 +1,18 @@
 from ecoindex.backend import get_api_version
 from ecoindex.backend.routers import router
 from ecoindex.backend.services.cache import cache
-from ecoindex.database import db
+from ecoindex.database.engine import init_db
 from fastapi import FastAPI
 from fastapi.concurrency import asynccontextmanager
 
 
 def init_app():
-    db.init()
     cache.init()
 
     @asynccontextmanager
     async def lifespan(app: FastAPI):
-        await db.create_all()
+        await init_db()
         yield
-        await db._session.close()
 
     app = FastAPI(
         title="Ecoindex API",

bases/ecoindex/backend/routers/ecoindex.py

@@ -13,6 +13,7 @@
 from ecoindex.backend.services.ecoindex import get_badge, get_latest_result_by_url
 from ecoindex.backend.utils import get_sort_parameters, get_status_code
 from ecoindex.config.settings import Settings
+from ecoindex.database.engine import get_session
 from ecoindex.database.models import (
     ApiEcoindex,
     EcoindexSearchResults,
@@ -33,6 +34,7 @@
 from fastapi.exceptions import HTTPException
 from fastapi.params import Query
 from fastapi.responses import FileResponse, RedirectResponse
+from sqlmodel.ext.asyncio.session import AsyncSession
 
 router = APIRouter(prefix="/{version}/ecoindexes", tags=["Ecoindex"])
 
@@ -67,8 +69,10 @@ async def get_ecoindex_analysis_list(
             )
         ),
     ] = ["date:desc"],
+    session: AsyncSession = Depends(get_session),
 ) -> PageApiEcoindexes:
     ecoindexes = await get_ecoindex_result_list_db(
+        session=session,
         date_from=date_range.date_from,
         date_to=date_range.date_to,
         host=host,
@@ -80,6 +84,7 @@ async def get_ecoindex_analysis_list(
         ),  # type: ignore
     )
     total_results = await get_count_analysis_db(
+        session=session,
         version=version,
         date_from=date_range.date_from,
         date_to=date_range.date_to,
@@ -106,6 +111,7 @@ async def get_ecoindex_analysis_list(
 async def get_latest_results(
     response: Response,
     parameters: Annotated[BffParameters, Depends(bff_parameters)],
+    session: AsyncSession = Depends(get_session),
 ) -> EcoindexSearchResults:
     """
     This returns the latest results for a given url. This feature is used by the Ecoindex
@@ -114,7 +120,10 @@ async def get_latest_results(
     If the url is not found in the database, the response status code will be 404.
     """
     latest_result = await get_latest_result_by_url(
-        url=parameters.url, refresh=parameters.refresh, version=parameters.version
+        session=session,
+        url=parameters.url,
+        refresh=parameters.refresh,
+        version=parameters.version,
     )
 
     if latest_result.count == 0:
@@ -135,6 +144,7 @@ async def get_badge_enpoint(
     theme: Annotated[
         BadgeTheme, Query(description="Theme of the badge")
     ] = BadgeTheme.light,
+    session: AsyncSession = Depends(get_session),
 ) -> Response:
     """
     This returns the SVG badge of the given url. This feature is used by the Ecoindex
@@ -144,6 +154,7 @@ async def get_badge_enpoint(
     """
     return Response(
         content=await get_badge(
+            session=session,
             url=parameters.url,
             refresh=parameters.refresh,
             version=parameters.version,
@@ -161,6 +172,7 @@ async def get_badge_enpoint(
 )
 async def get_latest_result_redirect(
     parameters: Annotated[BffParameters, Depends(bff_parameters)],
+    session: AsyncSession = Depends(get_session),
 ) -> RedirectResponse:
     """
     This redirects to the latest results on the frontend website for the given url.
@@ -169,7 +181,10 @@ async def get_latest_result_redirect(
     If the url is not found in the database, the response status code will be 404.
     """
     latest_result = await get_latest_result_by_url(
-        url=parameters.url, refresh=parameters.refresh, version=parameters.version
+        session=session,
+        url=parameters.url,
+        refresh=parameters.refresh,
+        version=parameters.version,
     )
 
     if latest_result.count == 0:
@@ -194,8 +209,11 @@ async def get_latest_result_redirect(
 async def get_ecoindex_analysis_by_id(
     id: Annotated[UUID, Depends(id_parameter)],
     version: Annotated[Version, Depends(version_parameter)] = Version.v1,
+    session: AsyncSession = Depends(get_session),
 ) -> ApiEcoindex:
-    ecoindex = await get_ecoindex_result_by_id_db(id=id, version=version)
+    ecoindex = await get_ecoindex_result_by_id_db(
+        session=session, id=id, version=version
+    )
 
     if not ecoindex:
         raise HTTPException(

bases/ecoindex/backend/routers/health.py

@@ -1,7 +1,8 @@
-from ecoindex.database.engine import db
+from ecoindex.database.engine import get_session
 from ecoindex.models.api import HealthResponse
 from ecoindex.worker.health import is_worker_healthy
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
+from sqlmodel.ext.asyncio.session import AsyncSession
 
 router = APIRouter(prefix="/health", tags=["Infra"])
 
@@ -11,5 +12,5 @@
     path="",
     description="This returns the health of the service",
 )
-async def health_check() -> HealthResponse:
-    return HealthResponse(database=db._session.is_active, workers=is_worker_healthy())
+async def health_check(session: AsyncSession = Depends(get_session)) -> HealthResponse:
+    return HealthResponse(database=session.is_active, workers=is_worker_healthy())

bases/ecoindex/backend/routers/host.py

@@ -7,6 +7,7 @@
     version_parameter,
 )
 from ecoindex.backend.utils import check_quota, get_status_code
+from ecoindex.database.engine import get_session
 from ecoindex.database.repositories.host import get_count_hosts_db, get_host_list_db
 from ecoindex.models.api import Host, PageHosts
 from ecoindex.models.enums import Version
@@ -16,6 +17,7 @@
 from fastapi.param_functions import Query
 from fastapi.responses import Response
 from fastapi.routing import APIRouter
+from sqlmodel.ext.asyncio.session import AsyncSession
 
 router = APIRouter(prefix="/{version}/hosts", tags=["Host"])
 
@@ -45,8 +47,10 @@ async def get_host_list(
         description="Filter by partial host name (replaced by `host`)",
         deprecated=True,
     ),
+    session: AsyncSession = Depends(get_session),
 ) -> PageHosts:
     hosts = await get_host_list_db(
+        session=session,
         date_from=date_range.date_from,
         date_to=date_range.date_to,
         host=host or q,
@@ -56,7 +60,11 @@ async def get_host_list(
     )
 
     total_hosts = await get_count_hosts_db(
-        version=version, q=q, date_from=date_range.date_from, date_to=date_range.date_to
+        session=session,
+        version=version,
+        q=q,
+        date_from=date_range.date_from,
+        date_to=date_range.date_to,
     )
 
     response.status_code = await get_status_code(items=hosts, total=total_hosts)
@@ -83,11 +91,12 @@ async def get_host_list(
 async def get_daily_remaining(
     host: Annotated[str, Path(..., description="Exact matching host name")],
     version: Annotated[Version, Depends(version_parameter)] = Version.v1,
+    session: AsyncSession = Depends(get_session),
 ) -> Host:
     return Host(
         name=host,
-        remaining_daily_requests=await check_quota(host=host),
+        remaining_daily_requests=await check_quota(session=session, host=host),
         total_count=await get_count_hosts_db(
-            name=host, version=version, group_by_host=False
+            session=session, name=host, version=version, group_by_host=False
         ),
     )

bases/ecoindex/backend/routers/tasks.py

@@ -6,6 +6,7 @@
 from ecoindex.backend.dependencies import id_parameter
 from ecoindex.backend.utils import check_quota
 from ecoindex.config.settings import Settings
+from ecoindex.database.engine import get_session
 from ecoindex.models import WebPage
 from ecoindex.models.enums import TaskStatus
 from ecoindex.models.response_examples import example_daily_limit_response
@@ -14,6 +15,7 @@
 from ecoindex.worker_component import app as task_app
 from fastapi import APIRouter, Depends, HTTPException, Response, status
 from fastapi.params import Body
+from sqlmodel.ext.asyncio.session import AsyncSession
 
 router = APIRouter()
 
@@ -38,9 +40,12 @@ async def add_ecoindex_analysis_task(
         title="Web page to analyze defined by its url and its screen resolution",
         example=WebPage(url="https://www.ecoindex.fr", width=1920, height=1080),
     ),
+    session: AsyncSession = Depends(get_session),
 ) -> str:
     if Settings().DAILY_LIMIT_PER_HOST:
-        remaining_quota = await check_quota(host=web_page.get_url_host())
+        remaining_quota = await check_quota(
+            session=session, host=web_page.get_url_host()
+        )
         response.headers["X-Remaining-Daily-Requests"] = str(remaining_quota - 1)
 
     if (
@@ -53,7 +58,7 @@ async def add_ecoindex_analysis_task(
         )
 
     task_result = ecoindex_task.delay(
-        str(web_page.url), web_page.width, web_page.height
+        url=str(web_page.url), width=web_page.width, height=web_page.height
     )
 
     return task_result.id

bases/ecoindex/backend/services/ecoindex.py

@@ -7,10 +7,11 @@
 from ecoindex.models.sort import Sort
 from pydantic import AnyHttpUrl
 from requests import get
+from sqlmodel.ext.asyncio.session import AsyncSession
 
 
 async def get_latest_result_by_url(
-    url: AnyHttpUrl, refresh: bool, version: Version
+    session: AsyncSession, url: AnyHttpUrl, refresh: bool, version: Version
 ) -> EcoindexSearchResults:
     """
     Get the latest ecoindex result for a given url. This function will first try to find
@@ -35,6 +36,7 @@ async def get_latest_result_by_url(
         return EcoindexSearchResults(**json.loads(cached_results))
 
     ecoindexes = await get_ecoindex_result_list_db(
+        session=session,
         host=str(url.host),
         version=version,
         size=20,
@@ -72,7 +74,7 @@ async def get_latest_result_by_url(
 
 
 async def get_badge(
-    url: AnyHttpUrl, refresh: bool, version: Version, theme: str
+    session: AsyncSession, url: AnyHttpUrl, refresh: bool, version: Version, theme: str
 ) -> str:
     """
     Get the badge for a given url. This function will use the method `get_latest_result_by_url`.
@@ -89,7 +91,9 @@ async def get_badge(
     returns:
         str: the badge image
     """
-    results = await get_latest_result_by_url(url=url, refresh=refresh, version=version)
+    results = await get_latest_result_by_url(
+        session=session, url=url, refresh=refresh, version=version
+    )
 
     grade = results.latest_result.grade if results.latest_result else "unknown"
     ecoindex_cache = cache.set_cache_key(key=f"badge-{grade}-{theme}")

bases/ecoindex/backend/utils/__init__.py

@@ -12,6 +12,7 @@
 from ecoindex.models.sort import Sort
 from fastapi import HTTPException, status
 from pydantic import BaseModel
+from sqlmodel.ext.asyncio.session import AsyncSession
 
 
 async def format_exception_response(exception: Exception) -> ExceptionResponse:
@@ -78,15 +79,18 @@ async def get_sort_parameters(query_params: list[str], model: BaseModel) -> list
 
 
 async def check_quota(
+    session: AsyncSession,
     host: str,
 ) -> int | None:
     if not Settings().DAILY_LIMIT_PER_HOST:
         return None
 
-    count_daily_request_per_host = await get_count_daily_request_per_host(host=host)
+    count_daily_request_per_host = await get_count_daily_request_per_host(
+        session=session, host=host
+    )
 
     if count_daily_request_per_host >= Settings().DAILY_LIMIT_PER_HOST:
-        latest_result = await get_latest_result(host=host)
+        latest_result = await get_latest_result(session=session, host=host)
         raise QuotaExceededException(
             limit=Settings().DAILY_LIMIT_PER_HOST,
             host=host,

bases/ecoindex/worker/tasks.py

@@ -4,7 +4,7 @@
 
 from ecoindex.backend.utils import check_quota, format_exception_response
 from ecoindex.config.settings import Settings
-from ecoindex.database.engine import db
+from ecoindex.database.engine import get_session
 from ecoindex.database.exceptions.quota import QuotaExceededException
 from ecoindex.database.repositories.worker import save_ecoindex_result_db
 from ecoindex.exceptions.scraper import EcoindexScraperStatusException
@@ -21,8 +21,6 @@
 from ecoindex.worker_component import app
 from playwright._impl._errors import Error as WebDriverException
 
-db.init()
-
 
 @app.task(
     name="Make ecoindex analysis",
@@ -35,16 +33,24 @@
     dont_autoretry_for=[EcoindexScraperStatusException, TypeError],
 )
 def ecoindex_task(self, url: str, width: int, height: int) -> str:
-    queue_task_result = run(async_ecoindex_task(self, url, width, height))
+    queue_task_result = run(
+        async_ecoindex_task(self, url=url, width=width, height=height)
+    )
 
     return queue_task_result.model_dump_json()
 
 
 async def async_ecoindex_task(
-    self, url: str, width: int, height: int
+    self,
+    url: str,
+    width: int,
+    height: int,
 ) -> QueueTaskResult:
     try:
-        await check_quota(host=urlparse(url=url).netloc)
+        session_generator = get_session()
+        session = await session_generator.__anext__()
+
+        await check_quota(session=session, host=urlparse(url=url).netloc)
 
         ecoindex = await EcoindexScraper(
             url=url,
@@ -61,6 +67,7 @@ async def async_ecoindex_task(
         ).get_page_analysis()
 
         db_result = await save_ecoindex_result_db(
+            session=session,
             id=self.request.id,
             ecoindex_result=ecoindex,
         )
@@ -122,16 +129,14 @@ async def async_ecoindex_task(
         )
 
     except TypeError as exc:
-        error = exc.args[0]
-
         return QueueTaskResult(
             status=TaskStatus.FAILURE,
             error=QueueTaskError(
                 url=url,
                 exception=EcoindexContentTypeError.__name__,
                 status_code=520,
-                message=error["message"],
-                detail={"mimetype": error["mimetype"]},
+                message=exc.args[0],
+                detail={"mimetype": None},
             ),
         )
 

components/ecoindex/database/__init__.py

@@ -1,3 +0,0 @@
-from ecoindex.database.engine import db
-
-__all__ = ["db"]