Merge branch 'master' into 370-create-proper-enums-for-job-statuses

.dockerignore

@@ -8,5 +8,5 @@ samples/
 index/
 e2e-state
 mquery.ini
-src/mqueryfront/build
+src/mqueryfront/dist
 src/mqueryfront/node_modules

.github/workflows/test_code.yml

@@ -24,12 +24,16 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: '3.10'
-      - name: install mypy==0.790
-        run: pip3 install mypy==0.790
+      - name: install mypy==1.2.0
+        run: pip3 install mypy==1.2.0
       - name: install requirements
         run: pip3 install -r requirements.txt
-      - name: run mypy
-        run: mypy src
+      - name: run mypy on main files
+        run: mypy src/app.py src/daemon.py
+      - name: run mypy on tests
+        run: MYPYPATH=src/ mypy src/tests/
+      - name: run mypy on utils
+        run: MYPYPATH=src/ mypy src/utils/
   test_python_style:
     name: python flake8
     runs-on: ubuntu-latest

.gitignore

@@ -11,3 +11,5 @@ mquery.egg-info/
 package-lock.json
 build/
 postgres/
+node_modules/
+dist/

CONTRIBUTING.md

@@ -50,7 +50,7 @@ $ prettier --write src/mqueryfront/
 - Verify that there are no type errors with [mypy](http://mypy-lang.org/):
 
 ```bash
-$ pip install mypy==0.790
+$ pip install mypy==1.2.0
 $ mypy src
 ```
 

MANIFEST.in

@@ -1 +1 @@
-graft src/mqueryfront/build
+graft src/mqueryfront/dist

deploy/docker/dev.frontend.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16 AS build
+FROM node:18 AS build
 
 RUN npm install -g serve
 COPY src/mqueryfront /app

deploy/docker/web.Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16 AS build
+FROM node:18 AS build
 
 RUN npm install -g serve
 COPY src/mqueryfront /app
@@ -15,6 +15,6 @@ RUN ls /tmp/requirements*.txt | xargs -i,, pip --no-cache-dir install -r ,,
 
 COPY requirements.txt setup.py MANIFEST.in /usr/src/app/
 COPY src /usr/src/app/src/
-COPY --from=build "/app/build" "/usr/src/app/src/mqueryfront/build"
+COPY --from=build "/app/dist" "/usr/src/app/src/mqueryfront/dist"
 RUN pip3 install /usr/src/app
 CMD ["uvicorn", "mquery.app:app", "--host", "0.0.0.0", "--port", "5000"]

requirements.txt

@@ -1,28 +1,34 @@
-anyio==3.4.0
+alembic==1.11.1
+annotated-types==0.7.0
+anyio==4.6.0
 asgiref==3.4.1
-cachetools==4.2.4
-certifi==2021.10.8
-charset-normalizer==2.0.9
-click==8.0.3
+cachetools==5.5.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.3.2
+click==8.1.7
+cryptography==43.0.1
 Deprecated==1.2.13
-fastapi==0.70.0
-h11==0.12.0
-idna==3.3
-pydantic==1.10.14
-pyzmq==24.0.1
-redis==4.0.2
-requests==2.26.0
-sniffio==1.2.0
-starlette==0.16.0
-typing-extensions==4.9.0
-urllib3==1.26.7
-uvicorn==0.15.0
-wrapt==1.13.3
-yara-python==4.1.3
-yaramod==3.12.1
-PyJWT[crypto]==2.3.0
-rq==1.11.1
-typed-config==1.3.2
-sqlmodel==0.0.11
+fastapi==0.115.0
+h11==0.14.0
+idna==3.10
 psycopg2==2.9.9
-alembic==1.11.1
+pycparser==2.22
+pydantic==1.10.18
+pydantic_core==2.23.4
+PyJWT[crypto]==2.9.0
+pyzmq==26.2.0
+redis==5.0.8
+requests==2.32.2
+rq==1.16.2
+sniffio==1.3.1
+sqlmodel==0.0.11
+starlette==0.38.6
+typed-config==2.0.3
+types-requests==2.32.0.20240914
+typing_extensions==4.12.2
+urllib3==2.2.3
+uvicorn==0.30.6
+wrapt==1.16.0
+yara-python==4.5.1
+yaramod==3.23.0

setup.cfg

@@ -3,7 +3,7 @@ ignore = E501,W503,E203
 exclude = mqueryfront/
 
 [mypy]
-python_version = 3.6
+python_version = 3.10
 
 [mypy-yaramod.*]
 ignore_missing_imports = True

src/app.py

@@ -122,7 +122,7 @@ async def current_user(authorization: Optional[str] = Header(None)) -> User:
 async def add_headers(request: Request, call_next: Callable) -> Response:
     response = await call_next(request)
     response.headers["X-Frame-Options"] = "deny"
-    response.headers["Access-Control-Allow-Origin"] = request.client.host
+    response.headers["Access-Control-Allow-Origin"] = request.client.host  # type: ignore
     response.headers[
         "Access-Control-Allow-Headers"
     ] = "cache-control,x-requested-with,content-type,authorization"
@@ -575,7 +575,7 @@ def server() -> ServerSchema:
 
 @app.get("/query/{path}", include_in_schema=False)
 def serve_index(path: str) -> FileResponse:
-    return FileResponse(Path(__file__).parent / "mqueryfront/build/index.html")
+    return FileResponse(Path(__file__).parent / "mqueryfront/dist/index.html")
 
 
 @app.get("/recent", include_in_schema=False)
@@ -586,15 +586,16 @@ def serve_index(path: str) -> FileResponse:
 @app.get("/about", include_in_schema=False)
 def serve_index_sub() -> FileResponse:
     # Static routes are always publicly accessible without authorisation.
-    return FileResponse(Path(__file__).parent / "mqueryfront/build/index.html")
+    return FileResponse(Path(__file__).parent / "mqueryfront/dist/index.html")
 
 
 app.mount(
     "/",
     StaticFiles(
         directory=os.path.join(
-            os.path.dirname(__file__), "mqueryfront", "build"
+            os.path.dirname(__file__), "mqueryfront", "dist"
         ),
+        check_dir=False,
         html=True,
     ),
 )

src/lib/ursadb.py

@@ -53,7 +53,7 @@ def __execute(self, command: str, recv_timeout: int = 2000) -> Json:
     def query(
         self,
         query: str,
-        taints: List[str] = None,
+        taints: List[str] | None = None,
         dataset: Optional[str] = None,
     ) -> Json:
         command = "select "

src/lib/yaraparse.py

@@ -1,7 +1,7 @@
 import argparse
 import itertools
 import re
-from typing import Any, Dict, List, Match, Optional
+from typing import Any, Dict, List, Match, Optional, cast, Callable
 
 from yaramod import (  # type: ignore
     AllExpression,
@@ -17,7 +17,6 @@
     OfExpression,
     OrExpression,
     ParenthesesExpression,
-    PlainString,
     Regexp,
     RegexpConcat,
     RegexpGroup,
@@ -358,7 +357,7 @@ def ursify_plain_string(
         return ursa_ascii
 
 
-def ursify_xor_string(string: PlainString) -> UrsaExpression:
+def ursify_xor_string(string: String) -> UrsaExpression:
     text_ascii = string.pure_text
     xored_strings: List[UrsaExpression] = []
 
@@ -389,14 +388,14 @@ def ursify_string(string: String) -> Optional[UrsaExpression]:
         value_safe = string.pure_text.decode()
         return ursify_hex(value_safe)
     elif string.is_regexp:
-        return ursify_regex_string(string)
+        return ursify_regex_string(cast(Regexp, string))
 
     return None
 
 
 class RuleParseEngine:
     def __init__(
-        self, strings: Dict[str, str], rules: Dict[str, YaraRuleData]
+        self, strings: Dict[str, String], rules: Dict[str, YaraRuleData]
     ) -> None:
         self.strings = strings
         self.rules = rules
@@ -545,7 +544,7 @@ def str_in_expr(
     ) -> Optional[UrsaExpression]:
         return ursify_string(self.strings[condition.id])
 
-    CONDITION_HANDLERS = {
+    CONDITION_HANDLERS: Dict[type, Callable] = {
         AndExpression: and_expr,
         OrExpression: or_expr,
         ParenthesesExpression: pare_expr,

src/models/agentgroup.py

@@ -3,23 +3,17 @@
 from ..models.jobagent import JobAgent
 
 
-class AgentGroupBase(SQLModel):
+class AgentGroupView(SQLModel):
     name: str
     ursadb_url: str
     plugins_spec: Dict[str, Dict[str, str]] = Field(sa_column=Column(JSON))
     active_plugins: List[str] = Field(sa_column=Column(ARRAY(String)))
 
 
-class AgentGroup(AgentGroupBase, table=True):
+class AgentGroup(AgentGroupView, table=True):
     """Agent group is a group of processes working on a single
     file group, with a shared storage, and a single backing ursadb.
     """
 
     id: Union[int, None] = Field(default=None, primary_key=True)
     jobs: List["JobAgent"] = Relationship(back_populates="agent")
-
-
-class AgentGroupView(AgentGroupBase):
-    """Pydantic model used in the public API."""
-
-    pass

src/models/job.py

@@ -16,8 +16,8 @@ class JobStatus(enum.Enum):
     processing = "processing"
 
 
-class JobBase(SQLModel):
-    """Base class for entities related to mquery jobs."""
+class JobView(SQLModel):
+    """Public fields of mquery jobs."""
 
     id: str
     status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))  # type: ignore
@@ -43,16 +43,10 @@ class Config:
         arbitrary_types_allowed = True
 
 
-class Job(JobBase, table=True):
+class Job(JobView, table=True):
     """Job object in the database. Internal ID is an implementation detail."""
 
     internal_id: Union[int, None] = Field(default=None, primary_key=True)
 
     matches: List["Match"] = Relationship(back_populates="job")
     agents: List["JobAgent"] = Relationship(back_populates="job")
-
-
-class JobView(JobBase):
-    """Pydantic model used in the public API."""
-
-    pass

src/mqueryfront/public/index.html → src/mqueryfront/index.html

@@ -11,24 +11,16 @@
       manifest.json provides metadata used when your web app is added to the
       homescreen on Android. See https://developers.google.com/web/fundamentals/engage-and-retain/web-app-manifest/
     -->
-        <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
-        <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico" />
-        <!--
-      Notice the use of %PUBLIC_URL% in the tags above.
-      It will be replaced with the URL of the `public` folder during the build.
-      Only files inside the `public` folder can be referenced from the HTML.
-
-      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
-      work correctly both with client-side routing and a non-root public URL.
-      Learn how to configure a non-root public URL by running `npm run build`.
-    -->
+        <link rel="manifest" href="/manifest.json" />
+        <link rel="shortcut icon" href="/favicon.ico" />
         <title>mquery | CERT.PL>_</title>
     </head>
     <body>
         <noscript>
             You need to enable JavaScript to run this app.
         </noscript>
         <div id="root"></div>
+        <script type="module" src="/src/index.js"></script>
         <!--
       This HTML file is a template.
       If you open it directly in the browser, you will see an empty page.