diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index ff1aae71..0c711859 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -24,12 +24,16 @@ jobs: uses: actions/setup-python@v1 with: python-version: '3.10' - - name: install mypy==0.790 - run: pip3 install mypy==0.790 + - name: install mypy==1.2.0 + run: pip3 install mypy==1.2.0 - name: install requirements run: pip3 install -r requirements.txt - - name: run mypy - run: mypy src + - name: run mypy on main files + run: mypy src/app.py src/daemon.py + - name: run mypy on tests + run: MYPYPATH=src/ mypy src/tests/ + - name: run mypy on utils + run: MYPYPATH=src/ mypy src/utils/ test_python_style: name: python flake8 runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a82fdb3..285d210b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ $ prettier --write src/mqueryfront/ - Verify that there are no type errors with [mypy](http://mypy-lang.org/): ```bash -$ pip install mypy==0.790 +$ pip install mypy==1.2.0 $ mypy src ``` diff --git a/requirements.txt b/requirements.txt index 46389868..27da0663 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,28 +1,34 @@ -anyio==3.4.0 +alembic==1.11.1 +annotated-types==0.7.0 +anyio==4.6.0 asgiref==3.4.1 -cachetools==4.2.4 -certifi==2021.10.8 -charset-normalizer==2.0.9 -click==8.0.3 +cachetools==5.5.0 +certifi==2024.8.30 +cffi==1.17.1 +charset-normalizer==3.3.2 +click==8.1.7 +cryptography==43.0.1 Deprecated==1.2.13 -fastapi==0.70.0 -h11==0.12.0 -idna==3.3 -pydantic==1.10.14 -pyzmq==24.0.1 -redis==4.0.2 -requests==2.26.0 -sniffio==1.2.0 -starlette==0.16.0 -typing-extensions==4.9.0 -urllib3==1.26.7 -uvicorn==0.15.0 -wrapt==1.13.3 -yara-python==4.1.3 -yaramod==3.12.1 -PyJWT[crypto]==2.3.0 -rq==1.11.1 -typed-config==1.3.2 -sqlmodel==0.0.11 +fastapi==0.115.0 +h11==0.14.0 +idna==3.10 psycopg2==2.9.9 -alembic==1.11.1 +pycparser==2.22 +pydantic==1.10.18 +pydantic_core==2.23.4 +PyJWT[crypto]==2.9.0 +pyzmq==26.2.0 +redis==5.0.8 +requests==2.32.2 +rq==1.16.2 +sniffio==1.3.1 +sqlmodel==0.0.11 +starlette==0.38.6 +typed-config==2.0.3 +types-requests==2.32.0.20240914 +typing_extensions==4.12.2 +urllib3==2.2.3 +uvicorn==0.30.6 +wrapt==1.16.0 +yara-python==4.5.1 +yaramod==3.23.0 diff --git a/setup.cfg b/setup.cfg index f56b362d..2c4a37d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ ignore = E501,W503,E203 exclude = mqueryfront/ [mypy] -python_version = 3.6 +python_version = 3.10 [mypy-yaramod.*] ignore_missing_imports = True diff --git a/src/app.py b/src/app.py index eb42bc4f..c86d70c9 100644 --- a/src/app.py +++ b/src/app.py @@ -122,7 +122,7 @@ async def current_user(authorization: Optional[str] = Header(None)) -> User: async def add_headers(request: Request, call_next: Callable) -> Response: response = await call_next(request) response.headers["X-Frame-Options"] = "deny" - response.headers["Access-Control-Allow-Origin"] = request.client.host + response.headers["Access-Control-Allow-Origin"] = request.client.host # type: ignore response.headers[ "Access-Control-Allow-Headers" ] = "cache-control,x-requested-with,content-type,authorization" diff --git a/src/lib/ursadb.py b/src/lib/ursadb.py index 7e97125f..7e995b9f 100644 --- a/src/lib/ursadb.py +++ b/src/lib/ursadb.py @@ -53,7 +53,7 @@ def __execute(self, command: str, recv_timeout: int = 2000) -> Json: def query( self, query: str, - taints: List[str] = None, + taints: List[str] | None = None, dataset: Optional[str] = None, ) -> Json: command = "select " diff --git a/src/lib/yaraparse.py b/src/lib/yaraparse.py index 44be487d..b353520b 100644 --- a/src/lib/yaraparse.py +++ b/src/lib/yaraparse.py @@ -1,7 +1,7 @@ import argparse import itertools import re -from typing import Any, Dict, List, Match, Optional +from typing import Any, Dict, List, Match, Optional, cast, Callable from yaramod import ( # type: ignore AllExpression, @@ -17,7 +17,6 @@ OfExpression, OrExpression, ParenthesesExpression, - PlainString, Regexp, RegexpConcat, RegexpGroup, @@ -358,7 +357,7 @@ def ursify_plain_string( return ursa_ascii -def ursify_xor_string(string: PlainString) -> UrsaExpression: +def ursify_xor_string(string: String) -> UrsaExpression: text_ascii = string.pure_text xored_strings: List[UrsaExpression] = [] @@ -389,14 +388,14 @@ def ursify_string(string: String) -> Optional[UrsaExpression]: value_safe = string.pure_text.decode() return ursify_hex(value_safe) elif string.is_regexp: - return ursify_regex_string(string) + return ursify_regex_string(cast(Regexp, string)) return None class RuleParseEngine: def __init__( - self, strings: Dict[str, str], rules: Dict[str, YaraRuleData] + self, strings: Dict[str, String], rules: Dict[str, YaraRuleData] ) -> None: self.strings = strings self.rules = rules @@ -545,7 +544,7 @@ def str_in_expr( ) -> Optional[UrsaExpression]: return ursify_string(self.strings[condition.id]) - CONDITION_HANDLERS = { + CONDITION_HANDLERS: Dict[type, Callable] = { AndExpression: and_expr, OrExpression: or_expr, ParenthesesExpression: pare_expr, diff --git a/src/models/agentgroup.py b/src/models/agentgroup.py index 70fa06a0..b5bfa39e 100644 --- a/src/models/agentgroup.py +++ b/src/models/agentgroup.py @@ -3,23 +3,17 @@ from ..models.jobagent import JobAgent -class AgentGroupBase(SQLModel): +class AgentGroupView(SQLModel): name: str ursadb_url: str plugins_spec: Dict[str, Dict[str, str]] = Field(sa_column=Column(JSON)) active_plugins: List[str] = Field(sa_column=Column(ARRAY(String))) -class AgentGroup(AgentGroupBase, table=True): +class AgentGroup(AgentGroupView, table=True): """Agent group is a group of processes working on a single file group, with a shared storage, and a single backing ursadb. """ id: Union[int, None] = Field(default=None, primary_key=True) jobs: List["JobAgent"] = Relationship(back_populates="agent") - - -class AgentGroupView(AgentGroupBase): - """Pydantic model used in the public API.""" - - pass diff --git a/src/models/job.py b/src/models/job.py index 17db42ad..4fa93abd 100644 --- a/src/models/job.py +++ b/src/models/job.py @@ -6,8 +6,8 @@ from ..models.jobagent import JobAgent -class JobBase(SQLModel): - """Base class for entities related to mquery jobs.""" +class JobView(SQLModel): + """Public fields of mquery jobs.""" id: str status: str @@ -30,16 +30,10 @@ class JobBase(SQLModel): agents_left: int -class Job(JobBase, table=True): +class Job(JobView, table=True): """Job object in the database. Internal ID is an implementation detail.""" internal_id: Union[int, None] = Field(default=None, primary_key=True) matches: List["Match"] = Relationship(back_populates="job") agents: List["JobAgent"] = Relationship(back_populates="job") - - -class JobView(JobBase): - """Pydantic model used in the public API.""" - - pass diff --git a/src/schema.py b/src/schema.py index a4b285fb..5b2ab8ff 100644 --- a/src/schema.py +++ b/src/schema.py @@ -1,12 +1,12 @@ from enum import Enum -from typing import List, Dict, Optional +from typing import List, Dict, Optional, Sequence from pydantic import BaseModel, Field # type: ignore from .models.job import JobView from .models.agentgroup import AgentGroupView class JobsSchema(BaseModel): - jobs: List[JobView] + jobs: Sequence[JobView] class ConfigSchema(BaseModel): diff --git a/src/tasks.py b/src/tasks.py index edb9a426..5e3e105c 100644 --- a/src/tasks.py +++ b/src/tasks.py @@ -165,7 +165,7 @@ def make_agent(group_override: Optional[str] = None): if group_override is not None: group_id = group_override else: - group_id = get_current_job().origin + group_id = get_current_job().origin # type: ignore return Agent(group_id) diff --git a/src/utils/index.py b/src/utils/index.py index 88ab1399..3a211f95 100644 --- a/src/utils/index.py +++ b/src/utils/index.py @@ -287,7 +287,7 @@ def main() -> None: path_mount = args.path path = Path(args.path) - if not path.exists: + if not path.exists(): logging.error("Path (--path) %s does not exist.", args.path) return