From 41313cda40718968cf1abf4f7ff1c170751b6e4c Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Tue, 1 Oct 2024 14:03:12 +0200
Subject: [PATCH 01/25] Job status added

---
 src/db.py                                     | 14 ++++++-------
 .../versions/84796bdc2f77_add_jobstatus.py    | 21 +++++++++++++++++++
 src/models/job.py                             | 16 +++++++++++++-
 src/tasks.py                                  |  8 +++----
 4 files changed, 47 insertions(+), 12 deletions(-)
 create mode 100644 src/migrations/versions/84796bdc2f77_add_jobstatus.py

diff --git a/src/db.py b/src/db.py
index 47036d0a..02d40bcf 100644
--- a/src/db.py
+++ b/src/db.py
@@ -19,7 +19,7 @@
 
 from .models.agentgroup import AgentGroup
 from .models.configentry import ConfigEntry
-from .models.job import Job
+from .models.job import Job, JobStatus
 from .models.jobagent import JobAgent
 from .models.match import Match
 from .schema import MatchesSchema, ConfigSchema
@@ -65,7 +65,7 @@ def cancel_job(self, job: JobId, error=None) -> None:
             session.execute(
                 update(Job)
                 .where(Job.id == job)
-                .values(status="cancelled", finished=int(time()), error=error)
+                .values(status=JobStatus.cancelled, finished=int(time()), error=error)
             )
             session.commit()
 
@@ -87,7 +87,7 @@ def get_valid_jobs(self, username_filter: Optional[str]) -> List[Job]:
         with self.session() as session:
             query = (
                 select(Job)
-                .where(Job.status != "removed")
+                .where(Job.status != JobStatus.removed)
                 .order_by(col(Job.submitted).desc())
             )
             if username_filter:
@@ -98,7 +98,7 @@ def remove_query(self, job: JobId) -> None:
         """Sets the job status to removed."""
         with self.session() as session:
             session.execute(
-                update(Job).where(Job.id == job).values(status="removed")
+                update(Job).where(Job.id == job).values(status=JobStatus.removed)
             )
             session.commit()
 
@@ -147,7 +147,7 @@ def agent_finish_job(self, job: Job) -> None:
                 session.execute(
                     update(Job)
                     .where(Job.internal_id == job.internal_id)
-                    .values(finished=int(time()), status="done")
+                    .values(finished=int(time()), status=JobStatus.done)
                 )
             session.commit()
 
@@ -218,7 +218,7 @@ def init_job_datasets(self, job: JobId, num_datasets: int) -> None:
                 .values(
                     total_datasets=num_datasets,
                     datasets_left=num_datasets,
-                    status="processing",
+                    status=JobStatus.processing,
                 )
             )
             session.commit()
@@ -251,7 +251,7 @@ def create_search_task(
         with self.session() as session:
             obj = Job(
                 id=job,
-                status="new",
+                status=JobStatus.new,
                 rule_name=rule_name,
                 rule_author=rule_author,
                 raw_yara=raw_yara,
diff --git a/src/migrations/versions/84796bdc2f77_add_jobstatus.py b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
new file mode 100644
index 00000000..aa9e86c0
--- /dev/null
+++ b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
@@ -0,0 +1,21 @@
+"""add_jobstatus
+Revision ID: 84796bdc2f77
+Revises: dbb81bd4d47f
+Create Date: 2024-10-01 08:09:42.808911
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '84796bdc2f77'
+down_revision = 'dbb81bd4d47f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute('ALTER TABLE job ALTER COLUMN status TYPE jobstatus USING status::jobstatus;')
+
+
+def downgrade() -> None:
+    op.execute('ALTER TABLE job ALTER COLUMN status TYPE VARCHAR USING status::text;')
diff --git a/src/models/job.py b/src/models/job.py
index 17db42ad..5c7a4c0c 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -1,3 +1,6 @@
+import enum
+
+from sqlalchemy import Enum as PgEnum
 from sqlmodel import SQLModel, Field, ARRAY, String, Column, Relationship
 from typing import Optional, List, Union, TYPE_CHECKING
 
@@ -6,11 +9,19 @@
     from ..models.jobagent import JobAgent
 
 
+class JobStatus(str, enum.Enum):
+    done = "done"
+    new = "new"
+    cancelled = "cancelled"
+    removed = "removed"
+    processing = "processing"
+
+
 class JobBase(SQLModel):
     """Base class for entities related to mquery jobs."""
 
     id: str
-    status: str
+    status: JobStatus = Field(sa_column=PgEnum(JobStatus, name="jobstatus", create_type=True))
     error: Optional[str]
     rule_name: str
     rule_author: str
@@ -29,6 +40,9 @@ class JobBase(SQLModel):
     total_datasets: int
     agents_left: int
 
+    class Config:
+        arbitrary_types_allowed = True
+
 
 class Job(JobBase, table=True):
     """Job object in the database. Internal ID is an implementation detail."""
diff --git a/src/tasks.py b/src/tasks.py
index edb9a426..d7443f41 100644
--- a/src/tasks.py
+++ b/src/tasks.py
@@ -9,7 +9,7 @@
 from .util import make_sha256_tag
 from .config import app_config
 from .plugins import PluginManager
-from .models.job import Job
+from .models.job import Job, JobStatus
 from .models.match import Match
 from .lib.yaraparse import parse_yara, combine_rules
 from .lib.ursadb import Json, UrsaDb
@@ -182,7 +182,7 @@ def start_search(job_id: JobId) -> None:
     """
     with job_context(job_id) as agent:
         job = agent.db.get_job(job_id)
-        if job.status == "cancelled":
+        if job.status == JobStatus.cancelled:
             logging.info("Job was cancelled, returning...")
             return
 
@@ -232,7 +232,7 @@ def query_ursadb(job_id: JobId, dataset_id: str, ursadb_query: str) -> None:
     """Queries ursadb and creates yara scans tasks with file batches."""
     with job_context(job_id) as agent:
         job = agent.db.get_job(job_id)
-        if job.status == "cancelled":
+        if job.status == JobStatus.cancelled:
             logging.info("Job was cancelled, returning...")
             return
 
@@ -271,7 +271,7 @@ def run_yara_batch(job_id: JobId, iterator: str, batch_size: int) -> None:
     """Actually scans files, and updates a database with the results."""
     with job_context(job_id) as agent:
         job = agent.db.get_job(job_id)
-        if job.status == "cancelled":
+        if job.status == JobStatus.cancelled:
             logging.info("Job was cancelled, returning...")
             return
 

From f4bae197cd592d74a8240b1012775793c287dd6b Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Tue, 1 Oct 2024 18:14:05 +0200
Subject: [PATCH 02/25] Job status with alembic migrations

---
 src/db.py                                     |  10 +-
 .../versions/84796bdc2f77_add_jobstatus.py    |  15 +-
 src/migrations/versions/cbbba858deb0_init.py  | 162 ++++++++++--------
 .../versions/dbb81bd4d47f_add_jobagent.py     |  41 +++--
 src/models/job.py                             |   2 +-
 5 files changed, 138 insertions(+), 92 deletions(-)

diff --git a/src/db.py b/src/db.py
index 02d40bcf..0952e407 100644
--- a/src/db.py
+++ b/src/db.py
@@ -65,7 +65,11 @@ def cancel_job(self, job: JobId, error=None) -> None:
             session.execute(
                 update(Job)
                 .where(Job.id == job)
-                .values(status=JobStatus.cancelled, finished=int(time()), error=error)
+                .values(
+                    status=JobStatus.cancelled,
+                    finished=int(time()),
+                    error=error,
+                )
             )
             session.commit()
 
@@ -98,7 +102,9 @@ def remove_query(self, job: JobId) -> None:
         """Sets the job status to removed."""
         with self.session() as session:
             session.execute(
-                update(Job).where(Job.id == job).values(status=JobStatus.removed)
+                update(Job)
+                .where(Job.id == job)
+                .values(status=JobStatus.removed)
             )
             session.commit()
 
diff --git a/src/migrations/versions/84796bdc2f77_add_jobstatus.py b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
index aa9e86c0..1f60a001 100644
--- a/src/migrations/versions/84796bdc2f77_add_jobstatus.py
+++ b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
@@ -7,15 +7,22 @@
 
 
 # revision identifiers, used by Alembic.
-revision = '84796bdc2f77'
-down_revision = 'dbb81bd4d47f'
+revision = "84796bdc2f77"
+down_revision = "dbb81bd4d47f"
 branch_labels = None
 depends_on = None
 
 
 def upgrade() -> None:
-    op.execute('ALTER TABLE job ALTER COLUMN status TYPE jobstatus USING status::jobstatus;')
+    op.execute(
+        "CREATE TYPE jobstatus AS ENUM ('done', 'new', 'cancelled', 'removed', 'processing');"
+    )
+    op.execute(
+        "ALTER TABLE job ALTER COLUMN status TYPE jobstatus USING status::text::jobstatus;"
+    )
 
 
 def downgrade() -> None:
-    op.execute('ALTER TABLE job ALTER COLUMN status TYPE VARCHAR USING status::text;')
+    op.execute(
+        "ALTER TABLE job ALTER COLUMN status TYPE VARCHAR USING status::text;"
+    )
diff --git a/src/migrations/versions/cbbba858deb0_init.py b/src/migrations/versions/cbbba858deb0_init.py
index 808bfd8f..f69614e7 100644
--- a/src/migrations/versions/cbbba858deb0_init.py
+++ b/src/migrations/versions/cbbba858deb0_init.py
@@ -5,84 +5,110 @@
 """
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy import inspect
+
 import sqlmodel
 
+from config import app_config
 
 revision = "cbbba858deb0"
 down_revision = None
 branch_labels = None
 depends_on = None
 
+DATABASE_URL = app_config.database.url
+engine = sa.create_engine(DATABASE_URL)
+
 
 def upgrade() -> None:
-    op.create_table(
-        "agentgroup",
-        sa.Column("plugins_spec", sa.JSON(), nullable=True),
-        sa.Column("active_plugins", sa.ARRAY(sa.String()), nullable=True),
-        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-        sa.Column(
-            "ursadb_url", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.PrimaryKeyConstraint("id"),
-    )
-    op.create_table(
-        "configentry",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column(
-            "plugin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column("key", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-        sa.Column("value", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-        sa.PrimaryKeyConstraint("id"),
-    )
-    op.create_table(
-        "job",
-        sa.Column("taints", sa.ARRAY(sa.String()), nullable=True),
-        sa.Column("id", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-        sa.Column(
-            "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column("error", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-        sa.Column(
-            "rule_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column(
-            "rule_author", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column(
-            "raw_yara", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column("submitted", sa.Integer(), nullable=False),
-        sa.Column("finished", sa.Integer(), nullable=True),
-        sa.Column("files_limit", sa.Integer(), nullable=False),
-        sa.Column(
-            "reference", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column("files_processed", sa.Integer(), nullable=False),
-        sa.Column("files_matched", sa.Integer(), nullable=False),
-        sa.Column("files_in_progress", sa.Integer(), nullable=False),
-        sa.Column("total_files", sa.Integer(), nullable=False),
-        sa.Column("files_errored", sa.Integer(), nullable=False),
-        sa.Column("datasets_left", sa.Integer(), nullable=False),
-        sa.Column("total_datasets", sa.Integer(), nullable=False),
-        sa.Column("agents_left", sa.Integer(), nullable=False),
-        sa.Column("internal_id", sa.Integer(), nullable=False),
-        sa.PrimaryKeyConstraint("internal_id"),
-    )
-    op.create_table(
-        "match",
-        sa.Column("meta", sa.JSON(), nullable=True),
-        sa.Column("matches", sa.ARRAY(sa.String()), nullable=True),
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("job_id", sa.Integer(), nullable=False),
-        sa.Column("file", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["job_id"],
-            ["job.internal_id"],
-        ),
-        sa.PrimaryKeyConstraint("id"),
-    )
+    if not inspect(engine).has_table("agentgroup"):
+        op.create_table(
+            "agentgroup",
+            sa.Column("plugins_spec", sa.JSON(), nullable=True),
+            sa.Column("active_plugins", sa.ARRAY(sa.String()), nullable=True),
+            sa.Column(
+                "name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column(
+                "ursadb_url",
+                sqlmodel.sql.sqltypes.AutoString(),
+                nullable=False,
+            ),
+            sa.Column("id", sa.Integer(), nullable=False),
+            sa.PrimaryKeyConstraint("id"),
+        )
+    if not inspect(engine).has_table("configentry"):
+        op.create_table(
+            "configentry",
+            sa.Column("id", sa.Integer(), nullable=False),
+            sa.Column(
+                "plugin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column(
+                "key", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column(
+                "value", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.PrimaryKeyConstraint("id"),
+        )
+    if not inspect(engine).has_table("job"):
+        op.create_table(
+            "job",
+            sa.Column("taints", sa.ARRAY(sa.String()), nullable=True),
+            sa.Column(
+                "id", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column(
+                "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column(
+                "error", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            ),
+            sa.Column(
+                "rule_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column(
+                "rule_author",
+                sqlmodel.sql.sqltypes.AutoString(),
+                nullable=False,
+            ),
+            sa.Column(
+                "raw_yara", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column("submitted", sa.Integer(), nullable=False),
+            sa.Column("finished", sa.Integer(), nullable=True),
+            sa.Column("files_limit", sa.Integer(), nullable=False),
+            sa.Column(
+                "reference", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.Column("files_processed", sa.Integer(), nullable=False),
+            sa.Column("files_matched", sa.Integer(), nullable=False),
+            sa.Column("files_in_progress", sa.Integer(), nullable=False),
+            sa.Column("total_files", sa.Integer(), nullable=False),
+            sa.Column("files_errored", sa.Integer(), nullable=False),
+            sa.Column("datasets_left", sa.Integer(), nullable=False),
+            sa.Column("total_datasets", sa.Integer(), nullable=False),
+            sa.Column("agents_left", sa.Integer(), nullable=False),
+            sa.Column("internal_id", sa.Integer(), nullable=False),
+            sa.PrimaryKeyConstraint("internal_id"),
+        )
+    if not inspect(engine).has_table("match"):
+        op.create_table(
+            "match",
+            sa.Column("meta", sa.JSON(), nullable=True),
+            sa.Column("matches", sa.ARRAY(sa.String()), nullable=True),
+            sa.Column("id", sa.Integer(), nullable=False),
+            sa.Column("job_id", sa.Integer(), nullable=False),
+            sa.Column(
+                "file", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            ),
+            sa.ForeignKeyConstraint(
+                ["job_id"],
+                ["job.internal_id"],
+            ),
+            sa.PrimaryKeyConstraint("id"),
+        )
 
 
 def downgrade() -> None:
diff --git a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
index 3afb5166..ae7071ef 100644
--- a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
+++ b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
@@ -5,34 +5,41 @@
 """
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy import inspect
 
 
+from config import app_config
+
 # revision identifiers, used by Alembic.
 revision = "dbb81bd4d47f"
 down_revision = "cbbba858deb0"
 branch_labels = None
 depends_on = None
 
+DATABASE_URL = app_config.database.url
+engine = sa.create_engine(DATABASE_URL)
+
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "jobagent",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("task_in_progress", sa.Integer(), nullable=False),
-        sa.Column("job_id", sa.Integer(), nullable=False),
-        sa.Column("agent_id", sa.Integer(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["agent_id"],
-            ["agentgroup.id"],
-        ),
-        sa.ForeignKeyConstraint(
-            ["job_id"],
-            ["job.internal_id"],
-        ),
-        sa.PrimaryKeyConstraint("id"),
-    )
-    # ### end Alembic commands ###
+    if not inspect(engine).has_table("jobagent"):
+        op.create_table(
+            "jobagent",
+            sa.Column("id", sa.Integer(), nullable=False),
+            sa.Column("task_in_progress", sa.Integer(), nullable=False),
+            sa.Column("job_id", sa.Integer(), nullable=False),
+            sa.Column("agent_id", sa.Integer(), nullable=False),
+            sa.ForeignKeyConstraint(
+                ["agent_id"],
+                ["agentgroup.id"],
+            ),
+            sa.ForeignKeyConstraint(
+                ["job_id"],
+                ["job.internal_id"],
+            ),
+            sa.PrimaryKeyConstraint("id"),
+        )
+        # ### end Alembic commands ###
 
 
 def downgrade() -> None:
diff --git a/src/models/job.py b/src/models/job.py
index 5c7a4c0c..563d83b5 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -21,7 +21,7 @@ class JobBase(SQLModel):
     """Base class for entities related to mquery jobs."""
 
     id: str
-    status: JobStatus = Field(sa_column=PgEnum(JobStatus, name="jobstatus", create_type=True))
+    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))
     error: Optional[str]
     rule_name: str
     rule_author: str

From d028b53e3ca5db967f62e068af5f8c112b1db74b Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Tue, 1 Oct 2024 18:20:36 +0200
Subject: [PATCH 03/25] In INSTALL.md and README.md alembic command  added.

---
 INSTALL.md | 2 ++
 README.md  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/INSTALL.md b/INSTALL.md
index e813b9a4..41bb9f46 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -21,6 +21,7 @@ mkdir samples
 vim .env
 docker compose up --scale daemon=3  # this will take a while
 docker compose exec web python3 -m mquery.db
+docker compose exec web python3 -m alembic upgrade head
 ```
 
 - Good for testing mquery and production deployments on a single server
@@ -39,6 +40,7 @@ cd mquery
 vim .env
 docker compose -f docker-compose.dev.yml up  # this will take a while
 docker compose exec dev-web python3 -m mquery.db
+docker compose exec dev-web python3 -m alembic upgrade head
 ```
 
 - Good for development - all file changes will be picked up automatically.
diff --git a/README.md b/README.md
index 0803755a..f2eb3b38 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,7 @@ cd mquery
 vim .env  # optional - change samples and index directory locations
 docker compose up --scale daemon=3  # building the images will take a while
 docker compose exec web python3 -m mquery.db
+docker compose exec web python3 -m alembic upgrade head
 ```
 
 The web interface should be available at `http://localhost`.

From 1498a44520374a408cd444c7dcb7fb6a0132bcd5 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Tue, 1 Oct 2024 18:41:58 +0200
Subject: [PATCH 04/25] ignore enum type error

---
 src/models/job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/models/job.py b/src/models/job.py
index 563d83b5..a515803a 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -21,7 +21,7 @@ class JobBase(SQLModel):
     """Base class for entities related to mquery jobs."""
 
     id: str
-    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))
+    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))  # type: ignore
     error: Optional[str]
     rule_name: str
     rule_author: str

From 98621c79355cf8ce83b6e17d29380f1a4cf48bdb Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Tue, 1 Oct 2024 22:53:41 +0200
Subject: [PATCH 05/25] Creating jobstatus enum by job.py site

---
 src/migrations/versions/84796bdc2f77_add_jobstatus.py | 3 ---
 src/models/job.py                                     | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/migrations/versions/84796bdc2f77_add_jobstatus.py b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
index 1f60a001..e15f4670 100644
--- a/src/migrations/versions/84796bdc2f77_add_jobstatus.py
+++ b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
@@ -14,9 +14,6 @@
 
 
 def upgrade() -> None:
-    op.execute(
-        "CREATE TYPE jobstatus AS ENUM ('done', 'new', 'cancelled', 'removed', 'processing');"
-    )
     op.execute(
         "ALTER TABLE job ALTER COLUMN status TYPE jobstatus USING status::text::jobstatus;"
     )
diff --git a/src/models/job.py b/src/models/job.py
index a515803a..3c0a0e11 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -21,7 +21,7 @@ class JobBase(SQLModel):
     """Base class for entities related to mquery jobs."""
 
     id: str
-    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))  # type: ignore
+    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus", create_type=True))  # type: ignore
     error: Optional[str]
     rule_name: str
     rule_author: str

From e0ce4f226bbf6f05070a0e5ab7fe986aca9da4ce Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 2 Oct 2024 09:13:45 +0200
Subject: [PATCH 06/25] Creating jobstatus enum in migration file

---
 src/migrations/versions/84796bdc2f77_add_jobstatus.py | 3 +++
 src/models/job.py                                     | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/migrations/versions/84796bdc2f77_add_jobstatus.py b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
index e15f4670..1f60a001 100644
--- a/src/migrations/versions/84796bdc2f77_add_jobstatus.py
+++ b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
@@ -14,6 +14,9 @@
 
 
 def upgrade() -> None:
+    op.execute(
+        "CREATE TYPE jobstatus AS ENUM ('done', 'new', 'cancelled', 'removed', 'processing');"
+    )
     op.execute(
         "ALTER TABLE job ALTER COLUMN status TYPE jobstatus USING status::text::jobstatus;"
     )
diff --git a/src/models/job.py b/src/models/job.py
index 3c0a0e11..a515803a 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -21,7 +21,7 @@ class JobBase(SQLModel):
     """Base class for entities related to mquery jobs."""
 
     id: str
-    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus", create_type=True))  # type: ignore
+    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))  # type: ignore
     error: Optional[str]
     rule_name: str
     rule_author: str

From 5525bdb7feb18cefab1d1ba595b9b37215ac4889 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 00:13:39 +0200
Subject: [PATCH 07/25] After review

---
 .../versions/84796bdc2f77_add_jobstatus.py    |  28 ---
 src/migrations/versions/cbbba858deb0_init.py  | 174 +++++++++---------
 .../versions/dbb81bd4d47f_add_jobagent.py     |  33 ++--
 ...a27b4390_modify_job_status_type_to_enum.py |  31 ++++
 src/models/job.py                             |   3 +-
 5 files changed, 132 insertions(+), 137 deletions(-)
 delete mode 100644 src/migrations/versions/84796bdc2f77_add_jobstatus.py
 create mode 100644 src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py

diff --git a/src/migrations/versions/84796bdc2f77_add_jobstatus.py b/src/migrations/versions/84796bdc2f77_add_jobstatus.py
deleted file mode 100644
index 1f60a001..00000000
--- a/src/migrations/versions/84796bdc2f77_add_jobstatus.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""add_jobstatus
-Revision ID: 84796bdc2f77
-Revises: dbb81bd4d47f
-Create Date: 2024-10-01 08:09:42.808911
-"""
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = "84796bdc2f77"
-down_revision = "dbb81bd4d47f"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.execute(
-        "CREATE TYPE jobstatus AS ENUM ('done', 'new', 'cancelled', 'removed', 'processing');"
-    )
-    op.execute(
-        "ALTER TABLE job ALTER COLUMN status TYPE jobstatus USING status::text::jobstatus;"
-    )
-
-
-def downgrade() -> None:
-    op.execute(
-        "ALTER TABLE job ALTER COLUMN status TYPE VARCHAR USING status::text;"
-    )
diff --git a/src/migrations/versions/cbbba858deb0_init.py b/src/migrations/versions/cbbba858deb0_init.py
index f69614e7..86cc27f4 100644
--- a/src/migrations/versions/cbbba858deb0_init.py
+++ b/src/migrations/versions/cbbba858deb0_init.py
@@ -5,8 +5,6 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy import inspect
-
 import sqlmodel
 
 from config import app_config
@@ -21,94 +19,90 @@
 
 
 def upgrade() -> None:
-    if not inspect(engine).has_table("agentgroup"):
-        op.create_table(
-            "agentgroup",
-            sa.Column("plugins_spec", sa.JSON(), nullable=True),
-            sa.Column("active_plugins", sa.ARRAY(sa.String()), nullable=True),
-            sa.Column(
-                "name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column(
-                "ursadb_url",
-                sqlmodel.sql.sqltypes.AutoString(),
-                nullable=False,
-            ),
-            sa.Column("id", sa.Integer(), nullable=False),
-            sa.PrimaryKeyConstraint("id"),
-        )
-    if not inspect(engine).has_table("configentry"):
-        op.create_table(
-            "configentry",
-            sa.Column("id", sa.Integer(), nullable=False),
-            sa.Column(
-                "plugin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column(
-                "key", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column(
-                "value", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.PrimaryKeyConstraint("id"),
-        )
-    if not inspect(engine).has_table("job"):
-        op.create_table(
-            "job",
-            sa.Column("taints", sa.ARRAY(sa.String()), nullable=True),
-            sa.Column(
-                "id", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column(
-                "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column(
-                "error", sqlmodel.sql.sqltypes.AutoString(), nullable=True
-            ),
-            sa.Column(
-                "rule_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column(
-                "rule_author",
-                sqlmodel.sql.sqltypes.AutoString(),
-                nullable=False,
-            ),
-            sa.Column(
-                "raw_yara", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column("submitted", sa.Integer(), nullable=False),
-            sa.Column("finished", sa.Integer(), nullable=True),
-            sa.Column("files_limit", sa.Integer(), nullable=False),
-            sa.Column(
-                "reference", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.Column("files_processed", sa.Integer(), nullable=False),
-            sa.Column("files_matched", sa.Integer(), nullable=False),
-            sa.Column("files_in_progress", sa.Integer(), nullable=False),
-            sa.Column("total_files", sa.Integer(), nullable=False),
-            sa.Column("files_errored", sa.Integer(), nullable=False),
-            sa.Column("datasets_left", sa.Integer(), nullable=False),
-            sa.Column("total_datasets", sa.Integer(), nullable=False),
-            sa.Column("agents_left", sa.Integer(), nullable=False),
-            sa.Column("internal_id", sa.Integer(), nullable=False),
-            sa.PrimaryKeyConstraint("internal_id"),
-        )
-    if not inspect(engine).has_table("match"):
-        op.create_table(
-            "match",
-            sa.Column("meta", sa.JSON(), nullable=True),
-            sa.Column("matches", sa.ARRAY(sa.String()), nullable=True),
-            sa.Column("id", sa.Integer(), nullable=False),
-            sa.Column("job_id", sa.Integer(), nullable=False),
-            sa.Column(
-                "file", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-            ),
-            sa.ForeignKeyConstraint(
-                ["job_id"],
-                ["job.internal_id"],
-            ),
-            sa.PrimaryKeyConstraint("id"),
-        )
+    op.create_table(
+        "agentgroup",
+        sa.Column("plugins_spec", sa.JSON(), nullable=True),
+        sa.Column("active_plugins", sa.ARRAY(sa.String()), nullable=True),
+        sa.Column(
+            "name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "ursadb_url",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+        ),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "configentry",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "plugin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "key", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "value", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "job",
+        sa.Column("taints", sa.ARRAY(sa.String()), nullable=True),
+        sa.Column(
+            "id", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "error", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "rule_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "rule_author",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+        ),
+        sa.Column(
+            "raw_yara", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("submitted", sa.Integer(), nullable=False),
+        sa.Column("finished", sa.Integer(), nullable=True),
+        sa.Column("files_limit", sa.Integer(), nullable=False),
+        sa.Column(
+            "reference", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("files_processed", sa.Integer(), nullable=False),
+        sa.Column("files_matched", sa.Integer(), nullable=False),
+        sa.Column("files_in_progress", sa.Integer(), nullable=False),
+        sa.Column("total_files", sa.Integer(), nullable=False),
+        sa.Column("files_errored", sa.Integer(), nullable=False),
+        sa.Column("datasets_left", sa.Integer(), nullable=False),
+        sa.Column("total_datasets", sa.Integer(), nullable=False),
+        sa.Column("agents_left", sa.Integer(), nullable=False),
+        sa.Column("internal_id", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("internal_id"),
+    )
+    op.create_table(
+        "match",
+        sa.Column("meta", sa.JSON(), nullable=True),
+        sa.Column("matches", sa.ARRAY(sa.String()), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("job_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "file", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["job_id"],
+            ["job.internal_id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
 
 
 def downgrade() -> None:
diff --git a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
index ae7071ef..755caf80 100644
--- a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
+++ b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
@@ -22,23 +22,22 @@
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    if not inspect(engine).has_table("jobagent"):
-        op.create_table(
-            "jobagent",
-            sa.Column("id", sa.Integer(), nullable=False),
-            sa.Column("task_in_progress", sa.Integer(), nullable=False),
-            sa.Column("job_id", sa.Integer(), nullable=False),
-            sa.Column("agent_id", sa.Integer(), nullable=False),
-            sa.ForeignKeyConstraint(
-                ["agent_id"],
-                ["agentgroup.id"],
-            ),
-            sa.ForeignKeyConstraint(
-                ["job_id"],
-                ["job.internal_id"],
-            ),
-            sa.PrimaryKeyConstraint("id"),
-        )
+    op.create_table(
+        "jobagent",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("task_in_progress", sa.Integer(), nullable=False),
+        sa.Column("job_id", sa.Integer(), nullable=False),
+        sa.Column("agent_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["agent_id"],
+            ["agentgroup.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["job_id"],
+            ["job.internal_id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
         # ### end Alembic commands ###
 
 
diff --git a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
new file mode 100644
index 00000000..9ddbe9b0
--- /dev/null
+++ b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
@@ -0,0 +1,31 @@
+"""Modify_job status type to Enum
+Revision ID: f552a27b4390
+Revises: dbb81bd4d47f
+Create Date: 2024-10-08 08:58:25.299726
+"""
+from alembic import op
+import sqlalchemy as sa
+from alembic_postgresql_enum import TableReference
+
+# revision identifiers, used by Alembic.
+revision = 'f552a27b4390'
+down_revision = 'dbb81bd4d47f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    sa.Enum('done', 'new', 'cancelled', 'processing', name='jobstatus').create(op.get_bind())
+    op.sync_enum_values('public', 'jobstatus', ['done', 'new', 'cancelled', 'processing'],
+                        [TableReference(table_schema='public', table_name='job', column_name='status')],
+                        enum_values_to_rename=[])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.sync_enum_values('public', 'jobstatus', ['done', 'new', 'cancelled', 'removed', 'processing'],
+                        [TableReference(table_schema='public', table_name='job', column_name='status')],
+                        enum_values_to_rename=[])
+    # ### end Alembic commands ###
diff --git a/src/models/job.py b/src/models/job.py
index a515803a..96d3ede4 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -9,11 +9,10 @@
     from ..models.jobagent import JobAgent
 
 
-class JobStatus(str, enum.Enum):
+class JobStatus(enum.Enum):
     done = "done"
     new = "new"
     cancelled = "cancelled"
-    removed = "removed"
     processing = "processing"
 
 

From db4e0e94846d1866f70daab2ab4638ac63178fbf Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 00:16:35 +0200
Subject: [PATCH 08/25] lint

---
 src/migrations/versions/cbbba858deb0_init.py         | 3 ---
 src/migrations/versions/dbb81bd4d47f_add_jobagent.py | 6 ------
 2 files changed, 9 deletions(-)

diff --git a/src/migrations/versions/cbbba858deb0_init.py b/src/migrations/versions/cbbba858deb0_init.py
index 86cc27f4..80dc7f6e 100644
--- a/src/migrations/versions/cbbba858deb0_init.py
+++ b/src/migrations/versions/cbbba858deb0_init.py
@@ -14,9 +14,6 @@
 branch_labels = None
 depends_on = None
 
-DATABASE_URL = app_config.database.url
-engine = sa.create_engine(DATABASE_URL)
-
 
 def upgrade() -> None:
     op.create_table(
diff --git a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
index 755caf80..6ccfdb55 100644
--- a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
+++ b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
@@ -5,20 +5,14 @@
 """
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy import inspect
 
 
-from config import app_config
-
 # revision identifiers, used by Alembic.
 revision = "dbb81bd4d47f"
 down_revision = "cbbba858deb0"
 branch_labels = None
 depends_on = None
 
-DATABASE_URL = app_config.database.url
-engine = sa.create_engine(DATABASE_URL)
-
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###

From 02ae1027ac87ff065f8ae406255494ef6dd67438 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 01:09:10 +0200
Subject: [PATCH 09/25] Added recognized enum changes and removing removed
 status

---
 requirements.txt                              | 56 +++++++++----------
 src/db.py                                     |  4 +-
 src/migrations/env.py                         |  1 +
 ...a27b4390_modify_job_status_type_to_enum.py |  3 +-
 src/models/job.py                             |  2 +-
 5 files changed, 30 insertions(+), 36 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 27da0663..46389868 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,34 +1,28 @@
-alembic==1.11.1
-annotated-types==0.7.0
-anyio==4.6.0
+anyio==3.4.0
 asgiref==3.4.1
-cachetools==5.5.0
-certifi==2024.8.30
-cffi==1.17.1
-charset-normalizer==3.3.2
-click==8.1.7
-cryptography==43.0.1
+cachetools==4.2.4
+certifi==2021.10.8
+charset-normalizer==2.0.9
+click==8.0.3
 Deprecated==1.2.13
-fastapi==0.115.0
-h11==0.14.0
-idna==3.10
-psycopg2==2.9.9
-pycparser==2.22
-pydantic==1.10.18
-pydantic_core==2.23.4
-PyJWT[crypto]==2.9.0
-pyzmq==26.2.0
-redis==5.0.8
-requests==2.32.2
-rq==1.16.2
-sniffio==1.3.1
+fastapi==0.70.0
+h11==0.12.0
+idna==3.3
+pydantic==1.10.14
+pyzmq==24.0.1
+redis==4.0.2
+requests==2.26.0
+sniffio==1.2.0
+starlette==0.16.0
+typing-extensions==4.9.0
+urllib3==1.26.7
+uvicorn==0.15.0
+wrapt==1.13.3
+yara-python==4.1.3
+yaramod==3.12.1
+PyJWT[crypto]==2.3.0
+rq==1.11.1
+typed-config==1.3.2
 sqlmodel==0.0.11
-starlette==0.38.6
-typed-config==2.0.3
-types-requests==2.32.0.20240914
-typing_extensions==4.12.2
-urllib3==2.2.3
-uvicorn==0.30.6
-wrapt==1.16.0
-yara-python==4.5.1
-yaramod==3.23.0
+psycopg2==2.9.9
+alembic==1.11.1
diff --git a/src/db.py b/src/db.py
index 0952e407..299a7fa4 100644
--- a/src/db.py
+++ b/src/db.py
@@ -91,7 +91,7 @@ def get_valid_jobs(self, username_filter: Optional[str]) -> List[Job]:
         with self.session() as session:
             query = (
                 select(Job)
-                .where(Job.status != JobStatus.removed)
+                .where(Job.status != JobStatus.cancelled)
                 .order_by(col(Job.submitted).desc())
             )
             if username_filter:
@@ -104,7 +104,7 @@ def remove_query(self, job: JobId) -> None:
             session.execute(
                 update(Job)
                 .where(Job.id == job)
-                .values(status=JobStatus.removed)
+                .values(status=JobStatus.cancelled)
             )
             session.commit()
 
diff --git a/src/migrations/env.py b/src/migrations/env.py
index 68b19a69..80f50f03 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -1,3 +1,4 @@
+import alembic_postgresql_enum  # type: ignore
 from sqlalchemy import create_engine
 from alembic import context
 from sqlmodel import SQLModel
diff --git a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
index 9ddbe9b0..bc847b6c 100644
--- a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
+++ b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
@@ -16,8 +16,7 @@
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    sa.Enum('done', 'new', 'cancelled', 'processing', name='jobstatus').create(op.get_bind())
-    op.sync_enum_values('public', 'jobstatus', ['done', 'new', 'cancelled', 'processing'],
+    op.sync_enum_values('public', 'jobstatus', ['done', 'new', 'cancelled', 'removed', 'processing'],
                         [TableReference(table_schema='public', table_name='job', column_name='status')],
                         enum_values_to_rename=[])
     # ### end Alembic commands ###
diff --git a/src/models/job.py b/src/models/job.py
index 1a2fdbea..bcb8e1ca 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -20,7 +20,7 @@ class JobView(SQLModel):
     """Public fields of mquery jobs."""
 
     id: str
-    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))  # type: ignore
+    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus", create_type=True))  # type: ignore
     error: Optional[str]
     rule_name: str
     rule_author: str

From 690051dde3668f614417c7afb8faab30e8f90375 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 10:37:14 +0200
Subject: [PATCH 10/25] with nmp errors

---
 requirements.txt                              | 56 ++++++++++---------
 src/migrations/env.py                         |  1 -
 src/migrations/versions/cbbba858deb0_init.py  |  2 -
 .../versions/dbb81bd4d47f_add_jobagent.py     |  2 +-
 ...a27b4390_modify_job_status_type_to_enum.py | 33 ++++++++---
 src/mqueryfront/src/utils.js                  |  7 +--
 6 files changed, 59 insertions(+), 42 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 46389868..305158fe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,28 +1,34 @@
-anyio==3.4.0
+alembic==1.11.1
+annotated-types==0.7.0
+anyio==4.6.0
 asgiref==3.4.1
-cachetools==4.2.4
-certifi==2021.10.8
-charset-normalizer==2.0.9
-click==8.0.3
+cachetools==5.5.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.3.2
+click==8.1.7
+cryptography==43.0.1
 Deprecated==1.2.13
-fastapi==0.70.0
-h11==0.12.0
-idna==3.3
-pydantic==1.10.14
-pyzmq==24.0.1
-redis==4.0.2
-requests==2.26.0
-sniffio==1.2.0
-starlette==0.16.0
-typing-extensions==4.9.0
-urllib3==1.26.7
-uvicorn==0.15.0
-wrapt==1.13.3
-yara-python==4.1.3
-yaramod==3.12.1
-PyJWT[crypto]==2.3.0
-rq==1.11.1
-typed-config==1.3.2
-sqlmodel==0.0.11
+fastapi==0.115.0
+h11==0.14.0
+idna==3.10
 psycopg2==2.9.9
-alembic==1.11.1
+pycparser==2.22
+pydantic==1.10.18
+pydantic_core==2.23.4
+PyJWT[crypto]==2.9.0
+pyzmq==26.2.0
+redis==5.0.8
+requests==2.32.2
+rq==1.16.2
+sniffio==1.3.1
+sqlmodel==0.0.11
+starlette==0.38.6
+typed-config==2.0.3
+types-requests==2.32.0.20240914
+typing_extensions==4.12.2
+urllib3==2.2.3
+uvicorn==0.30.6
+wrapt==1.16.0
+yara-python==4.5.1
+yaramod==3.23.0
\ No newline at end of file
diff --git a/src/migrations/env.py b/src/migrations/env.py
index 80f50f03..68b19a69 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -1,4 +1,3 @@
-import alembic_postgresql_enum  # type: ignore
 from sqlalchemy import create_engine
 from alembic import context
 from sqlmodel import SQLModel
diff --git a/src/migrations/versions/cbbba858deb0_init.py b/src/migrations/versions/cbbba858deb0_init.py
index 80dc7f6e..69a19634 100644
--- a/src/migrations/versions/cbbba858deb0_init.py
+++ b/src/migrations/versions/cbbba858deb0_init.py
@@ -7,8 +7,6 @@
 import sqlalchemy as sa
 import sqlmodel
 
-from config import app_config
-
 revision = "cbbba858deb0"
 down_revision = None
 branch_labels = None
diff --git a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
index 6ccfdb55..3afb5166 100644
--- a/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
+++ b/src/migrations/versions/dbb81bd4d47f_add_jobagent.py
@@ -32,7 +32,7 @@ def upgrade() -> None:
         ),
         sa.PrimaryKeyConstraint("id"),
     )
-        # ### end Alembic commands ###
+    # ### end Alembic commands ###
 
 
 def downgrade() -> None:
diff --git a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
index bc847b6c..6e88809c 100644
--- a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
+++ b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
@@ -4,27 +4,42 @@
 Create Date: 2024-10-08 08:58:25.299726
 """
 from alembic import op
-import sqlalchemy as sa
 from alembic_postgresql_enum import TableReference
 
 # revision identifiers, used by Alembic.
-revision = 'f552a27b4390'
-down_revision = 'dbb81bd4d47f'
+revision = "f552a27b4390"
+down_revision = "dbb81bd4d47f"
 branch_labels = None
 depends_on = None
 
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.sync_enum_values('public', 'jobstatus', ['done', 'new', 'cancelled', 'removed', 'processing'],
-                        [TableReference(table_schema='public', table_name='job', column_name='status')],
-                        enum_values_to_rename=[])
+    op.sync_enum_values(
+        "public",
+        "jobstatus",
+        ["done", "new", "cancelled", "removed", "processing"],
+        [
+            TableReference(
+                table_schema="public", table_name="job", column_name="status"
+            )
+        ],
+        enum_values_to_rename=[],
+    )
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.sync_enum_values('public', 'jobstatus', ['done', 'new', 'cancelled', 'removed', 'processing'],
-                        [TableReference(table_schema='public', table_name='job', column_name='status')],
-                        enum_values_to_rename=[])
+    op.sync_enum_values(
+        "public",
+        "jobstatus",
+        ["done", "new", "cancelled", "removed", "processing"],
+        [
+            TableReference(
+                table_schema="public", table_name="job", column_name="status"
+            )
+        ],
+        enum_values_to_rename=[],
+    )
     # ### end Alembic commands ###
diff --git a/src/mqueryfront/src/utils.js b/src/mqueryfront/src/utils.js
index b64e4bea..a7d8c309 100644
--- a/src/mqueryfront/src/utils.js
+++ b/src/mqueryfront/src/utils.js
@@ -1,12 +1,11 @@
 export const isStatusFinished = (status) =>
-    ["done", "cancelled", "removed"].includes(status);
+    ["done", "cancelled"].includes(status);
 
 const statusClassMap = {
     done: "success",
     new: "info",
     processing: "info",
-    cancelled: "danger",
-    removed: "dark",
+    cancelled: "danger"
 };
 
 export const isAuthEnabled = (config) =>
@@ -15,7 +14,7 @@ export const isAuthEnabled = (config) =>
 export const openidLoginUrl = (config) => {
     if (config["openid_url"] === null || config["openid_client_id"] === null) {
         // Defensive programming - config keys can be null.
-        return "#";
+        return "#";gi
     }
     const login_url = new URL(config["openid_url"] + "/auth");
     login_url.searchParams.append("client_id", config["openid_client_id"]);

From ef9e9d59e84144365950d805c36def2bfaba3e3b Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 10:39:09 +0200
Subject: [PATCH 11/25] delete removed

---
 .../0820dad794e6_delete_removed_status.py     | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 src/migrations/versions/0820dad794e6_delete_removed_status.py

diff --git a/src/migrations/versions/0820dad794e6_delete_removed_status.py b/src/migrations/versions/0820dad794e6_delete_removed_status.py
new file mode 100644
index 00000000..ce419551
--- /dev/null
+++ b/src/migrations/versions/0820dad794e6_delete_removed_status.py
@@ -0,0 +1,46 @@
+"""Delete removed status
+Revision ID: 0820dad794e6
+Revises: f552a27b4390
+Create Date: 2024-10-08 22:41:38.868185
+"""
+from alembic import op
+from alembic_postgresql_enum import TableReference
+
+# revision identifiers, used by Alembic.
+revision = "0820dad794e6"
+down_revision = "f552a27b4390"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute("UPDATE job SET status = 'cancelled' WHERE status = 'removed';")
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.sync_enum_values(
+        "public",
+        "jobstatus",
+        ["done", "new", "cancelled", "processing"],
+        [
+            TableReference(
+                table_schema="public", table_name="job", column_name="status"
+            )
+        ],
+        enum_values_to_rename=[],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.sync_enum_values(
+        "public",
+        "jobstatus",
+        ["done", "new", "cancelled", "removed", "processing"],
+        [
+            TableReference(
+                table_schema="public", table_name="job", column_name="status"
+            )
+        ],
+        enum_values_to_rename=[],
+    )
+    # ### end Alembic commands ###

From 8556a7515ab6cdd75300014bf48abd7f1147b62d Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 10:58:56 +0200
Subject: [PATCH 12/25] alembic_postgresql_enum for automatic enum type
 recognize

---
 requirements.txt      | 3 ++-
 src/migrations/env.py | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 305158fe..5ceb863b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -31,4 +31,5 @@ urllib3==2.2.3
 uvicorn==0.30.6
 wrapt==1.16.0
 yara-python==4.5.1
-yaramod==3.23.0
\ No newline at end of file
+yaramod==3.23.0
+alembic_postgresql_enum
\ No newline at end of file
diff --git a/src/migrations/env.py b/src/migrations/env.py
index 68b19a69..a5501150 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -1,3 +1,4 @@
+import alembic_postgresql_enum # noqa: ignore=F401
 from sqlalchemy import create_engine
 from alembic import context
 from sqlmodel import SQLModel

From 71f42efbc38c809ecb3622afccc16663ab50df4d Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 10:59:08 +0200
Subject: [PATCH 13/25] lint

---
 src/mqueryfront/src/utils.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mqueryfront/src/utils.js b/src/mqueryfront/src/utils.js
index a7d8c309..3bde2acd 100644
--- a/src/mqueryfront/src/utils.js
+++ b/src/mqueryfront/src/utils.js
@@ -5,7 +5,7 @@ const statusClassMap = {
     done: "success",
     new: "info",
     processing: "info",
-    cancelled: "danger"
+    cancelled: "danger",
 };
 
 export const isAuthEnabled = (config) =>
@@ -14,7 +14,8 @@ export const isAuthEnabled = (config) =>
 export const openidLoginUrl = (config) => {
     if (config["openid_url"] === null || config["openid_client_id"] === null) {
         // Defensive programming - config keys can be null.
-        return "#";gi
+        return "#";
+        gi;
     }
     const login_url = new URL(config["openid_url"] + "/auth");
     login_url.searchParams.append("client_id", config["openid_client_id"]);

From 5d78c6b7551296e2af2ade2618f01a9b693b0fb3 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 11:19:20 +0200
Subject: [PATCH 14/25] Create jobstatus enum type in migration file.

---
 .../versions/f552a27b4390_modify_job_status_type_to_enum.py | 6 ++----
 src/models/job.py                                           | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
index 6e88809c..ebae1dd6 100644
--- a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
+++ b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
@@ -4,6 +4,7 @@
 Create Date: 2024-10-08 08:58:25.299726
 """
 from alembic import op
+import sqlalchemy as sa
 from alembic_postgresql_enum import TableReference
 
 # revision identifiers, used by Alembic.
@@ -14,7 +15,7 @@
 
 
 def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
+    sa.Enum('done', 'new', 'cancelled', 'removed', 'processing', name='jobstatus').create(op.get_bind())
     op.sync_enum_values(
         "public",
         "jobstatus",
@@ -26,11 +27,9 @@ def upgrade() -> None:
         ],
         enum_values_to_rename=[],
     )
-    # ### end Alembic commands ###
 
 
 def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
     op.sync_enum_values(
         "public",
         "jobstatus",
@@ -42,4 +41,3 @@ def downgrade() -> None:
         ],
         enum_values_to_rename=[],
     )
-    # ### end Alembic commands ###
diff --git a/src/models/job.py b/src/models/job.py
index bcb8e1ca..1a2fdbea 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -20,7 +20,7 @@ class JobView(SQLModel):
     """Public fields of mquery jobs."""
 
     id: str
-    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus", create_type=True))  # type: ignore
+    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))  # type: ignore
     error: Optional[str]
     rule_name: str
     rule_author: str

From 9b83297ad4cbbdfa3b59f8694953107ac22b7e29 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Wed, 9 Oct 2024 11:25:54 +0200
Subject: [PATCH 15/25] black

---
 src/migrations/env.py                         |  2 +-
 src/migrations/versions/cbbba858deb0_init.py  | 24 +++++--------------
 ...a27b4390_modify_job_status_type_to_enum.py |  4 +++-
 3 files changed, 10 insertions(+), 20 deletions(-)

diff --git a/src/migrations/env.py b/src/migrations/env.py
index a5501150..e33ec076 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -1,4 +1,4 @@
-import alembic_postgresql_enum # noqa: ignore=F401
+import alembic_postgresql_enum  # noqa: ignore=F401
 from sqlalchemy import create_engine
 from alembic import context
 from sqlmodel import SQLModel
diff --git a/src/migrations/versions/cbbba858deb0_init.py b/src/migrations/versions/cbbba858deb0_init.py
index 69a19634..218a3a65 100644
--- a/src/migrations/versions/cbbba858deb0_init.py
+++ b/src/migrations/versions/cbbba858deb0_init.py
@@ -18,9 +18,7 @@ def upgrade() -> None:
         "agentgroup",
         sa.Column("plugins_spec", sa.JSON(), nullable=True),
         sa.Column("active_plugins", sa.ARRAY(sa.String()), nullable=True),
-        sa.Column(
-            "name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.Column(
             "ursadb_url",
             sqlmodel.sql.sqltypes.AutoString(),
@@ -35,26 +33,18 @@ def upgrade() -> None:
         sa.Column(
             "plugin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
-        sa.Column(
-            "key", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column(
-            "value", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
+        sa.Column("key", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("value", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.PrimaryKeyConstraint("id"),
     )
     op.create_table(
         "job",
         sa.Column("taints", sa.ARRAY(sa.String()), nullable=True),
-        sa.Column(
-            "id", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
+        sa.Column("id", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.Column(
             "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
-        sa.Column(
-            "error", sqlmodel.sql.sqltypes.AutoString(), nullable=True
-        ),
+        sa.Column("error", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
         sa.Column(
             "rule_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
@@ -89,9 +79,7 @@ def upgrade() -> None:
         sa.Column("matches", sa.ARRAY(sa.String()), nullable=True),
         sa.Column("id", sa.Integer(), nullable=False),
         sa.Column("job_id", sa.Integer(), nullable=False),
-        sa.Column(
-            "file", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
+        sa.Column("file", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.ForeignKeyConstraint(
             ["job_id"],
             ["job.internal_id"],
diff --git a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
index ebae1dd6..fda50076 100644
--- a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
+++ b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
@@ -15,7 +15,9 @@
 
 
 def upgrade() -> None:
-    sa.Enum('done', 'new', 'cancelled', 'removed', 'processing', name='jobstatus').create(op.get_bind())
+    sa.Enum(
+        "done", "new", "cancelled", "removed", "processing", name="jobstatus"
+    ).create(op.get_bind())
     op.sync_enum_values(
         "public",
         "jobstatus",

From e909efe60e55f875f1dc07ab93ab3fc6928aa3c4 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Thu, 17 Oct 2024 10:08:19 +0200
Subject: [PATCH 16/25] After revie - job status as a enum, match and jobagent
 with ON DELETE CASCADE

---
 src/db.py                                     | 18 ++----
 src/migrations/env.py                         |  1 -
 .../0820dad794e6_delete_removed_status.py     | 46 ----------------
 .../versions/6b495d5a4855_add_jobstatus.py    | 41 ++++++++++++++
 src/migrations/versions/cbbba858deb0_init.py  |  9 +--
 ...a27b4390_modify_job_status_type_to_enum.py | 45 ---------------
 ...n_delete_cascade_for_jobagent_and_match.py | 55 +++++++++++++++++++
 src/models/job.py                             |  7 ++-
 src/models/jobagent.py                        |  5 +-
 src/models/match.py                           |  5 +-
 10 files changed, 118 insertions(+), 114 deletions(-)
 delete mode 100644 src/migrations/versions/0820dad794e6_delete_removed_status.py
 create mode 100644 src/migrations/versions/6b495d5a4855_add_jobstatus.py
 delete mode 100644 src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
 create mode 100644 src/migrations/versions/f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py

diff --git a/src/db.py b/src/db.py
index 299a7fa4..0b173c35 100644
--- a/src/db.py
+++ b/src/db.py
@@ -15,6 +15,7 @@
     and_,
     update,
     col,
+    delete,
 )
 
 from .models.agentgroup import AgentGroup
@@ -87,25 +88,18 @@ def get_job(self, job: JobId) -> Job:
             return self.__get_job(session, job)
 
     def get_valid_jobs(self, username_filter: Optional[str]) -> List[Job]:
-        """Retrieves valid (accessible and not removed) jobs from the database."""
+        """Retrieves valid (accessible) jobs from the database."""
         with self.session() as session:
-            query = (
-                select(Job)
-                .where(Job.status != JobStatus.cancelled)
-                .order_by(col(Job.submitted).desc())
-            )
+            query = select(Job).order_by(col(Job.submitted).desc())
             if username_filter:
                 query = query.where(Job.rule_author == username_filter)
             return session.exec(query).all()
 
     def remove_query(self, job: JobId) -> None:
-        """Sets the job status to removed."""
+        """Delete the job, linked match and job agent from the database."""
         with self.session() as session:
-            session.execute(
-                update(Job)
-                .where(Job.id == job)
-                .values(status=JobStatus.cancelled)
-            )
+            delete_query = delete(Job).where(Job.id == job)
+            session.execute(delete_query)
             session.commit()
 
     def add_match(self, job: JobId, match: Match) -> None:
diff --git a/src/migrations/env.py b/src/migrations/env.py
index e33ec076..68b19a69 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -1,4 +1,3 @@
-import alembic_postgresql_enum  # noqa: ignore=F401
 from sqlalchemy import create_engine
 from alembic import context
 from sqlmodel import SQLModel
diff --git a/src/migrations/versions/0820dad794e6_delete_removed_status.py b/src/migrations/versions/0820dad794e6_delete_removed_status.py
deleted file mode 100644
index ce419551..00000000
--- a/src/migrations/versions/0820dad794e6_delete_removed_status.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""Delete removed status
-Revision ID: 0820dad794e6
-Revises: f552a27b4390
-Create Date: 2024-10-08 22:41:38.868185
-"""
-from alembic import op
-from alembic_postgresql_enum import TableReference
-
-# revision identifiers, used by Alembic.
-revision = "0820dad794e6"
-down_revision = "f552a27b4390"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.execute("UPDATE job SET status = 'cancelled' WHERE status = 'removed';")
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.sync_enum_values(
-        "public",
-        "jobstatus",
-        ["done", "new", "cancelled", "processing"],
-        [
-            TableReference(
-                table_schema="public", table_name="job", column_name="status"
-            )
-        ],
-        enum_values_to_rename=[],
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.sync_enum_values(
-        "public",
-        "jobstatus",
-        ["done", "new", "cancelled", "removed", "processing"],
-        [
-            TableReference(
-                table_schema="public", table_name="job", column_name="status"
-            )
-        ],
-        enum_values_to_rename=[],
-    )
-    # ### end Alembic commands ###
diff --git a/src/migrations/versions/6b495d5a4855_add_jobstatus.py b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
new file mode 100644
index 00000000..3afb5166
--- /dev/null
+++ b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
@@ -0,0 +1,41 @@
+"""add jobagent
+Revision ID: dbb81bd4d47f
+Revises: cbbba858deb0
+Create Date: 2024-05-29 13:13:03.980030
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "dbb81bd4d47f"
+down_revision = "cbbba858deb0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "jobagent",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("task_in_progress", sa.Integer(), nullable=False),
+        sa.Column("job_id", sa.Integer(), nullable=False),
+        sa.Column("agent_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["agent_id"],
+            ["agentgroup.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["job_id"],
+            ["job.internal_id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("jobagent")
+    # ### end Alembic commands ###
diff --git a/src/migrations/versions/cbbba858deb0_init.py b/src/migrations/versions/cbbba858deb0_init.py
index 218a3a65..808bfd8f 100644
--- a/src/migrations/versions/cbbba858deb0_init.py
+++ b/src/migrations/versions/cbbba858deb0_init.py
@@ -7,6 +7,7 @@
 import sqlalchemy as sa
 import sqlmodel
 
+
 revision = "cbbba858deb0"
 down_revision = None
 branch_labels = None
@@ -20,9 +21,7 @@ def upgrade() -> None:
         sa.Column("active_plugins", sa.ARRAY(sa.String()), nullable=True),
         sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.Column(
-            "ursadb_url",
-            sqlmodel.sql.sqltypes.AutoString(),
-            nullable=False,
+            "ursadb_url", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
         sa.Column("id", sa.Integer(), nullable=False),
         sa.PrimaryKeyConstraint("id"),
@@ -49,9 +48,7 @@ def upgrade() -> None:
             "rule_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
         sa.Column(
-            "rule_author",
-            sqlmodel.sql.sqltypes.AutoString(),
-            nullable=False,
+            "rule_author", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
         sa.Column(
             "raw_yara", sqlmodel.sql.sqltypes.AutoString(), nullable=False
diff --git a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py b/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
deleted file mode 100644
index fda50076..00000000
--- a/src/migrations/versions/f552a27b4390_modify_job_status_type_to_enum.py
+++ /dev/null
@@ -1,45 +0,0 @@
-"""Modify_job status type to Enum
-Revision ID: f552a27b4390
-Revises: dbb81bd4d47f
-Create Date: 2024-10-08 08:58:25.299726
-"""
-from alembic import op
-import sqlalchemy as sa
-from alembic_postgresql_enum import TableReference
-
-# revision identifiers, used by Alembic.
-revision = "f552a27b4390"
-down_revision = "dbb81bd4d47f"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    sa.Enum(
-        "done", "new", "cancelled", "removed", "processing", name="jobstatus"
-    ).create(op.get_bind())
-    op.sync_enum_values(
-        "public",
-        "jobstatus",
-        ["done", "new", "cancelled", "removed", "processing"],
-        [
-            TableReference(
-                table_schema="public", table_name="job", column_name="status"
-            )
-        ],
-        enum_values_to_rename=[],
-    )
-
-
-def downgrade() -> None:
-    op.sync_enum_values(
-        "public",
-        "jobstatus",
-        ["done", "new", "cancelled", "removed", "processing"],
-        [
-            TableReference(
-                table_schema="public", table_name="job", column_name="status"
-            )
-        ],
-        enum_values_to_rename=[],
-    )
diff --git a/src/migrations/versions/f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py b/src/migrations/versions/f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py
new file mode 100644
index 00000000..1fc886b9
--- /dev/null
+++ b/src/migrations/versions/f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py
@@ -0,0 +1,55 @@
+"""on delete cascade for jobagent and match
+Revision ID: f654d7e4fcc7
+Revises: 6b495d5a4855
+Create Date: 2024-10-17 07:16:34.262079
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "f654d7e4fcc7"
+down_revision = "6b495d5a4855"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_constraint("jobagent_job_id_fkey", "jobagent", type_="foreignkey")
+    op.create_foreign_key(
+        constraint_name="jobagent_job_id_fkey",
+        source_table="jobagent",
+        referent_table="job",
+        local_cols=["job_id"],
+        remote_cols=["internal_id"],
+        ondelete="CASCADE",
+    )
+
+    op.drop_constraint("match_job_id_fkey", "match", type_="foreignkey")
+    op.create_foreign_key(
+        constraint_name="match_job_id_fkey",
+        source_table="match",
+        referent_table="job",
+        local_cols=["job_id"],
+        remote_cols=["internal_id"],
+        ondelete="CASCADE",
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint("jobagent_job_id_fkey", "jobagent", type_="foreignkey")
+    op.create_foreign_key(
+        constraint_name="jobagent_job_id_fkey",
+        source_table="jobagent",
+        referent_table="job",
+        local_cols=["job_id"],
+        remote_cols=["internal_id"],
+    )
+
+    op.drop_constraint("match_job_id_fkey", "match", type_="foreignkey")
+    op.create_foreign_key(
+        constraint_name="match_job_id_fkey",
+        source_table="match",
+        referent_table="job",
+        local_cols=["job_id"],
+        remote_cols=["internal_id"],
+    )
diff --git a/src/models/job.py b/src/models/job.py
index 1a2fdbea..34e5f893 100644
--- a/src/models/job.py
+++ b/src/models/job.py
@@ -1,6 +1,7 @@
 import enum
 
-from sqlalchemy import Enum as PgEnum
+from sqlalchemy.dialects import postgresql
+
 from sqlmodel import SQLModel, Field, ARRAY, String, Column, Relationship
 from typing import Optional, List, Union, TYPE_CHECKING
 
@@ -19,8 +20,10 @@ class JobStatus(enum.Enum):
 class JobView(SQLModel):
     """Public fields of mquery jobs."""
 
+    __table_args__ = {"extend_existing": True}
+
     id: str
-    status: JobStatus = Field(sa_type=PgEnum(JobStatus, name="jobstatus"))  # type: ignore
+    status: JobStatus = Field(sa_column=Column(postgresql.ENUM(JobStatus, name="jobstatus")))  # type: ignore
     error: Optional[str]
     rule_name: str
     rule_author: str
diff --git a/src/models/jobagent.py b/src/models/jobagent.py
index f40313de..a8041ea3 100644
--- a/src/models/jobagent.py
+++ b/src/models/jobagent.py
@@ -1,3 +1,4 @@
+from sqlalchemy import Column, ForeignKey
 from sqlmodel import SQLModel, Field, Relationship
 from typing import Union, TYPE_CHECKING
 
@@ -12,7 +13,9 @@ class JobAgent(SQLModel, table=True):
     id: Union[int, None] = Field(default=None, primary_key=True)
     task_in_progress: int
 
-    job_id: int = Field(foreign_key="job.internal_id")
+    job_id: int = Field(
+        sa_column=Column(ForeignKey("job.internal_id", ondelete="CASCADE"))
+    )
     job: "Job" = Relationship(back_populates="agents")
 
     agent_id: int = Field(foreign_key="agentgroup.id")
diff --git a/src/models/match.py b/src/models/match.py
index 23980822..7e089688 100644
--- a/src/models/match.py
+++ b/src/models/match.py
@@ -1,3 +1,4 @@
+from sqlalchemy import ForeignKey
 from sqlmodel import SQLModel, Field, ARRAY, String, Column, JSON, Relationship
 from typing import List, Union, Dict, Any
 
@@ -15,5 +16,7 @@ class Match(SQLModel, table=True):
     # A list of yara rules matched to this file
     matches: List[str] = Field(sa_column=Column(ARRAY(String)))
 
-    job_id: int = Field(foreign_key="job.internal_id")
+    job_id: int = Field(
+        sa_column=Column(ForeignKey("job.internal_id", ondelete="CASCADE"))
+    )
     job: Job = Relationship(back_populates="matches")

From 855380ba34ed10ebd7176d9f34b9d3d1e1518d7c Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Thu, 17 Oct 2024 10:37:10 +0200
Subject: [PATCH 17/25] 'version' is obsolete

---
 docker-compose.dev.yml | 1 -
 docker-compose.yml     | 1 -
 2 files changed, 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a078ff9a..ab32cfd3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   dev-frontend:
     build:
diff --git a/docker-compose.yml b/docker-compose.yml
index 722c9ca2..b48ff248 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   web:
     restart: always

From 34a5d2dbbc9f19786fdf68ceee01f726ae148a11 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Thu, 17 Oct 2024 10:43:11 +0200
Subject: [PATCH 18/25] 'version' is obsolete

---
 docker-compose.dev.yml | 1 +
 docker-compose.yml     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ab32cfd3..a078ff9a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,3 +1,4 @@
+version: '3'
 services:
   dev-frontend:
     build:
diff --git a/docker-compose.yml b/docker-compose.yml
index b48ff248..722c9ca2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,3 +1,4 @@
+version: '3'
 services:
   web:
     restart: always

From 930b064f5bc4028dee6956041235a4a081f398f9 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Thu, 17 Oct 2024 10:50:00 +0200
Subject: [PATCH 19/25] fix migrations

---
 .../versions/6b495d5a4855_add_jobstatus.py    | 54 ++++++++++---------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/src/migrations/versions/6b495d5a4855_add_jobstatus.py b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
index 3afb5166..6153d8b8 100644
--- a/src/migrations/versions/6b495d5a4855_add_jobstatus.py
+++ b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
@@ -1,41 +1,43 @@
-"""add jobagent
-Revision ID: dbb81bd4d47f
-Revises: cbbba858deb0
-Create Date: 2024-05-29 13:13:03.980030
+"""add jobstatus
+Revision ID: 6b495d5a4855
+Revises: dbb81bd4d47f
+Create Date: 2024-10-15 08:17:30.036531
 """
 from alembic import op
 import sqlalchemy as sa
 
+from models.job import JobStatus
 
 # revision identifiers, used by Alembic.
-revision = "dbb81bd4d47f"
-down_revision = "cbbba858deb0"
+revision = "6b495d5a4855"
+down_revision = "dbb81bd4d47f"
 branch_labels = None
 depends_on = None
 
+job_status = sa.Enum(JobStatus, name="jobstatus")
+
 
 def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "jobagent",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("task_in_progress", sa.Integer(), nullable=False),
-        sa.Column("job_id", sa.Integer(), nullable=False),
-        sa.Column("agent_id", sa.Integer(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["agent_id"],
-            ["agentgroup.id"],
-        ),
-        sa.ForeignKeyConstraint(
-            ["job_id"],
-            ["job.internal_id"],
-        ),
-        sa.PrimaryKeyConstraint("id"),
+    op.execute("DELETE FROM job WHERE status = 'removed';")
+
+    job_status.create(op.get_bind())
+    op.alter_column(
+        "job",
+        "status",
+        existing_type=sa.VARCHAR(),
+        type_=job_status,
+        postgresql_using="status::jobstatus",
+        nullable=True,
     )
-    # ### end Alembic commands ###
 
 
 def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table("jobagent")
-    # ### end Alembic commands ###
+    op.alter_column(
+        "job",
+        "status",
+        existing_type=job_status,
+        type_=sa.VARCHAR(),
+        nullable=False,
+    )
+
+    op.execute("DROP TYPE IF EXISTS jobstatus")

From 4bb1b8931eb0907baed38ebe53ac100da01949e3 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Thu, 17 Oct 2024 11:32:00 +0200
Subject: [PATCH 20/25] .

---
 docker-compose.dev.yml | 1 -
 docker-compose.yml     | 1 -
 2 files changed, 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a078ff9a..ab32cfd3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   dev-frontend:
     build:
diff --git a/docker-compose.yml b/docker-compose.yml
index 722c9ca2..b48ff248 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   web:
     restart: always

From beabd176a4de10ca83f5307d9e26299af7fa78e1 Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Thu, 17 Oct 2024 11:35:41 +0200
Subject: [PATCH 21/25] .

---
 docker-compose.dev.yml | 1 +
 docker-compose.yml     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ab32cfd3..a078ff9a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,3 +1,4 @@
+version: '3'
 services:
   dev-frontend:
     build:
diff --git a/docker-compose.yml b/docker-compose.yml
index b48ff248..722c9ca2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,3 +1,4 @@
+version: '3'
 services:
   web:
     restart: always

From a63cdf166cb9dc3e8bab011eb30145dbb27a0de0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Jura?= <michal.jura@nask.pl>
Date: Thu, 17 Oct 2024 17:19:42 +0200
Subject: [PATCH 22/25] fix import in add_jobstatus migration

---
 src/migrations/versions/6b495d5a4855_add_jobstatus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/migrations/versions/6b495d5a4855_add_jobstatus.py b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
index 6153d8b8..18e3cb9c 100644
--- a/src/migrations/versions/6b495d5a4855_add_jobstatus.py
+++ b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
@@ -6,7 +6,7 @@
 from alembic import op
 import sqlalchemy as sa
 
-from models.job import JobStatus
+from mquery.models.job import JobStatus # type: ignore # noqa
 
 # revision identifiers, used by Alembic.
 revision = "6b495d5a4855"

From 7835dce20dfc45c0749c4d722b24f8e2b4761c73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Jura?= <michal.jura@nask.pl>
Date: Thu, 17 Oct 2024 17:27:17 +0200
Subject: [PATCH 23/25] black

---
 src/migrations/versions/6b495d5a4855_add_jobstatus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/migrations/versions/6b495d5a4855_add_jobstatus.py b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
index 18e3cb9c..b522f71a 100644
--- a/src/migrations/versions/6b495d5a4855_add_jobstatus.py
+++ b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
@@ -6,7 +6,7 @@
 from alembic import op
 import sqlalchemy as sa
 
-from mquery.models.job import JobStatus # type: ignore # noqa
+from mquery.models.job import JobStatus  # type: ignore # noqa
 
 # revision identifiers, used by Alembic.
 revision = "6b495d5a4855"

From 3db4bfa46c11cf658430d8fc2004deed5d4295cb Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Mon, 21 Oct 2024 17:11:42 +0200
Subject: [PATCH 24/25] After review - migration modified

---
 requirements.txt                              |  1 -
 .../versions/6b495d5a4855_add_jobstatus.py    | 43 -------------------
 ...l_removed_status_create_enum_jobstatus.py} | 37 +++++++++++++---
 src/mqueryfront/src/utils.js                  |  1 -
 4 files changed, 31 insertions(+), 51 deletions(-)
 delete mode 100644 src/migrations/versions/6b495d5a4855_add_jobstatus.py
 rename src/migrations/versions/{f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py => 6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py} (64%)

diff --git a/requirements.txt b/requirements.txt
index 5ceb863b..27da0663 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -32,4 +32,3 @@ uvicorn==0.30.6
 wrapt==1.16.0
 yara-python==4.5.1
 yaramod==3.23.0
-alembic_postgresql_enum
\ No newline at end of file
diff --git a/src/migrations/versions/6b495d5a4855_add_jobstatus.py b/src/migrations/versions/6b495d5a4855_add_jobstatus.py
deleted file mode 100644
index b522f71a..00000000
--- a/src/migrations/versions/6b495d5a4855_add_jobstatus.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""add jobstatus
-Revision ID: 6b495d5a4855
-Revises: dbb81bd4d47f
-Create Date: 2024-10-15 08:17:30.036531
-"""
-from alembic import op
-import sqlalchemy as sa
-
-from mquery.models.job import JobStatus  # type: ignore # noqa
-
-# revision identifiers, used by Alembic.
-revision = "6b495d5a4855"
-down_revision = "dbb81bd4d47f"
-branch_labels = None
-depends_on = None
-
-job_status = sa.Enum(JobStatus, name="jobstatus")
-
-
-def upgrade() -> None:
-    op.execute("DELETE FROM job WHERE status = 'removed';")
-
-    job_status.create(op.get_bind())
-    op.alter_column(
-        "job",
-        "status",
-        existing_type=sa.VARCHAR(),
-        type_=job_status,
-        postgresql_using="status::jobstatus",
-        nullable=True,
-    )
-
-
-def downgrade() -> None:
-    op.alter_column(
-        "job",
-        "status",
-        existing_type=job_status,
-        type_=sa.VARCHAR(),
-        nullable=False,
-    )
-
-    op.execute("DROP TYPE IF EXISTS jobstatus")
diff --git a/src/migrations/versions/f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py b/src/migrations/versions/6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py
similarity index 64%
rename from src/migrations/versions/f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py
rename to src/migrations/versions/6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py
index 1fc886b9..f8a4dd2b 100644
--- a/src/migrations/versions/f654d7e4fcc7_on_delete_cascade_for_jobagent_and_match.py
+++ b/src/migrations/versions/6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py
@@ -1,17 +1,20 @@
-"""on delete cascade for jobagent and match
-Revision ID: f654d7e4fcc7
-Revises: 6b495d5a4855
-Create Date: 2024-10-17 07:16:34.262079
+"""add jobstatus
+Revision ID: 6b495d5a4855
+Revises: dbb81bd4d47f
+Create Date: 2024-10-15 08:17:30.036531
 """
 from alembic import op
+import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = "f654d7e4fcc7"
-down_revision = "6b495d5a4855"
+revision = "6b495d5a4855"
+down_revision = "dbb81bd4d47f"
 branch_labels = None
 depends_on = None
 
+job_status = sa.Enum("done", "new", "cancelled", "processing", name="jobstatus")
+
 
 def upgrade() -> None:
     op.drop_constraint("jobagent_job_id_fkey", "jobagent", type_="foreignkey")
@@ -34,8 +37,30 @@ def upgrade() -> None:
         ondelete="CASCADE",
     )
 
+    op.execute("DELETE FROM job WHERE status = 'removed';")
+
+    job_status.create(op.get_bind())
+    op.alter_column(
+        "job",
+        "status",
+        existing_type=sa.VARCHAR(),
+        type_=job_status,
+        postgresql_using="status::jobstatus",
+        nullable=True,
+    )
+
 
 def downgrade() -> None:
+    op.alter_column(
+        "job",
+        "status",
+        existing_type=job_status,
+        type_=sa.VARCHAR(),
+        nullable=False,
+    )
+
+    op.execute("DROP TYPE IF EXISTS jobstatus")
+
     op.drop_constraint("jobagent_job_id_fkey", "jobagent", type_="foreignkey")
     op.create_foreign_key(
         constraint_name="jobagent_job_id_fkey",
diff --git a/src/mqueryfront/src/utils.js b/src/mqueryfront/src/utils.js
index 3bde2acd..3d054721 100644
--- a/src/mqueryfront/src/utils.js
+++ b/src/mqueryfront/src/utils.js
@@ -15,7 +15,6 @@ export const openidLoginUrl = (config) => {
     if (config["openid_url"] === null || config["openid_client_id"] === null) {
         // Defensive programming - config keys can be null.
         return "#";
-        gi;
     }
     const login_url = new URL(config["openid_url"] + "/auth");
     login_url.searchParams.append("client_id", config["openid_client_id"]);

From d9da4c573350ef2a3bb75714621ee34ede5fcd5d Mon Sep 17 00:00:00 2001
From: michalkrzem <michal_krzem@wp.pl>
Date: Mon, 21 Oct 2024 17:15:35 +0200
Subject: [PATCH 25/25] black

---
 ...gent_match_and_del_removed_status_create_enum_jobstatus.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/migrations/versions/6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py b/src/migrations/versions/6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py
index f8a4dd2b..5a57cb63 100644
--- a/src/migrations/versions/6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py
+++ b/src/migrations/versions/6b495d5a4855_cascade_jobagent_match_and_del_removed_status_create_enum_jobstatus.py
@@ -13,7 +13,9 @@
 branch_labels = None
 depends_on = None
 
-job_status = sa.Enum("done", "new", "cancelled", "processing", name="jobstatus")
+job_status = sa.Enum(
+    "done", "new", "cancelled", "processing", name="jobstatus"
+)
 
 
 def upgrade() -> None: