Store downstream Koji builds in DB (#2244)

TODO:

- [x] Openshift tests
- [x] TODO in code
- [ ] followup dashboard PR (the API was changed so this breaks
dashboard)


Fixes #1889 
Fixes #2243 

Merge after packit/packit#2146


---

RELEASE NOTES BEGIN
The downstream Koji builds are now stored in the database as well and
exposed via `/api/koji-builds` endpoint.
RELEASE NOTES END

alembic/versions/99841c3f8ba8_adjust_kojibuildtargetmodel.py

@@ -0,0 +1,40 @@
+"""Adjust KojiBuildTargetModel
+
+Revision ID: 99841c3f8ba8
+Revises: db64a37ff1c6
+Create Date: 2023-11-02 15:19:28.923208
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "99841c3f8ba8"
+down_revision = "db64a37ff1c6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column("koji_build_targets", "build_id", new_column_name="task_id")
+    op.execute(
+        "ALTER INDEX ix_koji_build_targets_build_id RENAME TO ix_koji_build_targets_task_id"
+    )
+    op.add_column(
+        "koji_build_targets", sa.Column("build_logs_urls", sa.JSON(), nullable=True)
+    )
+    # we can drop this, the logs URLs pointed to invalid URLs
+    op.drop_column("koji_build_targets", "build_logs_url")
+
+
+def downgrade():
+    op.alter_column("koji_build_targets", "task_id", new_column_name="build_id")
+    op.execute(
+        "ALTER INDEX ix_koji_build_targets_task_id RENAME TO ix_koji_build_targets_build_id"
+    )
+    op.add_column(
+        "koji_build_targets",
+        sa.Column("build_logs_url", sa.VARCHAR(), autoincrement=False, nullable=True),
+    )
+    op.drop_column("koji_build_targets", "build_logs_urls")

packit_service/models.py

@@ -1629,7 +1629,7 @@ def get_all_by_build_id(
         cls, build_id: Union[str, int]
     ) -> Iterable["CoprBuildTargetModel"]:
         if isinstance(build_id, int):
-            # See the comment in get_by_build_id()
+            # See the comment in get_by_task_id()
             build_id = str(build_id)
         return sa_session().query(CoprBuildTargetModel).filter_by(build_id=build_id)
 
@@ -1805,7 +1805,7 @@ class KojiBuildTargetModel(GroupAndTargetModelConnector, Base):
 
     __tablename__ = "koji_build_targets"
     id = Column(Integer, primary_key=True)
-    build_id = Column(String, index=True)  # koji build id
+    task_id = Column(String, index=True)  # ID of the Koji build task
 
     # what's the build status?
     status = Column(String)
@@ -1814,7 +1814,8 @@ class KojiBuildTargetModel(GroupAndTargetModelConnector, Base):
     # URL to koji web ui for the particular build
     web_url = Column(String)
     # url to koji build logs
-    build_logs_url = Column(String)
+    # dictionary with archs and links, e.g. {"x86_64": "my-url"}
+    build_logs_urls = Column(JSON)
     # datetime.utcnow instead of datetime.utcnow() because its an argument to the function
     # so it will run when the koji build is initiated, not when the table is made
     build_submitted_time = Column(DateTime, default=datetime.utcnow)
@@ -1838,19 +1839,19 @@ def set_status(self, status: str):
             self.status = status
             session.add(self)
 
-    def set_build_logs_url(self, build_logs: str):
+    def set_build_logs_urls(self, build_logs: dict):
         with sa_session_transaction() as session:
-            self.build_logs_url = build_logs
+            self.build_logs_urls = build_logs
             session.add(self)
 
     def set_web_url(self, web_url: str):
         with sa_session_transaction() as session:
             self.web_url = web_url
             session.add(self)
 
-    def set_build_id(self, build_id: str):
+    def set_task_id(self, task_id: str):
         with sa_session_transaction() as session:
-            self.build_id = build_id
+            self.task_id = task_id
             session.add(self)
 
     def set_build_start_time(self, build_start_time: Optional[DateTime]):
@@ -1873,6 +1874,11 @@ def set_scratch(self, value: bool):
             self.scratch = value
             session.add(self)
 
+    def set_data(self, data: dict):
+        with sa_session_transaction() as session:
+            self.data = data
+            session.add(self)
+
     def get_srpm_build(self) -> Optional["SRPMBuildModel"]:
         # All SRPMBuild models for all the runs have to be same.
         return (
@@ -1899,39 +1905,27 @@ def get_range(cls, first: int, last: int) -> Iterable["KojiBuildTargetModel"]:
         )
 
     @classmethod
-    def get_all_by_build_id(
-        cls, build_id: Union[str, int]
-    ) -> Iterable["KojiBuildTargetModel"]:
-        """
-        Returns all builds with that build_id, irrespective of target.
-        """
-        if isinstance(build_id, int):
-            # See the comment in get_by_build_id()
-            build_id = str(build_id)
-        return sa_session().query(KojiBuildTargetModel).filter_by(build_id=build_id)
-
-    @classmethod
-    def get_by_build_id(
-        cls, build_id: Union[str, int], target: Optional[str] = None
+    def get_by_task_id(
+        cls, task_id: Union[str, int], target: Optional[str] = None
     ) -> Optional["KojiBuildTargetModel"]:
         """
         Returns the first build matching the build_id and optionally the target.
         """
-        if isinstance(build_id, int):
+        if isinstance(task_id, int):
             # PG is pesky about this:
             #   LINE 3: WHERE koji_builds.build_id = 1245767 AND koji_builds.target ...
             #   HINT:  No operator matches the given name and argument type(s).
             #   You might need to add explicit type casts.
-            build_id = str(build_id)
-        query = sa_session().query(KojiBuildTargetModel).filter_by(build_id=build_id)
+            task_id = str(task_id)
+        query = sa_session().query(KojiBuildTargetModel).filter_by(task_id=task_id)
         if target:
             query = query.filter_by(target=target)
         return query.first()
 
     @classmethod
     def create(
         cls,
-        build_id: Optional[str],
+        task_id: Optional[str],
         web_url: Optional[str],
         target: str,
         status: str,
@@ -1940,7 +1934,7 @@ def create(
     ) -> "KojiBuildTargetModel":
         with sa_session_transaction() as session:
             build = cls()
-            build.build_id = build_id
+            build.task_id = task_id
             build.status = status
             build.web_url = web_url
             build.target = target
@@ -1958,7 +1952,7 @@ def get(
         build_id: str,
         target: str,
     ) -> Optional["KojiBuildTargetModel"]:
-        return cls.get_by_build_id(build_id, target)
+        return cls.get_by_task_id(build_id, target)
 
     def __repr__(self):
         return (
@@ -2911,7 +2905,7 @@ def get_all_by_build_id(
     ) -> Iterable["VMImageBuildTargetModel"]:
         """Returns all builds with that build_id, irrespective of target"""
         if isinstance(build_id, int):
-            # See the comment in get_by_build_id()
+            # See the comment in get_by_task_id()
             build_id = str(build_id)
         return sa_session().query(VMImageBuildTargetModel).filter_by(build_id=build_id)
 

packit_service/service/api/koji_builds.py

@@ -32,13 +32,14 @@ def get(self):
         for build in KojiBuildTargetModel.get_range(first, last):
             build_dict = {
                 "packit_id": build.id,
-                "build_id": build.build_id,
+                "task_id": build.task_id,
+                "scratch": build.scratch,
                 "status": build.status,
                 "build_submitted_time": optional_timestamp(build.build_submitted_time),
                 "chroot": build.target,
                 "web_url": build.web_url,
                 # from old data, sometimes build_logs_url is same and sometimes different to web_url
-                "build_logs_url": build.build_logs_url,
+                "build_logs_url": build.build_logs_urls,
                 "pr_id": build.get_pr_id(),
                 "branch_name": build.get_branch_name(),
                 "release": build.get_release_tag(),
@@ -77,16 +78,17 @@ def get(self, id):
             )
 
         build_dict = {
-            "build_id": build.build_id,
+            "task_id": build.task_id,
             "status": build.status,
             "chroot": build.target,
+            "scratch": build.scratch,
             "build_start_time": optional_timestamp(build.build_start_time),
             "build_finished_time": optional_timestamp(build.build_finished_time),
             "build_submitted_time": optional_timestamp(build.build_submitted_time),
             "commit_sha": build.commit_sha,
             "web_url": build.web_url,
             # from old data, sometimes build_logs_url is same and sometimes different to web_url
-            "build_logs_url": build.build_logs_url,
+            "build_logs_url": build.build_logs_urls,
             "srpm_build_id": build.get_srpm_build().id,
             "run_ids": sorted(run.id for run in build.group_of_targets.runs),
         }

packit_service/service/api/projects.py

@@ -167,7 +167,7 @@ def get(self, forge, namespace, repo_name):
 
             for build in pr.get_koji_builds():
                 build_info = {
-                    "build_id": build.build_id,
+                    "task_id": build.task_id,
                     "chroot": build.target,
                     "status": build.status,
                     "web_url": build.web_url,
@@ -275,7 +275,7 @@ def get(self, forge, namespace, repo_name):
 
             for build in branch.get_koji_builds():
                 build_info = {
-                    "build_id": build.build_id,
+                    "task_id": build.task_id,
                     "chroot": build.target,
                     "status": build.status,
                     "web_url": build.web_url,

packit_service/utils.py

@@ -5,6 +5,7 @@
 from datetime import datetime, timezone
 from io import StringIO
 from logging import StreamHandler
+from re import search
 from typing import List, Tuple
 
 from packit.config import JobConfig, PackageConfig
@@ -204,3 +205,20 @@ def get_packit_commands_from_comment(
             return packit_command
 
     return []
+
+
+def get_koji_task_id_and_url_from_stdout(stdout: str):
+    task_id, task_url = None, None
+
+    task_id_match = search(pattern=r"Created task: (\d+)", string=stdout)
+    if task_id_match:
+        task_id = int(task_id_match.group(1))
+
+    task_url_match = search(
+        pattern=r"(https://.+/koji/taskinfo\?taskID=\d+)",
+        string=stdout,
+    )
+    if task_url_match:
+        task_url = task_url_match.group(0)
+
+    return task_id, task_url