diff --git a/packit_service/worker/checker/koji.py b/packit_service/worker/checker/koji.py index 89c959a57c..e10a0a570f 100644 --- a/packit_service/worker/checker/koji.py +++ b/packit_service/worker/checker/koji.py @@ -7,6 +7,7 @@ KOJI_PRODUCTION_BUILDS_ISSUE, PERMISSIONS_ERROR_WRITE_OR_ADMIN, ) +from packit_service.models import SidetagModel from packit_service.worker.checker.abstract import Checker from packit_service.worker.events import ( MergeRequestGitlabEvent, @@ -55,3 +56,8 @@ def pre_check(self) -> bool: return False return True + + +class SidetagExists(Checker): + def pre_check(self) -> bool: + return SidetagModel.get_by_koji_name(self.data.tag_name) is not None diff --git a/packit_service/worker/handlers/abstract.py b/packit_service/worker/handlers/abstract.py index b9145a18de..ba9a8aa930 100644 --- a/packit_service/worker/handlers/abstract.py +++ b/packit_service/worker/handlers/abstract.py @@ -186,8 +186,9 @@ class TaskName(str, enum.Enum): github_fas_verification = "task.github_fas_verification" vm_image_build = "task.run_vm_image_build_handler" vm_image_build_result = "task.run_vm_image_build_result_handler" - pull_from_upstream = "pull_from_upstream" + pull_from_upstream = "task.pull_from_upstream" check_onboarded_projects = "task.check_onboarded_projects" + koji_build_tag = "task.koji_build_tag" class Handler(PackitAPIProtocol, Config): diff --git a/packit_service/worker/handlers/distgit.py b/packit_service/worker/handlers/distgit.py index c2df113221..1b7f63bd82 100644 --- a/packit_service/worker/handlers/distgit.py +++ b/packit_service/worker/handlers/distgit.py @@ -75,6 +75,7 @@ IssueCommentEvent, IssueCommentGitlabEvent, ) +from packit_service.worker.events.koji import KojiBuildTagEvent from packit_service.worker.events.new_hotness import NewHotnessUpdateEvent from packit_service.worker.handlers.abstract import ( JobHandler, @@ -877,6 +878,7 @@ def report_in_issue_repository(self, errors: dict[str, str]) -> None: @run_for_comment(command="koji-build") @reacts_to(event=PushPagureEvent) @reacts_to(event=PullRequestCommentPagureEvent) +@reacts_to(event=KojiBuildTagEvent) class DownstreamKojiBuildHandler( AbstractDownstreamKojiBuildHandler, ConfigFromEventMixin, @@ -922,6 +924,11 @@ def get_trigger_type_description(self) -> str: f"Fedora Koji build was triggered " f"by push with sha {self.data.commit_sha}." ) + elif self.data.event_type == KojiBuildTagEvent.__name__: + trigger_type_description += ( + f"Fedora Koji build was triggered " + f"by tagging of build {self.data.build_id} to {self.data.koji_tag_name}." + ) return trigger_type_description @staticmethod diff --git a/packit_service/worker/handlers/koji.py b/packit_service/worker/handlers/koji.py index c7f99ccb86..3f5c1fd9d0 100644 --- a/packit_service/worker/handlers/koji.py +++ b/packit_service/worker/handlers/koji.py @@ -6,23 +6,34 @@ """ import logging +from os import getenv from datetime import datetime from typing import Optional, Tuple, Type +from celery import signature + from ogr.abstract import GitProject from packit.config import ( JobConfig, JobType, ) from packit.config.package_config import PackageConfig +from packit.constants import DISTGIT_INSTANCES +from packit.utils.koji_helper import KojiHelper +from packit_service.config import PackageConfigGetter from packit_service.constants import ( KojiBuildState, ) from packit_service.constants import KojiTaskState +from packit_service.utils import ( + dump_job_config, + dump_package_config, +) from packit_service.models import ( AbstractProjectObjectDbType, KojiBuildTargetModel, ProjectEventModel, + SidetagModel, ) from packit_service.service.urls import ( get_koji_build_info_url, @@ -31,6 +42,7 @@ from packit_service.worker.checker.koji import ( PermissionOnKoji, IsJobConfigTriggerMatching, + SidetagExists, ) from packit_service.worker.events import ( CheckRerunCommitEvent, @@ -45,7 +57,7 @@ AbstractPRCommentEvent, ReleaseGitlabEvent, ) -from packit_service.worker.events.koji import KojiBuildEvent +from packit_service.worker.events.koji import KojiBuildEvent, KojiBuildTagEvent from packit_service.worker.handlers.abstract import ( JobHandler, TaskName, @@ -354,3 +366,85 @@ def run(self): self.build.set_build_logs_urls(koji_build_logs) return TaskResults(success=True, details={"msg": msg}) + + +@configured_as(job_type=JobType.koji_build_tag) +@reacts_to(event=KojiBuildTagEvent) +class KojiBuildTagHandler( + JobHandler, ConfigFromEventMixin, PackitAPIWithDownstreamMixin +): + task_name = TaskName.koji_build_tag + + _koji_helper: Optional[KojiHelper] = None + + @property + def koji_helper(self): + if not self._koji_helper: + self._koji_helper = KojiHelper() + return self._koji_helper + + @staticmethod + def get_checkers() -> Tuple[Type[Checker], ...]: + return (SidetagExists,) + + def run(self) -> TaskResults: + dg_base_url = getenv("DISTGIT_URL", DISTGIT_INSTANCES["fedpkg"].url) + sidetag = SidetagModel.get_by_koji_name(self.data.tag_name) + sidetag_group = sidetag.sidetag_group.name + builds = self.koji_helper.get_builds_in_tag(self.data.tag_name) + tagged_packages = {b["package_name"] for b in builds} + logger.debug(f"Packages tagged into {self.data.tag_name}: {tagged_packages}") + + packages_to_trigger = set() + for package_name in tagged_packages: + distgit_project_url = f"{dg_base_url}rpms/{package_name}" + project = self.service_config.get_project(url=distgit_project_url) + packages_config = PackageConfigGetter.get_package_config_from_repo( + base_project=None, + project=project, + pr_id=None, + reference=None, + fail_when_missing=True, + ) + for job in packages_config.get_job_views(): + if ( + job.type == JobType.koji_build + and job.sidetag_group == sidetag_group + ): + packages_to_trigger.update(job.dependents) + logger.debug(f"Packages to trigger: {packages_to_trigger}") + + for package_name in packages_to_trigger: + distgit_project_url = f"{dg_base_url}rpms/{package_name}" + project = self.service_config.get_project(url=distgit_project_url) + packages_config = PackageConfigGetter.get_package_config_from_repo( + base_project=None, + project=project, + pr_id=None, + reference=None, + fail_when_missing=True, + ) + for job in packages_config.get_job_views(): + if ( + job.type == JobType.koji_build + and job.sidetag_group == sidetag_group + ): + if not set(job.dependencies) <= tagged_packages: + logger.debug( + f"Dependencies of {job.type} not satisfied, " + f"missing: {set(job.dependencies) - tagged_packages}" + ) + continue + event_dict = self.data.get_dict() + event_dict["git_ref"] = sidetag.target + signature( + TaskName.downstream_koji_build.value, + kwargs={ + "event": event_dict, + "package_config": dump_package_config(packages_config), + "job_config": dump_job_config(job), + }, + ).apply_async() + + msg = f"Tag {self.data.tag_name} event handled." + return TaskResults(success=True, details={"msg": msg}) diff --git a/packit_service/worker/tasks.py b/packit_service/worker/tasks.py index bf59d454fe..631882b76e 100644 --- a/packit_service/worker/tasks.py +++ b/packit_service/worker/tasks.py @@ -72,7 +72,10 @@ PullFromUpstreamHandler, ) from packit_service.worker.handlers.forges import GithubFasVerificationHandler -from packit_service.worker.handlers.koji import KojiBuildReportHandler +from packit_service.worker.handlers.koji import ( + KojiBuildReportHandler, + KojiBuildTagHandler, +) from packit_service.worker.helpers.build.babysit import ( check_copr_build, check_pending_copr_builds, @@ -557,6 +560,12 @@ def babysit_vm_image_build(self, build_id: int): ) +@celery_app.task(name=TaskName.koji_build_tag, base=TaskWithRetry) +def run_koji_build_tag_handler(event: dict, package_config: dict, job_config: dict): + handler = KojiBuildTagHandler(package_config=None, job_config=None, event=event) + return get_handlers_task_results(handler.run_job(), event) + + def get_handlers_task_results(results: dict, event: dict) -> dict: # include original event to provide more info return {"job": results, "event": event} diff --git a/tests/conftest.py b/tests/conftest.py index 58e7684652..40cdc40889 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -550,6 +550,12 @@ def koji_build_completed_epel8(): return load_the_message_from_file(outfile) +@pytest.fixture() +def koji_build_tagged(): + with open(DATA_DIR / "fedmsg" / "koji_build_tagged.json") as outfile: + return load_the_message_from_file(outfile) + + def pytest_assertrepr_compare(op, left, right): if isinstance(left, JobConfig) and isinstance(right, JobConfig) and op == "==": from packit.schema import JobConfigSchema diff --git a/tests/data/fedmsg/koji_build_tagged.json b/tests/data/fedmsg/koji_build_tagged.json new file mode 100644 index 0000000000..5a05f315b6 --- /dev/null +++ b/tests/data/fedmsg/koji_build_tagged.json @@ -0,0 +1,26 @@ +{ + "body": { + "base_url": "https://koji.fedoraproject.org", + "build_id": 1234567, + "instance": "primary", + "name": "python-specfile", + "owner": "nforro", + "release": "1.fc40", + "tag": "f40-build-side-12345", + "tag_id": 12345, + "user": "nforro", + "version": "0.31.0" + }, + "headers": { + "fedora_messaging_rpm_python-specfile": true, + "fedora_messaging_schema": "koji_fedoramessaging.tag.TagV1", + "fedora_messaging_severity": 10, + "fedora_messaging_user_nforro": true, + "priority": 2, + "sent-at": "2024-06-20T15:04:24+00:00" + }, + "id": "0b2addd5-e1b8-4ebe-9e2a-9b1b21a8bbad", + "priority": 2, + "queue": null, + "topic": "org.fedoraproject.prod.buildsys.tag" +} diff --git a/tests/integration/test_listen_to_fedmsg.py b/tests/integration/test_listen_to_fedmsg.py index 3b9e8ab073..7f363fed4f 100644 --- a/tests/integration/test_listen_to_fedmsg.py +++ b/tests/integration/test_listen_to_fedmsg.py @@ -24,6 +24,7 @@ from packit.config.requirements import RequirementsConfig, LabelRequirementsConfig from packit.copr_helper import CoprHelper from packit.local_project import LocalProject +from packit.utils.koji_helper import KojiHelper from packit_service.config import PackageConfigGetter, ServiceConfig from packit_service.constants import COPR_API_FAIL_STATE, DEFAULT_RETRY_LIMIT from packit_service.models import ( @@ -37,6 +38,7 @@ BuildStatus, ProjectReleaseModel, GitBranchModel, + SidetagModel, ) from packit_service.service.urls import ( get_copr_build_info_url, @@ -45,6 +47,7 @@ ) from packit_service.worker.helpers.build.copr_build import CoprBuildJobHelper from packit_service.worker.events import AbstractCoprBuildEvent, KojiTaskEvent +from packit_service.worker.events.koji import KojiBuildTagEvent from packit_service.worker.handlers import CoprBuildEndHandler from packit_service.worker.jobs import SteveJobs from packit_service.worker.monitoring import Pushgateway @@ -53,6 +56,7 @@ run_copr_build_end_handler, run_copr_build_start_handler, run_koji_build_report_handler, + run_koji_build_tag_handler, run_testing_farm_handler, ) from packit_service.worker.helpers.testing_farm import TestingFarmJobHelper @@ -166,6 +170,56 @@ def pc_koji_build_pr(): ) +@pytest.fixture(scope="module") +def pc_koji_build_tag_specfile(): + return PackageConfig( + jobs=[ + JobConfig( + type=JobType.koji_build, + trigger=JobConfigTriggerType.commit, + sidetag_group="test", + dependents=["packit"], + packages={ + "specfile": CommonPackageConfig( + _targets=["fedora-all"], + specfile_path="python-specfile.spec", + ) + }, + ), + ], + packages={ + "specfile": CommonPackageConfig( + specfile_path="python-specfile.spec", + ), + }, + ) + + +@pytest.fixture(scope="module") +def pc_koji_build_tag_packit(): + return PackageConfig( + jobs=[ + JobConfig( + type=JobType.koji_build, + trigger=JobConfigTriggerType.koji_build, + sidetag_group="test", + dependencies=["python-specfile"], + packages={ + "packit": CommonPackageConfig( + _targets=["fedora-all"], + specfile_path="packit.spec", + ) + }, + ), + ], + packages={ + "packit": CommonPackageConfig( + specfile_path="packit.spec", + ), + }, + ) + + @pytest.fixture(scope="module") def pc_build_push(): return PackageConfig( @@ -2493,6 +2547,47 @@ def test_koji_build_end(koji_build_scratch_end, pc_koji_build_pr, koji_build_pr) assert first_dict_value(results["job"])["success"] +def test_koji_build_tag( + koji_build_tagged, pc_koji_build_tag_specfile, pc_koji_build_tag_packit +): + flexmock(KojiHelper).should_receive("get_build_info").with_args(1234567).and_return( + {"task_id": 7654321} + ) + + flexmock(KojiBuildTagEvent).should_receive("get_packages_config").and_return( + pc_koji_build_tag_specfile + ) + + flexmock(SidetagModel).should_receive("get_by_koji_name").with_args( + "f40-build-side-12345" + ).and_return(flexmock(sidetag_group=flexmock(name="test"), target="f40")) + + flexmock(KojiHelper).should_receive("get_builds_in_tag").with_args( + "f40-build-side-12345" + ).and_return([{"package_name": "python-specfile"}]) + + flexmock(PackageConfigGetter).should_receive( + "get_package_config_from_repo" + ).and_return(pc_koji_build_tag_specfile).and_return(pc_koji_build_tag_packit) + + flexmock(Signature).should_receive("apply_async").once() + flexmock(celery_group).should_receive("apply_async").once() + + processing_results = SteveJobs().process_message(koji_build_tagged) + event_dict, job, job_config, package_config = get_parameters_from_results( + processing_results + ) + assert json.dumps(event_dict) + + results = run_koji_build_tag_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + + assert first_dict_value(results["job"])["success"] + + def test_srpm_build_end(srpm_build_end, pc_build_pr, srpm_build_model): pr = flexmock(source_project=flexmock()) flexmock(GithubProject).should_receive("is_private").and_return(False) diff --git a/tests/unit/test_jobs.py b/tests/unit/test_jobs.py index effc0332e4..e8eb79a000 100644 --- a/tests/unit/test_jobs.py +++ b/tests/unit/test_jobs.py @@ -46,7 +46,11 @@ VMImageBuildResultEvent, AbstractCoprBuildEvent, ) -from packit_service.worker.events.koji import KojiBuildEvent, AbstractKojiEvent +from packit_service.worker.events.koji import ( + KojiBuildEvent, + KojiBuildTagEvent, + AbstractKojiEvent, +) from packit_service.worker.handlers import ( CoprBuildEndHandler, CoprBuildStartHandler, @@ -60,7 +64,10 @@ ) from packit_service.worker.handlers.bodhi import CreateBodhiUpdateHandler from packit_service.worker.handlers.distgit import DownstreamKojiBuildHandler -from packit_service.worker.handlers.koji import KojiBuildReportHandler +from packit_service.worker.handlers.koji import ( + KojiBuildReportHandler, + KojiBuildTagHandler, +) from packit_service.worker.jobs import SteveJobs, get_handlers_for_check_rerun from packit_service.worker.result import TaskResults @@ -893,6 +900,19 @@ {KojiBuildHandler}, id="Upstream Koji build on release on GitLab", ), + pytest.param( + KojiBuildTagEvent, + flexmock(job_config_trigger_type=JobConfigTriggerType.koji_build), + [ + JobConfig( + type=JobType.koji_build_tag, + trigger=JobConfigTriggerType.koji_build, + packages={"package": CommonPackageConfig()}, + ), + ], + {KojiBuildTagHandler}, + id="Koji build tagged", + ), ], ) def test_get_handlers_for_event(event_cls, db_project_object, jobs, result): @@ -909,7 +929,9 @@ def db_project_object(self): @property def packages_config(self): - return flexmock(get_job_views=lambda: jobs) + return flexmock( + get_job_views=lambda: jobs, packages={"package": CommonPackageConfig()} + ) event = Event() flexmock(ServiceConfig).should_receive("get_service_config").and_return(