From f653ac90ee4747e7f23c329d8eeb76105671a595 Mon Sep 17 00:00:00 2001 From: Martin Basti Date: Mon, 18 Nov 2024 23:07:37 +0100 Subject: [PATCH] cachi2: support both Cachito and Cachi2 User/admin may specify which version of remote sources should be used (or used by default): - 1: Cachito (current default) - 2: Cachi2 Build pipeline will then use the right task based on condition using result from the init task. STONEBLD-2591 Signed-off-by: Martin Basti --- atomic_reactor/cli/parser.py | 3 ++ atomic_reactor/config.py | 5 +++ atomic_reactor/inner.py | 2 + atomic_reactor/plugins/check_user_settings.py | 12 +++++ atomic_reactor/schemas/config.json | 6 +++ atomic_reactor/source.py | 1 + atomic_reactor/tasks/binary.py | 2 + tekton/pipelines/binary-container.yaml | 45 +++++++++++++++++++ tekton/tasks/binary-container-init.yaml | 12 ++++- tests/stubs.py | 1 + 10 files changed, 88 insertions(+), 1 deletion(-) diff --git a/atomic_reactor/cli/parser.py b/atomic_reactor/cli/parser.py index 22ebdbc5f..58a717a64 100644 --- a/atomic_reactor/cli/parser.py +++ b/atomic_reactor/cli/parser.py @@ -86,6 +86,9 @@ def parse_args(args: Optional[Sequence[str]] = None) -> dict: binary_container_init.set_defaults(func=task.binary_container_init) binary_container_init.add_argument("--platforms-result", metavar="FILE", default=None, help="file to write final platforms result") + binary_container_init.add_argument("--remote-sources-version-result", metavar="FILE", + default=None, + help="file to write final remote-sources version result") binary_container_cachito = tasks.add_parser( "binary-container-cachito", diff --git a/atomic_reactor/config.py b/atomic_reactor/config.py index dd2df3a15..2ec1819f1 100644 --- a/atomic_reactor/config.py +++ b/atomic_reactor/config.py @@ -187,6 +187,7 @@ class ReactorConfigKeys(object): OPERATOR_MANIFESTS_KEY = 'operator_manifests' IMAGE_SIZE_LIMIT_KEY = 'image_size_limit' BUILDER_CA_BUNDLE_KEY = 'builder_ca_bundle' + REMOTE_SOURCES_DEFAULT_VERSION = 'remote_sources_default_version' class ODCSConfig(object): @@ -511,3 +512,7 @@ def image_size_limit(self): @property def builder_ca_bundle(self): return self._get_value(ReactorConfigKeys.BUILDER_CA_BUNDLE_KEY, fallback=None) + + @property + def remote_sources_default_version(self): + return self._get_value(ReactorConfigKeys.REMOTE_SOURCES_DEFAULT_VERSION, fallback=1) diff --git a/atomic_reactor/inner.py b/atomic_reactor/inner.py index d61c264b6..2eabb11c6 100644 --- a/atomic_reactor/inner.py +++ b/atomic_reactor/inner.py @@ -462,6 +462,7 @@ def __init__( plugin_files: Optional[List[str]] = None, keep_plugins_running: bool = False, platforms_result: Optional[str] = None, + remote_sources_version_result: Optional[str] = None, annotations_result: Optional[str] = None, ): """ @@ -493,6 +494,7 @@ def __init__( self.source = source or DummySource(None, None) self.user_params = user_params or self._default_user_params.copy() self.platforms_result = platforms_result + self.remote_sources_version_result = remote_sources_version_result self.annotations_result = annotations_result self.keep_plugins_running = keep_plugins_running diff --git a/atomic_reactor/plugins/check_user_settings.py b/atomic_reactor/plugins/check_user_settings.py index ccffccc22..bd3e532f0 100644 --- a/atomic_reactor/plugins/check_user_settings.py +++ b/atomic_reactor/plugins/check_user_settings.py @@ -109,9 +109,21 @@ def validate_user_config_files(self): read_fetch_artifacts_url(self.workflow) read_content_sets(self.workflow) + def resolve_remote_sources_version(self) -> int: + """Resolve which version of remote sources should be used""" + version = self.workflow.source.config.remote_sources_version + if not version: + version = self.workflow.conf.remote_sources_default_version + + self.log.info("Remote sources version to be used: %d", version) + if self.workflow.remote_sources_version_result: + with open(self.workflow.remote_sources_version_result, 'w') as f: + f.write(f"{version}") + def run(self): """ run the plugin """ self.dockerfile_checks() self.validate_user_config_files() + self.resolve_remote_sources_version() diff --git a/atomic_reactor/schemas/config.json b/atomic_reactor/schemas/config.json index 472f46d1d..d0edcf9e9 100644 --- a/atomic_reactor/schemas/config.json +++ b/atomic_reactor/schemas/config.json @@ -708,6 +708,12 @@ } }, "additionalProperties": false + }, + "remote_sources_default_version": { + "description": "Default version of remote sources resolving which will be used when user doesn't explicitly specify it", + "type": "number", + "minimum": 1, + "maximum": 2 } }, "definitions": { diff --git a/atomic_reactor/source.py b/atomic_reactor/source.py index 4b2d2609b..7d9975238 100644 --- a/atomic_reactor/source.py +++ b/atomic_reactor/source.py @@ -72,6 +72,7 @@ def __init__(self, build_path): self.compose.pop('inherit', None) self.remote_source = self.data.get('remote_source') self.remote_sources = self.data.get('remote_sources') + self.remote_sources_version = self.data.get('remote_sources_version', 1) self.operator_manifests = self.data.get('operator_manifests') self.platforms = self.data.get('platforms') or {'not': [], 'only': []} diff --git a/atomic_reactor/tasks/binary.py b/atomic_reactor/tasks/binary.py index 6289169a4..9d52d192c 100644 --- a/atomic_reactor/tasks/binary.py +++ b/atomic_reactor/tasks/binary.py @@ -27,6 +27,7 @@ class InitTaskParams(TaskParams): """Binary container init task parameters""" platforms_result: Optional[str] + remote_sources_version_result: Optional[str] @dataclass(frozen=True) @@ -68,6 +69,7 @@ def prepare_workflow(self) -> inner.DockerBuildWorkflow: plugins_conf=self.plugins_conf, keep_plugins_running=self.keep_plugins_running, platforms_result=self._params.platforms_result, + remote_sources_version_result=self._params.remote_sources_version_result, ) return workflow diff --git a/tekton/pipelines/binary-container.yaml b/tekton/pipelines/binary-container.yaml index af2d17142..949efee48 100644 --- a/tekton/pipelines/binary-container.yaml +++ b/tekton/pipelines/binary-container.yaml @@ -7,6 +7,9 @@ spec: - name: osbs-image description: The location of the OSBS builder image (FQDN pullspec) type: string + - name: cachi2-image + description: The location of the Cachi2 image (FQDN pullspec) + type: string - name: user-params type: string description: User parameters in JSON format @@ -140,11 +143,53 @@ spec: value: "$(context.pipelineRun.name)" - name: user-params value: '$(params.user-params)' + when: + - input: "$(tasks.binary-container-init.results.remote_sources_version_result)" + operator: in + values: ["1"] + timeout: "0" + + - name: binary-container-cachi2 + runAfter: + - binary-container-init + taskRef: + name: binary-container-cachi2-0-1 + workspaces: + - name: ws-build-dir + workspace: ws-container + subPath: build-dir + - name: ws-context-dir + workspace: ws-container + subPath: context-dir + - name: ws-home-dir + workspace: ws-home-dir + - name: ws-registries-secret + workspace: ws-registries-secret + - name: ws-koji-secret + workspace: ws-koji-secret + - name: ws-reactor-config-map + workspace: ws-reactor-config-map + - name: ws-autobot-keytab + workspace: ws-autobot-keytab + params: + - name: osbs-image + value: "$(params.osbs-image)" + - name: cachi2-image + value: "$(params.cachi2-image)" + - name: pipeline-run-name + value: "$(context.pipelineRun.name)" + - name: user-params + value: '$(params.user-params)' + when: + - input: "$(tasks.binary-container-init.results.remote_sources_version_result)" + operator: in + values: ["2"] timeout: "0" - name: binary-container-prebuild runAfter: - binary-container-cachito + - binary-container-cachi2 taskRef: resolver: git params: diff --git a/tekton/tasks/binary-container-init.yaml b/tekton/tasks/binary-container-init.yaml index a54d9de5b..9da13b85f 100644 --- a/tekton/tasks/binary-container-init.yaml +++ b/tekton/tasks/binary-container-init.yaml @@ -27,6 +27,7 @@ spec: results: - name: platforms_result + - name: remote_sources_version_result stepTemplate: env: @@ -46,4 +47,13 @@ spec: cpu: 395m script: | set -x - atomic-reactor -v task --user-params='$(params.user-params)' --build-dir=$(workspaces.ws-build-dir.path) --context-dir=$(workspaces.ws-context-dir.path) --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml --namespace=$(context.taskRun.namespace) --pipeline-run-name="$(params.pipeline-run-name)" binary-container-init --platforms-result=$(results.platforms_result.path) + atomic-reactor -v task \ + --user-params='$(params.user-params)' \ + --build-dir=$(workspaces.ws-build-dir.path) \ + --context-dir=$(workspaces.ws-context-dir.path) \ + --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml \ + --namespace=$(context.taskRun.namespace) \ + --pipeline-run-name="$(params.pipeline-run-name)" \ + binary-container-init \ + --platforms-result=$(results.platforms_result.path) \ + --remote-sources-version-result=$(results.remote_sources_version_result.path) diff --git a/tests/stubs.py b/tests/stubs.py index 3c9a9d087..6c9bf2aa1 100644 --- a/tests/stubs.py +++ b/tests/stubs.py @@ -12,6 +12,7 @@ class StubConfig(object): image_build_method = None release_env_var = None + remote_sources_version = 1 class StubSource(object):