From 619ba31594381a59b27bf3a33c626b366f22deb2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 7 Feb 2025 15:39:39 +0100 Subject: [PATCH 1/6] Fix recalculate expected files for movies renderMain_1.mov caused issues as was recalculated by frames provided into multiple files. --- .../validate_expected_and_rendered_files.py | 65 +++++++++++-------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py index 516f5b559d..b6b80b8ca2 100644 --- a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py +++ b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py @@ -5,6 +5,7 @@ import clique from ayon_core.pipeline import PublishValidationError +from ayon_core.lib.transcoding import VIDEO_EXTENSIONS class ValidateExpectedFiles(pyblish.api.InstancePlugin): @@ -41,34 +42,10 @@ def process(self, instance): staging_dir = repre["stagingDir"] existing_files = self._get_existing_files(staging_dir) - if self.allow_user_override: - # We always check for user override because the user might have - # also overridden the Job frame list to be longer than the - # originally submitted frame range - # todo: We should first check if Job frame range was overridden - # at all so we don't unnecessarily override anything - collection_or_filename = self._get_collection(expected_files) - job_expected_files = self._get_job_expected_files( - collection_or_filename, frame_list) - - job_files_diff = job_expected_files.difference(expected_files) - if job_files_diff: - self.log.debug( - "Detected difference in expected output files from " - "Deadline job. Assuming an updated frame list by the " - "user. Difference: {}".format(sorted(job_files_diff)) - ) - - # Update the representation expected files - self.log.info("Update range from actual job range " - "to frame list: {}".format(frame_list)) - # single item files must be string not list - repre["files"] = (sorted(job_expected_files) - if len(job_expected_files) > 1 else - list(job_expected_files)[0]) - - # Update the expected files - expected_files = job_expected_files + is_video = f'.{repre["ext"]}' in VIDEO_EXTENSIONS + if self.allow_user_override and not is_video: + expected_files = self._recalculate_expected_files( + expected_files, frame_list, repre) # We don't use set.difference because we do allow other existing # files to be in the folder that we might not want to use. @@ -84,6 +61,36 @@ def process(self, instance): ) ) + def _recalculate_expected_files(self, expected_files, frame_list, repre): + # We always check for user override because the user might have + # also overridden the Job frame list to be longer than the + # originally submitted frame range + # todo: We should first check if Job frame range was overridden + # at all so we don't unnecessarily override anything + collection_or_filename = self._get_collection(expected_files) + job_expected_files = self._get_job_expected_files( + collection_or_filename, frame_list) + + job_files_diff = job_expected_files.difference(expected_files) + if job_files_diff: + self.log.debug( + "Detected difference in expected output files from " + "Deadline job. Assuming an updated frame list by the " + "user. Difference: {}".format(sorted(job_files_diff)) + ) + + # Update the representation expected files + self.log.info("Update range from actual job range " + "to frame list: {}".format(frame_list)) + # single item files must be string not list + repre["files"] = (sorted(job_expected_files) + if len(job_expected_files) > 1 else + list(job_expected_files)[0]) + + # Update the expected files + expected_files = job_expected_files + return expected_files + def _get_dependent_job_ids(self, instance): """Returns list of dependent job ids from instance metadata.json @@ -180,9 +187,11 @@ def _get_collection(self, files) -> "Iterable[str]": collections, remainder = clique.assemble( files, minimum_items=1, patterns=patterns) if collections: + self.log.info(f"collections::{collections}") return collections[0] else: # No sequence detected, we assume single frame + self.log.info(f"remainder::{remainder}") return remainder[0] def _get_job_info(self, instance, job_id): From f8a6e2ef4b0e0041506eac55fab44158b6ad330c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 7 Feb 2025 18:24:58 +0100 Subject: [PATCH 2/6] Change to positive check for image type Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../publish/global/validate_expected_and_rendered_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py index b6b80b8ca2..72b83262be 100644 --- a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py +++ b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py @@ -42,8 +42,8 @@ def process(self, instance): staging_dir = repre["stagingDir"] existing_files = self._get_existing_files(staging_dir) - is_video = f'.{repre["ext"]}' in VIDEO_EXTENSIONS - if self.allow_user_override and not is_video: + is_image = f'.{repre["ext"]}' in IMAGE_EXTENSIONS + if self.allow_user_override and is_image: expected_files = self._recalculate_expected_files( expected_files, frame_list, repre) From 6c68cb1c5c24e74ff759088d63ad0bf9fc60f8a5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 7 Feb 2025 18:25:06 +0100 Subject: [PATCH 3/6] Change to positive check for image type Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../publish/global/validate_expected_and_rendered_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py index 72b83262be..01eb595c1a 100644 --- a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py +++ b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py @@ -5,7 +5,7 @@ import clique from ayon_core.pipeline import PublishValidationError -from ayon_core.lib.transcoding import VIDEO_EXTENSIONS +from ayon_core.lib.transcoding import IMAGE_EXTENSIONS class ValidateExpectedFiles(pyblish.api.InstancePlugin): From b2268c6ae5450b65044c364cac8f098460146051 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 11 Feb 2025 11:10:26 +0100 Subject: [PATCH 4/6] Removed dev logging --- .../publish/global/validate_expected_and_rendered_files.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py index b6b80b8ca2..b3875bce65 100644 --- a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py +++ b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py @@ -187,11 +187,9 @@ def _get_collection(self, files) -> "Iterable[str]": collections, remainder = clique.assemble( files, minimum_items=1, patterns=patterns) if collections: - self.log.info(f"collections::{collections}") return collections[0] else: # No sequence detected, we assume single frame - self.log.info(f"remainder::{remainder}") return remainder[0] def _get_job_info(self, instance, job_id): From 091d74d9d029bd4e329d4b377e8c51a3fbf51089 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 11 Feb 2025 12:33:38 +0100 Subject: [PATCH 5/6] Extension query more safe It was highlighed that `repre["ext"]` might not be required. This should be safer. --- .../global/validate_expected_and_rendered_files.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py index d353abc485..e30d0e2fe7 100644 --- a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py +++ b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py @@ -42,7 +42,7 @@ def process(self, instance): staging_dir = repre["stagingDir"] existing_files = self._get_existing_files(staging_dir) - is_image = f'.{repre["ext"]}' in IMAGE_EXTENSIONS + is_image = self._is_image(repre, expected_files) if self.allow_user_override and is_image: expected_files = self._recalculate_expected_files( expected_files, frame_list, repre) @@ -61,6 +61,14 @@ def process(self, instance): ) ) + def _is_image(self, repre, expected_files): + ext = repre.get("ext") # repre.ext might not be required + if not ext: + _, ext = os.path.splitext(expected_files[0]) + ext = ext.lstrip(".") + + return f'.{ext}' in IMAGE_EXTENSIONS + def _recalculate_expected_files(self, expected_files, frame_list, repre): # We always check for user override because the user might have # also overridden the Job frame list to be longer than the From 42695b8132a8e9ce2c57910d6430b33f2c1fde5b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 12 Feb 2025 17:30:12 +0100 Subject: [PATCH 6/6] Revert "Extension query more safe" This reverts commit 091d74d9d029bd4e329d4b377e8c51a3fbf51089. --- .../global/validate_expected_and_rendered_files.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py index e30d0e2fe7..d353abc485 100644 --- a/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py +++ b/client/ayon_deadline/plugins/publish/global/validate_expected_and_rendered_files.py @@ -42,7 +42,7 @@ def process(self, instance): staging_dir = repre["stagingDir"] existing_files = self._get_existing_files(staging_dir) - is_image = self._is_image(repre, expected_files) + is_image = f'.{repre["ext"]}' in IMAGE_EXTENSIONS if self.allow_user_override and is_image: expected_files = self._recalculate_expected_files( expected_files, frame_list, repre) @@ -61,14 +61,6 @@ def process(self, instance): ) ) - def _is_image(self, repre, expected_files): - ext = repre.get("ext") # repre.ext might not be required - if not ext: - _, ext = os.path.splitext(expected_files[0]) - ext = ext.lstrip(".") - - return f'.{ext}' in IMAGE_EXTENSIONS - def _recalculate_expected_files(self, expected_files, frame_list, repre): # We always check for user override because the user might have # also overridden the Job frame list to be longer than the