diff --git a/client/ayon_deadline_cloud/addon.py b/client/ayon_deadline_cloud/addon.py index 9dbd7b3..80db55b 100644 --- a/client/ayon_deadline_cloud/addon.py +++ b/client/ayon_deadline_cloud/addon.py @@ -2,12 +2,18 @@ from __future__ import annotations import os -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional -from ayon_core.addon import AYONAddon, IPluginPaths +import click +from ayon_core.addon import AYONAddon, IPluginPaths, click_wrap + +from ayon_deadline_cloud.api.publish import publish_content from .version import __version__ +if TYPE_CHECKING: + from logging import Logger + DEADLINE_CLOUD_ADDON_ROOT = os.path.dirname(os.path.abspath(__file__)) @@ -15,6 +21,7 @@ class DeadlineCloudAddon(AYONAddon, IPluginPaths): """Deadline Cloud Addon for AYON.""" name = "deadline_cloud" version = __version__ + log: Logger @staticmethod def add_implementation_envs( @@ -32,9 +39,9 @@ def add_implementation_envs( # Deadline Cloud configuration file path for the client to use. # env["DEADLINE_CONFIG_FILE_PATH"] = ... - @staticmethod - def get_publish_plugin_paths( - host_name: Optional[str] = None # noqa: ARG004 + def get_publish_plugin_paths( # noqa: PLR6301 + self, + host_name: str, ) -> list[str]: # ty:ignore[invalid-method-override] """Return list of paths to publish plugins. @@ -49,10 +56,10 @@ def get_publish_plugin_paths( return [os.path.join( DEADLINE_CLOUD_ADDON_ROOT, "plugins", "publish")] - @staticmethod - def get_create_plugin_paths( - host_name: Optional[str] = None - ) -> list[str]: # ty:ignore[invalid-method-override] + def get_create_plugin_paths( # noqa: PLR6301 + self, + host_name: str, + ) -> list[str]: """Return list of paths to creator plugins. Args: @@ -67,17 +74,123 @@ def get_create_plugin_paths( DEADLINE_CLOUD_ADDON_ROOT, "plugins", "create", host_name or "global")] - @staticmethod - def get_launch_hook_paths(app: str) -> list[str]: # noqa: ARG004 - """Return list of paths to launch hooks. + def _publish( # noqa: PLR0913, PLR0917 + self, + path: str, + folder_path: str, + project_name: str, + user_name: str, + product_base_type: str, + task_name: Optional[str], + host_name: Optional[str], + source_file: Optional[str], + ) -> None: + """Publish the result of a Deadline Cloud processed job. Args: - app: Name of the host application - to get specific hook paths. + path: Path to the folder containing the job + result to publish. + folder_path: Folder path for the context on AYON. + project_name: Name of the project associated with the job result. + user_name: Name of the user who submitted the job. + product_base_type: Base name of the product to publish. + Note that currently it is overridden down the line + with hardcoded `render` - in the future, product base type + should be passed correctly to support other publish + types. + task_name: Optional name of the task associated with + the job result. + host_name: Optional name of the host application associated with + the job result. + source_file: Optional path to a source file related to the job + result, which might be used for validation or as part of + the publishing process. - Returns: - List of paths to launch hooks. + """ + # TODO(antirotor): Implement this method to trigger publishing of the + # result of a Deadline Cloud processed job. This might involve + # collecting the output files from the job, validating them, and then + # moving them to their final destination or registering them in AYON. + self.log.debug( + "publish called with arguments: " + "folder_path=%s, project_name=%s, " + "user_name=%s, product_base_name=%s, task_name=%s, " + "host_name=%s, source_file=%s", + folder_path, project_name, user_name, + product_base_type, task_name, host_name, source_file + ) + + publish_content( + path=path, + project_name=project_name, + folder_path=folder_path, + user_name=user_name, + task_name=task_name, + host_name=host_name, + source_file=source_file, + ) + + def _cli_main(self) -> None: + """Add CLI commands to this addon.""" + + def cli(self, addon_click_group: click.Group) -> None: + """CLI interface. + + Args: + addon_click_group: Click group to add commands to. """ - return [os.path.join( - DEADLINE_CLOUD_ADDON_ROOT, "hooks")] + cli_main = click_wrap.group( + self._cli_main, + name=self.name, + help="Deadline Cloud commands", + ) + + cli_main.command( + self._publish, + name="publish", + help=( + "Publish the result of a Deadline Cloud " + "processed job." + ), + ).option( + "-f", + "--folder-path", + type=click.STRING, + required=True, + ).option( + "-t", + "--task-name", + type=click.STRING, + required=False, + ).option( + "-p", + "--project-name", + type=click.STRING, + required=True, + ).option( + "-u", + "--user-name", + type=click.STRING, + required=True, + ).option( + "--host-name", + type=click.STRING, + required=False, + ).option( + "--product-base-type", + type=click.STRING, + required=True + ).option( + "-s", + "--source-file", + type=click.Path(exists=True, file_okay=True, dir_okay=False), + ) + + cli_main.argument( + "path", + nargs=1, + type=click.Path(exists=True, file_okay=False, dir_okay=True), + ) + + addon_click_group.add_command(cli_main.to_click_obj()) diff --git a/client/ayon_deadline_cloud/api/datatypes.py b/client/ayon_deadline_cloud/api/datatypes.py new file mode 100644 index 0000000..9c96529 --- /dev/null +++ b/client/ayon_deadline_cloud/api/datatypes.py @@ -0,0 +1,35 @@ +"""Data classes.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Union + +if TYPE_CHECKING: + from ayon_core.pipeline.traits import Representation as TraitRepresentation + + +@dataclass +class StandardRepresentation: + """Representation dataclass.""" + name: str + ext: str + files: Union[str, list[str]] + stagingDir: str + + +@dataclass +class InstanceData: + """Instance dataclass.""" + publish: bool + active: bool + label: str + name: str + productName: str + productBaseType: str + family: str + families: list[str] + folderPath: str + task: str + variant: str + standard_representations: list[StandardRepresentation] + trait_representations: list[TraitRepresentation] diff --git a/client/ayon_deadline_cloud/api/publish.py b/client/ayon_deadline_cloud/api/publish.py new file mode 100644 index 0000000..c8d92c3 --- /dev/null +++ b/client/ayon_deadline_cloud/api/publish.py @@ -0,0 +1,102 @@ +"""Publishing Deadline Cloud Jobs.""" +from __future__ import annotations + +import contextlib +from typing import Optional + +import ayon_api +import pyblish.api +import pyblish.util +from ayon_core.pipeline import install_ayon_plugins +from ayon_core.pipeline.publish import publish_plugins_discover + + +def publish_content( # noqa: PLR0913, PLR0917 + path: str, + project_name: str, + folder_path: str, + user_name: str, + task_name: Optional[str] = None, + host_name: Optional[str] = None, + source_file: Optional[str] = None, + ) -> None: + """Publish content. + + This function bootstraps pyblish to run, collect the + rendered files and publish them using appropriate pipeline. + + Args: + path: Path to the folder where the content is to be published. + project_name: Name of the project. + folder_path: Path to the folder where the content is to be published. + user_name: Name of the user who submitted the job. + task_name: Name of the task to be published. + host_name: Name of the host to be published. + source_file: Path to the source file. + + Raises: + ValueError: If the folder or task does not exist. + RuntimeError: If the publishing process fails. + + """ + # Make public ayon api behave as other user + # - this works only if public ayon api is using service user: + # ayon-python-api does not have public api function to find + # out if is used service user. So we need to have try-except. + con = ayon_api.get_server_api_connection() + with contextlib.suppress(ValueError): + con.set_default_service_username(user_name) + + # check if folder exists + folder_entity = ayon_api.get_folder_by_path( + project_name=project_name, + folder_path=folder_path, + ) + if not folder_entity: + msg = ( + f"Unable to find folder '{folder_path}' in " + f"project '{project_name}'." + ) + raise ValueError(msg) + + # check if task exists + if task_name: + task_entity = ayon_api.get_task_by_name( + project_name=project_name, + folder_id=folder_entity["id"], + task_name=task_name, + ) + if not task_entity: + msg = ( + f"Unable to find task '{task_name}' in " + f"folder '{folder_path}' in project '{project_name}'." + ) + raise ValueError(msg) + + pyblish_context = pyblish.api.Context() + pyblish_context.data["hostName"] = "workflow" + pyblish_context.data["projectName"] = project_name + pyblish_context.data["folderPath"] = folder_path + pyblish_context.data["outputPath"] = path + + if task_name: + pyblish_context.data["taskName"] = task_name + + if host_name: + pyblish_context.data["hostName"] = host_name + + if source_file: + pyblish_context.data["sourceFile"] = source_file + + pyblish.api.register_host("shell") + + install_ayon_plugins() + discover_result = publish_plugins_discover() + publish_plugins = discover_result.plugins + + for result in pyblish.util.publish_iter( + context=pyblish_context, + plugins=publish_plugins, + ): + if result["error"]: + raise RuntimeError(repr(result)) diff --git a/client/ayon_deadline_cloud/plugins/publish/add_publish_job_step.py b/client/ayon_deadline_cloud/plugins/publish/add_publish_job_step.py new file mode 100644 index 0000000..5fe73c1 --- /dev/null +++ b/client/ayon_deadline_cloud/plugins/publish/add_publish_job_step.py @@ -0,0 +1,183 @@ +"""Add publishing to the job template. + +This plugin gets `instance.data["deadline_cloud_job_data"]["job_template"]` +and iterates over "steps". + +First, it needs to add hostRequirement defined in the Settings to +differentiate between machines that can render and those that can publish. + +See the discussion here: + https://github.com/ynput/ayon-deadline-cloud/issues/8 + +Then it needs to add publish step after the render step with the dependency +on the previous rendering steps. + +""" +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Any, ClassVar + +import pyblish.api +from ayon_core.pipeline.publish import PublishError + +if TYPE_CHECKING: + from logging import Logger + + +class AddPublishingStep(pyblish.api.InstancePlugin): + """Add publishing to the job template.""" + label = "Add Publishing Step to the Job Template" + # make sure it runs after the data is collected + order = pyblish.api.IntegratorOrder + targets: ClassVar[list[str]] = ["local"] + families: ClassVar[list[str]] = ["deadline_cloud"] + log: Logger + + def process(self, instance: pyblish.api.Instance) -> None: + """Process this instance. + + Args: + instance (pyblish.api.Instance): Instance. + + Raises: + PublishError: If job template is missing data. + + """ + ayon_settings = instance.context.data.get("project_settings", {}) + dc_settings = ayon_settings.get("deadline_cloud", {}) + render_roles: list[str] = dc_settings.get( + "render_host_requirement_roles", ["render"]) + publish_roles: list[str] = dc_settings.get( + "publish_host_requirement_roles", ["publish"]) + + self.log.debug( + "Adding render roles '%s' and " + "publish roles '%s' to the job template", + render_roles, + publish_roles, + ) + + job_template = instance.data["deadline_cloud_job_data"]["job_template"] + try: + steps: list[dict] = job_template["steps"] + except KeyError as e: + msg = "Job template is missing 'steps' key" + raise PublishError(msg) from e + + self._add_render_roles_to_steps(render_roles, steps) + self._add_publishing_step(publish_roles, steps) + + def _add_render_roles_to_steps( + self, render_roles: list[str], steps: list[dict]) -> None: + """Add render roles to steps. + + This method adds `render_roles` to the render steps in the + job template. Since there is no (easy) way to find out if the step is + rendering or not, we assume that all steps currently are. Therefore, + this has to run before adding publishing step. + + Args: + render_roles (list[str]): The render role names. + steps (list): Steps. + + """ + # this has to be set on either the fleet or the specific + # worker machine + render_role_attr: dict[str, Any] = { + "name": "attr.role", + "anyOf": render_roles, + } + for step in steps: + host_requirements = step.get("hostRequirements") + if host_requirements is None: + host_requirements = step["hostRequirements"] = {} + attributes: list[dict[str, Any]] = host_requirements.get( + "attributes", [] + ) + if not attributes: + host_requirements["attributes"] = attributes + if not any( + a.get("name") == render_role_attr["name"] + and a.get("anyOf") == render_role_attr["anyOf"] + for a in attributes + ): + self.log.debug("adding 'render' role to host " + "requirement for the step '%s'", step["name"]) + attributes.append(render_role_attr) + + @staticmethod + def _add_publishing_step( + publish_role: list[str], + steps: list[dict]) -> None: + """Add publishing step to the job template. + + Args: + publish_role (str): The publishing role name. + steps (list): Steps. + + """ + render_step_names: list[str] = [s["name"] for s in steps] + publishing_step = { + "name": "publish to AYON", + "description": "Publish rendering result to AYON", + "dependencies": [ + {"dependsOn": name} for name in render_step_names + ], + "hostRequirements": { + "attributes": [{"name": "attr.role", "anyOf": publish_role}] + }, + "stepEnvironments": [ + { + "name": "CondaEnv", + "description": "Set to disable conda", + "variables": { + "DISABLE_CONDA_ENV": "true", + }, + }, + { + "name": "AYONEnv", + "description": "Set AYON env", + "variables": { + "AYON_STUDIO_BUNDLE_NAME": ( + os.environ["AYON_STUDIO_BUNDLE_NAME"] + ), + "AYON_BUNDLE_NAME": os.environ["AYON_BUNDLE_NAME"], + }, + }, + ], + "script": { + "embeddedFiles": [ + { + "name": "Publish", + "filename": "ayon_publish.sh", + "type": "TEXT", + "data": """ +#!/bin/bash +set -xeuo pipefail + +echo "Running publish step for AYON Deadline Cloud addon..." +ayon addon deadline_cloud publish \ + --folder "{{Param.folderPath}}" \ + --task-name "{{Param.taskName}}" \ + --project-name "{{Param.projectName}}" \ + --user-name "{{Param.userName}}" \ + --host-name "{{Param.hostName}}" \ + --product-base-type "{{Param.productBaseType}}" \ + --source-file "{{Param.sourceFile}}" \ + "{{Param.OutputFilePath}}" + """, + } + ], + "actions": { + "onRun": { + "command": "bash", + "args": [ + "{{Task.File.Publish}}", + ], + } + }, + }, + } + + steps.append(publishing_step) diff --git a/client/ayon_deadline_cloud/plugins/publish/collect_deadline_cloud_job_data.py b/client/ayon_deadline_cloud/plugins/publish/collect_deadline_cloud_job_data.py index 8cf0cbf..69089d3 100644 --- a/client/ayon_deadline_cloud/plugins/publish/collect_deadline_cloud_job_data.py +++ b/client/ayon_deadline_cloud/plugins/publish/collect_deadline_cloud_job_data.py @@ -3,7 +3,7 @@ import dataclasses import os -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Optional import pyblish.api from ayon_core.lib import TextDef @@ -41,6 +41,37 @@ class CollectDeadlineCloudJobData( hosts: ClassVar[list[str]] = ["maya"] log: Logger + @staticmethod + def add_ayon_context_parameter( + param_defs: list[dict[str, Any]], + name: str, + default: str, + description: Optional[str], + ) -> None: + """Add an AYON context string parameter. + + Adds STRING parameter to template with AYON + namespace. + + Args: + param_defs: List of parameter definitions to append to. + name: Name of the parameter (without namespace). + default: Default value for the parameter. + description: Optional description for the parameter. + + """ + param = { + "name": f"{name}", + "type": "STRING", + "userInterface": { + "control": "HIDDEN", + }, + "default": f"{default}", + } + if description is not None: + param["description"] = description + param_defs.append(param) + @classmethod def get_attr_defs_for_instance( cls, @@ -63,7 +94,7 @@ def get_attr_defs_for_instance( ) ] - def process(self, instance: pyblish.api.Instance) -> None: + def process(self, instance: pyblish.api.Instance) -> None: # noqa: PLR0915 """Collect job data from Deadline Submitter UI. Args: @@ -82,9 +113,94 @@ def process(self, instance: pyblish.api.Instance) -> None: pv_by_name: dict[str, dict] = { pv["name"]: pv for pv in parameter_values } + + template_param_defs = job_template.get("parameterDefinitions", []) + template_param_names = { p["name"] for p in job_template.get("parameterDefinitions", []) } + # inject AYON context and other data used by the publishing step + if "folderPath" not in template_param_names: + self.add_ayon_context_parameter( + template_param_defs, + name="folderPath", + default=instance.data["folderPath"], + description="AYON folder path for this job", + ) + template_param_names.add("folderPath") + self.log.debug( + "adding folder path: %s", instance.data["folderPath"]) + if "taskName" not in template_param_names: + self.add_ayon_context_parameter( + template_param_defs, + name="taskName", + default=instance.data["task"], + description="AYON task for this job", + ) + template_param_names.add("taskName") + self.log.debug( + "adding task name: %s", instance.data["task"] + ) + if "projectName" not in template_param_names: + self.add_ayon_context_parameter( + template_param_defs, + name="projectName", + default=instance.context.data["projectName"], + description="AYON project for this job", + ) + template_param_names.add("projectName") + self.log.debug( + "adding project name: %s", + instance.context.data["projectName"] + ) + if "userName" not in template_param_names: + self.add_ayon_context_parameter( + template_param_defs, + name="userName", + default=instance.context.data["user"], + description="AYON user name for this job", + ) + template_param_names.add("userName") + self.log.debug( + "adding user name: %s", + instance.context.data["user"] + ) + if "hostName" not in template_param_names: + self.add_ayon_context_parameter( + template_param_defs, + name="hostName", + default=instance.context.data["hostName"], + description="AYON host name for this job", + ) + template_param_names.add("hostName") + self.log.debug( + "adding host name: %s", + instance.context.data["hostName"] + ) + if "sourceFile" not in template_param_names: + self.add_ayon_context_parameter( + template_param_defs, + name="sourceFile", + default=instance.context.data.get("currentFile", ""), + description="Source file path from the host for this job", + ) + template_param_names.add("sourceFile") + self.log.debug( + "adding source file: %s", + instance.context.data.get("currentFile", "") + ) + if "productBaseType" not in template_param_names: + self.add_ayon_context_parameter( + template_param_defs, + name="productBaseType", + default=instance.data.get("productBaseType", ""), + description="Product base type for this job", + ) + template_param_names.add("productBaseType") + self.log.debug( + "adding product base type: %s", + instance.data.get("productBaseType", "") + ) instance_attrs = instance.data.get("creator_attributes", {}) self._apply_instance_attrs( diff --git a/client/ayon_deadline_cloud/plugins/publish/create_instances_from_files.py b/client/ayon_deadline_cloud/plugins/publish/create_instances_from_files.py new file mode 100644 index 0000000..08551be --- /dev/null +++ b/client/ayon_deadline_cloud/plugins/publish/create_instances_from_files.py @@ -0,0 +1,543 @@ +"""Create publishing instance from provided files.""" +from __future__ import annotations + +import mimetypes +import os +from contextlib import suppress +from dataclasses import asdict +from pathlib import Path +from typing import ClassVar, NamedTuple + +import ayon_api +import clique +import pyblish.api +from ayon_core.pipeline import KnownPublishError +from ayon_core.pipeline.create import get_product_name +from ayon_core.pipeline.publish import ( + add_trait_representations, +) +from ayon_core.pipeline.traits import ( + FileLocation, + FileLocations, + FrameRanged, + Image, + MimeType, +) +from ayon_core.pipeline.traits import ( + Representation as TraitRepresentation, +) +from ayon_deadline_cloud.api.datatypes import ( + InstanceData, + StandardRepresentation, +) + + +class RepresentationTuple(NamedTuple): + """Representation named tuple.""" + standard: StandardRepresentation + trait: TraitRepresentation + + +# Minimum number of characters a common prefix must have (after stripping +# trailing separators) to be considered "meaningful" for grouping. +_MIN_PREFIX_LEN = 2 + + +def _common_stem_prefix(stems: list[str]) -> str: + """Return the longest common leading substring shared by all *stems*. + + Unlike ``os.path.commonprefix`` this helper is not affected by any Ruff + path-related lint rules, and the character-by-character comparison is + intentional here because we are comparing file *stems*, not path + components. + + Args: + stems: list of filename stems (without extension). + + Returns: + str: Longest common prefix, or empty string if there is none. + + """ + if not stems: + return "" + prefix = stems[0] + for stem in stems[1:]: + while not stem.startswith(prefix): + prefix = prefix[:-1] + if not prefix: + return "" + return prefix + + +class CreateInstancesFromFiles(pyblish.api.ContextPlugin): + """Create publishing instances from provided files.""" + + label = "Create publishing instances from files" + # must run as soon as possible + order = pyblish.api.CollectorOrder - 0.5 + hosts: ClassVar[list[str]] = ["shell"] + targets: ClassVar[list[str]] = ["farm"] + + def __init__(self): + """Constructor.""" + self._context: pyblish.api.Context + self._task_entity = None + self._folder_entity = None + + def process(self, context: pyblish.api.Context) -> None: + """Create publishing instances from provided files. + + Args: + context: pyblish context. + + Raises: + KnownPublishError: When folder or task cannot be found in AYON. + + """ + if not context.data.get("outputPath"): + msg = "Unable to find output path in context." + raise KnownPublishError(msg) + + username = ( + context.data.get("user") or os.environ.get("AYON_USERNAME") + ) + if username: + # ayon-python-api does not have public api function to find + # out if is used service user. So we need to have try-except. + con = ayon_api.get_server_api_connection() + with suppress(ValueError): + con.set_default_service_username(username) + + folder_path: str = context.data["folderPath"] + project_name: str = context.data["projectName"] + self._folder_entity = ayon_api.get_folder_by_path( + project_name=project_name, + folder_path=folder_path, + ) + if not self._folder_entity: + msg = ( + f"Unable to find folder '{folder_path}' in project" + f" '{project_name}'." + ) + raise KnownPublishError(msg) + + self._task_entity = None + if context.data.get("task"): + self._task_entity = ayon_api.get_task_by_name( + project_name=project_name, + folder_id=self._folder_entity["id"], + task_name=context.data["task"], + ) + if not self._task_entity: + msg = ( + f"Unable to find task '{context.data['ask']}' " + f"in folder '{folder_path}' " + f"in project '{project_name}'." + ) + raise KnownPublishError(msg) + + self._context = context + + instances = self.get_instances( + Path(context.data["outputPath"])) + + for instance in instances: + pyblish_instance = context.create_instance( + name=instance.name) + pyblish_instance.data.update(asdict(instance)) + add_trait_representations( + pyblish_instance, + pyblish_instance.data.pop( + "trait_representations") + ) + + pyblish_instance.data["representations"] = ( + pyblish_instance.data.pop("standard_representations") + ) + + @staticmethod + def _make_trait_representation( + reminder_path: Path) -> TraitRepresentation: + """Build a trait-based Representation for a single reminder file. + + Guesses MimeType from the file extension and adds Image trait + when the mime type indicates an image. + + Args: + reminder_path: Path to the reminder file + + Returns: + TraitRepresentation: trait-based representation for the file. + + """ + mime_type_str, _ = mimetypes.guess_type(reminder_path) + traits: list = [FileLocation(file_path=reminder_path)] + if mime_type_str: + traits.append(MimeType(mime_type=mime_type_str)) + if mime_type_str.startswith("image/"): + traits.append(Image()) + return TraitRepresentation(name=reminder_path.stem, traits=traits) + + @staticmethod + def _make_standard_representation( + reminder_path: Path) -> StandardRepresentation: + """Build a standard Representation for a single reminder file. + + Args: + reminder_path (Path): Path to the reminder file. + + Returns: + StandardRepresentation: standard representation for the file. + + """ + return StandardRepresentation( + name=reminder_path.stem, + ext=reminder_path.suffix, + files=reminder_path.name, + stagingDir=reminder_path.parent.as_posix(), + ) + + def _make_representations( + self, + reminder_path: Path + ) -> RepresentationTuple: + """Build both trait-based and standard Representations. + + Args: + reminder_path: Path to the reminder file. + + Returns: + RepresentationTuple: tuple + with both representation types. + + """ + return RepresentationTuple( + self._make_standard_representation(reminder_path), + self._make_trait_representation(reminder_path) + ) + + @staticmethod + def _make_collection_trait_representation( + col: clique.Collection, + staging_dir: Path, + name: str, + ) -> TraitRepresentation: + """Build a trait-based Representation for a file sequence collection. + + Derives frame range from the collection indexes and guesses MimeType + from the file extension. An ``Image`` trait is appended when the + mime type indicates an image. + + Args: + col: clique.Collection describing the sequence. + staging_dir: Directory that contains the sequence files. + name: Name to assign to the representation. + + Returns: + TraitRepresentation: trait-based representation for the sequence. + + """ + file_locations = [ + FileLocation(file_path=staging_dir / Path(f).name) + for f in col + ] + ext = col.tail.lstrip(".") + mime_type_str, _ = mimetypes.guess_type(f"file.{ext}") + + frame_start = min(col.indexes) + frame_end = max(col.indexes) + + traits: list = [ + FileLocations(file_paths=file_locations), + FrameRanged(frame_start=frame_start, frame_end=frame_end), + ] + if mime_type_str: + traits.append(MimeType(mime_type=mime_type_str)) + if mime_type_str.startswith("image/"): + traits.append(Image()) + return TraitRepresentation(name=name, traits=traits) + + @staticmethod + def _make_collection_standard_representation( + col: clique.Collection, + staging_dir: Path, + name: str, + ) -> StandardRepresentation: + """Build a standard Representation for a file sequence collection. + + Args: + col: clique.Collection describing the sequence. + staging_dir: Directory that contains the sequence files. + name: Name to assign to the representation. + + Returns: + StandardRepresentation: standard representation for the sequence. + + """ + files = [Path(f).name for f in col] + return StandardRepresentation( + name=name, + ext=col.tail.lstrip("."), + files=files, + stagingDir=staging_dir.as_posix(), + ) + + def _make_collection_representations( + self, + col: clique.Collection, + staging_dir: Path, + name: str, + ) -> RepresentationTuple: + """Build trait-based and standard Representations for a collection. + + Args: + col: clique.Collection describing the sequence. + staging_dir: Directory that contains the sequence files. + name: Name to assign to the representation. + + Returns: + RepresentationTuple: tuple with both representation types. + + """ + return RepresentationTuple( + self._make_collection_standard_representation( + col, staging_dir, name), + self._make_collection_trait_representation( + col, staging_dir, name), + ) + + def _handle_collections( + self, + cols: list, + ) -> list[InstanceData]: + """Handle file sequence collections. + + Collections that share the same sequence pattern (head, padding, and + frame indexes) but differ only in their file extension (e.g. ``.exr`` + vs ``.png``) are grouped into a single instance with one + representation per extension. + + Args: + cols: list of ``clique.Collection`` objects returned by + ``clique.assemble``. + + Returns: + list[InstanceData]: Created instances, one per unique sequence. + + """ + if not cols: + return [] + + # Group collections whose only difference is the file extension. + # Key = (head, padding, frozenset of indexes) — collections in the + # same group share the same sequence pattern and therefore belong to + # the same product version. + groups: dict[tuple, list] = {} + for col in cols: + key = (col.head, col.padding, frozenset(col.indexes)) + groups.setdefault(key, []).append(col) + + instances: list[InstanceData] = [] + for (head, _padding, _indexes), group_cols in groups.items(): + # Strip trailing separators from the head to get the clean file + # stem, then split into staging directory and variant name. + head_path = Path(head.rstrip("._- ")) + staging_dir = head_path.parent + variant = head_path.name # e.g. "render" from "/path/to/render" + + representations = [ + self._make_collection_representations( + col, staging_dir, variant) + for col in group_cols + ] + + product_name = get_product_name( + project_name=self._context.data["projectName"], + folder_entity=self._folder_entity, + task_entity=self._task_entity, + product_base_type="render", + product_type="render", + host_name="deadline_cloud", + variant=variant, + ) + + instances.append( + InstanceData( + publish=True, + active=True, + label="", + name=variant, + family="render", + families=["render"], + folderPath=self._context.data["folderPath"], + task=self._context.data["task"], + variant=variant, + productBaseType="render", + productName=product_name, + trait_representations=[ + r.trait for r in representations], + standard_representations=[ + r.standard for r in representations], + ) + ) + + return instances + + def _handle_reminders( + self, + path: Path, + reminders: list[str]) -> list[InstanceData]: + """Handle reminder files that are not part of any sequence. + + We'll do some fuzzy logic here - process the file name - if + there is a common part at the beginning, collect them as multiple + representations under one product version (one instance). + + If their names is completely different, we'll create separate + instances for each of them. + + We create both trait-based representations and regular ones. For + traits, we'll do some guessing for basic traits. Another plugin can + add more, based on more sophisticated detection (for example using + OpenImageIO tools, etc.) + + Args: + path: root path for the reminder files + reminders: list of reminder file names + (not full paths, just names) that are not part + of any sequence. + + Returns: + list[InstanceData]: Created instances + + """ + if not reminders: + return [] + + # make reminders list with full paths + reminder_paths = [(path / r) for r in reminders] + + stems = [Path(r).stem for r in reminder_paths] + common_prefix = _common_stem_prefix(stems).rstrip("_-. ") + + if len(common_prefix) >= _MIN_PREFIX_LEN: + # All files share a meaningful common prefix -> one instance + # with multiple representations (one per file). + + representations = [ + self._make_representations(r) + for r in reminder_paths + ] + + product_name = get_product_name( + project_name=self._context.data["projectName"], + folder_entity=self._folder_entity, + task_entity=self._task_entity, + product_base_type="render", + product_type="render", + host_name="deadline_cloud", + variant=common_prefix, + ) + + return [ + InstanceData( + publish=True, + active=True, + label="", + name=common_prefix, + family="render", + families=["render"], + folderPath=self._context.data["folderPath"], + task=self._context.data["task"], + variant=common_prefix, + productBaseType="render", + productName=product_name, + trait_representations=[ + r.trait for r in representations], + standard_representations=[ + r.standard for r in representations + ], + ) + ] + # there is no correlation between the file names, create separate + # instance for each of them + instances = [] + for reminder_path in reminder_paths: + representations = self._make_representations( + reminder_path + ) + + product_name = get_product_name( + project_name=self._context.data["projectName"], + folder_entity=self._folder_entity, + task_entity=self._task_entity, + product_base_type="render", + product_type="render", + host_name="deadline_cloud", + variant=representations.trait.name, + ) + + instances.append( + InstanceData( + publish=True, + active=True, + label="", + name=representations.trait.name, + family="render", + families=["render"], + folderPath=self._context.data["folderPath"], + task=self._context.data["task"], + variant=representations.trait.name, + productBaseType="render", + productName=product_name, + trait_representations=[representations.trait], + standard_representations=[representations.standard], + ) + ) + return instances + + def get_instances(self, path: Path) -> list[InstanceData]: + """Get publishing instances from provided path. + + Path has to contain one file or a sequence of files. + + Args: + path: Path + + Returns: + list[InstanceData]: List of publishing instances. + + Raises: + KnownPublishError: When file path is not a file or a sequence. + + """ + cols: list[clique.Collection] = [] + if path.is_file(): + # we'll use the same logic as if it is a single reminder + rems = [path.name] + else: + files = list(path.glob("*")) + if not files: + msg = f"Provided path {path} has no files." + raise KnownPublishError(msg) + + cols: list[clique.Collection] + rems: list[str] + cols, rems = clique.assemble( + [f.as_posix() for f in files] + ) + + instances: list[InstanceData] = [] + + # First, process reminders. This is just a list of file names. + # We'll do some fuzzy logic here - process the file name - if + # there is a common part at the beginning, collect them as multiple + # representations under one product version (one instance). + # If their names is completely different, we'll create separate + # instances for each of them. + if rems: + instances = self._handle_reminders(path, rems) + if cols: + instances += self._handle_collections(cols) + + return instances diff --git a/pyproject.toml b/pyproject.toml index b4047c9..9335974 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,13 +12,10 @@ license = "Apache-2.0" readme = "README.md" requires-python = ">=3.11.9" dependencies = [ + "clique", "pre-commit", "ruff ~= 0.15.0", "ty", -] - -[project.optional-dependencies] -dev = [ "pytest", "codespell", "pyblish-base>=1.8.12", @@ -31,5 +28,6 @@ extra-paths = [ "../deadline-cloud-for-maya/plugin_env/scripts", "../ayon-core/.venv/Lib/site-packages", "../ayon-docker/backend", + "../ayon-python-api", "client" ] diff --git a/ruff.toml b/ruff.toml index 4c6f12a..825d707 100644 --- a/ruff.toml +++ b/ruff.toml @@ -62,7 +62,6 @@ ignore = [ "S404", # suspicious-subprocess-import: subprocess module can be insecure "PLC0415", # import-outside-top-level: import must be on top of the file "CPY001", # missing-copyright-notice: missing copyright header - ] # Allow fix for all enabled rules (when `--fix`) is provided. @@ -98,3 +97,7 @@ docstring-code-format = false # This only has an effect when the `docstring-code-format` setting is # enabled. docstring-code-line-length = "dynamic" + +[lint.per-file-ignores] +# mixed case for variable names (needed by pyblish context and instance +"client/*/api/datatypes.py" = ["N815"] diff --git a/server/settings.py b/server/settings.py index b21b61a..00c4b50 100644 --- a/server/settings.py +++ b/server/settings.py @@ -9,6 +9,8 @@ "conda_packages": "", "conda_channels": "deadline-cloud", "extra_job_parameters": [], + "publish_host_requirement_roles": ["publish"], + "render_host_requirement_roles": ["render"], } @@ -82,3 +84,23 @@ class DeadlineCloudSettings(BaseSettingsModel): "template's parameterDefinitions." ), ) + + publish_host_requirement_roles: list[str] = SettingsField( + default_factory=list, + title="Publish Host Requirement Roles", + description=( + "When submitting job to Deadline Cloud, the publishing step " + "will run only on machine (or fleet) that has this set as " + "Worker Capability." + ), + ) + + render_host_requirement_roles: list[str] = SettingsField( + default_factory=list, + title="render Host Requirement Roles", + description=( + "When submitting job to Deadline Cloud, the render step " + "will run only on machine (or fleet) that has this set as " + "Worker Capability." + ), + )