diff --git a/client/ayon_sitesync/addon.py b/client/ayon_sitesync/addon.py index 3d9c88a..8b0397f 100644 --- a/client/ayon_sitesync/addon.py +++ b/client/ayon_sitesync/addon.py @@ -32,6 +32,7 @@ SyncStatus, SiteAlreadyPresentError, SiteSyncStatus, + get_linked_representation_id, ) SYNC_ADDON_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -160,7 +161,8 @@ def add_site( site_name=None, file_id=None, force=False, - status=SiteSyncStatus.QUEUED + status=SiteSyncStatus.QUEUED, + link_type="reference", ): """Adds new site to representation to be synced. @@ -171,6 +173,8 @@ def add_site( Use 'force' to reset existing site. + If link_type is provided, also adds site to all linked representations. + Args: project_name (str): Project name. representation_id (str): Representation id. @@ -179,6 +183,8 @@ def add_site( force (bool): Reset site if exists. status (SiteSyncStatus): Current status, default SiteSyncStatus.QUEUED + link_type (str): Type of link to follow (e.g. 'reference'). + If provided, will also add site to all linked representations. Raises: SiteAlreadyPresentError: If adding already existing site and @@ -196,6 +202,78 @@ def add_site( project_name, representation_id ) + files = representation.get("files", []) + if not files: + self.log.debug("No files for {}".format(representation_id)) + return + + # Collect all representation IDs to process (original + linked) + representation_ids = [representation_id] + + self._add_site_to_representation( + project_name, + representation_id, + site_name, + file_id, + force, + status + ) + + # If link_type is provided, find all linked representations + if link_type: + linked_repre_ids = get_linked_representation_id( + project_name, representation, link_type + ) + self.log.debug( + "Found {} linked representations for {}".format( + len(linked_repre_ids), representation_id + ) + ) + # Add site to each representation + for repre_id in linked_repre_ids: + try: + self._add_site_to_representation( + project_name, + repre_id, + site_name, + file_id, + force, + status + ) + except SiteAlreadyPresentError: + self.log.warning( + f"Site {site_name} already present on {repre_id}" + ) + + + def _add_site_to_representation( + self, + project_name, + representation_id, + site_name, + file_id, + force, + status + ): + """Internal method to add site to a single representation. + + Args: + project_name (str): Project name. + representation_id (str): Representation id. + site_name (str): Site name of configured site. + file_id (str): File id. + force (bool): Reset site if exists. + status (SiteSyncStatus): Current status. + + Raises: + SiteAlreadyPresentError: If adding already existing site and + not 'force' + ValueError: other errors (repre not found, misconfiguration) + """ + representation = get_representation_by_id( + project_name, representation_id + ) + files = representation.get("files", []) if not files: self.log.debug("No files for {}".format(representation_id)) @@ -242,16 +320,22 @@ def remove_site( project_name, representation_id, site_name, - remove_local_files=False + remove_local_files=False, + link_type="reference", ): """Removes site for particular representation in project. + If link_type is provided, also removes site from all linked representations. + Also removes the remote site from project settings for all representations. + Args: project_name (str): project name (must match DB) representation_id (str): MongoDB _id value site_name (str): name of configured and active site remove_local_files (bool): remove only files for 'local_id' site + link_type (str): Type of link to follow (e.g. 'reference'). + If provided, will also remove site from all linked representations. Raises: ValueError: Throws if any issue. @@ -260,6 +344,66 @@ def remove_site( if not self.get_sync_project_setting(project_name): raise ValueError("Project not configured") + representation = get_representation_by_id( + project_name, representation_id + ) + + # Collect all representation IDs to process (original + linked) + representation_ids = [representation_id] + + # If link_type is provided, find all linked representations + if link_type: + linked_repre_ids = get_linked_representation_id( + project_name, representation, link_type + ) + representation_ids.extend(linked_repre_ids) + self.log.debug( + "Found {} linked representations for {}".format( + len(linked_repre_ids), representation_id + ) + ) + + # Get remote site from project settings + remote_site = self.get_remote_site(project_name) + + # Remove site from each representation (both local and remote) + for repre_id in representation_ids: + # Remove local site + self._remove_site_from_representation( + project_name, + repre_id, + site_name, + remove_local_files + ) + # Remove remote site if different from local site + if remote_site and remote_site != site_name: + self._remove_site_from_representation( + project_name, + repre_id, + remote_site, + remove_local_files + ) + + def _remove_site_from_representation( + self, + project_name, + representation_id, + site_name, + remove_local_files=False + ): + """Internal method to remove site from a single representation. + + Args: + project_name (str): project name (must match DB) + representation_id (str): MongoDB _id value + site_name (str): name of configured and active site + remove_local_files (bool): remove only files for 'local_id' + site + + Raises: + ValueError: Throws if any issue. + + """ sync_info = self.get_repre_sync_state( project_name, representation_id, @@ -781,30 +925,36 @@ def get_site_root_overrides( {"work": "c:/projects_local"} """ - # Validate that site name is valid - if site_name not in ("studio", "local"): - # Consider local site id as 'local' - if site_name != get_local_site_id(): - raise ValueError(( - "Root overrides are available only for" - " default sites not for \"{}\"" - ).format(site_name)) - site_name = "local" - sitesync_settings = self.get_sync_project_setting(project_name) roots = {} if not sitesync_settings["enabled"]: return roots local_project_settings = sitesync_settings["local_setting"] + # look for local roots overrides if site_name == "local": for root_info in local_project_settings["local_roots"]: roots[root_info["name"]] = root_info["path"] + # check if there are roots in Studio settings + # (background process doesn't have local settings, + # but it should have roots for local site in Studio settings) + if not roots: + for setting_site_name, site_info in sitesync_settings["sites"].items(): + if setting_site_name == site_name: + site_roots = site_info.get("root") + if not site_roots: + continue + if isinstance(site_roots, dict): + roots = site_roots + else: + for root_info in site_roots: + platform_key = platform.system().lower() + roots[root_info["name"]] = root_info[platform_key] return roots def get_local_normalized_site(self, site_name): - """Normlize local site name. + """Normalize local site name. Return 'local' if 'site_name' is local id. @@ -815,7 +965,13 @@ def get_local_normalized_site(self, site_name): str: Normalized site name. """ - if site_name == get_local_site_id(): + studio_site_names = self._transform_sites_from_settings( + self.sync_studio_settings + ).keys() + if ( + site_name not in studio_site_names and + site_name == get_local_site_id() + ): site_name = self.LOCAL_SITE return site_name @@ -1252,6 +1408,8 @@ def _transform_sites_from_settings(self, settings): """Transforms list of 'sites' from Setting to dict. It processes both System and Project Settings as they have same format. + Returns: + dict[str, dict]: {'site_name': {site_info}...} """ sites = {} if not self.enabled: diff --git a/client/ayon_sitesync/plugins/publish/integrate_site_sync.py b/client/ayon_sitesync/plugins/publish/integrate_site_sync.py index 51f499c..819f74f 100644 --- a/client/ayon_sitesync/plugins/publish/integrate_site_sync.py +++ b/client/ayon_sitesync/plugins/publish/integrate_site_sync.py @@ -21,6 +21,10 @@ class IntegrateSiteSync(pyblish.api.InstancePlugin): order = pyblish.api.IntegratorOrder + 0.2 label = "Integrate Site Sync state" + @classmethod + def apply_settings(cls, project_settings): + cls.enabled = project_settings["sitesync"]["enabled"] + def process(self, instance): published_representations = instance.data.get( "published_representations") diff --git a/client/ayon_sitesync/providers/lib.py b/client/ayon_sitesync/providers/lib.py index 7c5cabd..8f2744e 100644 --- a/client/ayon_sitesync/providers/lib.py +++ b/client/ayon_sitesync/providers/lib.py @@ -1,6 +1,7 @@ from .gdrive import GDriveHandler from .dropbox import DropboxHandler from .local_drive import LocalDriveHandler +from .resilio import ResilioHandler from .sftp import SFTPHandler from .rclone import RCloneHandler @@ -11,7 +12,6 @@ class ProviderFactory: Each new implementation needs to be registered and added to Providers enum. """ - def __init__(self): self.providers = {} # {'PROVIDER_LABEL: {cls, int},..} @@ -107,4 +107,5 @@ class and batch limit. factory.register_provider(DropboxHandler.CODE, DropboxHandler, 10) factory.register_provider(LocalDriveHandler.CODE, LocalDriveHandler, 50) factory.register_provider(SFTPHandler.CODE, SFTPHandler, 20) +factory.register_provider(ResilioHandler.CODE, ResilioHandler, 50) factory.register_provider(RCloneHandler.CODE, RCloneHandler, 20) diff --git a/client/ayon_sitesync/providers/local_drive.py b/client/ayon_sitesync/providers/local_drive.py index 93382d3..c4b0280 100644 --- a/client/ayon_sitesync/providers/local_drive.py +++ b/client/ayon_sitesync/providers/local_drive.py @@ -4,11 +4,11 @@ import threading import time -from ayon_core.lib import Logger +from ayon_core.addon import AddonsManager +from ayon_core.lib import Logger, get_local_site_id from ayon_core.pipeline import Anatomy -from .abstract_provider import AbstractProvider -from ayon_core.addon import AddonsManager +from .abstract_provider import AbstractProvider log = Logger.get_logger("SiteSync") @@ -136,19 +136,19 @@ def get_roots_config(self, anatomy=None): {"root": {"root_ONE": "value", "root_TWO":"value}} Format is importing for usage of python's format ** approach """ - site_name = self._normalize_site_name(self.site_name) + manager = AddonsManager() + sitesync_addon = manager.get_enabled_addon("sitesync") + if not sitesync_addon: + raise RuntimeError("No SiteSync addon") + + site_name = sitesync_addon.get_local_normalized_site(self.site_name) if not anatomy: - anatomy = Anatomy(self.project_name, - site_name) + anatomy = Anatomy(self.project_name, site_name) # TODO cleanup when Anatomy will implement siteRoots method roots = anatomy.roots root_values = [root.value for root in roots.values()] if not all(root_values): - manager = AddonsManager() - sitesync_addon = manager.get_enabled_addon("sitesync") - if not sitesync_addon: - raise RuntimeError("No SiteSync addon") roots = sitesync_addon._get_project_root_overrides_by_site_id( self.project_name, site_name) @@ -206,9 +206,3 @@ def _mark_progress( except FileNotFoundError: pass time.sleep(0.5) - - def _normalize_site_name(self, site_name): - """Transform user id to 'local' for Local settings""" - if site_name != 'studio': - return 'local' - return site_name diff --git a/client/ayon_sitesync/providers/resilio.py b/client/ayon_sitesync/providers/resilio.py new file mode 100644 index 0000000..2fc61df --- /dev/null +++ b/client/ayon_sitesync/providers/resilio.py @@ -0,0 +1,431 @@ +import os.path +import time +from sys import platform +import platform +import secrets + +from ayon_core.lib import Logger +from ayon_core.pipeline import Anatomy +from ayon_sitesync.providers.abstract_provider import AbstractProvider +from ayon_sitesync.providers.vendor.resilio import ConnectApi, Path, Job + +log = Logger.get_logger("SiteSync") + + +class ResilioHandler(AbstractProvider): + CODE = "resilio" + LABEL = "Resilio" + + # Error codes to ignore (not raise exception for) + # TODO expose as Settings maybe + IGNORE_ERRORS = { + # currently separate job per repre >> to same folder + "SE_SM_DUPLICATE_FOLDER", + } + + _log = None + + def __init__(self, project_name, site_name, tree=None, presets=None): + self.active = False + self.project_name = project_name + self.site_name = site_name + self._conn = None + self.root = None + + self.presets = presets + if not self.presets: + self.log.info( + "Sync Server: There are no presets for {}.".format(site_name) + ) + return + + if not self.presets.get("enabled"): + self.log.debug( + "Sync Server: Site {} not enabled for {}.".format( + site_name, project_name + ) + ) + return + + host = self.presets.get("host", "") + if not host: + msg = "Sync Server: No host to Resilio Management Console" + self.log.info(msg) + return + + port = self.presets.get("port", "") + if not port: + msg = "Sync Server: No port to Resilio Management Console" + self.log.info(msg) + return + + token = self.presets.get("token", "") + if not token: + msg = ( + "Sync Server: No access token for to " + "Resilio Management Console" + ) + self.log.info(msg) + return + + agent_id = self.presets.get("agent_id", "") + if not agent_id: + msg = ( + "Sync Server: No agent id for to " + "Resilio Management Console" + ) + self.log.info(msg) + return + self.agent_id = agent_id + + address = f"{host}:{port}" + self._conn = ConnectApi(address, token) + + def is_active(self): + """ + Returns True if provider is activated, eg. has working credentials. + Returns: + (boolean) + """ + return self.presets.get("enabled") and self._conn is not None + + def upload_file( + self, + source_path, + target_path, + addon, + project_name, + file, + repre_status, + site_name, + overwrite=False + ): + """ + Copy file from 'source_path' to 'target_path' on provider. + Use 'overwrite' boolean to rewrite existing file on provider + + Args: + source_path (string): absolute path on provider + target_path (string): absolute path with or without name of the file + addon (SiteSyncAddon): addon instance to call update_db on + project_name (str): + file (dict): info about uploaded file (matches structure from db) + repre_status (dict): complete representation containing + sync progress + site_name (str): target site name + overwrite (boolean): replace existing file + Returns: + (string) file_id of created/modified file , + throws FileExistsError, FileNotFoundError exceptions + """ + src_agent_id = self._get_site_agent_id(addon, project_name, "active") + trg_agent_id = self._get_site_agent_id(addon, project_name, "remote") + + job_data = self._build_job_data( + source_path, + src_agent_id, + target_path, + trg_agent_id + ) + + return self._upload_download_process( + project_name, + addon, + file, + repre_status, + site_name, + target_path, + job_data, + "local" + ) + + def download_file( + self, + source_path, + local_path, + addon, + project_name, + file, + repre_status, + site_name, + overwrite=False + ): + """ + Download file from provider into local system + + Args: + source_path (string): absolute path on provider + local_path (string): absolute path with or without name of the file + addon (SiteSyncAddon): addon instance to call update_db on + project_name (str): + file (dict): info about uploaded file (matches structure from db) + repre_status (dict): complete representation containing + sync progress + site_name (str): site name + overwrite (boolean): replace existing file + Returns: + (string) file_id of created/modified file , + throws FileExistsError, FileNotFoundError exceptions + """ + src_agent_id = self._get_site_agent_id(addon, project_name, "remote") + trg_agent_id = self._get_site_agent_id(addon, project_name, "active") + + if src_agent_id == trg_agent_id: + raise ValueError( + f"Source and target agent cannot be the same ({src_agent_id}" + ) + + job_data = self._build_job_data( + source_path, + src_agent_id, + local_path, + trg_agent_id + ) + + return self._upload_download_process( + project_name, + addon, + file, + repre_status, + site_name, + local_path, + job_data, + "remote" + ) + + def delete_file(self, path): + """ + Deletes file from 'path'. Expects path to specific file. + + Args: + path (string): absolute path to particular file + + Returns: + None + """ + raise NotImplementedError("This provider does not support folders") + + def list_folder(self, folder_path): + """ + List all files and subfolders of particular path non-recursively. + Args: + folder_path (string): absolut path on provider + + Returns: + (list) + """ + pass + + def create_folder(self, folder_path): + """ + Create all nonexistent folders and subfolders in 'path'. + + Args: + path (string): absolute path + + Returns: + (string) folder id of lowest subfolder from 'path' + """ + # Resilio creates folder path automatically + return os.path.basename(folder_path) + + def get_tree(self): + """ + Creates folder structure for providers which do not provide + tree folder structure (GDrive has no accessible tree structure, + only parents and their parents) + """ + pass + + def get_roots_config(self, anatomy=None): + """ + Returns root values for path resolving + + Takes value from Anatomy which takes values from Settings + overridden by Local Settings + + Returns: + (dict) - {"root": {"root": "/My Drive"}} + OR + {"root": {"root_ONE": "value", "root_TWO":"value}} + Format is importing for usage of python's format ** approach + """ + platform_name = platform.system().lower() + root_configs = {} + for root_info in self.presets["root"]: + root_configs[root_info["name"]] = root_info.get(platform_name) + return {"root": root_configs} + + def resolve_path(self, path, root_config=None, anatomy=None): + """ + Replaces all root placeholders with proper values + + Args: + path(string): root[work]/folder... + root_config (dict): {'work': "c:/..."...} + anatomy (Anatomy): object of Anatomy + Returns: + (string): proper url + """ + if not root_config: + root_config = self.get_roots_config(anatomy) + + if root_config and not root_config.get("root"): + root_config = {"root": root_config} + + try: + if not root_config: + raise KeyError + + path = path.format(**root_config) + except KeyError: + try: + path = anatomy.fill_root(path) + except KeyError: + msg = "Error in resolving local root from anatomy" + self.log.error(msg) + raise ValueError(msg) + + return path + + def _get_site_agent_id(self, addon, project_name, side): + """Get agent_id for a specific site from project settings. + + Args: + addon: SiteSyncAddon instance + project_name: Project name + side: active | remote + + Returns: + int: Agent ID for the site + + Raises: + ValueError: If site configuration or agent_id not found + """ + project_settings = addon.sync_project_settings[project_name] + local_setting = project_settings["local_setting"] + sites = project_settings.get("sites", {}) + + site_name = local_setting[f"{side}_site"] + + if site_name == "local": + return local_setting["resilio"]["agent_id"] + + site_config = sites.get(site_name, {}) + if not site_config: + msg = (f"Sync Server: No configuration found for site '{site_name}'" + f" in project '{project_name}'.") + self.log.error(msg) + raise ValueError(msg) + + agent_id = site_config.get("agent_id") + if not agent_id: + msg = (f"Sync Server: No agent_id configured for site '{site_name}'" + f" in project '{project_name}'.") + self.log.error(msg) + raise ValueError(msg) + + return agent_id + + def _build_job_data( + self, + source_path, + source_agent_id, + target_path, + target_agent_id + ): + """Build job data for Resilio sync operation. + + Args: + source_path: Path on source agent + source_agent_id: Source agent ID + target_path: Path on target agent + target_agent_id: Target agent ID + + Returns: + dict: Job data configuration + """ + source_path = os.path.normpath(source_path) + target_path = os.path.normpath(target_path) + job_id =f"{os.path.basename(source_path)}_{secrets.token_hex(3)}" + return { + "name": f"Sync Job via API {job_id}", + "description": "Created using the connect_api module", + "type": "distribution", + "agents": [ + { + "id": source_agent_id, + "path": Path(source_path).get_object(), + "permission": "rw" + }, + { + "id": target_agent_id, + "path": Path(os.path.dirname(target_path)).get_object(), + "permission": "ro" + } + ] + } + + def _upload_download_process( + self, + project_name, + addon, + file, + repre_status, + site_name, + target_path, + job_data, + side + ): + new_job = Job(self._conn, job_data) + new_job.save() + + self.log.debug(f"Job '{new_job.name}' created successfully.") + job_run_id = new_job.start() + + last_tick = None + job_run = None + while ( + job_run is None or + job_run.status not in ["finished", "failed", "aborted"] + ): + time.sleep(10) + job_run = self._conn.get_job_run(job_run_id) + + if addon.is_representation_paused( + repre_status["representationId"], + check_parents=True, + project_name=project_name): + raise ValueError("Paused during process, please redo.") + + progress_value = ( + float(job_run.attrs["transferred"] / job_run.attrs["size_total"]) + if job_run.attrs["size_total"] + else 0.0 + ) + + if not last_tick or \ + time.time() - last_tick >= addon.LOG_PROGRESS_SEC: + last_tick = time.time() + progress_value_log = min(int(progress_value * 100), 100) + self.log.debug("Uploaded %d%%." % progress_value_log) + addon.update_db( + project_name=project_name, + new_file_id=None, + file=file, + repre_status=repre_status, + site_name=site_name, + side=side, + progress=progress_value + ) + # Filter out ignorable errors + if job_run.errors: + filtered_errors = [ + err for err in job_run.errors + if err.get("code_str") not in self.IGNORE_ERRORS + ] + if filtered_errors: + raise ValueError(filtered_errors) + + if job_run.status == "finished": + return target_path diff --git a/client/ayon_sitesync/providers/resources/resilio.png b/client/ayon_sitesync/providers/resources/resilio.png new file mode 100644 index 0000000..8490630 Binary files /dev/null and b/client/ayon_sitesync/providers/resources/resilio.png differ diff --git a/client/ayon_sitesync/providers/vendor/readme.md b/client/ayon_sitesync/providers/vendor/readme.md new file mode 100644 index 0000000..0e31a56 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/readme.md @@ -0,0 +1,5 @@ +Temporary vendorized python-management-console-api-module from +https://github.com/resilio-inc/python-management-console-api-module + +Should be moved to `ayon-sitesync/client/pyproject.toml` after the +development is done and the module is ready to be used. diff --git a/client/ayon_sitesync/providers/vendor/resilio/__init__.py b/client/ayon_sitesync/providers/vendor/resilio/__init__.py new file mode 100644 index 0000000..3fd4cc2 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/__init__.py @@ -0,0 +1,17 @@ +""" +Resilio Connect API python module +===== + +This module allows you to control and interact with the Management Console +server using HTTPS protocol through its REST API. +""" + +from .api import ConnectApi + +from .models import Agent, Group, Job, JobRun + +from .errors import * + +from .constants import * + +from .utils import Path, Script \ No newline at end of file diff --git a/client/ayon_sitesync/providers/vendor/resilio/api.py b/client/ayon_sitesync/providers/vendor/resilio/api.py new file mode 100644 index 0000000..f2d386c --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/api.py @@ -0,0 +1,302 @@ +import requests +from .base_connection import BaseConnection +from .models import Agent, Group, Job, JobRun + + +class ConnectApi(BaseConnection): + """ + Create adapter for the Management Console API + + This class is used to call top level API such as list/create/delete entities. + + Parameters + ---------- + address : str + Base URL for all API requests which includes https protocol, hosname and port + token : str + API token + verify : bool, optional + Verify SSL certificate (default is False) + """ + + def __init__(self, address, token, verify=False): + super(ConnectApi, self).__init__(address, token, verify) + + if not verify: + from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + + # Agents + def get_agents(self): + """ + Get all agents + + Parameters + ---------- + None + + Returns + ------- + list + List of agents as Agent models + """ + + agents_attrs = self._get_agents() + return [Agent(self, attrs) for attrs in agents_attrs] + + def get_agent(self, agent_id): + """ + Get agent by id + + Parameters + ---------- + agent_id : int + ID of agent to get + + Returns + ------- + Agent + Agent model corresponding to specified ID + """ + + attrs = self._get_agent(agent_id) + return Agent(self, attrs) + + def disconnect_agent(self, agent): + """ + Disconnect agent from the Management Console by id + + Parameters + ---------- + agent : Agent or int + Agent model or agent ID representing agent to disconnect + """ + + agent_id = agent.id if isinstance(agent, Agent) else agent + self._delete_agent(agent_id) + + # Groups + def create_group(self, name, agents, description=''): + """ + Create group + + Parameters + ---------- + name : str + Name of the group + agents : list + List of Agent models + description : str, optional + Description of the group (default is empty string) + + Returns + ------- + Group + Created Group model + + Example + ------- + ```py + api = ConnectApi('https://localhost:8443', '') + servers = [] + for agent in api.get_agents(): + if agent.name.startswith('srv_'): + servers.append(agent) + api.create_group('Servers', servers, 'Servers group') + ``` + """ + + attrs = { + 'name': name, + 'description': description, + 'agents': [{'id': a.id} for a in agents] + } + group_id = self._create_group(attrs) + + group = Group(self, dict(id=group_id)) + group.fetch() + return group + + def get_groups(self): + """ + Get all groups + + Parameters + ---------- + None + + Returns + ------- + list + List of groups as Group models + """ + + groups_attrs = self._get_groups() + return [Group(self, attrs) for attrs in groups_attrs] + + def get_group(self, group_id): + """ + Get group by id + + Parameters + ---------- + group_id : int + ID of group to get + + Returns + ------- + Group + Group model corresponding to specified ID + """ + + attrs = self._get_group(group_id) + return Group(self, attrs) + + def delete_group(self, group): + """ + Delete group from the Management Console + + Parameters + ---------- + group : Group or int + Group model or group ID representing group to delete + """ + + group_id = group.id if isinstance(group, Group) else group + self._delete_group(group_id) + + # Jobs + def new_job(self, job_type, name, description=''): + """ + Create new job + + Parameters + ---------- + job_type : JobType + Job type. See JobType in constants for possible values + name : str + Job name + description : str, optional + Job description (default is empty string) + + Example + ------- + ```py + api = ConnectApi('https://localhost:8443', '') + agents = api.get_agents() + + src_group = api.create_group('src.group', [agents[0]]) + dst_group = api.create_group('dst.group', agents[1:]) + + job = api.new_job(JobType.DISTRIBUTION, 'Deploy dataset') + + job.add_source_group(src_group, Path('source')) + job.add_destination_group(dst_group, Path('dest')) + + job.add_trigger('post_download', { + 'linux': 'Linux command', + 'win': 'Windows command', + 'osx': 'OS X command' + }) + + job.save() + job.start() + ``` + """ + + attrs = { + 'type': job_type, + 'name': name, + 'description': description + } + + return Job(self, attrs) + + def get_jobs(self, name=None, wildcard_name_pattern=None, is_active=None, ever_run=None): + """ + Get all jobs + + Parameters + ---------- + name: str, optional + wildcard_name_pattern: str, optional + is_active: bool, optional + ever_run: bool, optional + + Returns + ------- + list + List of jobs as Job models + """ + params = {} + if name is not None: + params['name'] = name + if wildcard_name_pattern is not None: + params['wildcard_name_pattern'] = wildcard_name_pattern + if is_active is not None: + params['is_active'] = is_active + if ever_run is not None: + params['ever_run'] = ever_run + + jobs_attrs = self._get_json('/jobs', params=params) + return [Job(self, attrs) for attrs in jobs_attrs] + + def get_job(self, job_id): + """ + Get job by id + + Parameters + ---------- + job_id : int + ID of job to get + + Returns + ------- + Job + Job model corresponding to specified ID + """ + + attrs = self._get_job(job_id) + return Job(self, attrs) + + def delete_job(self, job): + """ + Delete job from the Management Console + + Parameters + ---------- + job : Job or int + Job model or job ID representing job to delete + """ + + job_id = job.id if isinstance(job, Job) else job + self._delete_job(job_id) + + def get_job_run(self, job_run_id): + """ + Get job run by ID + + Parameters + ---------- + job_run_id : int + Job run ID + + Returns + ------- + JobRun + Job run model corresponding to specified ID + """ + + attrs = self._get_job_run(job_run_id) + return JobRun(self, attrs) + + def test_connection(self): + """ + Test API connection + + Parameters + ---------- + None + """ + + self._get_jobs() \ No newline at end of file diff --git a/client/ayon_sitesync/providers/vendor/resilio/base_connection.py b/client/ayon_sitesync/providers/vendor/resilio/base_connection.py new file mode 100644 index 0000000..dc2545d --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/base_connection.py @@ -0,0 +1,172 @@ +import requests +from .errors import * + + +try: + from json import JSONDecodeError +except ImportError: + JSONDecodeError = ValueError + + +BASE_API_URL = '/api/v2' + + +def api_request(func): + def wrapper(self, url, *args, **kwargs): + kwargs['headers'] = { + 'Authorization': 'Token ' + self._token, + 'Content-Type': 'application/json' + } + kwargs['verify'] = self._verify + + url = self._base_url + url + + try: + response = func(self, url, *args, **kwargs) + except requests.RequestException as e: + raise ApiConnectionError('Connection to Management Console failed', e) + + if response.status_code >= 400: + try: + error_json = response.json() + message = error_json['message'] if 'message' in error_json else '' + data = error_json['data'] if 'data' in error_json else None + except JSONDecodeError: + message = response.text + data = None + + if response.status_code == 401: + raise ApiUnauthorizedError(message) + else: + raise ApiError(message, status_code=response.status_code, data=data) + + return response + return wrapper + + +class BaseConnection(object): + """ + Base class for interaction with Management Console API + + Don't use it directly + + See also + -------- + ConnectApi + """ + + def __init__(self, address, token, verify): + self._token = token + self._address = address + self._base_url = address + BASE_API_URL + self._verify = verify + + # Request methods + @api_request + def _get(self, *args, **kwargs): + return requests.get(*args, **kwargs) + + @api_request + def _post(self, *args, **kwargs): + return requests.post(*args, **kwargs) + + @api_request + def _put(self, *args, **kwargs): + return requests.put(*args, **kwargs) + + @api_request + def _delete(self, *args, **kwargs): + return requests.delete(*args, **kwargs) + + # Helpers + def _create(self, *args, **kwargs): + r = self._post(*args, **kwargs) + try: + return r.json()['id'] + except JSONDecodeError as e: + raise ApiError('Response is not a json: ' + r.text, e) + + def _get_json(self, *args, **kwargs): + r = self._get(*args, **kwargs) + try: + return r.json() + except JSONDecodeError as e: + raise ApiError('Response is not a json: ' + r.text, e) + + # Agents + def _get_agents(self): + return self._get_json('/agents') + + def _get_agent(self, agent_id): + return self._get_json('/agents/%d' % agent_id) + + def _update_agent(self, agent_id, attrs): + self._put('/agents/%d' % agent_id, json=attrs) + + def _get_agent_config(self): + return self._get_json('/agents/config') + + def _delete_agent(self, agent_id): + self._delete('/agents/%d' % agent_id) + + + # Jobs + def _get_jobs(self): + return self._get_json('/jobs') + + def _get_job(self, job_id): + return self._get_json('/jobs/%d' % job_id) + + def _create_job(self, attrs): + return self._create('/jobs', json=attrs) + + def _update_job(self, job_id, attrs): + self._put('/jobs/%d' % job_id, json=attrs) + + def _delete_job(self, job_id): + self._delete('/jobs/%d' % job_id) + + def _start_job(self, job_id): + params = { 'job_id': job_id } + return self._create('/runs', json=params) + + + # Job runs + def _get_job_run(self, job_run_id): + return self._get_json('/runs/%d' % job_run_id) + + def _get_last_job_run(self, job_id): + job_runs = self._get_json('/runs?limit=1&job_id=%d&sort=finish_time' % job_id) + return job_runs['data'][0] if len(job_runs['data']) else None + + def _stop_job_run(self, job_run_id): + self._put('/runs/%d/stop' % job_run_id) + + def _pause_job_run(self, job_run_id): + self._put('/runs/%d/pause' % job_run_id) + + def _resume_job_run(self, job_run_id): + self._put('/runs/%d/resume' % job_run_id) + + def _get_agent_status(self, job_run_id, agent_id): + return self._get_json('/runs/%d/agents/%d' % (job_run_id, agent_id)) + + def _get_agents_statuses(self, job_run_id): + return self._get_json('/runs/%d/agents' % job_run_id) + + + # Groups + def _get_groups(self): + return self._get_json('/groups') + + def _get_group(self, group_id): + return self._get_json('/groups/%d' % group_id) + + def _create_group(self, attrs): + return self._create('/groups', json=attrs) + + def _update_group(self, group_id, attrs): + self._put('/groups/%d' % group_id, json=attrs) + + def _delete_group(self, group_id): + self._delete('/groups/%d' % group_id) diff --git a/client/ayon_sitesync/providers/vendor/resilio/constants.py b/client/ayon_sitesync/providers/vendor/resilio/constants.py new file mode 100644 index 0000000..f2b53f8 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/constants.py @@ -0,0 +1,57 @@ +class Permission: + READ_ONLY = 'ro' + DESTINATION = 'ro' + READ_WRITE = 'rw' + SOURCE = 'rw' + SELECTIVE_RO = 'sro' + SELECTIVE_RW = 'srw' + + +class JobType: + SYNC = 'sync' + DISTRIBUTION = 'distribution' + CONSOLIDATION = 'consolidation' + SCRIPT = 'script' + + +class SchedulerType: + ONCE = 'once' + MANUALLY = 'manually' + MINUTES = 'minutes' + HOURLY = 'hourly' + DAILY = 'daily' + WEEKLY = 'weekly' + + +class JobRunStatus: + WORKING = 'working' + FINISHED = 'finished' + ABORTED = 'aborted' + TIMEOUT = 'timeout' + PAUSED = 'paused' + + +class AgentOS: + WIN32 = 'win32' + WIN64 = 'win64' + MAC = 'mac' + LINUX = 'linux' + ANDROID = 'android' + IOS = 'iOS' + WINDOWS_PHONE = 'Wp8' + + @classmethod + def is_windows(cls, os): + return os in [cls.WIN64, cls.WIN32] + + @classmethod + def is_linux(cls, os): + return os == cls.LINUX + + @classmethod + def is_mac(cls, os): + return os == cls.MAC + + @classmethod + def is_mobile(cls, os): + return os in [cls.ANDROID, cls.IOS, cls.WINDOWS_PHONE] diff --git a/client/ayon_sitesync/providers/vendor/resilio/errors.py b/client/ayon_sitesync/providers/vendor/resilio/errors.py new file mode 100644 index 0000000..1ee7e01 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/errors.py @@ -0,0 +1,50 @@ + +class ErrorCodes: + SE_JOB_NO_SOURCE_GROUP = 10001 + SE_JOB_NO_DESTINATION_GROUP = 10002 + SE_JOB_NO_GROUPS = 10003 + SE_JOB_SAME_DESTINATION_PATH = 10004 + SE_JOB_UNRESOLVED_TAGS = 10005 + SE_JOB_NO_CLIENTS = 10006 + SE_JOB_NO_SOURCE_CLIENTS = 10007 + SE_JOB_NO_DESTINATION_CLIENTS = 10008 + SE_AGENT_DISK_FULL = 10200 + SE_SERVER_EXCESSIVE_TIME_DIFF = 10300 + + +class ApiError(Exception): + def __init__(self, message, status_code=None, cause=None, data=None): + super(ApiError, self).__init__(message) + self.cause = cause + self.message = message + self.status_code = status_code + self.data = data + + def _same_destination_path_error(self): + if not self.data or not isinstance(self.data, list): + return None + + for d in self.data: + if d['code'] == ErrorCodes.SE_JOB_SAME_DESTINATION_PATH: + return '{} ({})'.format(d['message'], d['description']) + return None + + def error_name(self): + return 'Api error' + + def __str__(self): + same_destination_error = self._same_destination_path_error() + if same_destination_error: + return '{}: {}'.format(self.error_name(), same_destination_error) + else: + return '{}: {}'.format(self.error_name(), self.message) + + +class ApiConnectionError(ApiError): + def error_name(self): + return 'Connection error' + + +class ApiUnauthorizedError(ApiError): + def error_name(self): + return 'Unauthorized error' \ No newline at end of file diff --git a/client/ayon_sitesync/providers/vendor/resilio/models/__init__.py b/client/ayon_sitesync/providers/vendor/resilio/models/__init__.py new file mode 100644 index 0000000..f563ebb --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/models/__init__.py @@ -0,0 +1,4 @@ +from .agent import Agent +from .group import Group +from .job import Job +from .job_run import JobRun diff --git a/client/ayon_sitesync/providers/vendor/resilio/models/agent.py b/client/ayon_sitesync/providers/vendor/resilio/models/agent.py new file mode 100644 index 0000000..67e9769 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/models/agent.py @@ -0,0 +1,112 @@ +from datetime import datetime +from .base_model import BaseModel + + +class Agent(BaseModel): + """ + Data model for agent representation + + Parameters + ---------- + api : BaseConnection + Adapter for all API requests + data : dict + Object attributes + """ + + def __init__(self, api, data): + super(Agent, self).__init__(api, data) + + def __str__(self): + return '{}[{}]'.format(self._attrs['name'], self._attrs['ip']) + + def fetch(self): + """ + Pull updated model from the Management Console + + Parameters + ---------- + None + """ + + self._attrs = self._get_agent(self.id) + + @property + def name(self): + """Name""" + + return self._attrs['name'] + + @property + def deviceid(self): + """Peer ID""" + + return self._attrs['deviceid'] + + @property + def online(self): + """Online status""" + + return self._attrs['online'] + + @property + def ip(self): + """IP address""" + + if 'ip' not in self._attrs: + return None + return self._attrs['ip'] + + @property + def group_ids(self): + """IDs of groups to which agent is added""" + + return [g['id'] for g in self._attrs['groups']] + + @property + def status(self): + """Status""" + + return self._attrs['status'] + + @property + def wan_enabled(self): + """WAN optimization""" + + return self._attrs['wan_enabled'] + + @property + def last_seen(self): + """ + Last connect or disconnect time + + See also + -------- + Agent.online + """ + + return datetime.fromtimestamp(self._attrs['last_seen']) + + @property + def os(self): + """ + Operating system + + See also + -------- + AgentOS + """ + + return self._attrs['os'] + + @property + def errors(self): + """Errors""" + + return self._attrs['errors'] + + @property + def tags(self): + """Tags""" + + return self._attrs['tags'] diff --git a/client/ayon_sitesync/providers/vendor/resilio/models/base_model.py b/client/ayon_sitesync/providers/vendor/resilio/models/base_model.py new file mode 100644 index 0000000..c793f76 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/models/base_model.py @@ -0,0 +1,67 @@ +from ..base_connection import BaseConnection +from ..errors import ApiError + + +class BaseModel(BaseConnection): + """ + Base class for all data models + + Parameters + ---------- + api : BaseConnection + Adapter for all API requests + data : dict + Object attributes + """ + + def __init__(self, api, data): + super(BaseModel, self).__init__(api._address, api._token, api._verify) + self._attrs = data if data else {} + + def save(self): + """ + Push model to the Management Console + + Parameters + ---------- + None + """ + + raise NotImplementedError() + + def fetch(self): + """ + Pull updated model from the Management Console + + Parameters + ---------- + None + """ + + raise NotImplementedError() + + @property + def created(self): + """Does model exist on the Management Console""" + + return 'id' in self._attrs + + @property + def attrs(self): + """Model attributes""" + + return self._attrs + + @property + def id(self): + """Model ID""" + + if 'id' not in self._attrs: + raise ApiError('No id field. The model is not saved.') + return self._attrs['id'] + + def __eq__(self, other): + if isinstance(other, self.__class__) and self.created and other.created: + return self.id == other.id + else: + return False \ No newline at end of file diff --git a/client/ayon_sitesync/providers/vendor/resilio/models/group.py b/client/ayon_sitesync/providers/vendor/resilio/models/group.py new file mode 100644 index 0000000..db89636 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/models/group.py @@ -0,0 +1,71 @@ +from .base_model import BaseModel + + +class Group(BaseModel): + """ + Data model for group representation + + Parameters + ---------- + api : BaseConnection + Adapter for all API requests + data : dict + Object attributes + """ + + def __init__(self, api, data): + super(Group, self).__init__(api, data) + + def __str__(self): + return '{}'.format(self._attrs['name']) + + def save(self): + """ + Push model to the Management Console + + Parameters + ---------- + None + """ + + if not self.created: + model_id = self._create_group(self._attrs) + self._attrs['id'] = model_id + else: + self._update_group(self._attrs) + self.fetch() + + def fetch(self): + """ + Pull updated model from the Management Console + + Parameters + ---------- + None + """ + + self._attrs = self._get_group(self.id) + + @property + def name(self): + """Name""" + + return self._attrs['name'] + + @property + def description(self): + """Description""" + + return self._attrs['description'] + + @property + def agents_ids(self): + """IDs of agents in the group""" + + return [a['id'] for a in self._attrs['agents']] + + @property + def jobs_ids(self): + """IDs of jobs to which group is added""" + + return [j['id'] for j in self._attrs['jobs']] diff --git a/client/ayon_sitesync/providers/vendor/resilio/models/job.py b/client/ayon_sitesync/providers/vendor/resilio/models/job.py new file mode 100644 index 0000000..7cc22af --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/models/job.py @@ -0,0 +1,392 @@ +from datetime import datetime +from .base_model import BaseModel +from ..constants import Permission, SchedulerType, JobType +from ..utils import Path + + +class Job(BaseModel): + """ + Data model for job representation + + Parameters + ---------- + api : BaseConnection + Adapter for all API requests + data : dict + Object attributes + + See also + -------- + ConnectApi.new_job + """ + + def __init__(self, api, data): + super(Job, self).__init__(api, data) + + if 'groups' not in self._attrs: + self._attrs['groups'] = [] + + if 'settings' not in self._attrs: + self._attrs['settings'] = {} + + if (self._attrs['type'] != JobType.SYNC) and ('scheduler' not in self._attrs): + self._attrs['scheduler'] = { 'type': SchedulerType.MANUALLY } + + def __str__(self): + if not self.created: + return 'Unsaved job' + else: + return 'Job[{}]'.format(self.id) + + def save(self): + """ + Push model to the Management Console + + Parameters + ---------- + None + """ + + if not self.created: + job_id = self._create_job(self._attrs) + self._attrs['id'] = job_id + else: + new_attrs = {} + update_attrs = ['name', 'description', 'groups', 'triggers', 'script', + 'scheduler', 'settings', 'post_command_local_time', 'profile_id'] + + for a in update_attrs: + if a in self._attrs: + new_attrs[a] = self._attrs[a] + + self._update_job(self.id, new_attrs) + self.fetch() + + def fetch(self): + """ + Pull updated model from the Management Console + + Parameters + ---------- + None + """ + + self._attrs = self._get_job(self.id) + + def start(self): + """ + Start job + + Parameters + ---------- + None + + Returns + ------- + int + Job run ID + """ + + return self._start_job(self.id) + + def stop(self): + """ + Stop last job run + + Parameters + ---------- + None + """ + + job_run_id = self.last_run_id + if job_run_id: + self._stop_job_run(job_run_id) + + @property + def last_run_id(self): + """Last job run id. 0 if there is no job run""" + + jr = self._get_last_job_run(self.id) + return jr['id'] if jr else 0 + + # Groups + def add_group(self, group, path, permission): + """ + Add group to the job + + Parameters + ---------- + group : Group + Group model to be added to the job + path : Path + Path object containing paths for all platforms + permission : str + Permission of the group. See `Permission` in constants for possible values + """ + + assert isinstance(path, Path) + + group_job = { + 'id': group.id, + 'permission': permission, + 'path': path.get_object() + } + + self._attrs['groups'].append(group_job) + + def add_source_group(self, group, path): + """ + Add group to the job with RW permission + + Parameters + ---------- + group : Group + Group model to be added to the job + path : Path + Path object containing paths for all platforms + """ + + self.add_group(group, path, Permission.SOURCE) + + def add_destination_group(self, group, path): + """ + Add group to the job with RO permission + + Parameters + ---------- + group : Group + Group model to be added to the job + path : Path + Path object containing paths for all platforms + """ + + self.add_group(group, path, Permission.DESTINATION) + + def get_groups_ids(self): + """ + Get list of group IDs participating in the job + + Parameters + ---------- + None + """ + + return [g['id'] for g in self._attrs['groups']] + + + # Scheduler + def _set_scheduler_params(self, start, finish): + assert self._attrs['type'] != JobType.SYNC, 'Sync job does not support scheduler' + + if start is not None: + self._attrs['scheduler']['start'] = start + elif 'start' in self._attrs['scheduler']: + del self._attrs['scheduler']['start'] + + if finish is not None: + self._attrs['scheduler']['finish'] = finish + elif 'finish' in self._attrs['scheduler']: + del self._attrs['scheduler']['finish'] + + def schedule_once(self, time): + """ + Set job scheduler to run the job only once + + Parameters + ---------- + time : int + Timestamp in seconds when to start + """ + + self._attrs['scheduler'] = { + 'type': SchedulerType.ONCE, + 'time': time + } + self._set_scheduler_params(None, None) + + def schedule_manually(self): + """ + Disable job scheduler + + Parameters + ---------- + None + """ + + self._attrs['scheduler'] = { + 'type': SchedulerType.MANUALLY + } + self._set_scheduler_params(None, None) + + def schedule_minutes(self, every, start=None, finish=None): + """ + Set job scheduler to run the job every N minutes + + Parameters + ---------- + every : int + Run every N minutes (minimum value is 5 minutes) + start : int, optional + Scheduler start time. Timestamp in seconds + finish : int, optional + Scheduler finish time. Timestamp in seconds + """ + + self._attrs['scheduler'] = { + 'type': SchedulerType.MINUTES, + 'every': every + } + self._set_scheduler_params(start, finish) + + def schedule_hourly(self, every, start=None, finish=None): + """ + Set job scheduler to run the job every N hours + + Parameters + ---------- + every : int + Run every N hours + start : int, optional + Scheduler start time. Timestamp in seconds + finish : int, optional + Scheduler finish time. Timestamp in seconds + """ + + self._attrs['scheduler'] = { + 'type': SchedulerType.HOURLY, + 'every': every + } + self._set_scheduler_params(start, finish) + + def schedule_daily(self, every, time, start=None, finish=None): + """ + Set job scheduler to run the job every N days + + Parameters + ---------- + every : int + Run every N days + time : int + Number of seconds since midnight + start : int, optional + Scheduler start time. Timestamp in seconds + finish : int, optional + Scheduler finish time. Timestamp in seconds + """ + + self._attrs['scheduler'] = { + 'type': SchedulerType.DAILY, + 'every': every, + 'time': time + } + self._set_scheduler_params(start, finish) + + def schedule_weekly(self, days, time, start=None, finish=None): + """ + Set job scheduler to run the job weekly + + Parameters + ---------- + days : int[] + Array of days of week to run. Encoded in numbers from 0 to 6, where 0 is Sunday + time : int[] + Array of seconds since midnight to run at + Time is not related to day of week. Job can be started several times a day + start : int, optional + Scheduler start time. Timestamp in seconds + finish : int, optional + Scheduler finish time. Timestamp in seconds + """ + + self._attrs['scheduler'] = { + 'type': SchedulerType.WEEKLY, + 'days': days, + 'time': time + } + self._set_scheduler_params(start, finish) + + # Triggers + def add_trigger(self, trigger, script): + """ + Set script for consolidation or distribution job + + Parameters + ---------- + trigger : str + Trigger name. Possible values: pre_indexing, post_download, complete + script : Script + Script object which contains scripts for all platforms + """ + + assert self.type in [JobType.CONSOLIDATION, JobType.DISTRIBUTION], 'Job doesn\'t supports triggers' + assert trigger in ['pre_indexing', 'post_download', 'complete'], 'Invalid trigger name' + + if 'triggers' not in self._attrs: + self._attrs['triggers'] = {} + self._attrs['triggers'][trigger] = script.get_object() + + def set_script(self, script): + """ + Set script for script job + + Parameters + ---------- + script : Script + Script object which contains scripts for all platforms + """ + + assert self.type == JobType.SCRIPT, 'Only script job supports script' + + self._attrs['script'] = script.get_object() + + @property + def type(self): + """Job type""" + + return self._attrs['type'] + + @property + def name(self): + """Name""" + + return self._attrs['name'] + + @property + def description(self): + """Description""" + + return self._attrs['description'] + + @property + def errors(self): + """Errors list""" + + return self._attrs['errors'] + + @property + def groups(self): + """ + Groups in the job. + Note that these are NOT Group objects, but dicts with group attributes. + """ + + return self._attrs['groups'] + + @property + def last_start_time(self): + """Last start time""" + + return datetime.fromtimestamp(self._attrs['last_start_time']) + + @property + def settings(self): + """Job settings""" + + return self._attrs['settings'] + + @property + def groups_ids(self): + """IDs of groups in the job""" + + return self.get_groups_ids() + diff --git a/client/ayon_sitesync/providers/vendor/resilio/models/job_run.py b/client/ayon_sitesync/providers/vendor/resilio/models/job_run.py new file mode 100644 index 0000000..9771f62 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/models/job_run.py @@ -0,0 +1,165 @@ +from .base_model import BaseModel +from ..constants import JobRunStatus + +class JobRun(BaseModel): + """ + Data model for job run representation + + Parameters + ---------- + api : BaseConnection + Adapter for all API requests + data : dict + Object attributes + + See also + -------- + ConnectApi.get_job_run + """ + + def __init__(self, api, data): + super().__init__(api, data) + + def __str__(self): + return 'JobRun[{}]'.format(self.id) + + def fetch(self): + """ + Pull updated model from the Management Console + + Parameters + ---------- + None + """ + + self._attrs = self._get_job_run(self.id) + + def stop(self): + """ + Stop job run + + Parameters + ---------- + None + """ + self._stop_job_run(self.id) + + def pause(self): + """ + Pause job run + + Parameters + ---------- + None + """ + if not self.paused: + self._pause_job_run(self.id) + self.fetch() + + def resume(self): + """ + Resume job run + + Parameters + ---------- + None + """ + if self.paused: + self._resume_job_run(self.id) + self.fetch() + + @property + def is_synced(self): + """Is job run finished""" + + return self._attrs['status'] == JobRunStatus.FINISHED + + @property + def download_percent(self): + """Data download progress""" + + if self._attrs['size_total'] == 0: + return 0 + + if self._attrs['agents_total'] < 2: + return 100 + + expected = self._attrs['size_total'] * (self._attrs['agents_total'] - 1) + completed = self._attrs['size_completed'] + + return min(int((completed / expected) * 100), 100) + + @property + def active(self): + """Is job run active""" + + return self._attrs['active'] + + @property + def paused(self): + """Is job run paused""" + + return self._attrs['status'] == JobRunStatus.PAUSED + + # Agent status + def get_agents_statuses(self): + """ + Get all agents statuses + + Paramaters + ---------- + None + + Returns + ------- + list + List of dicts with detailed agents status + """ + + return self._get_agents_statuses(self.id)['data'] + + def get_agent_status(self, agent_id): + """ + Get status of particular agent + + Parameters + ---------- + agent_id : int + Agent ID + + Returns + ------- + dict + Detailed agent's status + """ + return self._get_agent_status(self.id, agent_id) + + @property + def status(self): + """Job run status""" + + return self._attrs['status'] + + @property + def total_size(self): + """Total size in bytes""" + + return self._attrs['size_total'] + + @property + def total_files(self): + """Total size in files""" + + return self._attrs['files_total'] + + @property + def down_speed(self): + """Download speed""" + + return self._attrs['down_speed'] + + @property + def errors(self): + """Errors""" + + return self._attrs['errors'] diff --git a/client/ayon_sitesync/providers/vendor/resilio/utils.py b/client/ayon_sitesync/providers/vendor/resilio/utils.py new file mode 100644 index 0000000..b0b14b4 --- /dev/null +++ b/client/ayon_sitesync/providers/vendor/resilio/utils.py @@ -0,0 +1,104 @@ + +class Path: + """ + Script model for all platforms + + Paramaters + ---------- + path : str + Path to be used for all platforms + macro : str + Macro for relative path + + Use public variables `linux`, `win`, `osx` and `android` to set path for corresponding platform + """ + + FOLDERS_STORAGE = '%FOLDERS_STORAGE%' + + def __init__(self, path=None, macro=None): + self.linux = path if path else '' + self.win = path if path else '' + self.osx = path if path else '' + self.android = path if path else '' + + self.macro = macro + + def get_object(self): + attrs = { + 'linux': self.linux, + 'win': self.win, + 'osx': self.osx, + 'android': self.android + } + + if self.macro is not None: + attrs['macro'] = self.macro + + return attrs + + def __eq__(self, other): + if isinstance(other, self.__class__): + return hash(self) == hash(other) + else: + return False + + def __hash__(self): + return hash(self.linux) ^ hash(self.win) ^ hash(self.osx) ^ hash(self.android) ^ hash(self.macro) + +class Script: + """ + Script model for all platforms + + Possible platforms are defined in Script.Platform + + Parameters + ---------- + scripts : dict, optional + Initial scripts in format {'linux':'Linux script', 'win':'Windows script', 'osx':'Mac script'} + Default value is {} + """ + + class Platform: + LINUX = 'linux' + WINDOWS = 'win' + MAC = 'osx' + + def __init__(self, scripts={}): + self._platforms = {} + + self.set_script(self.Platform.LINUX, scripts[self.Platform.LINUX] if self.Platform.LINUX in scripts else '') + self.set_script(self.Platform.WINDOWS, scripts[self.Platform.WINDOWS] if self.Platform.WINDOWS in scripts else '') + self.set_script(self.Platform.MAC, scripts[self.Platform.MAC] if self.Platform.MAC in scripts else '') + + def set_script(self, platform, script, shell=None, ext=None): + """ + Set script for particular platform + + Parameters + ---------- + platform : Script.Platform + Platform name + script : str + Content of the script for specified platform + shell : str, optional + Shell to be used for the script. Default values: + Linux: /bin/sh + Windows: cmd.exe /Q /C + Mac: /bin/sh + ext : str, optional + Extention of the script file. Default values: + Linux: sh + Windows: cmd + Mac: sh + """ + + assert platform in [self.Platform.LINUX, self.Platform.WINDOWS, self.Platform.MAC], 'Unknown platform' + + self._platforms[platform] = { 'script': script } + if shell: + self._platforms[platform]['shell'] = shell + if ext: + self._platforms[platform]['ext'] = ext + + def get_object(self): + return self._platforms diff --git a/client/ayon_sitesync/utils.py b/client/ayon_sitesync/utils.py index 0dbb339..b98b6f3 100644 --- a/client/ayon_sitesync/utils.py +++ b/client/ayon_sitesync/utils.py @@ -59,10 +59,9 @@ class EditableScopes: def get_linked_representation_id( - project_name, - repre_entity, - link_type, - max_depth=None + project_name, + repre_entity, + link_type="reference", ): """Returns list of linked ids of particular type (if provided). @@ -72,15 +71,12 @@ def get_linked_representation_id( version back to representations. Todos: - Missing depth query. Not sure how it did find more representations - in depth, probably links to version? This function should probably live in sitesync addon? Args: project_name (str): Name of project where look for links. repre_entity (dict[str, Any]): Representation entity. link_type (str): Type of link (e.g. 'reference', ...). - max_depth (int): Limit recursion level. Default: 0 Returns: List[ObjectId] Linked representation ids. @@ -90,22 +86,17 @@ def get_linked_representation_id( return [] version_id = repre_entity["versionId"] - if max_depth is None or max_depth == 0: - max_depth = 1 link_types = None if link_type: link_types = [link_type] - # Store already found version ids to avoid recursion, and also to store - # output -> Don't forget to remove 'version_id' at the end!!! + # Store already found version ids to avoid infinite recursion linked_version_ids = {version_id} - # Each loop of depth will reset this variable + # Each loop will find new versions linked to current versions versions_to_check = {version_id} - for _ in range(max_depth): - if not versions_to_check: - break + while versions_to_check: versions_links = get_versions_links( project_name, versions_to_check, @@ -119,12 +110,17 @@ def get_linked_representation_id( if link["entityType"] != "version": continue entity_id = link["entityId"] - linked_version_ids.add(entity_id) - versions_to_check.add(entity_id) + # Only add if not already visited + if entity_id not in linked_version_ids: + linked_version_ids.add(entity_id) + versions_to_check.add(entity_id) + + # Remove the original version_id from results + linked_version_ids.discard(version_id) - linked_version_ids.remove(version_id) if not linked_version_ids: return [] + representations = get_representations( project_name, version_ids=linked_version_ids, diff --git a/client/ayon_sitesync/version.py b/client/ayon_sitesync/version.py index 81c21bf..c8ddf61 100644 --- a/client/ayon_sitesync/version.py +++ b/client/ayon_sitesync/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'sitesync' version.""" -__version__ = "1.2.6+dev" +__version__ = "1.2.6+dev.2" diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a3c3ba9..2ad2c7c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "sitesync-addon", - "version": "1.2.6+dev", + "version": "1.2.6+dev.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sitesync-addon", - "version": "1.2.6+dev", + "version": "1.2.6+dev.2", "dependencies": { "@types/styled-components": "^5.1.25", "@ynput/ayon-react-addon-provider": "^0.0.6", diff --git a/frontend/package.json b/frontend/package.json index e549720..5d71705 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "sitesync-addon", "private": true, - "version": "1.2.6+dev", + "version": "1.2.6+dev.2", "type": "module", "scripts": { "dev": "vite", @@ -34,5 +34,6 @@ "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-simple-import-sort": "^10.0.0", "prettier": "^3.0.3" - } + }, + "packageManager": "yarn@1.22.21+sha1.1959a18351b811cdeedbd484a8f86c3cc3bbaf72" } diff --git a/frontend/src/main.jsx b/frontend/src/main.jsx index 3167272..39f7fca 100644 --- a/frontend/src/main.jsx +++ b/frontend/src/main.jsx @@ -14,7 +14,7 @@ const AddonWrapper = () => { // const addonName = useContext(AddonContext).addonName const addonName = 'sitesync' // const addonVersion = useContext(AddonContext).addonVersion - const addonVersion = '1.2.6+dev' + const addonVersion = '1.2.6+dev.2' const accessToken = useContext(AddonContext).accessToken const projectName = useContext(AddonContext).projectName const userName = useContext(AddonContext).userName diff --git a/package.py b/package.py index 0d58ae9..4f1b947 100644 --- a/package.py +++ b/package.py @@ -2,7 +2,7 @@ """Package declaring addon version.""" name = "sitesync" title = "SiteSync" -version = "1.2.6+dev" +version = "1.2.6+dev.2" client_dir = "ayon_sitesync" ayon_required_addons = { diff --git a/pyproject.toml b/pyproject.toml index a22a31a..14113e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "SiteSync" -version = "1.2.6+dev" +version = "1.2.6+dev.2" description = "SiteSync Addon" authors = ["Ynput s.r.o. "] license = "MIT License" diff --git a/scripts/set_user.py b/scripts/set_user.py new file mode 100644 index 0000000..7610826 --- /dev/null +++ b/scripts/set_user.py @@ -0,0 +1,57 @@ +SERVER_URL = "https://localhost:5000/" # FILL +SERVICE_API_KEY = "999aaaaaacddddd" # FILL +PROJECT_NAME = "resilio_sync" # FILL + +USER_NAME = "test" # FILL, - use service user name for background process +SITE_NAME = "test-site-name" # FILL, - use ‘us-cache’ for background process + +ACTIVE_SITE = "us-cache" # FILL, +REMOTE_SITE = "africa-studio" # FILL + + +skeleton = { + "local_setting": { + "active_site": ACTIVE_SITE, + "remote_site": REMOTE_SITE + }, + "local_roots": [] +} + + +from ayon_api import ServerAPI, get_client_version,get_bundles + +# Create connection with service API key +api = ServerAPI( + SERVER_URL, + token=SERVICE_API_KEY, + site_id=SITE_NAME, + client_version=get_client_version() +) + +bundles = get_bundles() + +# Find the production bundle +production_bundle = next( + (bundle for bundle in bundles["bundles"] if bundle["isProduction"]), + None +) + +if not production_bundle: + raise ValueError("No production bundle found, stopping") + +sitesync_version = production_bundle["addons"].get("sitesync") + +# Run commands as a specific user temporarily +with api.as_username(USER_NAME): + api.get_info() # necessary to register site if not present + response = api.get(f"addons/sitesync/{sitesync_version}/rawOverrides/{PROJECT_NAME}?site_id={SITE_NAME}") + + + response.raise_for_status("Cannot get site settings") + site_settings = response.data + site_settings.update(skeleton) + + + response = api.put(f"addons/sitesync/{sitesync_version}/rawOverrides/{PROJECT_NAME}?site_id={SITE_NAME}", **site_settings,) + response.raise_for_status("Cannot save site settings") + print(f"Site settings saved for {USER_NAME} {SITE_NAME}") \ No newline at end of file diff --git a/server/settings/providers/resilio.py b/server/settings/providers/resilio.py new file mode 100644 index 0000000..2dc2af5 --- /dev/null +++ b/server/settings/providers/resilio.py @@ -0,0 +1,47 @@ +from pydantic import Field + +from ayon_server.settings import BaseSettingsModel +from ayon_server.settings.anatomy.roots import Root, default_roots + + +class ResilioSubmodel(BaseSettingsModel): + """Specific settings for Resilio sites. + + token: API token for site + root: root folder on Resilio + """ + _layout = "expanded" + + host: str = Field( + "", + title="Resilio Management Console host name", + scope=["studio", "project"], + description="Domain name or IP of sftp server", + ) + + port: int = Field( + 0, + title="Resilio Management Console port", + scope=["studio", "project"], + placeholder="8443" + ) + + token: str = Field( + "", + title="Access token", + scope=["studio", "project", "site"], + description="API access token", + ) + + agent_id: int = Field( + 0, + title="Agent id", + scope=["studio", "project"], + ) + + roots: list[Root] = Field( + default=default_roots, + scope=["studio", "project"], + title="Roots", + description="Setup root paths for the project", + ) diff --git a/server/settings/settings.py b/server/settings/settings.py index 3acdb29..4cbab89 100644 --- a/server/settings/settings.py +++ b/server/settings/settings.py @@ -11,6 +11,7 @@ from .providers.gdrive import GoogleDriveSubmodel from .providers.dropbox import DropboxSubmodel from .providers.sftp import SFTPSubmodel +from .providers.resilio import ResilioSubmodel from .providers.rclone import RCloneSubmodel if typing.TYPE_CHECKING: @@ -21,8 +22,10 @@ class GeneralSubmodel(BaseSettingsModel): """Properties for loop and module configuration""" retry_cnt: int = Field(3, title="Retry Count") loop_delay: int = Field(60, title="Loop Delay") - always_accessible_on: list[str] = Field([], - title="Always accessible on sites") + always_accessible_on: list[str] = Field( + [], + title="Always accessible on sites" + ) active_site: str = Field("studio", title="User Default Active Site") remote_site: str = Field("studio", title="User Default Remote Site") @@ -68,10 +71,13 @@ def provider_resolver(): "local_drive": "Local Drive", "dropbox": "Dropbox", "sftp": "SFTP", + "resilio": "Resilio", "rclone": "Rclone" } - return [{"value": f"{key}", "label": f"{label}"} - for key, label in provider_dict.items()] + return [ + {"value": f"{key}", "label": f"{label}"} + for key, label in provider_dict.items() + ] async def defined_sited_enum_resolver( @@ -84,8 +90,10 @@ async def defined_sited_enum_resolver( return [] if project_name: - settings = await addon.get_project_settings(project_name=project_name, - variant=settings_variant) + settings = await addon.get_project_settings( + project_name=project_name, + variant=settings_variant + ) else: settings = await addon.get_studio_settings(variant=settings_variant) @@ -99,6 +107,22 @@ async def defined_sited_enum_resolver( provider_enum = provider_resolver() +class ResilioLocalSubmodel(BaseSettingsModel): + """Configure Resilio credentials for sync from local site""" + token: str = Field( + "", + title="Access token", + scope=["site"], + description="API access token", + ) + + agent_id: int = Field( + 0, + title="Agent id", + scope=["site"], + ) + + class SitesSubmodel(BaseSettingsModel): """Configured additional sites and properties for their providers""" _layout = "expanded" @@ -136,13 +160,20 @@ class SitesSubmodel(BaseSettingsModel): default_factory=SFTPSubmodel, scope=["studio", "project", "site"] ) + resilio: ResilioSubmodel = Field( + default_factory=ResilioSubmodel, + scope=["studio", "project"] + ) rclone: RCloneSubmodel = Field( default_factory=RCloneSubmodel, scope=["studio", "project", "site"] ) - name: str = Field(..., title="Site name", - scope=["studio", "project", "site"]) + name: str = Field( + ..., + title="Site name", + scope=["studio", "project", "site"] + ) @validator("name") def validate_name(cls, value): @@ -152,15 +183,19 @@ def validate_name(cls, value): class LocalSubmodel(BaseSettingsModel): """Select your local and remote site""" - active_site: str = Field("", - title="My Active Site", - scope=["site"], - enum_resolver=defined_sited_enum_resolver) + active_site: str = Field( + "", + title="My Active Site", + scope=["site"], + enum_resolver=defined_sited_enum_resolver + ) - remote_site: str = Field("", - title="My Remote Site", - scope=["site"], - enum_resolver=defined_sited_enum_resolver) + remote_site: str = Field( + "", + title="My Remote Site", + scope=["site"], + enum_resolver=defined_sited_enum_resolver + ) local_roots: list[RootSubmodel] = Field( default=default_roots, @@ -169,6 +204,12 @@ class LocalSubmodel(BaseSettingsModel): description="Overrides for local root(s)." ) + resilio: ResilioLocalSubmodel = Field( + title="Resilio credentials for local site", + default_factory=ResilioLocalSubmodel, + scope=["site"] + ) + class SiteSyncSettings(BaseSettingsModel): """Settings for synchronization process"""