diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..998b54c --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,28 @@ +# Project Guidelines + +## Project Context + +XTS Core is a Python CLI for running declarative command workflows defined in `.xts` YAML files. +Prioritize the alias-based workflow described in [README.md](../README.md) over direct file execution. +Target Linux environments and Python 3.10+. + +## Architecture + +- Keep changes aligned with the existing split between CLI entrypoints, alias management, argument parsing, and plugins under `src/xts_core/`. +- Treat `.xts` files as the user-facing workflow definition format and Python modules as the execution/runtime layer. +- Keep plugin work isolated to `src/xts_core/plugins/` and follow the existing `BaseXTSPlugin` abstraction. +- Prefer small, local changes over broad refactors unless the task explicitly requires restructuring. + +## Build And Test + +- Set up a local environment with `python -m venv .venv`, activate it, then run `pip install -e .`. +- Install runtime dependencies from the project metadata rather than duplicating dependency definitions. +- Run the focused test slice first when possible, then `pytest test/` for broader validation. +- If a change affects CLI behavior, validate it with a targeted `pytest` run before expanding scope. + +## Conventions + +- Preserve the existing Apache 2.0 header format in Python source files. +- Use helpers from `xts_core.utils` for user-facing terminal output instead of adding ad hoc `print()` calls. +- Keep YAML examples and `.xts` fixtures readable, minimal, and consistent with [examples/hello_world.xts](/home/ubuntu/TEST/test-automation/xts_core/worktrees/bash_completion_AI/examples/hello_world.xts). +- Link contributors to [CONTRIBUTING.md](/home/ubuntu/TEST/test-automation/xts_core/worktrees/bash_completion_AI/CONTRIBUTING.md) for branch naming, issue linkage, and pull request workflow. \ No newline at end of file diff --git a/.github/instructions/python-conventions.instructions.md b/.github/instructions/python-conventions.instructions.md new file mode 100644 index 0000000..69b31de --- /dev/null +++ b/.github/instructions/python-conventions.instructions.md @@ -0,0 +1,14 @@ +--- +description: "Use when editing Python source in src/xts_core, adding CLI behavior, extending alias management, or implementing plugins. Covers project-specific Python conventions and library usage." +applyTo: "src/**/*.py" +--- + +# Python Conventions + +- Preserve the existing Apache 2.0 file header when editing or creating Python source files. +- Add a module docstring for new modules and keep public function or class docstrings concise and concrete. +- Prefer project helpers from `xts_core.utils` for user-facing output instead of raw `print()`. +- Follow the existing CLI style: keep argument parsing explicit, keep validation close to the command entrypoint, and avoid hidden side effects. +- Keep plugin implementations under `src/xts_core/plugins/` and subclass `BaseXTSPlugin` when adding new plugins. +- Reuse the project's current libraries and patterns before introducing new dependencies. In particular, stay consistent with `argparse`, `yaml`, `requests`, and `rich` usage already present in the codebase. +- Keep functions focused and local. Prefer straightforward control flow over indirection. \ No newline at end of file diff --git a/.github/instructions/test-conventions.instructions.md b/.github/instructions/test-conventions.instructions.md new file mode 100644 index 0000000..d06e980 --- /dev/null +++ b/.github/instructions/test-conventions.instructions.md @@ -0,0 +1,13 @@ +--- +description: "Use when adding or updating pytest coverage, CLI tests, alias tests, or plugin tests in the test suite. Covers fixtures, mocking, and repository test patterns." +applyTo: "test/**/*.py" +--- + +# Test Conventions + +- Write tests with `pytest` and the existing fixture style used in the `test/` directory. +- Use `tmp_path` for temporary files and directories instead of writing into real user locations. +- Use `monkeypatch` to isolate environment-dependent behavior such as `HOME` and XTS cache paths. +- Mock external systems and network calls with `unittest.mock.patch` or pytest fixtures so tests stay deterministic. +- Keep tests narrow and behavior-focused. Name them for the observable behavior being validated. +- When adding coverage for a module, prefer extending the nearest existing test file unless a new file gives a clearer split. \ No newline at end of file diff --git a/.github/instructions/xts-yaml-format.instructions.md b/.github/instructions/xts-yaml-format.instructions.md new file mode 100644 index 0000000..7c439c6 --- /dev/null +++ b/.github/instructions/xts-yaml-format.instructions.md @@ -0,0 +1,14 @@ +--- +description: "Use when creating or editing .xts workflow files, examples, or YAML command definitions for XTS. Covers command structure, nesting, passthrough parameters, and formatting expectations." +applyTo: "**/*.xts" +--- + +# XTS YAML Format + +- Follow the command layout shown in [examples/hello_world.xts](../../examples/hello_world.xts). +- Use top-level keys as command groups and nested keys as executable commands or subcommands. +- Provide a `description` for each runnable command so CLI help stays useful. +- Use `command` as either a single shell command string or a list of commands to run in sequence. +- Use `params.passthrough: true` only when the command is intended to forward extra CLI arguments. +- Keep YAML indentation to two spaces and do not use tabs. +- Ignore unrelated data structures unless the task explicitly requires documenting or validating ignored sections. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f893c9a..7595725 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,38 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + [project] name = "xts" version = "2.0.0" dependencies = [ "rich>=13.7.1", - "yaml_runner @ git+https://github.com/rdkcentral/yaml_runner.git@2.1.0", "pyinstaller>=6.12.0", "requests>=2.32.3", - "rich-argparse>=1.7.2" + "rich-argparse>=1.7.2", + "importlib_resources>=7.1.0", + "yaml_runner @ git+https://github.com/rdkcentral/yaml_runner.git@2.2.0", + "argparse_completion @ git+https://github.com/rdkcentral/argparse_completion.git@1.0.0" ] [tool.setuptools.packages.find] -include = ["xts_core", "xts_core.plugins"] +include = ["xts_core*"] where = ["src"] namespaces = false +# --- Include non-Python files --- +[tool.setuptools.package-data] +# If your data folder is *inside* xts_core, use this: +xts_core = ["data/**/*"] + +# --- Include data outside packages --- +[tool.setuptools] +include-package-data = true + +# This ensures data/ is included in sdist/wheel +[tool.setuptools.data-files] +# Installs data into a shared location in site-packages +"xts_core_data" = ["data/**/*"] + [project.scripts] xts-install = "xts_core.install:main" xts = "xts_core.xts:main" diff --git a/src/xts_core/data/xts_bash_completion.sh b/src/xts_core/data/xts_bash_completion.sh new file mode 100644 index 0000000..3ea5cd6 --- /dev/null +++ b/src/xts_core/data/xts_bash_completion.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +function _xts_runner_completion() +{ + local IFS=' + ' + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _XTS_COMPLETE=complete_bash $1 ) ) + return 0 +} + +complete -o default -F _xts_runner_completion xts diff --git a/src/xts_core/install.py b/src/xts_core/install.py index e46b051..72bf17a 100644 --- a/src/xts_core/install.py +++ b/src/xts_core/install.py @@ -22,14 +22,19 @@ #* ****************************************************************************** import datetime +import importlib_resources import os from pathlib import Path import platform import subprocess import tempfile +import xts_core from .utils import info, error, warning +SCRIPT_PATH = Path(__file__).absolute() +SCRIPT_DIR = Path(os.path.dirname(SCRIPT_PATH)) + def main(): if 'Linux' in platform.platform(): _linux_install() @@ -37,8 +42,6 @@ def main(): error('Unfortunately XTS does not currently support your OS') def _linux_install(): - script_path = Path(__file__).absolute() - script_dir = Path(os.path.dirname(script_path)) user_home = Path('.').home() xts_dir = user_home.joinpath(Path('.xts')) bin_dir = xts_dir.joinpath(Path('bin')) @@ -50,7 +53,7 @@ def _linux_install(): os.makedirs(log_dir, exist_ok=True) result = subprocess.run(['pyinstaller', '--onefile', - f'{script_dir}/xts.py', + f'{SCRIPT_DIR}/xts.py', '--name', 'xts', '--distpath', @@ -79,10 +82,43 @@ def _linux_install(): 'Raise an issue on xts_core for more help:\n'+ 'https://github.com/rdkcentral/xts_core/issues/new?template=01-bugs.yml') else: - info('XTS has been installed successfully.\n'+ - 'Please run the following commands:\n' + - f'\texport PATH="{bin_dir}:$PATH"\n' + - 'To permanently install this, add the export command as a line in your ~/.bashrc file') + shell = os.environ.get('SHELL','') + match shell: + case '/bin/bash': + _install_bash(user_home) + info('XTS has been installed successfully.\n') + case _: + warning('XTS has been installed successfully, but we could not automatically add xts to your PATH.\n' + + 'Please add the following line to your shell config file:\n' + + f'export PATH="{user_home.joinpath(Path(".xts/bin"))}:$PATH"\n') + +def _install_bash(user_home: Path): + if (bashrc:=user_home.joinpath(Path('.bashrc'))).exists(): + with open(user_home.joinpath(Path('.bashrc')), 'a') as f: + f.write('\n# XTS PATH\n') + f.write(f'export PATH="{user_home.joinpath(Path(".xts/bin"))}:$PATH"\n') + _install_bash_completion(bashrc) + elif (bash_profile:=user_home.joinpath(Path('.bash_profile'))).exists(): + with open(user_home.joinpath(Path('.bash_profile')), 'a') as f: + f.write('\n# XTS PATH\n') + f.write(f'export PATH="{user_home.joinpath(Path(".xts/bin"))}:$PATH"\n') + _install_bash_completion(bash_profile) + else: + warning('Could not find .bashrc or .bash_profile to add xts to PATH. Please add the following line to your shell config file:\n' + + f'export PATH="{user_home.joinpath(Path(".xts/bin"))}:$PATH"\n') + +def _install_bash_completion(rc_file: Path): + user_home = rc_file.parent + bash_completion_dir = user_home.joinpath(Path('.bash_completion.d')) + if not bash_completion_dir.exists(): + os.makedirs(bash_completion_dir) + bash_completion_script = bash_completion_dir.joinpath(Path('xts_bash_completion.sh')) + with importlib_resources.open_text(xts_core,'data/xts_bash_completion.sh', encoding='utf-8') as src, \ + open(bash_completion_script, 'w') as dst: + dst.write(src.read()) + with open(rc_file, 'a') as f: + f.write('\n# XTS bash completion\n') + f.write(f'source {bash_completion_dir}/xts_bash_completion.sh\n') if __name__ == '__main__': main() \ No newline at end of file diff --git a/src/xts_core/plugins/xts_allocator_client.py b/src/xts_core/plugins/xts_allocator_client.py index 41ebd32..07c6d55 100755 --- a/src/xts_core/plugins/xts_allocator_client.py +++ b/src/xts_core/plugins/xts_allocator_client.py @@ -506,7 +506,7 @@ def _update_slot(self, state, owner_email]): plugin_utils.error('At least one field must be provided for update.') - sys.exit(1) + raise SystemExit(1) # Required fields payload = { 'slot_id': slot_id diff --git a/src/xts_core/xts.py b/src/xts_core/xts.py index 4940bfc..5b4316e 100755 --- a/src/xts_core/xts.py +++ b/src/xts_core/xts.py @@ -37,7 +37,6 @@ import os import re import sys -import json import yaml try: @@ -47,6 +46,7 @@ import yaml.scanner from yaml_runner import YamlRunner +import argparse_completion try: from .plugins import XTSAllocatorClient @@ -68,6 +68,8 @@ except ImportError: from xts_core.xts_arg_parser import XTSArgumentParser +from xts_core.xts_rich_help_formatter import XTSRichHelpFormatter + class XTS(): """ @@ -162,7 +164,7 @@ def _is_command_section(subdict: dict) -> bool: return command_sections - def _parse_first_arg(self): + def _setup_first_parser(self): """ Parse CLI arguments and set up argparse for all commands. The first argument must be either: @@ -174,63 +176,85 @@ def _parse_first_arg(self): list[str]: Remaining args starting with the command name, e.g. ["run", ...]. """ first_arg_parser = XTSArgumentParser(prog='xts', + formatter_class=XTSRichHelpFormatter, add_help=False) - first_arg_parser.add_argument('--alias', - action='store_true', - help='Add/Remove or list aliases', - dest='alias_option', - default=False) - args, remaining_args = first_arg_parser.parse_known_args(sys.argv[1:]) - - if args.alias_option: - xts_alias.run_alias_builtin(remaining_args) - raise SystemExit(0) - - if not remaining_args: - first_arg_parser.print_help() - raise SystemExit(0) - - alias_name = remaining_args[0] - resolved_xts_path = xts_alias.resolve_alias_to_xts_path(alias_name) - - if resolved_xts_path is None: - error( - f'Unknown alias "{alias_name}". ' - 'Use "xts --alias --list" to see available aliases.' - ) - raise SystemExit(1) - + first_arg_subparsers = first_arg_parser.add_subparsers(dest='alias_name', + metavar='') + known_aliases = sorted(list(xts_alias.load_aliases())) + if len(known_aliases) >= 1: + first_arg_subparsers.add_parser('alias_name', + aliases=known_aliases, + help='Alias to run commands from.', + add_help=False) + + alias_parser = first_arg_subparsers.add_parser('alias', + help='Manage aliases (add, list, remove)') + xts_alias.setup_alias_parser(alias_parser) + return first_arg_parser + + def _run_yaml_runner(self, alias:str, arguments:list[str]): + resolved_xts_path = xts_alias.resolve_alias_to_xts_path(alias) # load xts config remove alias name from argv before parsing self.xts_config = resolved_xts_path - - return remaining_args[1:] - - def run(self): - """Run the XTS app. - - Raises: - SystemExit: Raised when unrecogised arguments are given. - """ - args = self._parse_first_arg() - try: yaml_runner = YamlRunner( self._command_sections, - program='xts', + program=f'xts {alias}', hierarchical=True, fail_fast=True, parser_class=XTSArgumentParser ) - _, _, exit_code = yaml_runner.run(args) - sys.exit(sorted(exit_code)[-1]) - + _, _, exit_code = yaml_runner.run(arguments) + raise SystemExit(sorted(exit_code)[-1]) except Exception as e: - error( - 'An unrecognised command caused an error\n\n' - f'Command Args: [{" ".join(args)}]\n\n' - f'{str(e)}' - ) + error( + 'An unrecognised command caused an error\n\n' + f'Command Args: [{" ".join(arguments)}]\n\n' + f'{str(e)}') + + def _run_completion(self, arg_parser:XTSArgumentParser): + os.environ['_ARGPARSE_COMPLETE'] = os.getenv('_XTS_COMPLETE') + completion = argparse_completion.get_completion(arg_parser) + if 'alias_name' in completion: + completion.remove('alias_name') + if len(completion) == 0 and (comp_words:=(os.getenv('COMP_WORDS').split()))[1] in xts_alias.load_aliases().keys(): + os.environ['_YAML_RUNNER_COMPLETE'] = os.getenv('_XTS_COMPLETE') + alias = comp_words[1] + comp_words.remove('xts') + comp_words.remove(alias) + os.environ['COMP_WORDS'] = " ".join(comp_words) + self._run_yaml_runner(alias, comp_words) + else: + print('\n'.join(completion)) + + def run(self): + """Run the XTS app. + + Raises: + SystemExit: Raised when unrecogised arguments are given. + """ + parser = self._setup_first_parser() + if os.getenv('_XTS_COMPLETE'): + self._run_completion(parser) + raise SystemExit(0) + if len(sys.argv) <= 1: + parser.print_help() + raise SystemExit(0) + args, remaining_args = parser.parse_known_args() + args = vars(args) + alias_name = args.get('alias_name') + match alias_name: + case 'alias': + alias_name_subparser = list(filter(lambda x: x.dest == 'alias_name',parser._actions))[0] + alias_subparser = alias_name_subparser.choices.get('alias') + xts_alias.run_alias_builtin(alias_subparser) + case None|'alias_name': + parser.print_help() + raise SystemExit(0) + case _: + self._run_yaml_runner(alias_name, remaining_args) + def main(): XTS().run() diff --git a/src/xts_core/xts_alias.py b/src/xts_core/xts_alias.py index 8143d70..1d7fb32 100644 --- a/src/xts_core/xts_alias.py +++ b/src/xts_core/xts_alias.py @@ -33,13 +33,13 @@ - `xts --alias --refresh ` re-downloads or re-copies the file from its original source """ -import argparse -import os import hashlib -import shutil import json -import requests +import shutil +import sys +import os from pathlib import Path +import requests from urllib.parse import urlparse try: @@ -398,8 +398,33 @@ def refresh_alias(alias_name: str) -> tuple[str, str]: add_alias(alias_name, cached, source) return (alias_name, cached) +def setup_alias_parser(parser:XTSArgumentParser): + subparsers = parser.add_subparsers(dest='alias_action') + list_parser = subparsers.add_parser('list', + help='List all aliases.') + remove_parser = subparsers.add_parser('remove', aliases=['rm'], help='Remove an alias.') + remove_parser.add_argument('alias', + action='store', + help='Alias to remove.', + default=None, + choices=list(load_aliases().keys())) + refresh_parser = subparsers.add_parser('refresh', help='Refresh an alias from it\'s original source.') + refresh_parser.add_argument('alias', + action='store', + help='Alias to refresh.', + default=None, + metavar='ALIAS_NAME') + add_parser = subparsers.add_parser('add', help='Add an alias of an xts file URI.') + add_parser.add_argument('path', + action='store', + metavar='URI', + default=None, + help='URI of xts file to add.') + add_parser.add_argument('--name', + action='store', + help='Name to use for alias') -def run_alias_builtin(argv: list[str]) -> int: +def run_alias_builtin(alias_parser) -> int: """ Built-in alias CLI. @@ -413,86 +438,50 @@ def run_alias_builtin(argv: list[str]) -> int: Notes: - For directories, all *.xts files are added. """ - alias_parser = XTSArgumentParser(prog='xts --alias', add_help=True) - alias_parser.add_argument('uri', - action='store', - default=None, - help='URI of xts file to add or alias name', - nargs='?') - alias_parser.add_argument('--list', - action='store_true', - default=False, - help='List all aliases') - alias_parser.add_argument('--remove', '--rm', - action='store', - help='Remove alias', - default=None, - metavar='ALIAS_NAME', - dest='remove') - alias_parser.add_argument('--refresh', - action='store', - help='Refresh alias from original source', - default=None, - metavar='ALIAS_NAME', - dest='refresh') - alias_parser.add_argument('--add', - action='store', - metavar='URI', - default=None, - help='Add an alias of an xts file URI') - alias_parser.add_argument('--name', - action='store', - help='Name to use for alias') - # show help & current aliases if no args - if not argv: + if len(sys.argv) < 3: alias_parser.print_help() - print("\nCurrent aliases:") - list_aliases() - return 0 - - args = alias_parser.parse_args(argv) - - if args.list: - list_aliases() - return 0 - - if args.remove is not None: - if remove_alias(args.remove): - print(f"Removed alias: {args.remove}") - return 0 - print(f"Alias not found: {args.remove}") - return 2 - - if args.refresh is not None: - try: - name, cached_path = refresh_alias(args.refresh) - print(f"Refreshed alias: {name} -> {cached_path}") + raise SystemExit(0) + args_dict = vars(alias_parser.parse_args(sys.argv[2:])) + action = args_dict.get('alias_action') + match action: + case 'list': + list_aliases() return 0 - except (ValueError, FileNotFoundError) as e: - utils.error(str(e)) - return 2 - except Exception as e: - utils.error(f"Failed to refresh alias: {str(e)}") + + case'remove'|'rm': + if remove_alias(args_dict.get('alias')): + print(f"Removed alias: {args_dict.get('alias')}") + return 0 + print(f"Alias not found: {args_dict.get('alias')}") return 2 - if args.uri and args.add: - alias_parser.error('Provide the URI only once (either positional OR via "--add").') - return 2 - - input_value = args.add or args.uri - if not input_value: - alias_parser.print_help() - return 2 - - try: - added = add_alias_from_input(input_value, args.name) - except (FileNotFoundError, ValueError) as e: - alias_parser.error(str(e)) - return 2 - except Exception as e: - utils.error(f"Failed to add alias: {str(e)}") - return 2 + case 'refresh': + try: + name, cached_path = refresh_alias(args_dict.get('alias')) + print(f"Refreshed alias: {name} -> {cached_path}") + return 0 + except (ValueError, FileNotFoundError) as e: + utils.error(str(e)) + return 2 + except Exception as e: + utils.error(f"Failed to refresh alias: {str(e)}") + return 2 + + case 'add': + input_value = args_dict.get('path') + if not input_value: + alias_parser.print_help() + return 2 + + try: + added = add_alias_from_input(input_value, args_dict.get('name',None)) + except (FileNotFoundError, ValueError) as e: + alias_parser.error(str(e)) + return 2 + except Exception as e: + utils.error(f"Failed to add alias: {str(e)}") + return 2 - for k, v in added: - print(f"{k} -> {v}") - return 0 + for k, v in added: + print(f"{k} -> {v}") + return 0 diff --git a/src/xts_core/xts_arg_parser.py b/src/xts_core/xts_arg_parser.py index 6358cda..010b809 100644 --- a/src/xts_core/xts_arg_parser.py +++ b/src/xts_core/xts_arg_parser.py @@ -30,13 +30,13 @@ class XTSArgumentParser(argparse.ArgumentParser): """XTS Specific argument parser. """ - def __init__(self,*args, **kwargs): + def __init__(self,*args, formatter_class=RichHelpFormatter, **kwargs): kwargs.pop('formatter_class','') super().__init__(*args, **kwargs, - formatter_class=RichHelpFormatter) + formatter_class=formatter_class) def error(self, message): sys.stderr.write('error: %s\n' % message) self.print_help() - sys.exit(2) \ No newline at end of file + raise SystemExit(2) \ No newline at end of file diff --git a/src/xts_core/xts_rich_help_formatter.py b/src/xts_core/xts_rich_help_formatter.py new file mode 100644 index 0000000..f24412d --- /dev/null +++ b/src/xts_core/xts_rich_help_formatter.py @@ -0,0 +1,60 @@ +from argparse import _SubParsersAction, Action + +from rich_argparse import RichHelpFormatter, _lazy_rich as r + +from xts_core import xts_alias + +class XTSRichHelpFormatter(RichHelpFormatter): + """ + Custom formatter to display subcommands and aliases in separate sections. + """ + all_entries = [] + + @property + def _max_name_len(self) -> int: + if self.all_entries: + max_name_len = max(len(name) for name, _ in self.all_entries) + return max_name_len + self._current_indent + return 0 + + def add_arguments(self, actions): + # Override to inject custom subparser formatting + subcommands = [] + aliases = [] + non_subparser_actions = [] + for action in actions: + if isinstance(action, _SubParsersAction) and action.dest == 'alias_name': + known_aliases = list(xts_alias.load_aliases().keys()) + for choice, parser in list(action.choices.items()): + # The 'alias_name' subparser is a special case used to display aliases, + # so we skip it in the subcommands section + if choice == 'alias_name': + continue + choice_action = list(filter(lambda x: parser.prog == f'xts {x.dest}', action._choices_actions))[0] + choice_tuple = (choice, choice_action) + if choice in known_aliases: + aliases.append(choice_tuple) + else: + subcommands.append(choice_tuple) + else: + non_subparser_actions.append(action) + if non_subparser_actions: + super().add_arguments(non_subparser_actions) + elif subcommands or aliases: + # Suppress the enclosing "positional arguments" heading — we render our own sections + self._current_section.heading = None + self.all_entries = subcommands + aliases + if subcommands: + self._create_section('Subcommands', subcommands) + if aliases: + self._create_section('Aliases', aliases) + + def _create_section(self, heading:str, actions:list[tuple[str,Action]]): + self.start_section(heading) + self._action_max_length = max(self._action_max_length, self._max_name_len + self._current_indent) + for action_name, action in actions: + header = r.Text(action_name, style="argparse.args") + header.pad_left(self._current_indent) + help_text = r.Text(action.help, style="argparse.help") if action.help else None + self._current_section.rich_actions.append((header, help_text)) + self.end_section()