diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff4186..ba8dcfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # Changelog -## v0.4.1 - ? +## v0.5.0 - ? +- Add `git-ops project slice` command group to create, list and inspect the slices of a project. +- Address some compiler warnings in plugin annotations ## v0.4.0 - 2026-05-30 diff --git a/inmanta_git_ops/cli.py b/inmanta_git_ops/cli.py index 983f531..cd23853 100644 --- a/inmanta_git_ops/cli.py +++ b/inmanta_git_ops/cli.py @@ -21,12 +21,16 @@ import logging import os import pathlib +import re import subprocess import sys +import tempfile +from collections.abc import Mapping, Sequence import click import texttable +from inmanta.const import ENVIRON_FORCE_TTY from inmanta.module import ModuleV2, Project from inmanta_git_ops import const from inmanta_plugins.git_ops.store import SLICE_STORE_REGISTRY @@ -201,6 +205,181 @@ def project(inmanta_arg: list[str]) -> None: INMANTA_ARGS.extend(inmanta_arg) +def inmanta_compile_command(inmanta_compile_arg: Sequence[str]) -> list[str]: + """ + Construct the command line to run a compile on the current project. + + :param inmanta_compile_arg: Additional arguments to pass to the inmanta + compile command. + """ + return [ + sys.executable, + "-m", + "inmanta.app", + *INMANTA_ARGS, + "compile", + *inmanta_compile_arg, + ] + + +def run_compile( + inmanta_compile_arg: Sequence[str], + *, + compile_mode: str, +) -> None: + """ + Run a compile on the current project, in a subprocess, with the given + compile mode. + + :param inmanta_compile_arg: Additional arguments to pass to the inmanta + compile command. + :param compile_mode: The compile mode the compile should run in. + """ + subprocess.run( + inmanta_compile_command(inmanta_compile_arg), + check=True, + env={**os.environ, const.COMPILE_MODE_ENV_VAR: compile_mode}, + ) + + +# Environment variables, defined by inmanta-core, with which the user can +# set the logging configuration of the compiler. +COMPILER_LOGGING_CONTENT_ENV_VAR = "INMANTA_LOGGING_COMPILER_CONTENT" +COMPILER_LOGGING_ENV_VARS = [ + "INMANTA_LOGGING_COMPILER", + COMPILER_LOGGING_CONTENT_ENV_VAR, + "INMANTA_LOGGING_COMPILER_TMPL", + "INMANTA_CONFIG_LOGGING_CONFIG", + "INMANTA_CONFIG_LOGGING_CONFIG_CONTENT", + "INMANTA_CONFIG_LOGGING_CONFIG_TMPL", +] + +# Pattern matching the headers of the summary that the compiler prints at the +# end of every compile, and which shouldn't be part of a slice command output. +COMPILE_SUMMARY_HEADER_REGEX = re.compile( + r"=+ (SUCCESS|COMPILATION FAILURE|EXPORT FAILURE|EXCEPTION TRACE) =+" +) +ANSI_ESCAPE_SEQUENCE_REGEX = re.compile(r"\x1b\[[0-9;]*m") + + +def slice_compile_logging_config() -> str: + """ + Build the logging configuration for the slice command compiles: all the + compiler logs are sent to stderr, so that the compiler doesn't log + anything on stdout. The logs are colored, like the default console + logging of the compiler, when stderr is a tty. The inmanta.logging + logger is restricted to errors because it warns about the default cli + logging options it ignores when a logging config is provided. + """ + on_tty = sys.stderr.isatty() + return json.dumps( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "console": { + "()": "inmanta.logging.MultiLineFormatter", + "fmt": ( + "%(log_color)s%(name)-15s%(levelname)-8s%(reset)s%(blue)s%(message)s" + if on_tty + else "%(name)-15s%(levelname)-8s%(message)s" + ), + "log_colors": ( + { + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red", + } + if on_tty + else None + ), + "reset": on_tty, + "no_color": not on_tty, + "keep_logger_names": False, + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "console", + "level": "WARNING", + "stream": "ext://sys.stderr", + } + }, + "loggers": { + "inmanta.logging": {"level": "ERROR"}, + }, + "root": {"handlers": ["console"], "level": "WARNING"}, + } + ) + + +def run_slice_command_compile( + inmanta_compile_arg: Sequence[str], + *, + compile_mode: str, + env: Mapping[str, str], +) -> object: + """ + Run a slice command compile on the current project. The compiler is + configured to log to stderr, and any remaining compile output on stdout + is forwarded to stderr, without the compile summary banner. The result + of the command, written to the output file by the corresponding + finalizer, is read back and returned. + + :param inmanta_compile_arg: Additional arguments to pass to the inmanta + compile command. + :param compile_mode: The compile mode the compile should run in. + :param env: Additional environment variables to pass to the compile. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + output_file = pathlib.Path(tmp_dir) / "output.json" + compile_env = { + **os.environ, + const.COMPILE_MODE_ENV_VAR: compile_mode, + **env, + const.OUTPUT_FILE_ENV_VAR: str(output_file), + } + if not any(var in os.environ for var in COMPILER_LOGGING_ENV_VARS): + # The user didn't configure the compiler logging, make sure the + # compiler doesn't log anything on stdout + compile_env[COMPILER_LOGGING_CONTENT_ENV_VAR] = ( + slice_compile_logging_config() + ) + if sys.stderr.isatty(): + # The compile output goes through a pipe, let the compiler know + # its output still ends up on a tty, so it keeps its colors + compile_env[ENVIRON_FORCE_TTY] = "true" + + with subprocess.Popen( + inmanta_compile_command(inmanta_compile_arg), + env=compile_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) as process: + assert process.stdout is not None + for line in process.stdout: + plain_line = ANSI_ESCAPE_SEQUENCE_REGEX.sub("", line).strip() + if COMPILE_SUMMARY_HEADER_REGEX.fullmatch(plain_line): + # Drop the compile summary banner from the output + continue + sys.stderr.write(line) + + if process.returncode != 0: + raise click.ClickException("The compile failed (see logs above)") + + if not output_file.exists(): + raise click.ClickException( + "The compile didn't emit any result for the slice command. " + "Make sure the project model imports the git_ops module." + ) + + return json.loads(output_file.read_text()) + + @project.command("update") @click.option( "--inmanta-compile-arg", @@ -214,18 +393,7 @@ def update(inmanta_compile_arg: list[str]) -> None: Read each source slice, and update their content if processors need to resolve some values. Verify that the input data is correct, don't export any resource to the orchestrator. """ - subprocess.run( - [ - sys.executable, - "-m", - "inmanta.app", - *INMANTA_ARGS, - "compile", - *inmanta_compile_arg, - ], - check=True, - env={**os.environ, "INMANTA_GIT_OPS_COMPILE_MODE": const.COMPILE_UPDATE}, - ) + run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_UPDATE) @project.command("sync") @@ -242,18 +410,7 @@ def sync(inmanta_compile_arg: list[str]) -> None: by emitting a newer version of the slice or marking it as deleted. This will make sure the slice store is in sync with the source slices, and that the orchestrator will receive the expected resources when doing the next export. """ - subprocess.run( - [ - sys.executable, - "-m", - "inmanta.app", - *INMANTA_ARGS, - "compile", - *inmanta_compile_arg, - ], - check=True, - env={**os.environ, "INMANTA_GIT_OPS_COMPILE_MODE": const.COMPILE_SYNC}, - ) + run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_SYNC) @project.command("prune") @@ -269,19 +426,168 @@ def prune(inmanta_compile_arg: list[str]) -> None: Remove from the slice store all active slices which have a more recent version or which are deleted. """ - subprocess.run( - [ - sys.executable, - "-m", - "inmanta.app", - *INMANTA_ARGS, - "compile", - *inmanta_compile_arg, - ], - check=True, - env={**os.environ, "INMANTA_GIT_OPS_COMPILE_MODE": const.COMPILE_PRUNE}, + run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_PRUNE) + + +@project.group("slice") +def slice() -> None: + """ + Commands to manage individual slices of the current Inmanta project. + """ + pass + + +@slice.command("create") +@click.option( + "--store", + type=str, + help="The name of the store in which the slice should be created.", + envvar=const.SLICE_STORE_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--name", + type=str, + help="The name of the slice to create, used as the source file name.", + envvar=const.SLICE_NAME_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--extension", + type=click.Choice(["json", "yaml"]), + default="json", + help="The format of the created slice file.", + envvar=const.SLICE_EXTENSION_ENV_VAR, + show_default=True, + show_envvar=True, +) +@click.option( + "--inmanta-compile-arg", + multiple=True, + help="Additional arguments to pass to the inmanta compile command.", +) +def create( + store: str, + name: str, + extension: str, + inmanta_compile_arg: list[str], +) -> None: + """ + Scaffold a new source slice file for the given store. + + The created file contains all the properties of the store's schema: the + required ones with a placeholder value that should be replaced by the user, + the others pre-filled with their default value. The path of the created + file is printed to stdout. + """ + path = run_slice_command_compile( + inmanta_compile_arg, + compile_mode=const.COMPILE_SLICE_CREATE, + env={ + const.SLICE_STORE_ENV_VAR: store, + const.SLICE_NAME_ENV_VAR: name, + const.SLICE_EXTENSION_ENV_VAR: extension, + }, + ) + click.echo(path) + + +@slice.command("inspect") +@click.option( + "--store", + type=str, + help="The name of the store in which the slice is defined.", + envvar=const.SLICE_STORE_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--name", + type=str, + help="The name of the slice to inspect.", + envvar=const.SLICE_NAME_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--inmanta-compile-arg", + multiple=True, + help="Additional arguments to pass to the inmanta compile command.", +) +def inspect(store: str, name: str, inmanta_compile_arg: list[str]) -> None: + """ + Dump the fully-resolved view of a single slice as JSON. + + The output matches the view of the slice during an update compile: the merged + current and previous attributes, with operation/path markers, and the version + the slice would be assigned. + """ + result = run_slice_command_compile( + inmanta_compile_arg, + compile_mode=const.COMPILE_SLICE_INSPECT, + env={ + const.SLICE_STORE_ENV_VAR: store, + const.SLICE_NAME_ENV_VAR: name, + }, + ) + click.echo(json.dumps(result, indent=2)) + + +@slice.command("list") +@click.option( + "--store", + type=str, + default=None, + help="Only list the slices of the store with this name.", + envvar=const.SLICE_STORE_ENV_VAR, + show_envvar=True, +) +@click.option( + "--format", + type=click.Choice(["table", "json"]), + default="table", + help="Output format", +) +@click.option( + "--inmanta-compile-arg", + multiple=True, + help="Additional arguments to pass to the inmanta compile command.", +) +def list_slices( + store: str | None, + format: str, + inmanta_compile_arg: list[str], +) -> None: + """ + List the slices of the current project, optionally filtered by store. + + Slices present in either the source folder or the active store are listed, + with the version they would be assigned during an update compile. + """ + slices = run_slice_command_compile( + inmanta_compile_arg, + compile_mode=const.COMPILE_SLICE_LIST, + env={const.SLICE_STORE_ENV_VAR: store} if store is not None else {}, ) + if format == "json": + click.echo(json.dumps(slices, indent=2)) + return + + elif format == "table": + table = texttable.Texttable() + table.header(["Store", "Name", "Version", "Deleted"]) + + for s in slices: + table.add_row([s["store_name"], s["name"], s["version"], s["deleted"]]) + + click.echo(table.draw()) + + else: + raise click.BadParameter(f"Unsupported format {format}") + if __name__ == "__main__": cli() diff --git a/inmanta_git_ops/const.py b/inmanta_git_ops/const.py index 9548886..bc79ee1 100644 --- a/inmanta_git_ops/const.py +++ b/inmanta_git_ops/const.py @@ -21,7 +21,15 @@ import pydantic -type CompileMode = typing.Literal["update", "sync", "export", "prune"] +type CompileMode = typing.Literal[ + "update", + "sync", + "export", + "prune", + "slice-create", + "slice-list", + "slice-inspect", +] COMPILE_UPDATE = "update" @@ -29,6 +37,18 @@ COMPILE_EXPORT = "export" COMPILE_PRUNE = "prune" COMPILE_EMPTY = "empty" +COMPILE_SLICE_CREATE = "slice-create" +COMPILE_SLICE_LIST = "slice-list" +COMPILE_SLICE_INSPECT = "slice-inspect" + +# The compile modes triggered by the `git-ops project slice` commands. During +# these compiles, no slice is emitted in the model, the command logic is +# executed in a finalizer instead. +COMPILE_SLICE_COMMANDS = [ + COMPILE_SLICE_CREATE, + COMPILE_SLICE_LIST, + COMPILE_SLICE_INSPECT, +] COMPILE_MODE_ENV_VAR = "INMANTA_GIT_OPS_COMPILE_MODE" COMPILE_MODE_ADAPTER = pydantic.TypeAdapter(CompileMode) @@ -36,6 +56,26 @@ os.getenv(COMPILE_MODE_ENV_VAR, COMPILE_EXPORT) ) +# Options for the slice command compiles, passed to the compile subprocess +# by the cli, the same way the compile mode is. +SLICE_STORE_ENV_VAR = "INMANTA_GIT_OPS_SLICE_STORE" +SLICE_STORE: str | None = os.getenv(SLICE_STORE_ENV_VAR) + +SLICE_NAME_ENV_VAR = "INMANTA_GIT_OPS_SLICE_NAME" +SLICE_NAME: str | None = os.getenv(SLICE_NAME_ENV_VAR) + +SLICE_EXTENSION_ENV_VAR = "INMANTA_GIT_OPS_SLICE_EXTENSION" +SLICE_EXTENSION: str = os.getenv(SLICE_EXTENSION_ENV_VAR, "json") + +# File in which the result of a slice command should be written by the +# finalizer executing it, so that the cli process can read it back. +OUTPUT_FILE_ENV_VAR = "INMANTA_GIT_OPS_OUTPUT_FILE" +OUTPUT_FILE: str | None = os.getenv(OUTPUT_FILE_ENV_VAR) + SLICE_CREATE = "create" SLICE_UPDATE = "update" SLICE_DELETE = "delete" + +# Placeholder used for all the required property values of a newly scaffolded +# slice, which the user must replace with real values. +SLICE_PLACEHOLDER = "" diff --git a/inmanta_plugins/git_ops/__init__.py b/inmanta_plugins/git_ops/__init__.py index 513d4f5..c28092c 100644 --- a/inmanta_plugins/git_ops/__init__.py +++ b/inmanta_plugins/git_ops/__init__.py @@ -46,6 +46,11 @@ def unroll_slices( """ from inmanta_plugins.git_ops import store + if const.COMPILE_MODE in const.COMPILE_SLICE_COMMANDS: + # Slice command compile, no slice should be emitted in the model, + # the command logic is handled in a finalizer + return [] + all_slices = store.get_store(store_name).get_all_slices() return [asdict(s) for s in all_slices] diff --git a/inmanta_plugins/git_ops/processors.py b/inmanta_plugins/git_ops/processors.py index 711cf65..0abacf9 100644 --- a/inmanta_plugins/git_ops/processors.py +++ b/inmanta_plugins/git_ops/processors.py @@ -116,7 +116,9 @@ def get_elements( @plugin def join_used_values( - *used_values: typing.Annotated[typing.Callable[[], Collection[object]], object], + *used_values: typing.Annotated[ + typing.Callable[[], Collection[object]], ModelType["any"] + ], ) -> typing.Annotated[typing.Callable[[], Collection[object]], ModelType["any"]]: """ Join multiple used values collectors into one. @@ -135,7 +137,9 @@ def unique_integer( path: str, previous_value: int | None = None, *, - used_integers: typing.Annotated[typing.Callable[[], Collection[int]], object], + used_integers: typing.Annotated[ + typing.Callable[[], Collection[int]], ModelType["any"] + ], range_start: int = 0, range_stop: int = 1000, refresh: bool = False, diff --git a/inmanta_plugins/git_ops/slice.py b/inmanta_plugins/git_ops/slice.py index bf083f4..8db85ee 100644 --- a/inmanta_plugins/git_ops/slice.py +++ b/inmanta_plugins/git_ops/slice.py @@ -33,6 +33,7 @@ from pydantic.json_schema import SkipJsonSchema import inmanta.ast.type as inmanta_type +from inmanta_git_ops import const LOGGER = logging.getLogger(__name__) @@ -583,6 +584,55 @@ class EmbeddedSliceObjectABC(pydantic.BaseModel): exclude_if=slice_update, ) + @classmethod + def scaffold(cls) -> dict: + """ + Build a template attributes dict for this slice object. The dict + contains all the properties of the schema: the required ones (those + without a default value) get a placeholder value the user must + replace, the others are pre-filled with their default value. + Mandatory relations towards embedded slice objects (single embedded + slice annotation, even when the field has a default) are scaffolded + recursively, any other required value (primitive, list, union, ...) + gets the placeholder string. + """ + attributes: dict = {} + for attribute, info in cls.model_fields.items(): + if info.exclude_if is slice_update: + # Model-only attribute, it is not part of the slice source files + continue + + annotation = info.annotation + if inspect.isclass(annotation) and issubclass( + annotation, EmbeddedSliceObjectABC + ): + # Mandatory relation towards an embedded slice object, always + # scaffold it recursively: even when the field has a default, + # the embedded object may have required properties of its own + attributes[attribute] = annotation.scaffold() + continue + + if not info.is_required(): + # Optional attribute, pre-fill it with its default value + try: + default = info.get_default(call_default_factory=True) + except pydantic.ValidationError: + # The default can not be constructed (e.g. a factory + # building an embedded object with required properties), + # the user will have to provide the value + attributes[attribute] = const.SLICE_PLACEHOLDER + continue + + with exclude_model_values(): + attributes[attribute] = pydantic.TypeAdapter( + annotation + ).dump_python(default, mode="json") + continue + + attributes[attribute] = const.SLICE_PLACEHOLDER + + return attributes + @classmethod def entity_schema(cls) -> SliceEntitySchema: """ diff --git a/inmanta_plugins/git_ops/store.py b/inmanta_plugins/git_ops/store.py index dea2a4a..4c02d53 100644 --- a/inmanta_plugins/git_ops/store.py +++ b/inmanta_plugins/git_ops/store.py @@ -24,7 +24,7 @@ import re import typing from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import asdict, dataclass import pydantic import yaml @@ -543,15 +543,23 @@ def load_current_slices(self) -> dict[str, Slice]: active_slices = self.load_active_slices() + # In these compile modes, the current version of the slices is read + # from the source folder, like for an update compile + source_view = const.COMPILE_MODE in [ + const.COMPILE_UPDATE, + const.COMPILE_SYNC, + const.COMPILE_SLICE_LIST, + const.COMPILE_SLICE_INSPECT, + ] + slices = set(active_slices.keys()) - if const.COMPILE_MODE in [const.COMPILE_UPDATE, const.COMPILE_SYNC]: - # Activating compile, we need to look at the source of the - # slices too + if source_view: + # We need to look at the source of the slices too slices |= self.load_source_slices().keys() self.current_slices: dict[str, Slice] = {} for s in slices: - if const.COMPILE_MODE in [const.COMPILE_UPDATE, const.COMPILE_SYNC]: + if source_view: self.current_slices[s] = self.load_source_slices()[s] else: self.current_slices[s] = self.get_latest_slice(s) @@ -636,6 +644,11 @@ def load_slices(self) -> dict[str, Slice]: self.slices: dict[str, Slice] = {} for s, current in self.load_current_slices().items(): if current.deleted: + if s not in previous_slices: + # The slice never had any undeleted version, there is + # nothing to delete, skip it + continue + # We need to get the attributes of the last undeleted # version, otherwise we don't know what we have to delete attributes = merge_attributes( @@ -866,6 +879,41 @@ def update(self) -> None: json.dumps(self.schema.model_json_schema(), indent=2) ) + def create_slice(self, name: str, *, extension: str = "json") -> SliceFile: + """ + Scaffold a new source slice file for this store. The file contains + all the properties of the store's schema: the required ones with a + placeholder value that the user must replace, the others pre-filled + with their default value. Refuse to create the slice if a source + slice with the same name already exists. + + :param name: The name of the slice to create, it becomes the name + of the source file. + :param extension: The extension (and format) of the created file. + """ + if not name or name.startswith(".") or "/" in name or "@" in name: + raise ValueError( + f"Invalid slice name: {name!r}. A slice name can not be empty, " + "start with a dot, or contain any '/' or '@' character." + ) + + source_slice_files = self.load_source_slice_files() + if name in source_slice_files: + raise ValueError( + f"A source slice named {name} already exists in store {self.name} " + f"at {source_slice_files[name].path}" + ) + + slice_file = SliceFile( + path=self.source_path / f"{name}.{extension}", + name=name, + version=None, + extension=extension, + schema=self.schema, + ) + slice_file.write_raw(self.schema.scaffold()) + return slice_file + def clear(self) -> None: """ Clear the cache of slices in memory. @@ -1033,6 +1081,62 @@ def get_store(store_name: str) -> SliceStore[slice.SliceObjectABC]: return SLICE_STORE_REGISTRY[store_name] +def write_command_output(result: object) -> None: + """ + Write the result of a slice command to the output file designated by + the cli process. If no output file is set (manual compile), print the + result to stdout instead. + """ + serialized = json.dumps(result, indent=2) + if const.OUTPUT_FILE is not None: + pathlib.Path(const.OUTPUT_FILE).write_text(serialized) + else: + print(serialized) + + +@finalizer +def run_slice_command() -> None: + """ + At the end of the compile, if the compile was triggered by a + `git-ops project slice` command, execute the command logic and emit + its result. + """ + if const.COMPILE_MODE not in const.COMPILE_SLICE_COMMANDS: + return + + if const.COMPILE_MODE == const.COMPILE_SLICE_LIST: + # List the slices of one store, or of all the registered stores + stores = ( + [get_store(const.SLICE_STORE)] + if const.SLICE_STORE is not None + else list(SLICE_STORE_REGISTRY.values()) + ) + write_command_output( + [ + {k: v for k, v in asdict(s).items() if k != "attributes"} + for store in stores + for s in store.get_all_slices() + ] + ) + return + + # The create and inspect commands always target one specific slice + if const.SLICE_STORE is None or const.SLICE_NAME is None: + raise ValueError( + f"The {const.COMPILE_MODE} compile requires both {const.SLICE_STORE_ENV_VAR} " + f"and {const.SLICE_NAME_ENV_VAR} environment variables to be set" + ) + + store = get_store(const.SLICE_STORE) + if const.COMPILE_MODE == const.COMPILE_SLICE_CREATE: + slice_file = store.create_slice( + const.SLICE_NAME, extension=const.SLICE_EXTENSION + ) + write_command_output(str(slice_file.path)) + else: + write_command_output(asdict(store.get_one_slice(const.SLICE_NAME))) + + @finalizer def persist_store() -> None: """ diff --git a/setup.cfg b/setup.cfg index adccda2..f52da87 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = inmanta-module-git-ops -version = 0.4.1 +version = 0.5.0 description = Declarative parametrization, extension and validation of Inmanta DSL models. long_description = file: README.md long_description_content_type = text/markdown diff --git a/tests/test_cli.py b/tests/test_cli.py index 5ab0e69..198bf2c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,6 +21,8 @@ import pathlib import subprocess +import yaml + def test_basics() -> None: example_path = pathlib.Path(__file__).parent.parent / "docs/example" @@ -46,3 +48,162 @@ def test_basics() -> None: check=True, env={"INMANTA_GIT_OPS_MODULE_PATH": str(example_path), **os.environ}, ) + + +def test_project_slice_commands(tmp_path: pathlib.Path) -> None: + # Set up a minimal inmanta project using the example module, the modules + # are resolved from the python environment running the test + project_path = tmp_path / "project" + project_path.mkdir() + (project_path / "project.yml").write_text(yaml.safe_dump({"name": "test-project"})) + (project_path / "main.cf").write_text( + "\n".join( + [ + "import example::slices::fs", + "import example::slices::fs::unroll", + "import example::slices::simple", + "import example::slices::recursive", + ] + ) + ) + + def git_ops(*args: str, expect_failure: bool = False) -> str: + result = subprocess.run( + ["git-ops", "project", *args], + cwd=project_path, + env={**os.environ}, + text=True, + capture_output=True, + ) + if expect_failure: + assert result.returncode != 0, result.stdout + result.stderr + else: + assert result.returncode == 0, result.stdout + result.stderr + if args[0] == "slice": + # The compile summary banner is dropped from the output of the + # slice commands + assert "SUCCESS" not in result.stderr, result.stderr + assert "FAILURE" not in result.stderr, result.stderr + return result.stdout + + # No slices yet + assert json.loads(git_ops("slice", "list", "--format", "json")) == [] + + # Scaffold a new fs slice, the created file contains a placeholder for + # each required property of the store schema, and the default value of + # the non-required ones. The created path, printed to stdout, is + # relative to the directory the command is invoked from. + created = git_ops("slice", "create", "--store", "fs", "--name", "test-folder") + fs_source = project_path / "files" / "fs" / "test-folder.json" + assert project_path / created.strip() == fs_source + assert json.loads(fs_source.read_text()) == { + "name": "", + "root": "", + "permissions": "770", + "owner": None, + "group": None, + "type": "folder", + "content": [], + } + + # Refuse to overwrite an existing source slice + git_ops( + "slice", + "create", + "--store", + "fs", + "--name", + "test-folder", + expect_failure=True, + ) + + # Refuse to create a slice in an unknown store + git_ops( + "slice", "create", "--store", "unknown", "--name", "test", expect_failure=True + ) + + # Scaffold a yaml slice, with a mandatory embedded relation + created = git_ops( + "slice", + "create", + "--store", + "recursive", + "--name", + "rec", + "--extension", + "yaml", + ) + rec_source = project_path / "files" / "recursive" / "rec.yaml" + assert project_path / created.strip() == rec_source + assert yaml.safe_load(rec_source.read_text()) == { + "name": "", + "description": None, + "unique_id": None, + "embedded_required": { + "name": "", + "description": None, + "unique_id": None, + "recursive_slice": [], + }, + "embedded_optional": None, + "embedded_sequence": [], + } + rec_source.unlink() + + # Fill in the placeholders of the fs slice + attributes = json.loads(fs_source.read_text()) + attributes["name"] = "folder" + attributes["root"] = "/tmp" + fs_source.write_text(json.dumps(attributes)) + + # The slice now shows up in the list, with the version it would be + # assigned during an update compile + assert json.loads(git_ops("slice", "list", "--format", "json")) == [ + {"name": "test-folder", "store_name": "fs", "version": 1, "deleted": False} + ] + + # The table output contains the slice details + table = git_ops("slice", "list") + assert "Store" in table + assert "fs" in table + assert "test-folder" in table + + # Inspect the slice, the output is the fully resolved merged slice + inspected = json.loads( + git_ops("slice", "inspect", "--store", "fs", "--name", "test-folder") + ) + assert inspected["name"] == "test-folder" + assert inspected["store_name"] == "fs" + assert inspected["version"] == 1 + assert inspected["deleted"] is False + assert inspected["attributes"]["operation"] == "create" + assert inspected["attributes"]["path"] == "." + assert inspected["attributes"]["version"] == 1 + assert inspected["attributes"]["slice_store"] == "fs" + assert inspected["attributes"]["slice_name"] == "test-folder" + assert inspected["attributes"]["name"] == "folder" + assert inspected["attributes"]["root"] == "/tmp" + assert inspected["attributes"]["content"] == [] + + # Inspecting a slice that doesn't exist fails + git_ops( + "slice", "inspect", "--store", "fs", "--name", "missing", expect_failure=True + ) + + # Sync the slice to the active store, then delete its source file + git_ops("sync") + fs_source.unlink() + + # The slice is still listed, as a deleted slice with a new version + assert json.loads(git_ops("slice", "list", "--format", "json")) == [ + {"name": "test-folder", "store_name": "fs", "version": 2, "deleted": True} + ] + + # An empty source slice which never had any active version is ignored + empty_source = project_path / "files" / "simple" / "empty.json" + empty_source.parent.mkdir(parents=True, exist_ok=True) + empty_source.write_text("{}") + assert ( + json.loads(git_ops("slice", "list", "--store", "simple", "--format", "json")) + == [] + ) diff --git a/tests/test_slice_schema.py b/tests/test_slice_schema.py index 21c0a2b..5c41cd2 100644 --- a/tests/test_slice_schema.py +++ b/tests/test_slice_schema.py @@ -16,7 +16,16 @@ Contact: edvgui@gmail.com """ +import typing +from collections.abc import Sequence + +import pydantic +from inmanta_plugins.example.slices.fs import File, RootFolder from inmanta_plugins.example.slices.recursive import EmbeddedSlice, Slice +from inmanta_plugins.example.slices.simple import Slice as SimpleSlice + +from inmanta_git_ops import const +from inmanta_plugins.git_ops import slice def test_basics() -> None: @@ -86,3 +95,117 @@ def test_basics() -> None: # The embedded slice should extend the EmbeddedSliceObjectABC assert [b.name for b in embedded_schema.base_entities] == ["NamedSlice"] + + +def test_scaffold() -> None: + # The scaffold of a slice contains all its properties (including the + # inherited ones): the required ones with a placeholder value, the + # others pre-filled with their default value + assert RootFolder.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "root": const.SLICE_PLACEHOLDER, + "permissions": "770", + "owner": None, + "group": None, + "type": "folder", + "content": [], + } + assert SimpleSlice.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "description": None, + "unique_id": None, + "some_number": 0.0, + "some_flag": False, + "some_list": [], + "some_dict": {}, + } + + # Model-only attributes (such as previous_content) are not part of the + # scaffold, as they are not part of the slice source files + assert File.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "permissions": "770", + "owner": None, + "group": None, + "type": "file", + "content": "", + } + + # Mandatory embedded relations are scaffolded recursively + assert Slice.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "description": None, + "unique_id": None, + "embedded_required": { + "name": const.SLICE_PLACEHOLDER, + "description": None, + "unique_id": None, + "recursive_slice": [], + }, + "embedded_optional": None, + "embedded_sequence": [], + } + + +def test_scaffold_embedded_defaults() -> None: + class Container(slice.EmbeddedSliceObjectABC): + image: str + cpus: float = 1.0 + + class Service(slice.SliceObjectABC): + name: str + container: Container = pydantic.Field(default_factory=Container) + replicas: Sequence[Container] = pydantic.Field( + default_factory=lambda: [Container(image="alpine")] + ) + invalid: Sequence[Container] = pydantic.Field( + default_factory=lambda: [Container()] + ) + + assert Service.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + # A mandatory embedded relation is scaffolded recursively even when + # the field has a default: the factory can not construct the embedded + # object as it has required properties of its own + "container": { + "image": const.SLICE_PLACEHOLDER, + "cpus": 1.0, + }, + # Embedded objects in a default value are serialized like the slice + # files, without the model-only attributes + "replicas": [{"image": "alpine", "cpus": 1.0}], + # A default that can not be constructed falls back to the placeholder + "invalid": const.SLICE_PLACEHOLDER, + } + + +def test_scaffold_discriminated_union() -> None: + class A(slice.EmbeddedSliceObjectABC): + type: typing.Literal["a"] = "a" + required_value: str + + class B(slice.EmbeddedSliceObjectABC): + type: typing.Literal["b"] = "b" + + AB = typing.Annotated[A | B, pydantic.Field(discriminator="type")] + + class Root(slice.SliceObjectABC): + name: str + item: AB + fallback: AB = pydantic.Field(default_factory=B) + broken: AB = pydantic.Field(default_factory=A) + items: Sequence[AB] = pydantic.Field(default_factory=lambda: [B()]) + + assert Root.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + # A required union relation gets the placeholder: the scaffold can + # not pick a union member for the user + "item": const.SLICE_PLACEHOLDER, + # Union members in a default value are serialized like the slice + # files, with their discriminator + "fallback": {"type": "b"}, + "items": [{"type": "b"}], + # A default union member that can not be constructed falls back to + # the placeholder + "broken": const.SLICE_PLACEHOLDER, + }