From cec058e8ed1d91f41217defb48decfc001aafe48 Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Tue, 2 Jun 2026 21:22:52 +0200 Subject: [PATCH 1/9] Add git-ops project slice command group (create, list, inspect) Add three new commands to manage individual slices of a project from the cli: - `git-ops project slice create`: scaffold a new source slice file containing a placeholder for each required property of the store's schema, recursing into mandatory embedded relations. - `git-ops project slice list`: list the slices of one or all stores, as a table or as json, with the version they would be assigned during an update compile. - `git-ops project slice inspect`: dump the fully-resolved merged view of a single slice as json. Each command triggers a dedicated compile mode during which no slice is emitted in the model, the command logic runs in a finalizer which sends its result back to the cli process through a temporary output file. The compile output is redirected to stderr so the command result on stdout stays clean for piping. Also guard the merge of deleted slices against source slices that never had any active version, which previously raised a KeyError. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 1 + docs/design/project-slice-commands.md | 121 ++++++++++++ inmanta_git_ops/cli.py | 271 ++++++++++++++++++++++---- inmanta_git_ops/const.py | 42 +++- inmanta_plugins/git_ops/__init__.py | 5 + inmanta_plugins/git_ops/slice.py | 27 +++ inmanta_plugins/git_ops/store.py | 113 ++++++++++- tests/test_cli.py | 138 +++++++++++++ tests/test_slice_schema.py | 20 ++ 9 files changed, 697 insertions(+), 41 deletions(-) create mode 100644 docs/design/project-slice-commands.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff4186..0b31859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## v0.4.1 - ? +- Add `git-ops project slice` command group to create, list and inspect the slices of a project. ## v0.4.0 - 2026-05-30 diff --git a/docs/design/project-slice-commands.md b/docs/design/project-slice-commands.md new file mode 100644 index 0000000..ac840fc --- /dev/null +++ b/docs/design/project-slice-commands.md @@ -0,0 +1,121 @@ +# Feature Request: `git-ops project slice` command set + +## Summary + +Extend the `git-ops` CLI with a new command group, `git-ops project slice`, that lets +users manage individual slices of a project from the command line without having to +hand-edit slice files or write throw-away scripts. + +The group introduces three sub-commands: + +| Command | Purpose | +| --------- | ----------------------------------------------------------------------- | +| `create` | Scaffold a new source slice file for a given store. | +| `list` | List all slices of a project, optionally filtered by store. | +| `inspect` | Dump the fully-resolved view of a single slice as JSON. | + +These commands live alongside the existing project commands (`update`, `sync`, +`prune`) in `inmanta_git_ops/cli.py`, under the existing `@cli.group("project")`. + +## Background + +A few concepts from the existing codebase are relevant (see +`inmanta_plugins/git_ops/store.py` and `inmanta_plugins/git_ops/slice.py`): + +- **Store** (`SliceStore`): a named collection of slices, defined in a project's + modules and registered in `SLICE_STORE_REGISTRY` on import. Each store has a + pydantic `schema` (a subclass of `slice.SliceObjectABC`) and a source folder + (`store._folder`, e.g. `inmanta:///files/fs/`). +- **Source slice**: an unversioned, user-editable file in the store's source folder + (`.json` / `.yaml`). The file name (without extension/version) is the + slice name. +- **Active slice**: a versioned, committed snapshot in the active folder + (`inmanta:///git_ops/active//@v.json`). A slice may have + several active versions. +- **Latest / oldest version**: `SliceStore.get_latest_slice(name)` returns the highest + active version; the set of active versions is available via + `load_active_slices()[name]`. +- **In sync**: a source slice is "in sync" when its content matches the latest active + version. `load_source_slices()` reflects this: when the source content equals the + latest active attributes, the source slice keeps the latest version; otherwise it is + emitted as `latest.version + 1` (i.e. out of sync, pending a `sync`). +- **`get_one_slice(name)`**: returns the fully-resolved, merged `Slice` (current vs. + previous, with `operation`/`path` markers, `version`, `slice_store`, `slice_name`). + +## Requirements + +### 1. `git-ops project slice create` + +Scaffold a brand-new **source** slice file and place it in the correct store's source +folder. + +- Inputs (each resolvable via CLI argument, environment variable, or interactive + prompt when missing — following the existing `click` option pattern with + `envvar=...` / `show_envvar=True`): + - **store** — name of the target store (must exist in `SLICE_STORE_REGISTRY`). + - **name** — the slice name; becomes the source file name. + - **format/extension** — `json` or `yaml` (default `json`, matching the format the + rest of the tooling writes). +- The created file must contain **all required properties** of the store's schema, with + **explicit placeholders** wherever the user must still supply a value. Required + properties are those without a default (the schema's `required` list, e.g. `name` and + `root` for the `fs` store) plus mandatory nested relations (cardinality_min ≥ 1). + Placeholders should be obvious and self-describing (e.g. a sentinel string carrying + the attribute's description) so the file does not silently validate with bogus data. +- The file is written into the store's source folder (`store.source_path`) under + `.`, with no version suffix. +- The command should refuse to overwrite an existing source slice with the same name. +- The command should seed placeholders for required property values (the user edits + the file afterwards), **not** prompt for each required value. + +### 2. `git-ops project slice inspect` + +Inspect a single slice. + +- Inputs: **store** and **name** (CLI arg / env var / prompt, as above). +- Output: the JSON serialization of `SliceStore.get_one_slice(name)` — the + fully-resolved, merged slice (attributes with `operation`/`path` markers, `version`, + `slice_store`, `slice_name`). +- A missing slice should surface a clear error (`get_one_slice` raises `LookupError`). + +### 3. `git-ops project slice list` + +List the slices of the current project. + +- **Filter** by store via `--store ` (optional; default lists every registered + store). +- **Output format** configurable via `--format {table,json}` (default `table`), mirroring + `git-ops module store list`. +- Each slice "row" contains the same details as the inspect command, except for the attributes. +- Rows should cover slices that exist in either the active store or the source folder. + A slice with no active version yet (never synced) and a slice deleted from source but + still active are both meaningful states the output should represent sensibly. + +## Technical details + +1. **Compiler context.** Similarly to other commands (like prune) a + compile must be running in order to find the available stores and slices. Each of + these commands will trigger a special type of compiles, which doesn't emit any + slice in the model (similarly to prune). All other logic should be handled in + a finalizer, which checks for the current compile mode. +2. **Slice view.** list and inspect command should run in "update" mode, except that + no slice should be emitted in the model. But the output version and attributes + should match the same as an update compile. +3. **Placeholder representation.** Use as placeholder for all values, + regardless of their types. +4. **Reuse of formatting.** `list` should reuse the existing `texttable` + `json.dumps` + pattern already used by `git-ops module store list` for consistency. + +## Out of scope + +- Editing/updating existing slice content (covered by hand-editing + `update`/`sync`). +- Deleting slices, pruning, or any active-store mutation (already covered by `prune`). +- Schema migrations. + +## Affected files (anticipated) + +- `inmanta_git_ops/cli.py` — new `@project.group("slice")` and its three commands. +- `inmanta_plugins/git_ops/store.py` — possibly small helpers if store loading / + oldest-version lookup needs to be exposed cleanly. +- `tests/test_cli.py` — CLI invocation tests against `docs/example` (the `fs`, + `simple`, `recursive` stores). diff --git a/inmanta_git_ops/cli.py b/inmanta_git_ops/cli.py index 983f531..69d0431 100644 --- a/inmanta_git_ops/cli.py +++ b/inmanta_git_ops/cli.py @@ -23,6 +23,9 @@ import pathlib import subprocess import sys +import tempfile +import typing +from collections.abc import Mapping, Sequence import click import texttable @@ -201,18 +204,22 @@ def project(inmanta_arg: list[str]) -> None: INMANTA_ARGS.extend(inmanta_arg) -@project.command("update") -@click.option( - "--inmanta-compile-arg", - multiple=True, - help="Additional arguments to pass to the inmanta compile command.", -) -def update(inmanta_compile_arg: list[str]) -> None: +def run_compile( + inmanta_compile_arg: Sequence[str], + *, + compile_mode: str, + env: Mapping[str, str] | None = None, + stdout: typing.IO | None = None, +) -> None: """ - Update the source slices. - - Read each source slice, and update their content if processors need to resolve some values. - Verify that the input data is correct, don't export any resource to the orchestrator. + Run a compile on the current project, in a subprocess, with the given + compile mode. + + :param inmanta_compile_arg: Additional arguments to pass to the inmanta + compile command. + :param compile_mode: The compile mode the compile should run in. + :param env: Additional environment variables to pass to the compile. + :param stdout: Where to redirect the standard output of the compile. """ subprocess.run( [ @@ -224,10 +231,67 @@ def update(inmanta_compile_arg: list[str]) -> None: *inmanta_compile_arg, ], check=True, - env={**os.environ, "INMANTA_GIT_OPS_COMPILE_MODE": const.COMPILE_UPDATE}, + env={**os.environ, const.COMPILE_MODE_ENV_VAR: compile_mode, **(env or {})}, + stdout=stdout, ) +def run_slice_command_compile( + inmanta_compile_arg: Sequence[str], + *, + compile_mode: str, + env: Mapping[str, str], +) -> object: + """ + Run a slice command compile on the current project. The output of the + compile is redirected to stderr, and the result of the command, written + to the output file by the corresponding finalizer, is read back and + returned. + + :param inmanta_compile_arg: Additional arguments to pass to the inmanta + compile command. + :param compile_mode: The compile mode the compile should run in. + :param env: Additional environment variables to pass to the compile. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + output_file = pathlib.Path(tmp_dir) / "output.json" + try: + run_compile( + inmanta_compile_arg, + compile_mode=compile_mode, + env={**env, const.OUTPUT_FILE_ENV_VAR: str(output_file)}, + stdout=sys.stderr, + ) + except subprocess.CalledProcessError as e: + raise click.ClickException( + f"The compile failed (see logs above): {e}" + ) from e + + if not output_file.exists(): + raise click.ClickException( + "The compile didn't emit any result for the slice command. " + "Make sure the project model imports the git_ops module." + ) + + return json.loads(output_file.read_text()) + + +@project.command("update") +@click.option( + "--inmanta-compile-arg", + multiple=True, + help="Additional arguments to pass to the inmanta compile command.", +) +def update(inmanta_compile_arg: list[str]) -> None: + """ + Update the source slices. + + Read each source slice, and update their content if processors need to resolve some values. + Verify that the input data is correct, don't export any resource to the orchestrator. + """ + run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_UPDATE) + + @project.command("sync") @click.option( "--inmanta-compile-arg", @@ -242,18 +306,7 @@ def sync(inmanta_compile_arg: list[str]) -> None: by emitting a newer version of the slice or marking it as deleted. This will make sure the slice store is in sync with the source slices, and that the orchestrator will receive the expected resources when doing the next export. """ - subprocess.run( - [ - sys.executable, - "-m", - "inmanta.app", - *INMANTA_ARGS, - "compile", - *inmanta_compile_arg, - ], - check=True, - env={**os.environ, "INMANTA_GIT_OPS_COMPILE_MODE": const.COMPILE_SYNC}, - ) + run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_SYNC) @project.command("prune") @@ -269,19 +322,167 @@ def prune(inmanta_compile_arg: list[str]) -> None: Remove from the slice store all active slices which have a more recent version or which are deleted. """ - subprocess.run( - [ - sys.executable, - "-m", - "inmanta.app", - *INMANTA_ARGS, - "compile", - *inmanta_compile_arg, - ], - check=True, - env={**os.environ, "INMANTA_GIT_OPS_COMPILE_MODE": const.COMPILE_PRUNE}, + run_compile(inmanta_compile_arg, compile_mode=const.COMPILE_PRUNE) + + +@project.group("slice") +def slice() -> None: + """ + Commands to manage individual slices of the current Inmanta project. + """ + pass + + +@slice.command("create") +@click.option( + "--store", + type=str, + help="The name of the store in which the slice should be created.", + envvar=const.SLICE_STORE_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--name", + type=str, + help="The name of the slice to create, used as the source file name.", + envvar=const.SLICE_NAME_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--extension", + type=click.Choice(["json", "yaml"]), + default="json", + help="The format of the created slice file.", + envvar=const.SLICE_EXTENSION_ENV_VAR, + show_default=True, + show_envvar=True, +) +@click.option( + "--inmanta-compile-arg", + multiple=True, + help="Additional arguments to pass to the inmanta compile command.", +) +def create( + store: str, + name: str, + extension: str, + inmanta_compile_arg: list[str], +) -> None: + """ + Scaffold a new source slice file for the given store. + + The created file contains all the required properties of the store's schema, + with placeholder values that should be replaced by the user. The path of the + created file is printed to stdout. + """ + path = run_slice_command_compile( + inmanta_compile_arg, + compile_mode=const.COMPILE_SLICE_CREATE, + env={ + const.SLICE_STORE_ENV_VAR: store, + const.SLICE_NAME_ENV_VAR: name, + const.SLICE_EXTENSION_ENV_VAR: extension, + }, + ) + click.echo(path) + + +@slice.command("inspect") +@click.option( + "--store", + type=str, + help="The name of the store in which the slice is defined.", + envvar=const.SLICE_STORE_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--name", + type=str, + help="The name of the slice to inspect.", + envvar=const.SLICE_NAME_ENV_VAR, + prompt=True, + show_envvar=True, +) +@click.option( + "--inmanta-compile-arg", + multiple=True, + help="Additional arguments to pass to the inmanta compile command.", +) +def inspect(store: str, name: str, inmanta_compile_arg: list[str]) -> None: + """ + Dump the fully-resolved view of a single slice as JSON. + + The output matches the view of the slice during an update compile: the merged + current and previous attributes, with operation/path markers, and the version + the slice would be assigned. + """ + result = run_slice_command_compile( + inmanta_compile_arg, + compile_mode=const.COMPILE_SLICE_INSPECT, + env={ + const.SLICE_STORE_ENV_VAR: store, + const.SLICE_NAME_ENV_VAR: name, + }, + ) + click.echo(json.dumps(result, indent=2)) + + +@slice.command("list") +@click.option( + "--store", + type=str, + default=None, + help="Only list the slices of the store with this name.", + envvar=const.SLICE_STORE_ENV_VAR, + show_envvar=True, +) +@click.option( + "--format", + type=click.Choice(["table", "json"]), + default="table", + help="Output format", +) +@click.option( + "--inmanta-compile-arg", + multiple=True, + help="Additional arguments to pass to the inmanta compile command.", +) +def list_slices( + store: str | None, + format: str, + inmanta_compile_arg: list[str], +) -> None: + """ + List the slices of the current project, optionally filtered by store. + + Slices present in either the source folder or the active store are listed, + with the version they would be assigned during an update compile. + """ + slices = run_slice_command_compile( + inmanta_compile_arg, + compile_mode=const.COMPILE_SLICE_LIST, + env={const.SLICE_STORE_ENV_VAR: store} if store is not None else {}, ) + if format == "json": + click.echo(json.dumps(slices, indent=2)) + return + + elif format == "table": + table = texttable.Texttable() + table.header(["Store", "Name", "Version", "Deleted"]) + + for s in slices: + table.add_row([s["store_name"], s["name"], s["version"], s["deleted"]]) + + click.echo(table.draw()) + + else: + raise click.BadParameter(f"Unsupported format {format}") + if __name__ == "__main__": cli() diff --git a/inmanta_git_ops/const.py b/inmanta_git_ops/const.py index 9548886..bc79ee1 100644 --- a/inmanta_git_ops/const.py +++ b/inmanta_git_ops/const.py @@ -21,7 +21,15 @@ import pydantic -type CompileMode = typing.Literal["update", "sync", "export", "prune"] +type CompileMode = typing.Literal[ + "update", + "sync", + "export", + "prune", + "slice-create", + "slice-list", + "slice-inspect", +] COMPILE_UPDATE = "update" @@ -29,6 +37,18 @@ COMPILE_EXPORT = "export" COMPILE_PRUNE = "prune" COMPILE_EMPTY = "empty" +COMPILE_SLICE_CREATE = "slice-create" +COMPILE_SLICE_LIST = "slice-list" +COMPILE_SLICE_INSPECT = "slice-inspect" + +# The compile modes triggered by the `git-ops project slice` commands. During +# these compiles, no slice is emitted in the model, the command logic is +# executed in a finalizer instead. +COMPILE_SLICE_COMMANDS = [ + COMPILE_SLICE_CREATE, + COMPILE_SLICE_LIST, + COMPILE_SLICE_INSPECT, +] COMPILE_MODE_ENV_VAR = "INMANTA_GIT_OPS_COMPILE_MODE" COMPILE_MODE_ADAPTER = pydantic.TypeAdapter(CompileMode) @@ -36,6 +56,26 @@ os.getenv(COMPILE_MODE_ENV_VAR, COMPILE_EXPORT) ) +# Options for the slice command compiles, passed to the compile subprocess +# by the cli, the same way the compile mode is. +SLICE_STORE_ENV_VAR = "INMANTA_GIT_OPS_SLICE_STORE" +SLICE_STORE: str | None = os.getenv(SLICE_STORE_ENV_VAR) + +SLICE_NAME_ENV_VAR = "INMANTA_GIT_OPS_SLICE_NAME" +SLICE_NAME: str | None = os.getenv(SLICE_NAME_ENV_VAR) + +SLICE_EXTENSION_ENV_VAR = "INMANTA_GIT_OPS_SLICE_EXTENSION" +SLICE_EXTENSION: str = os.getenv(SLICE_EXTENSION_ENV_VAR, "json") + +# File in which the result of a slice command should be written by the +# finalizer executing it, so that the cli process can read it back. +OUTPUT_FILE_ENV_VAR = "INMANTA_GIT_OPS_OUTPUT_FILE" +OUTPUT_FILE: str | None = os.getenv(OUTPUT_FILE_ENV_VAR) + SLICE_CREATE = "create" SLICE_UPDATE = "update" SLICE_DELETE = "delete" + +# Placeholder used for all the required property values of a newly scaffolded +# slice, which the user must replace with real values. +SLICE_PLACEHOLDER = "" diff --git a/inmanta_plugins/git_ops/__init__.py b/inmanta_plugins/git_ops/__init__.py index 513d4f5..c28092c 100644 --- a/inmanta_plugins/git_ops/__init__.py +++ b/inmanta_plugins/git_ops/__init__.py @@ -46,6 +46,11 @@ def unroll_slices( """ from inmanta_plugins.git_ops import store + if const.COMPILE_MODE in const.COMPILE_SLICE_COMMANDS: + # Slice command compile, no slice should be emitted in the model, + # the command logic is handled in a finalizer + return [] + all_slices = store.get_store(store_name).get_all_slices() return [asdict(s) for s in all_slices] diff --git a/inmanta_plugins/git_ops/slice.py b/inmanta_plugins/git_ops/slice.py index bf083f4..ecc799f 100644 --- a/inmanta_plugins/git_ops/slice.py +++ b/inmanta_plugins/git_ops/slice.py @@ -33,6 +33,7 @@ from pydantic.json_schema import SkipJsonSchema import inmanta.ast.type as inmanta_type +from inmanta_git_ops import const LOGGER = logging.getLogger(__name__) @@ -583,6 +584,32 @@ class EmbeddedSliceObjectABC(pydantic.BaseModel): exclude_if=slice_update, ) + @classmethod + def scaffold(cls) -> dict: + """ + Build a template attributes dict for this slice object. The dict + contains all the required properties of the schema (those without + a default value), with a placeholder value the user must replace. + Mandatory relations towards embedded slice objects are scaffolded + recursively, any other value (primitive, list, union, ...) gets + the placeholder string. + """ + attributes: dict = {} + for attribute, info in cls.model_fields.items(): + if not info.is_required(): + continue + + annotation = info.annotation + if inspect.isclass(annotation) and issubclass( + annotation, EmbeddedSliceObjectABC + ): + # Mandatory relation towards an embedded slice object + attributes[attribute] = annotation.scaffold() + else: + attributes[attribute] = const.SLICE_PLACEHOLDER + + return attributes + @classmethod def entity_schema(cls) -> SliceEntitySchema: """ diff --git a/inmanta_plugins/git_ops/store.py b/inmanta_plugins/git_ops/store.py index dea2a4a..268ddef 100644 --- a/inmanta_plugins/git_ops/store.py +++ b/inmanta_plugins/git_ops/store.py @@ -24,7 +24,7 @@ import re import typing from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import asdict, dataclass import pydantic import yaml @@ -543,15 +543,23 @@ def load_current_slices(self) -> dict[str, Slice]: active_slices = self.load_active_slices() + # In these compile modes, the current version of the slices is read + # from the source folder, like for an update compile + source_view = const.COMPILE_MODE in [ + const.COMPILE_UPDATE, + const.COMPILE_SYNC, + const.COMPILE_SLICE_LIST, + const.COMPILE_SLICE_INSPECT, + ] + slices = set(active_slices.keys()) - if const.COMPILE_MODE in [const.COMPILE_UPDATE, const.COMPILE_SYNC]: - # Activating compile, we need to look at the source of the - # slices too + if source_view: + # We need to look at the source of the slices too slices |= self.load_source_slices().keys() self.current_slices: dict[str, Slice] = {} for s in slices: - if const.COMPILE_MODE in [const.COMPILE_UPDATE, const.COMPILE_SYNC]: + if source_view: self.current_slices[s] = self.load_source_slices()[s] else: self.current_slices[s] = self.get_latest_slice(s) @@ -636,6 +644,11 @@ def load_slices(self) -> dict[str, Slice]: self.slices: dict[str, Slice] = {} for s, current in self.load_current_slices().items(): if current.deleted: + if s not in previous_slices: + # The slice never had any undeleted version, there is + # nothing to delete, skip it + continue + # We need to get the attributes of the last undeleted # version, otherwise we don't know what we have to delete attributes = merge_attributes( @@ -866,6 +879,40 @@ def update(self) -> None: json.dumps(self.schema.model_json_schema(), indent=2) ) + def create_slice(self, name: str, *, extension: str = "json") -> SliceFile: + """ + Scaffold a new source slice file for this store. The file contains + all the required properties of the store's schema, with placeholder + values that the user must replace. Refuse to create the slice if a + source slice with the same name already exists. + + :param name: The name of the slice to create, it becomes the name + of the source file. + :param extension: The extension (and format) of the created file. + """ + if not name or name.startswith(".") or "/" in name or "@" in name: + raise ValueError( + f"Invalid slice name: {name!r}. A slice name can not be empty, " + "start with a dot, or contain any '/' or '@' character." + ) + + source_slice_files = self.load_source_slice_files() + if name in source_slice_files: + raise ValueError( + f"A source slice named {name} already exists in store {self.name} " + f"at {source_slice_files[name].path}" + ) + + slice_file = SliceFile( + path=self.source_path / f"{name}.{extension}", + name=name, + version=None, + extension=extension, + schema=self.schema, + ) + slice_file.write_raw(self.schema.scaffold()) + return slice_file + def clear(self) -> None: """ Clear the cache of slices in memory. @@ -1033,6 +1080,62 @@ def get_store(store_name: str) -> SliceStore[slice.SliceObjectABC]: return SLICE_STORE_REGISTRY[store_name] +def write_command_output(result: object) -> None: + """ + Write the result of a slice command to the output file designated by + the cli process. If no output file is set (manual compile), print the + result to stdout instead. + """ + serialized = json.dumps(result, indent=2) + if const.OUTPUT_FILE is not None: + pathlib.Path(const.OUTPUT_FILE).write_text(serialized) + else: + print(serialized) + + +@finalizer +def run_slice_command() -> None: + """ + At the end of the compile, if the compile was triggered by a + `git-ops project slice` command, execute the command logic and emit + its result. + """ + if const.COMPILE_MODE not in const.COMPILE_SLICE_COMMANDS: + return + + if const.COMPILE_MODE == const.COMPILE_SLICE_LIST: + # List the slices of one store, or of all the registered stores + stores = ( + [get_store(const.SLICE_STORE)] + if const.SLICE_STORE is not None + else list(SLICE_STORE_REGISTRY.values()) + ) + write_command_output( + [ + {k: v for k, v in asdict(s).items() if k != "attributes"} + for store in stores + for s in store.get_all_slices() + ] + ) + return + + # The create and inspect commands always target one specific slice + if const.SLICE_STORE is None or const.SLICE_NAME is None: + raise ValueError( + f"The {const.COMPILE_MODE} compile requires both {const.SLICE_STORE_ENV_VAR} " + f"and {const.SLICE_NAME_ENV_VAR} environment variables to be set" + ) + + store = get_store(const.SLICE_STORE) + if const.COMPILE_MODE == const.COMPILE_SLICE_CREATE: + slice_file = store.create_slice( + const.SLICE_NAME, extension=const.SLICE_EXTENSION + ) + write_command_output(str(slice_file.path)) + else: + write_command_output(asdict(store.get_one_slice(const.SLICE_NAME))) + + @finalizer def persist_store() -> None: """ diff --git a/tests/test_cli.py b/tests/test_cli.py index 5ab0e69..0692a06 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,6 +21,8 @@ import pathlib import subprocess +import yaml + def test_basics() -> None: example_path = pathlib.Path(__file__).parent.parent / "docs/example" @@ -46,3 +48,139 @@ def test_basics() -> None: check=True, env={"INMANTA_GIT_OPS_MODULE_PATH": str(example_path), **os.environ}, ) + + +def test_project_slice_commands(tmp_path: pathlib.Path) -> None: + # Set up a minimal inmanta project using the example module, the modules + # are resolved from the python environment running the test + project_path = tmp_path / "project" + project_path.mkdir() + (project_path / "project.yml").write_text(yaml.safe_dump({"name": "test-project"})) + (project_path / "main.cf").write_text( + "\n".join( + [ + "import example::slices::fs", + "import example::slices::fs::unroll", + "import example::slices::simple", + "import example::slices::recursive", + ] + ) + ) + + def git_ops(*args: str, expect_failure: bool = False) -> str: + result = subprocess.run( + ["git-ops", "project", *args], + cwd=project_path, + env={**os.environ}, + text=True, + capture_output=True, + ) + if expect_failure: + assert result.returncode != 0, result.stdout + result.stderr + else: + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout + + # No slices yet + assert json.loads(git_ops("slice", "list", "--format", "json")) == [] + + # Scaffold a new fs slice, the created file contains a placeholder for + # each required property of the store schema. The created path, printed + # to stdout, is relative to the directory the command is invoked from. + created = git_ops("slice", "create", "--store", "fs", "--name", "test-folder") + fs_source = project_path / "files" / "fs" / "test-folder.json" + assert project_path / created.strip() == fs_source + assert json.loads(fs_source.read_text()) == { + "name": "", + "root": "", + } + + # Refuse to overwrite an existing source slice + git_ops( + "slice", + "create", + "--store", + "fs", + "--name", + "test-folder", + expect_failure=True, + ) + + # Refuse to create a slice in an unknown store + git_ops( + "slice", "create", "--store", "unknown", "--name", "test", expect_failure=True + ) + + # Scaffold a yaml slice, with a mandatory embedded relation + created = git_ops( + "slice", + "create", + "--store", + "recursive", + "--name", + "rec", + "--extension", + "yaml", + ) + rec_source = project_path / "files" / "recursive" / "rec.yaml" + assert project_path / created.strip() == rec_source + assert yaml.safe_load(rec_source.read_text()) == { + "name": "", + "embedded_required": {"name": ""}, + } + rec_source.unlink() + + # Fill in the placeholders of the fs slice + fs_source.write_text(json.dumps({"name": "folder", "root": "/tmp"})) + + # The slice now shows up in the list, with the version it would be + # assigned during an update compile + assert json.loads(git_ops("slice", "list", "--format", "json")) == [ + {"name": "test-folder", "store_name": "fs", "version": 1, "deleted": False} + ] + + # The table output contains the slice details + table = git_ops("slice", "list") + assert "Store" in table + assert "fs" in table + assert "test-folder" in table + + # Inspect the slice, the output is the fully resolved merged slice + inspected = json.loads( + git_ops("slice", "inspect", "--store", "fs", "--name", "test-folder") + ) + assert inspected["name"] == "test-folder" + assert inspected["store_name"] == "fs" + assert inspected["version"] == 1 + assert inspected["deleted"] is False + assert inspected["attributes"]["operation"] == "create" + assert inspected["attributes"]["path"] == "." + assert inspected["attributes"]["version"] == 1 + assert inspected["attributes"]["slice_store"] == "fs" + assert inspected["attributes"]["slice_name"] == "test-folder" + assert inspected["attributes"]["name"] == "folder" + assert inspected["attributes"]["root"] == "/tmp" + assert inspected["attributes"]["content"] == [] + + # Inspecting a slice that doesn't exist fails + git_ops( + "slice", "inspect", "--store", "fs", "--name", "missing", expect_failure=True + ) + + # Sync the slice to the active store, then delete its source file + git_ops("sync") + fs_source.unlink() + + # The slice is still listed, as a deleted slice with a new version + assert json.loads(git_ops("slice", "list", "--format", "json")) == [ + {"name": "test-folder", "store_name": "fs", "version": 2, "deleted": True} + ] + + # An empty source slice which never had any active version is ignored + empty_source = project_path / "files" / "simple" / "empty.json" + empty_source.parent.mkdir(parents=True, exist_ok=True) + empty_source.write_text("{}") + assert ( + json.loads(git_ops("slice", "list", "--store", "simple", "--format", "json")) + == [] + ) diff --git a/tests/test_slice_schema.py b/tests/test_slice_schema.py index 21c0a2b..82ba3e4 100644 --- a/tests/test_slice_schema.py +++ b/tests/test_slice_schema.py @@ -16,7 +16,11 @@ Contact: edvgui@gmail.com """ +from inmanta_plugins.example.slices.fs import RootFolder from inmanta_plugins.example.slices.recursive import EmbeddedSlice, Slice +from inmanta_plugins.example.slices.simple import Slice as SimpleSlice + +from inmanta_git_ops import const def test_basics() -> None: @@ -86,3 +90,19 @@ def test_basics() -> None: # The embedded slice should extend the EmbeddedSliceObjectABC assert [b.name for b in embedded_schema.base_entities] == ["NamedSlice"] + + +def test_scaffold() -> None: + # The scaffold of a slice contains all its required properties (including + # the inherited ones), with a placeholder value + assert RootFolder.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "root": const.SLICE_PLACEHOLDER, + } + assert SimpleSlice.scaffold() == {"name": const.SLICE_PLACEHOLDER} + + # Mandatory embedded relations are scaffolded recursively + assert Slice.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "embedded_required": {"name": const.SLICE_PLACEHOLDER}, + } From ce5b040a2cdb724c6b7c03fd1f1e56252a6052fa Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Tue, 2 Jun 2026 21:54:50 +0200 Subject: [PATCH 2/9] Include non-required properties in slice scaffold The scaffold of a new slice now also contains the non-required properties of the store's schema, pre-filled with their default value, so the created file shows every available property. Model-only attributes (marked with exclude_if) stay out of the scaffold, as they are not part of the slice source files. Co-Authored-By: Claude Opus 4.8 --- docs/design/project-slice-commands.md | 2 ++ inmanta_git_ops/cli.py | 7 +++-- inmanta_plugins/git_ops/slice.py | 19 +++++++++--- inmanta_plugins/git_ops/store.py | 7 +++-- tests/test_cli.py | 26 +++++++++++++--- tests/test_slice_schema.py | 44 ++++++++++++++++++++++++--- 6 files changed, 86 insertions(+), 19 deletions(-) diff --git a/docs/design/project-slice-commands.md b/docs/design/project-slice-commands.md index ac840fc..47659c8 100644 --- a/docs/design/project-slice-commands.md +++ b/docs/design/project-slice-commands.md @@ -62,6 +62,8 @@ folder. `root` for the `fs` store) plus mandatory nested relations (cardinality_min ≥ 1). Placeholders should be obvious and self-describing (e.g. a sentinel string carrying the attribute's description) so the file does not silently validate with bogus data. + Non-required properties are included as well, pre-filled with their default value, so + the created file shows every available property. - The file is written into the store's source folder (`store.source_path`) under `.`, with no version suffix. - The command should refuse to overwrite an existing source slice with the same name. diff --git a/inmanta_git_ops/cli.py b/inmanta_git_ops/cli.py index 69d0431..56529c7 100644 --- a/inmanta_git_ops/cli.py +++ b/inmanta_git_ops/cli.py @@ -373,9 +373,10 @@ def create( """ Scaffold a new source slice file for the given store. - The created file contains all the required properties of the store's schema, - with placeholder values that should be replaced by the user. The path of the - created file is printed to stdout. + The created file contains all the properties of the store's schema: the + required ones with a placeholder value that should be replaced by the user, + the others pre-filled with their default value. The path of the created + file is printed to stdout. """ path = run_slice_command_compile( inmanta_compile_arg, diff --git a/inmanta_plugins/git_ops/slice.py b/inmanta_plugins/git_ops/slice.py index ecc799f..e56aaa0 100644 --- a/inmanta_plugins/git_ops/slice.py +++ b/inmanta_plugins/git_ops/slice.py @@ -588,15 +588,26 @@ class EmbeddedSliceObjectABC(pydantic.BaseModel): def scaffold(cls) -> dict: """ Build a template attributes dict for this slice object. The dict - contains all the required properties of the schema (those without - a default value), with a placeholder value the user must replace. + contains all the properties of the schema: the required ones (those + without a default value) get a placeholder value the user must + replace, the others are pre-filled with their default value. Mandatory relations towards embedded slice objects are scaffolded - recursively, any other value (primitive, list, union, ...) gets - the placeholder string. + recursively, any other required value (primitive, list, union, ...) + gets the placeholder string. """ attributes: dict = {} for attribute, info in cls.model_fields.items(): + if info.exclude_if is slice_update: + # Model-only attribute, it is not part of the slice source files + continue + if not info.is_required(): + # Optional attribute, pre-fill it with its default value + default = info.get_default(call_default_factory=True) + if isinstance(default, pydantic.BaseModel): + with exclude_model_values(): + default = default.model_dump(mode="json") + attributes[attribute] = default continue annotation = info.annotation diff --git a/inmanta_plugins/git_ops/store.py b/inmanta_plugins/git_ops/store.py index 268ddef..4c02d53 100644 --- a/inmanta_plugins/git_ops/store.py +++ b/inmanta_plugins/git_ops/store.py @@ -882,9 +882,10 @@ def update(self) -> None: def create_slice(self, name: str, *, extension: str = "json") -> SliceFile: """ Scaffold a new source slice file for this store. The file contains - all the required properties of the store's schema, with placeholder - values that the user must replace. Refuse to create the slice if a - source slice with the same name already exists. + all the properties of the store's schema: the required ones with a + placeholder value that the user must replace, the others pre-filled + with their default value. Refuse to create the slice if a source + slice with the same name already exists. :param name: The name of the slice to create, it becomes the name of the source file. diff --git a/tests/test_cli.py b/tests/test_cli.py index 0692a06..ef47d03 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -85,14 +85,20 @@ def git_ops(*args: str, expect_failure: bool = False) -> str: assert json.loads(git_ops("slice", "list", "--format", "json")) == [] # Scaffold a new fs slice, the created file contains a placeholder for - # each required property of the store schema. The created path, printed - # to stdout, is relative to the directory the command is invoked from. + # each required property of the store schema, and the default value of + # the non-required ones. The created path, printed to stdout, is + # relative to the directory the command is invoked from. created = git_ops("slice", "create", "--store", "fs", "--name", "test-folder") fs_source = project_path / "files" / "fs" / "test-folder.json" assert project_path / created.strip() == fs_source assert json.loads(fs_source.read_text()) == { "name": "", "root": "", + "permissions": "770", + "owner": None, + "group": None, + "type": "folder", + "content": [], } # Refuse to overwrite an existing source slice @@ -126,12 +132,24 @@ def git_ops(*args: str, expect_failure: bool = False) -> str: assert project_path / created.strip() == rec_source assert yaml.safe_load(rec_source.read_text()) == { "name": "", - "embedded_required": {"name": ""}, + "description": None, + "unique_id": None, + "embedded_required": { + "name": "", + "description": None, + "unique_id": None, + "recursive_slice": [], + }, + "embedded_optional": None, + "embedded_sequence": [], } rec_source.unlink() # Fill in the placeholders of the fs slice - fs_source.write_text(json.dumps({"name": "folder", "root": "/tmp"})) + attributes = json.loads(fs_source.read_text()) + attributes["name"] = "folder" + attributes["root"] = "/tmp" + fs_source.write_text(json.dumps(attributes)) # The slice now shows up in the list, with the version it would be # assigned during an update compile diff --git a/tests/test_slice_schema.py b/tests/test_slice_schema.py index 82ba3e4..b765a33 100644 --- a/tests/test_slice_schema.py +++ b/tests/test_slice_schema.py @@ -16,7 +16,7 @@ Contact: edvgui@gmail.com """ -from inmanta_plugins.example.slices.fs import RootFolder +from inmanta_plugins.example.slices.fs import File, RootFolder from inmanta_plugins.example.slices.recursive import EmbeddedSlice, Slice from inmanta_plugins.example.slices.simple import Slice as SimpleSlice @@ -93,16 +93,50 @@ def test_basics() -> None: def test_scaffold() -> None: - # The scaffold of a slice contains all its required properties (including - # the inherited ones), with a placeholder value + # The scaffold of a slice contains all its properties (including the + # inherited ones): the required ones with a placeholder value, the + # others pre-filled with their default value assert RootFolder.scaffold() == { "name": const.SLICE_PLACEHOLDER, "root": const.SLICE_PLACEHOLDER, + "permissions": "770", + "owner": None, + "group": None, + "type": "folder", + "content": [], + } + assert SimpleSlice.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "description": None, + "unique_id": None, + "some_number": 0.0, + "some_flag": False, + "some_list": [], + "some_dict": {}, + } + + # Model-only attributes (such as previous_content) are not part of the + # scaffold, as they are not part of the slice source files + assert File.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + "permissions": "770", + "owner": None, + "group": None, + "type": "file", + "content": "", } - assert SimpleSlice.scaffold() == {"name": const.SLICE_PLACEHOLDER} # Mandatory embedded relations are scaffolded recursively assert Slice.scaffold() == { "name": const.SLICE_PLACEHOLDER, - "embedded_required": {"name": const.SLICE_PLACEHOLDER}, + "description": None, + "unique_id": None, + "embedded_required": { + "name": const.SLICE_PLACEHOLDER, + "description": None, + "unique_id": None, + "recursive_slice": [], + }, + "embedded_optional": None, + "embedded_sequence": [], } From 32e383aca3351b0864f7d2eb38329ed634891663 Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Tue, 2 Jun 2026 22:01:10 +0200 Subject: [PATCH 3/9] Fix scaffold of mandatory embedded slices carrying a default Scaffolding a store whose schema has a mandatory embedded relation with a default factory (e.g. container: Container = Field(default_factory=Container)) failed when the embedded object has required properties of its own: the factory call raised a validation error. Embedded relations with a single embedded slice annotation are now always scaffolded recursively, even when the field has a default, the same way the entity schema always treats them as mandatory relations. Other default values are serialized through pydantic, so embedded objects nested in a default value are written like slice files, and a default that can not be constructed falls back to the placeholder. Co-Authored-By: Claude Opus 4.8 --- inmanta_plugins/git_ops/slice.py | 38 +++++++++++++++++++++----------- tests/test_slice_schema.py | 36 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/inmanta_plugins/git_ops/slice.py b/inmanta_plugins/git_ops/slice.py index e56aaa0..8db85ee 100644 --- a/inmanta_plugins/git_ops/slice.py +++ b/inmanta_plugins/git_ops/slice.py @@ -591,7 +591,8 @@ def scaffold(cls) -> dict: contains all the properties of the schema: the required ones (those without a default value) get a placeholder value the user must replace, the others are pre-filled with their default value. - Mandatory relations towards embedded slice objects are scaffolded + Mandatory relations towards embedded slice objects (single embedded + slice annotation, even when the field has a default) are scaffolded recursively, any other required value (primitive, list, union, ...) gets the placeholder string. """ @@ -601,23 +602,34 @@ def scaffold(cls) -> dict: # Model-only attribute, it is not part of the slice source files continue - if not info.is_required(): - # Optional attribute, pre-fill it with its default value - default = info.get_default(call_default_factory=True) - if isinstance(default, pydantic.BaseModel): - with exclude_model_values(): - default = default.model_dump(mode="json") - attributes[attribute] = default - continue - annotation = info.annotation if inspect.isclass(annotation) and issubclass( annotation, EmbeddedSliceObjectABC ): - # Mandatory relation towards an embedded slice object + # Mandatory relation towards an embedded slice object, always + # scaffold it recursively: even when the field has a default, + # the embedded object may have required properties of its own attributes[attribute] = annotation.scaffold() - else: - attributes[attribute] = const.SLICE_PLACEHOLDER + continue + + if not info.is_required(): + # Optional attribute, pre-fill it with its default value + try: + default = info.get_default(call_default_factory=True) + except pydantic.ValidationError: + # The default can not be constructed (e.g. a factory + # building an embedded object with required properties), + # the user will have to provide the value + attributes[attribute] = const.SLICE_PLACEHOLDER + continue + + with exclude_model_values(): + attributes[attribute] = pydantic.TypeAdapter( + annotation + ).dump_python(default, mode="json") + continue + + attributes[attribute] = const.SLICE_PLACEHOLDER return attributes diff --git a/tests/test_slice_schema.py b/tests/test_slice_schema.py index b765a33..ec90ac9 100644 --- a/tests/test_slice_schema.py +++ b/tests/test_slice_schema.py @@ -16,11 +16,15 @@ Contact: edvgui@gmail.com """ +from collections.abc import Sequence + +import pydantic from inmanta_plugins.example.slices.fs import File, RootFolder from inmanta_plugins.example.slices.recursive import EmbeddedSlice, Slice from inmanta_plugins.example.slices.simple import Slice as SimpleSlice from inmanta_git_ops import const +from inmanta_plugins.git_ops import slice def test_basics() -> None: @@ -140,3 +144,35 @@ def test_scaffold() -> None: "embedded_optional": None, "embedded_sequence": [], } + + +def test_scaffold_embedded_defaults() -> None: + class Container(slice.EmbeddedSliceObjectABC): + image: str + cpus: float = 1.0 + + class Service(slice.SliceObjectABC): + name: str + container: Container = pydantic.Field(default_factory=Container) + replicas: Sequence[Container] = pydantic.Field( + default_factory=lambda: [Container(image="alpine")] + ) + invalid: Sequence[Container] = pydantic.Field( + default_factory=lambda: [Container()] + ) + + assert Service.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + # A mandatory embedded relation is scaffolded recursively even when + # the field has a default: the factory can not construct the embedded + # object as it has required properties of its own + "container": { + "image": const.SLICE_PLACEHOLDER, + "cpus": 1.0, + }, + # Embedded objects in a default value are serialized like the slice + # files, without the model-only attributes + "replicas": [{"image": "alpine", "cpus": 1.0}], + # A default that can not be constructed falls back to the placeholder + "invalid": const.SLICE_PLACEHOLDER, + } From b023b491dbbf210ee34916448af7f3d49a8c2288 Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Tue, 2 Jun 2026 22:04:10 +0200 Subject: [PATCH 4/9] Add scaffold test coverage for discriminated unions Co-Authored-By: Claude Opus 4.8 --- tests/test_slice_schema.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_slice_schema.py b/tests/test_slice_schema.py index ec90ac9..5c41cd2 100644 --- a/tests/test_slice_schema.py +++ b/tests/test_slice_schema.py @@ -16,6 +16,7 @@ Contact: edvgui@gmail.com """ +import typing from collections.abc import Sequence import pydantic @@ -176,3 +177,35 @@ class Service(slice.SliceObjectABC): # A default that can not be constructed falls back to the placeholder "invalid": const.SLICE_PLACEHOLDER, } + + +def test_scaffold_discriminated_union() -> None: + class A(slice.EmbeddedSliceObjectABC): + type: typing.Literal["a"] = "a" + required_value: str + + class B(slice.EmbeddedSliceObjectABC): + type: typing.Literal["b"] = "b" + + AB = typing.Annotated[A | B, pydantic.Field(discriminator="type")] + + class Root(slice.SliceObjectABC): + name: str + item: AB + fallback: AB = pydantic.Field(default_factory=B) + broken: AB = pydantic.Field(default_factory=A) + items: Sequence[AB] = pydantic.Field(default_factory=lambda: [B()]) + + assert Root.scaffold() == { + "name": const.SLICE_PLACEHOLDER, + # A required union relation gets the placeholder: the scaffold can + # not pick a union member for the user + "item": const.SLICE_PLACEHOLDER, + # Union members in a default value are serialized like the slice + # files, with their discriminator + "fallback": {"type": "b"}, + "items": [{"type": "b"}], + # A default union member that can not be constructed falls back to + # the placeholder + "broken": const.SLICE_PLACEHOLDER, + } From 51ceef04132f92448b9db5e46c3f08efd8524469 Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Tue, 2 Jun 2026 22:10:51 +0200 Subject: [PATCH 5/9] Delete project-slice-commands.md --- docs/design/project-slice-commands.md | 123 -------------------------- 1 file changed, 123 deletions(-) delete mode 100644 docs/design/project-slice-commands.md diff --git a/docs/design/project-slice-commands.md b/docs/design/project-slice-commands.md deleted file mode 100644 index 47659c8..0000000 --- a/docs/design/project-slice-commands.md +++ /dev/null @@ -1,123 +0,0 @@ -# Feature Request: `git-ops project slice` command set - -## Summary - -Extend the `git-ops` CLI with a new command group, `git-ops project slice`, that lets -users manage individual slices of a project from the command line without having to -hand-edit slice files or write throw-away scripts. - -The group introduces three sub-commands: - -| Command | Purpose | -| --------- | ----------------------------------------------------------------------- | -| `create` | Scaffold a new source slice file for a given store. | -| `list` | List all slices of a project, optionally filtered by store. | -| `inspect` | Dump the fully-resolved view of a single slice as JSON. | - -These commands live alongside the existing project commands (`update`, `sync`, -`prune`) in `inmanta_git_ops/cli.py`, under the existing `@cli.group("project")`. - -## Background - -A few concepts from the existing codebase are relevant (see -`inmanta_plugins/git_ops/store.py` and `inmanta_plugins/git_ops/slice.py`): - -- **Store** (`SliceStore`): a named collection of slices, defined in a project's - modules and registered in `SLICE_STORE_REGISTRY` on import. Each store has a - pydantic `schema` (a subclass of `slice.SliceObjectABC`) and a source folder - (`store._folder`, e.g. `inmanta:///files/fs/`). -- **Source slice**: an unversioned, user-editable file in the store's source folder - (`.json` / `.yaml`). The file name (without extension/version) is the - slice name. -- **Active slice**: a versioned, committed snapshot in the active folder - (`inmanta:///git_ops/active//@v.json`). A slice may have - several active versions. -- **Latest / oldest version**: `SliceStore.get_latest_slice(name)` returns the highest - active version; the set of active versions is available via - `load_active_slices()[name]`. -- **In sync**: a source slice is "in sync" when its content matches the latest active - version. `load_source_slices()` reflects this: when the source content equals the - latest active attributes, the source slice keeps the latest version; otherwise it is - emitted as `latest.version + 1` (i.e. out of sync, pending a `sync`). -- **`get_one_slice(name)`**: returns the fully-resolved, merged `Slice` (current vs. - previous, with `operation`/`path` markers, `version`, `slice_store`, `slice_name`). - -## Requirements - -### 1. `git-ops project slice create` - -Scaffold a brand-new **source** slice file and place it in the correct store's source -folder. - -- Inputs (each resolvable via CLI argument, environment variable, or interactive - prompt when missing — following the existing `click` option pattern with - `envvar=...` / `show_envvar=True`): - - **store** — name of the target store (must exist in `SLICE_STORE_REGISTRY`). - - **name** — the slice name; becomes the source file name. - - **format/extension** — `json` or `yaml` (default `json`, matching the format the - rest of the tooling writes). -- The created file must contain **all required properties** of the store's schema, with - **explicit placeholders** wherever the user must still supply a value. Required - properties are those without a default (the schema's `required` list, e.g. `name` and - `root` for the `fs` store) plus mandatory nested relations (cardinality_min ≥ 1). - Placeholders should be obvious and self-describing (e.g. a sentinel string carrying - the attribute's description) so the file does not silently validate with bogus data. - Non-required properties are included as well, pre-filled with their default value, so - the created file shows every available property. -- The file is written into the store's source folder (`store.source_path`) under - `.`, with no version suffix. -- The command should refuse to overwrite an existing source slice with the same name. -- The command should seed placeholders for required property values (the user edits - the file afterwards), **not** prompt for each required value. - -### 2. `git-ops project slice inspect` - -Inspect a single slice. - -- Inputs: **store** and **name** (CLI arg / env var / prompt, as above). -- Output: the JSON serialization of `SliceStore.get_one_slice(name)` — the - fully-resolved, merged slice (attributes with `operation`/`path` markers, `version`, - `slice_store`, `slice_name`). -- A missing slice should surface a clear error (`get_one_slice` raises `LookupError`). - -### 3. `git-ops project slice list` - -List the slices of the current project. - -- **Filter** by store via `--store ` (optional; default lists every registered - store). -- **Output format** configurable via `--format {table,json}` (default `table`), mirroring - `git-ops module store list`. -- Each slice "row" contains the same details as the inspect command, except for the attributes. -- Rows should cover slices that exist in either the active store or the source folder. - A slice with no active version yet (never synced) and a slice deleted from source but - still active are both meaningful states the output should represent sensibly. - -## Technical details - -1. **Compiler context.** Similarly to other commands (like prune) a - compile must be running in order to find the available stores and slices. Each of - these commands will trigger a special type of compiles, which doesn't emit any - slice in the model (similarly to prune). All other logic should be handled in - a finalizer, which checks for the current compile mode. -2. **Slice view.** list and inspect command should run in "update" mode, except that - no slice should be emitted in the model. But the output version and attributes - should match the same as an update compile. -3. **Placeholder representation.** Use as placeholder for all values, - regardless of their types. -4. **Reuse of formatting.** `list` should reuse the existing `texttable` + `json.dumps` - pattern already used by `git-ops module store list` for consistency. - -## Out of scope - -- Editing/updating existing slice content (covered by hand-editing + `update`/`sync`). -- Deleting slices, pruning, or any active-store mutation (already covered by `prune`). -- Schema migrations. - -## Affected files (anticipated) - -- `inmanta_git_ops/cli.py` — new `@project.group("slice")` and its three commands. -- `inmanta_plugins/git_ops/store.py` — possibly small helpers if store loading / - oldest-version lookup needs to be exposed cleanly. -- `tests/test_cli.py` — CLI invocation tests against `docs/example` (the `fs`, - `simple`, `recursive` stores). From 787efe3aa7086113ce1e27a8688e71b5bccdcb01 Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Tue, 2 Jun 2026 22:19:31 +0200 Subject: [PATCH 6/9] Configure slice command compiles to never log on stdout Pass a logging configuration to the slice command compiles (through the INMANTA_LOGGING_COMPILER_CONTENT environment variable defined by inmanta-core) which sends all the compiler logs to stderr. The compiler no longer logs anything on stdout, which only carries the result of the slice command. The stdout redirection to stderr is kept as a safety net for anything writing to stdout outside of the logging framework. The logging configuration is only applied when the user didn't configure the compiler logging themselves, through one of the environment variables recognized by inmanta-core. A --logging-config argument passed to the compile takes precedence over it in any case. Co-Authored-By: Claude Opus 4.8 --- inmanta_git_ops/cli.py | 52 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/inmanta_git_ops/cli.py b/inmanta_git_ops/cli.py index 56529c7..4b7dbb4 100644 --- a/inmanta_git_ops/cli.py +++ b/inmanta_git_ops/cli.py @@ -236,6 +236,43 @@ def run_compile( ) +# Environment variables, defined by inmanta-core, with which the user can +# set the logging configuration of the compiler. +COMPILER_LOGGING_CONTENT_ENV_VAR = "INMANTA_LOGGING_COMPILER_CONTENT" +COMPILER_LOGGING_ENV_VARS = [ + "INMANTA_LOGGING_COMPILER", + COMPILER_LOGGING_CONTENT_ENV_VAR, + "INMANTA_LOGGING_COMPILER_TMPL", + "INMANTA_CONFIG_LOGGING_CONFIG", + "INMANTA_CONFIG_LOGGING_CONFIG_CONTENT", + "INMANTA_CONFIG_LOGGING_CONFIG_TMPL", +] + +# Logging configuration for the slice command compiles: send all the compiler +# logs to stderr, so that the compiler doesn't log anything on stdout. The +# inmanta.logging logger is restricted to errors because it warns about the +# default cli logging options it ignores when a logging config is provided. +SLICE_COMPILE_LOGGING_CONFIG = """ +version: 1 +disable_existing_loggers: false +formatters: + console: + format: "%(name)-25s%(levelname)-8s%(message)s" +handlers: + console: + class: logging.StreamHandler + formatter: console + level: WARNING + stream: ext://sys.stderr +loggers: + inmanta.logging: + level: ERROR +root: + handlers: [console] + level: WARNING +""" + + def run_slice_command_compile( inmanta_compile_arg: Sequence[str], *, @@ -243,10 +280,10 @@ def run_slice_command_compile( env: Mapping[str, str], ) -> object: """ - Run a slice command compile on the current project. The output of the - compile is redirected to stderr, and the result of the command, written - to the output file by the corresponding finalizer, is read back and - returned. + Run a slice command compile on the current project. The compiler is + configured to log to stderr, and any remaining output on stdout is + redirected to stderr too. The result of the command, written to the + output file by the corresponding finalizer, is read back and returned. :param inmanta_compile_arg: Additional arguments to pass to the inmanta compile command. @@ -255,11 +292,16 @@ def run_slice_command_compile( """ with tempfile.TemporaryDirectory() as tmp_dir: output_file = pathlib.Path(tmp_dir) / "output.json" + compile_env = {**env, const.OUTPUT_FILE_ENV_VAR: str(output_file)} + if not any(var in os.environ for var in COMPILER_LOGGING_ENV_VARS): + # The user didn't configure the compiler logging, make sure the + # compiler doesn't log anything on stdout + compile_env[COMPILER_LOGGING_CONTENT_ENV_VAR] = SLICE_COMPILE_LOGGING_CONFIG try: run_compile( inmanta_compile_arg, compile_mode=compile_mode, - env={**env, const.OUTPUT_FILE_ENV_VAR: str(output_file)}, + env=compile_env, stdout=sys.stderr, ) except subprocess.CalledProcessError as e: From 97a595d7589515b5c29f514cc805f493997142a2 Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Tue, 2 Jun 2026 22:31:19 +0200 Subject: [PATCH 7/9] Color slice command compile logs and drop the summary banner The logging configuration passed to the slice command compiles now mirrors the default console logging of the compiler: the logs sent to stderr keep their colors when stderr is a tty. The FORCE_TTY environment variable, recognized by inmanta-core, is also set in that case, so the rest of the compile output (error summary, explainers) keeps its styling even though it now goes through a pipe. The compile output is streamed to stderr through that pipe, and the success/failure banner of the compile summary is dropped on the way. The error message of a failed compile is preserved. Co-Authored-By: Claude Opus 4.8 --- inmanta_git_ops/cli.py | 168 ++++++++++++++++++++++++++++------------- tests/test_cli.py | 5 ++ 2 files changed, 120 insertions(+), 53 deletions(-) diff --git a/inmanta_git_ops/cli.py b/inmanta_git_ops/cli.py index 4b7dbb4..cd23853 100644 --- a/inmanta_git_ops/cli.py +++ b/inmanta_git_ops/cli.py @@ -21,15 +21,16 @@ import logging import os import pathlib +import re import subprocess import sys import tempfile -import typing from collections.abc import Mapping, Sequence import click import texttable +from inmanta.const import ENVIRON_FORCE_TTY from inmanta.module import ModuleV2, Project from inmanta_git_ops import const from inmanta_plugins.git_ops.store import SLICE_STORE_REGISTRY @@ -204,12 +205,27 @@ def project(inmanta_arg: list[str]) -> None: INMANTA_ARGS.extend(inmanta_arg) +def inmanta_compile_command(inmanta_compile_arg: Sequence[str]) -> list[str]: + """ + Construct the command line to run a compile on the current project. + + :param inmanta_compile_arg: Additional arguments to pass to the inmanta + compile command. + """ + return [ + sys.executable, + "-m", + "inmanta.app", + *INMANTA_ARGS, + "compile", + *inmanta_compile_arg, + ] + + def run_compile( inmanta_compile_arg: Sequence[str], *, compile_mode: str, - env: Mapping[str, str] | None = None, - stdout: typing.IO | None = None, ) -> None: """ Run a compile on the current project, in a subprocess, with the given @@ -218,21 +234,11 @@ def run_compile( :param inmanta_compile_arg: Additional arguments to pass to the inmanta compile command. :param compile_mode: The compile mode the compile should run in. - :param env: Additional environment variables to pass to the compile. - :param stdout: Where to redirect the standard output of the compile. """ subprocess.run( - [ - sys.executable, - "-m", - "inmanta.app", - *INMANTA_ARGS, - "compile", - *inmanta_compile_arg, - ], + inmanta_compile_command(inmanta_compile_arg), check=True, - env={**os.environ, const.COMPILE_MODE_ENV_VAR: compile_mode, **(env or {})}, - stdout=stdout, + env={**os.environ, const.COMPILE_MODE_ENV_VAR: compile_mode}, ) @@ -248,29 +254,66 @@ def run_compile( "INMANTA_CONFIG_LOGGING_CONFIG_TMPL", ] -# Logging configuration for the slice command compiles: send all the compiler -# logs to stderr, so that the compiler doesn't log anything on stdout. The -# inmanta.logging logger is restricted to errors because it warns about the -# default cli logging options it ignores when a logging config is provided. -SLICE_COMPILE_LOGGING_CONFIG = """ -version: 1 -disable_existing_loggers: false -formatters: - console: - format: "%(name)-25s%(levelname)-8s%(message)s" -handlers: - console: - class: logging.StreamHandler - formatter: console - level: WARNING - stream: ext://sys.stderr -loggers: - inmanta.logging: - level: ERROR -root: - handlers: [console] - level: WARNING -""" +# Pattern matching the headers of the summary that the compiler prints at the +# end of every compile, and which shouldn't be part of a slice command output. +COMPILE_SUMMARY_HEADER_REGEX = re.compile( + r"=+ (SUCCESS|COMPILATION FAILURE|EXPORT FAILURE|EXCEPTION TRACE) =+" +) +ANSI_ESCAPE_SEQUENCE_REGEX = re.compile(r"\x1b\[[0-9;]*m") + + +def slice_compile_logging_config() -> str: + """ + Build the logging configuration for the slice command compiles: all the + compiler logs are sent to stderr, so that the compiler doesn't log + anything on stdout. The logs are colored, like the default console + logging of the compiler, when stderr is a tty. The inmanta.logging + logger is restricted to errors because it warns about the default cli + logging options it ignores when a logging config is provided. + """ + on_tty = sys.stderr.isatty() + return json.dumps( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "console": { + "()": "inmanta.logging.MultiLineFormatter", + "fmt": ( + "%(log_color)s%(name)-15s%(levelname)-8s%(reset)s%(blue)s%(message)s" + if on_tty + else "%(name)-15s%(levelname)-8s%(message)s" + ), + "log_colors": ( + { + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red", + } + if on_tty + else None + ), + "reset": on_tty, + "no_color": not on_tty, + "keep_logger_names": False, + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "console", + "level": "WARNING", + "stream": "ext://sys.stderr", + } + }, + "loggers": { + "inmanta.logging": {"level": "ERROR"}, + }, + "root": {"handlers": ["console"], "level": "WARNING"}, + } + ) def run_slice_command_compile( @@ -281,9 +324,10 @@ def run_slice_command_compile( ) -> object: """ Run a slice command compile on the current project. The compiler is - configured to log to stderr, and any remaining output on stdout is - redirected to stderr too. The result of the command, written to the - output file by the corresponding finalizer, is read back and returned. + configured to log to stderr, and any remaining compile output on stdout + is forwarded to stderr, without the compile summary banner. The result + of the command, written to the output file by the corresponding + finalizer, is read back and returned. :param inmanta_compile_arg: Additional arguments to pass to the inmanta compile command. @@ -292,22 +336,40 @@ def run_slice_command_compile( """ with tempfile.TemporaryDirectory() as tmp_dir: output_file = pathlib.Path(tmp_dir) / "output.json" - compile_env = {**env, const.OUTPUT_FILE_ENV_VAR: str(output_file)} + compile_env = { + **os.environ, + const.COMPILE_MODE_ENV_VAR: compile_mode, + **env, + const.OUTPUT_FILE_ENV_VAR: str(output_file), + } if not any(var in os.environ for var in COMPILER_LOGGING_ENV_VARS): # The user didn't configure the compiler logging, make sure the # compiler doesn't log anything on stdout - compile_env[COMPILER_LOGGING_CONTENT_ENV_VAR] = SLICE_COMPILE_LOGGING_CONFIG - try: - run_compile( - inmanta_compile_arg, - compile_mode=compile_mode, - env=compile_env, - stdout=sys.stderr, + compile_env[COMPILER_LOGGING_CONTENT_ENV_VAR] = ( + slice_compile_logging_config() ) - except subprocess.CalledProcessError as e: - raise click.ClickException( - f"The compile failed (see logs above): {e}" - ) from e + if sys.stderr.isatty(): + # The compile output goes through a pipe, let the compiler know + # its output still ends up on a tty, so it keeps its colors + compile_env[ENVIRON_FORCE_TTY] = "true" + + with subprocess.Popen( + inmanta_compile_command(inmanta_compile_arg), + env=compile_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) as process: + assert process.stdout is not None + for line in process.stdout: + plain_line = ANSI_ESCAPE_SEQUENCE_REGEX.sub("", line).strip() + if COMPILE_SUMMARY_HEADER_REGEX.fullmatch(plain_line): + # Drop the compile summary banner from the output + continue + sys.stderr.write(line) + + if process.returncode != 0: + raise click.ClickException("The compile failed (see logs above)") if not output_file.exists(): raise click.ClickException( diff --git a/tests/test_cli.py b/tests/test_cli.py index ef47d03..198bf2c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -79,6 +79,11 @@ def git_ops(*args: str, expect_failure: bool = False) -> str: assert result.returncode != 0, result.stdout + result.stderr else: assert result.returncode == 0, result.stdout + result.stderr + if args[0] == "slice": + # The compile summary banner is dropped from the output of the + # slice commands + assert "SUCCESS" not in result.stderr, result.stderr + assert "FAILURE" not in result.stderr, result.stderr return result.stdout # No slices yet From ca43dfa2b8df413631acf209ed2e1b42dc9ead17 Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Thu, 4 Jun 2026 19:56:52 +0200 Subject: [PATCH 8/9] Less compiler warnings --- CHANGELOG.md | 3 ++- inmanta_plugins/git_ops/processors.py | 4 ++-- setup.cfg | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b31859..ba8dcfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # Changelog -## v0.4.1 - ? +## v0.5.0 - ? - Add `git-ops project slice` command group to create, list and inspect the slices of a project. +- Address some compiler warnings in plugin annotations ## v0.4.0 - 2026-05-30 diff --git a/inmanta_plugins/git_ops/processors.py b/inmanta_plugins/git_ops/processors.py index 711cf65..aefc942 100644 --- a/inmanta_plugins/git_ops/processors.py +++ b/inmanta_plugins/git_ops/processors.py @@ -116,7 +116,7 @@ def get_elements( @plugin def join_used_values( - *used_values: typing.Annotated[typing.Callable[[], Collection[object]], object], + *used_values: typing.Annotated[typing.Callable[[], Collection[object]], ModelType["any"]], ) -> typing.Annotated[typing.Callable[[], Collection[object]], ModelType["any"]]: """ Join multiple used values collectors into one. @@ -135,7 +135,7 @@ def unique_integer( path: str, previous_value: int | None = None, *, - used_integers: typing.Annotated[typing.Callable[[], Collection[int]], object], + used_integers: typing.Annotated[typing.Callable[[], Collection[int]], ModelType["any"]], range_start: int = 0, range_stop: int = 1000, refresh: bool = False, diff --git a/setup.cfg b/setup.cfg index adccda2..f52da87 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = inmanta-module-git-ops -version = 0.4.1 +version = 0.5.0 description = Declarative parametrization, extension and validation of Inmanta DSL models. long_description = file: README.md long_description_content_type = text/markdown From 892351d1a1c33a1d4d725ecdd5d049a714163cfe Mon Sep 17 00:00:00 2001 From: Guillaume Everarts de Velp Date: Thu, 4 Jun 2026 19:58:45 +0200 Subject: [PATCH 9/9] Fix linting --- inmanta_plugins/git_ops/processors.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/inmanta_plugins/git_ops/processors.py b/inmanta_plugins/git_ops/processors.py index aefc942..0abacf9 100644 --- a/inmanta_plugins/git_ops/processors.py +++ b/inmanta_plugins/git_ops/processors.py @@ -116,7 +116,9 @@ def get_elements( @plugin def join_used_values( - *used_values: typing.Annotated[typing.Callable[[], Collection[object]], ModelType["any"]], + *used_values: typing.Annotated[ + typing.Callable[[], Collection[object]], ModelType["any"] + ], ) -> typing.Annotated[typing.Callable[[], Collection[object]], ModelType["any"]]: """ Join multiple used values collectors into one. @@ -135,7 +137,9 @@ def unique_integer( path: str, previous_value: int | None = None, *, - used_integers: typing.Annotated[typing.Callable[[], Collection[int]], ModelType["any"]], + used_integers: typing.Annotated[ + typing.Callable[[], Collection[int]], ModelType["any"] + ], range_start: int = 0, range_stop: int = 1000, refresh: bool = False,