From 06f13c2dc0dca1440b4e167bbbfb322e07c07e98 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Wed, 3 Jun 2026 16:22:53 +0100 Subject: [PATCH 1/5] DOC-6711 initial config page rebuild --- .../reference/config-yaml-reference-v2.md | 11 + data/rdi-reference/cli.json | 3757 +++++++++++++++++ data/rdi-reference/config.json | 2227 ++++++++++ layouts/partials/rdi/render-schema.html | 274 ++ layouts/shortcodes/rdi-config-reference.html | 29 + 5 files changed, 6298 insertions(+) create mode 100644 content/integrate/redis-data-integration/reference/config-yaml-reference-v2.md create mode 100644 data/rdi-reference/cli.json create mode 100644 data/rdi-reference/config.json create mode 100644 layouts/partials/rdi/render-schema.html create mode 100644 layouts/shortcodes/rdi-config-reference.html diff --git a/content/integrate/redis-data-integration/reference/config-yaml-reference-v2.md b/content/integrate/redis-data-integration/reference/config-yaml-reference-v2.md new file mode 100644 index 0000000000..9e09c34523 --- /dev/null +++ b/content/integrate/redis-data-integration/reference/config-yaml-reference-v2.md @@ -0,0 +1,11 @@ +--- +Title: Redis Data Integration configuration file (rebuilt from data) +linkTitle: RDI configuration file (v2) +description: Redis Data Integration configuration file reference, rendered from data/rdi-reference/config.json +weight: 11 +alwaysopen: false +categories: ["redis-di"] +aliases: +--- + +{{% rdi-config-reference %}} diff --git a/data/rdi-reference/cli.json b/data/rdi-reference/cli.json new file mode 100644 index 0000000000..ef978392a5 --- /dev/null +++ b/data/rdi-reference/cli.json @@ -0,0 +1,3757 @@ +{ + "name": "redis-di", + "params": [ + { + "name": "version", + "param_type_name": "option", + "opts": ["--version"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show the version and exit.", + "is_flag": true, + "flag_value": true + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "A command line tool to manage & configure Redis Data Integration", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false, + "commands": { + "add-context": { + "name": "add-context", + "params": [ + { + "name": "context_name", + "param_type_name": "argument", + "opts": ["context-name"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null + }, + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_namespace", + "param_type_name": "option", + "opts": ["--rdi-namespace"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "rdi", + "envvar": "RDI_NAMESPACE", + "help": "RDI Kubernetes namespace", + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Adds a new context", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "configure-rdi": { + "name": "configure-rdi", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_namespace", + "param_type_name": "option", + "opts": ["--rdi-namespace"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "rdi", + "envvar": "RDI_NAMESPACE", + "help": "RDI Kubernetes namespace", + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "rdi_log_level", + "param_type_name": "option", + "opts": ["--rdi-log-level"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Log level for RDI components", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Configures RDI db connection credentials", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "delete-all-contexts": { + "name": "delete-all-contexts", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "force", + "param_type_name": "option", + "opts": ["--force", "-f"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Force operation. Skips verification prompts", + "is_flag": true, + "flag_value": true + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Deletes all contexts", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "delete-context": { + "name": "delete-context", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "context_name", + "param_type_name": "argument", + "opts": ["context-name"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null + }, + { + "name": "force", + "param_type_name": "option", + "opts": ["--force", "-f"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Force operation. Skips verification prompts", + "is_flag": true, + "flag_value": true + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Deletes a context", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "deploy": { + "name": "deploy", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "directory", + "param_type_name": "option", + "opts": ["--dir"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": ".", + "envvar": null, + "help": "Directory containing RDI configuration", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Deploys the RDI configurations including target", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "describe-job": { + "name": "describe-job", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "job_name", + "param_type_name": "argument", + "opts": ["job-name"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Describes a transformation engine's job", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "dump-support-package": { + "name": "dump-support-package", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_namespace", + "param_type_name": "option", + "opts": ["--rdi-namespace"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "rdi", + "envvar": "RDI_NAMESPACE", + "help": "RDI Kubernetes namespace", + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "directory", + "param_type_name": "option", + "opts": ["--dir"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": ".", + "envvar": null, + "help": "Directory where the support file should be generated", + "is_flag": false + }, + { + "name": "dump_rejected", + "param_type_name": "option", + "opts": ["--dump-rejected"], + "type": { + "param_type": "Int", + "name": "integer" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Dumps rejected records", + "is_flag": false + }, + { + "name": "trace_timeout", + "param_type_name": "option", + "opts": ["--trace-timeout"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 600, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Stops the trace after exceeding this timeout (in seconds)", + "is_flag": false + }, + { + "name": "max_change_records", + "param_type_name": "option", + "opts": ["--max-change-records"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": 10, + "envvar": null, + "help": "Maximum traced change records", + "is_flag": false + }, + { + "name": "trace_only_rejected", + "param_type_name": "option", + "opts": ["--trace-only-rejected"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Trace only rejected change records", + "is_flag": true, + "flag_value": true + }, + { + "name": "log_days", + "param_type_name": "option", + "opts": ["--log-days"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 0, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": 2, + "envvar": null, + "help": "Number of days to look back for log files", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Dumps RDI support package", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "get-rejected": { + "name": "get-rejected", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "max_records", + "param_type_name": "option", + "opts": ["--max-records"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Maximum rejected records per DLQ", + "is_flag": false + }, + { + "name": "oldest", + "param_type_name": "option", + "opts": ["--oldest", "-o"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Displays the oldest rejected records. If omitted, most recent records will be retrieved", + "is_flag": true, + "flag_value": true + }, + { + "name": "dlq_name", + "param_type_name": "option", + "opts": ["--dlq-name"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Only prints the rejected records for the specified DLQ (Dead Letter Queue) name", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Returns all the stored rejected entries", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "install": { + "name": "install", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "WARNING", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "file", + "param_type_name": "option", + "opts": ["-f", "--file"], + "type": { + "param_type": "Path", + "name": "file", + "exists": true, + "file_okay": true, + "dir_okay": false, + "writable": false, + "readable": true, + "allow_dash": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Path to a YAML configuration file for silent installation", + "is_flag": false + }, + { + "name": "installation_dir", + "param_type_name": "option", + "opts": ["--installation-dir"], + "type": { + "param_type": "Path", + "name": "directory", + "exists": false, + "file_okay": false, + "dir_okay": true, + "writable": false, + "readable": true, + "allow_dash": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Custom installation directory", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Installs RDI", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "list-contexts": { + "name": "list-contexts", + "params": [ + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Lists all saved contexts", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "list-jobs": { + "name": "list-jobs", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Lists transformation engine's jobs", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "monitor": { + "name": "monitor", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_namespace", + "param_type_name": "option", + "opts": ["--rdi-namespace"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "rdi", + "envvar": "RDI_NAMESPACE", + "help": "RDI Kubernetes namespace", + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "exporter_port", + "param_type_name": "option", + "opts": ["--exporter-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": 9121, + "envvar": null, + "help": "HTTP port to start the exporter on", + "is_flag": false + }, + { + "name": "collect_interval", + "param_type_name": "option", + "opts": ["--collect-interval"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 60, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": 5, + "envvar": null, + "help": "Metrics collection interval (seconds)", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Monitors RDI by collecting metrics and exporting to Prometheus", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "reset": { + "name": "reset", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "force", + "param_type_name": "option", + "opts": ["--force", "-f"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Force operation. Skips verification prompts", + "is_flag": true, + "flag_value": true + }, + { + "name": "pause_for_confirmation", + "param_type_name": "option", + "opts": ["--pause-for-confirmation"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Pause for user confirmation if manual shutdown of collector required", + "is_flag": true, + "flag_value": true + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Resets the pipeline into initial full sync mode", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "scaffold": { + "name": "scaffold", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "db_type", + "param_type_name": "option", + "opts": ["--db-type"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "cassandra", + "mariadb", + "mongodb", + "mysql", + "oracle", + "postgresql", + "snowflake", + "sqlserver", + "spanner" + ], + "case_sensitive": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "DB type", + "is_flag": false + }, + { + "name": "db_flavor", + "param_type_name": "option", + "opts": ["--db-flavor"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "mongodb-atlas", + "mongodb-replica-set", + "mongodb-sharded-cluster" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "DB flavor", + "is_flag": false + }, + { + "name": "directory", + "param_type_name": "option", + "opts": ["--dir"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Directory containing RDI configuration", + "is_flag": false + }, + { + "name": "preview", + "param_type_name": "option", + "opts": ["--preview"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Print the content of the scaffolded config file to CLI output", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Generates configuration files for RDI", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "set-context": { + "name": "set-context", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "context_name", + "param_type_name": "argument", + "opts": ["context-name"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Sets a context to be the active one", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "set-secret": { + "name": "set-secret", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_namespace", + "param_type_name": "option", + "opts": ["--rdi-namespace"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "rdi", + "envvar": "RDI_NAMESPACE", + "help": "RDI Kubernetes namespace", + "is_flag": false + }, + { + "name": "key", + "param_type_name": "argument", + "opts": ["key"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "RDI_REDIS_USERNAME", + "RDI_REDIS_PASSWORD", + "RDI_REDIS_CACERT", + "RDI_REDIS_CERT", + "RDI_REDIS_KEY", + "RDI_REDIS_KEY_PASSPHRASE", + "SOURCE_DB_USERNAME", + "SOURCE_DB_PASSWORD", + "SOURCE_DB_CACERT", + "SOURCE_DB_CERT", + "SOURCE_DB_KEY", + "SOURCE_DB_KEY_PASSWORD", + "TARGET_DB_USERNAME", + "TARGET_DB_PASSWORD", + "TARGET_DB_CACERT", + "TARGET_DB_CERT", + "TARGET_DB_KEY", + "TARGET_DB_KEY_PASSWORD", + "JWT_SECRET_KEY" + ], + "case_sensitive": true + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null + }, + { + "name": "value", + "param_type_name": "argument", + "opts": ["value"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Creates a secret of a specified key", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "start": { + "name": "start", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Starts the pipeline", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "status": { + "name": "status", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_namespace", + "param_type_name": "option", + "opts": ["--rdi-namespace"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "rdi", + "envvar": "RDI_NAMESPACE", + "help": "RDI Kubernetes namespace", + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "live", + "param_type_name": "option", + "opts": ["--live", "-l"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Live data flow monitoring", + "is_flag": true, + "flag_value": true + }, + { + "name": "page_number", + "param_type_name": "option", + "opts": ["--page-number", "-p"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Set the page number (live mode only)", + "is_flag": false + }, + { + "name": "page_size", + "param_type_name": "option", + "opts": ["--page-size", "-s"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Set the page size (live mode only)", + "is_flag": false + }, + { + "name": "ingested_only", + "param_type_name": "option", + "opts": ["--ingested-only", "-i"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Display ingested data streams (live mode only)", + "is_flag": true, + "flag_value": true + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Displays the status of the pipeline end to end", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "stop": { + "name": "stop", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Stops the pipeline", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "trace": { + "name": "trace", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "max_change_records", + "param_type_name": "option", + "opts": ["--max-change-records"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": null, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": 10, + "envvar": null, + "help": "Maximum traced change records", + "is_flag": false + }, + { + "name": "timeout", + "param_type_name": "option", + "opts": ["--timeout"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 600, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": 20, + "envvar": null, + "help": "Stops the trace after exceeding this timeout (in seconds)", + "is_flag": false + }, + { + "name": "trace_only_rejected", + "param_type_name": "option", + "opts": ["--trace-only-rejected"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Trace only rejected change records", + "is_flag": true, + "flag_value": true + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Starts a trace session for troubleshooting data transformation", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + }, + "upgrade": { + "name": "upgrade", + "params": [ + { + "name": "log_level", + "param_type_name": "option", + "opts": ["--log-level", "-l"], + "type": { + "param_type": "Choice", + "name": "choice", + "choices": [ + "TRACE", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "case_sensitive": false + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "INFO", + "envvar": null, + "help": null, + "is_flag": false + }, + { + "name": "rdi_namespace", + "param_type_name": "option", + "opts": ["--rdi-namespace"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": "rdi", + "envvar": "RDI_NAMESPACE", + "help": "RDI Kubernetes namespace", + "is_flag": false + }, + { + "name": "rdi_host", + "param_type_name": "option", + "opts": ["--rdi-host"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_HOST", + "help": "Host/IP of RDI Database", + "is_flag": false + }, + { + "name": "rdi_port", + "param_type_name": "option", + "opts": ["--rdi-port"], + "type": { + "param_type": "IntRange", + "name": "integer range", + "min": 1, + "max": 65535, + "min_open": false, + "max_open": false, + "clamp": false + }, + "required": true, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PORT", + "help": "Port of RDI Database", + "is_flag": false + }, + { + "name": "rdi_user", + "param_type_name": "option", + "opts": ["--rdi-user"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_USERNAME", + "help": "RDI Database Username", + "is_flag": false + }, + { + "name": "rdi_password", + "param_type_name": "option", + "opts": ["--rdi-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": "RDI_REDIS_PASSWORD", + "help": "RDI Database Password", + "is_flag": false + }, + { + "name": "rdi_key", + "param_type_name": "option", + "opts": ["--rdi-key"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Private key file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cert", + "param_type_name": "option", + "opts": ["--rdi-cert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Client certificate file to authenticate with", + "is_flag": false + }, + { + "name": "rdi_cacert", + "param_type_name": "option", + "opts": ["--rdi-cacert"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "CA certificate file to verify with", + "is_flag": false + }, + { + "name": "rdi_key_password", + "param_type_name": "option", + "opts": ["--rdi-key-password"], + "type": { + "param_type": "String", + "name": "text" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": null, + "envvar": null, + "help": "Password for unlocking an encrypted private key", + "is_flag": false + }, + { + "name": "force", + "param_type_name": "option", + "opts": ["--force", "-f"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Force operation. Skips verification prompts", + "is_flag": true, + "flag_value": true + }, + { + "name": "help", + "param_type_name": "option", + "opts": ["--help"], + "type": { + "param_type": "Bool", + "name": "boolean" + }, + "required": false, + "nargs": 1, + "multiple": false, + "default": false, + "envvar": null, + "help": "Show this message and exit.", + "is_flag": true, + "flag_value": true + } + ], + "help": "Upgrades RDI without losing data or downtime", + "epilog": null, + "short_help": null, + "hidden": false, + "deprecated": false + } + }, + "chain": false +} diff --git a/data/rdi-reference/config.json b/data/rdi-reference/config.json new file mode 100644 index 0000000000..759d3fc81e --- /dev/null +++ b/data/rdi-reference/config.json @@ -0,0 +1,2227 @@ +{ + "title": "Redis Data Integration Configuration File", + "description": "Configuration file for Redis Data Integration (RDI) source collectors and target connections.", + "type": ["object", "null"], + "errorMessage": { + "type": "Configuration must be either an object or null" + }, + "properties": { + "sources": { + "title": "Source collectors", + "description": "Source collectors that capture changes from upstream databases. Each key is a unique source identifier; the value configures one collector.", + "type": "object", + "errorMessage": { + "type": "Sources configuration must be an object containing named source collectors" + }, + "patternProperties": { + ".*": { + "properties": { + "connection": { + "title": "Source database connection", + "description": "Connection configuration for a non-Redis source database. The exact set of properties depends on the database type.", + "properties": { + "SQL database": { + "title": "SQL database", + "description": "Connection configuration for a supported SQL database.", + "type": "object", + "properties": { + "type": { + "title": "Database type", + "description": "SQL database engine.", + "type": "string", + "enum": [ + "mariadb", + "mysql", + "oracle", + "postgresql", + "sqlserver" + ], + "errorMessage": "Database type must be one of: db2, mariadb, mysql, oracle, postgresql, sqlserver" + }, + "host": { + "title": "Database host", + "description": "Hostname or IP address of the SQL database server.", + "type": "string", + "errorMessage": "Host must be a valid hostname or IP address" + }, + "port": { + "title": "Database port", + "description": "Network port on which the SQL database server is listening.", + "errorMessage": { + "oneOf": "Port must be either a number between 1-65535 or an environment variable reference", + "pattern": "Environment variable reference must be in ${VAR_NAME} format", + "minimum": "Port number must be at least 1", + "maximum": "Port number cannot exceed 65535" + }, + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "database": { + "title": "Database name", + "description": "Name of the database to connect to.", + "type": "string", + "errorMessage": "Database name must be a valid string" + }, + "user": { + "title": "Database user", + "description": "Username for authentication to the SQL database.", + "type": "string", + "errorMessage": "Database user must be a valid string" + }, + "password": { + "title": "Database password", + "description": "Password for authentication to the SQL database.", + "type": "string", + "errorMessage": "Database password must be a valid string" + } + }, + "additionalProperties": false, + "errorMessage": { + "oneOf": "Invalid database connection configuration" + }, + "examples": [ + { + "hr": { + "type": "postgresql", + "host": "localhost", + "port": 5432, + "database": "postgres", + "user": "postgres", + "password": "postgres" + } + }, + { + "my-oracle": { + "type": "oracle", + "host": "172.17.0.4", + "port": 1521, + "user": "c##dbzuser", + "password": "dbz" + } + } + ] + }, + "MongoDB": { + "title": "MongoDB", + "description": "Connection configuration for a MongoDB database.", + "type": "object", + "properties": { + "type": { + "title": "Database type", + "description": "Database type identifier. Always `mongodb` for this connection.", + "type": "string", + "const": "mongodb" + }, + "connection_string": { + "title": "Connection string", + "description": "MongoDB connection URI including host, port, and any connection options.", + "type": "string" + }, + "user": { + "title": "MongoDB user", + "description": "Username for authentication to MongoDB.", + "type": "string" + }, + "password": { + "title": "MongoDB password", + "description": "Password for authentication to MongoDB.", + "type": "string" + }, + "database": { + "title": "MongoDB databases", + "description": "Comma-separated list of MongoDB databases to monitor.", + "type": "string" + } + }, + "required": ["type", "connection_string"], + "additionalProperties": false, + "errorMessage": { + "required": "MongoDB connection string must be provided" + }, + "examples": [ + { + "mongodb-source": { + "type": "mongodb", + "connection_string": "mongodb://localhost:27017/?replicaSet=rs0", + "user": "debezium", + "password": "dbz", + "database": "db1,db2" + } + } + ] + }, + "Spanner": { + "title": "Spanner", + "description": "Connection configuration for a Google Cloud Spanner database.", + "type": "object", + "properties": { + "type": { + "title": "Database type", + "description": "Database type identifier. Always `spanner` for this connection.", + "type": "string", + "const": "spanner" + }, + "project_id": { + "title": "Spanner project ID", + "description": "Google Cloud project ID that hosts the Spanner instance.", + "type": "string" + }, + "instance_id": { + "title": "Spanner instance ID", + "description": "Spanner instance identifier within the project.", + "type": "string" + }, + "database_id": { + "title": "Spanner database ID", + "description": "Spanner database identifier within the instance.", + "type": "string" + }, + "emulator_host": { + "title": "Spanner emulator host", + "description": "Host and port of the Spanner emulator. Used for local development; leave unset against real Spanner.", + "type": "string" + }, + "use_credentials_file": { + "title": "Use credentials file", + "description": "When `true`, RDI authenticates using a service account credentials file; when `false`, it uses application default credentials.", + "type": "boolean", + "default": false, + "errorMessage": "Use credentials file flag must be a boolean value (true/false)" + }, + "change_streams": { + "title": "Change streams configuration", + "description": "Spanner change streams to capture, keyed by change stream name.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": ["object", "null"], + "properties": { + "retention_period_hours": { + "title": "Change stream retention period hours", + "description": "Retention period for the change stream, in hours.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$" + } + }, + "additionalProperties": false + }, + "errorMessage": { + "minProperties": "Change streams configuration cannot be empty" + } + } + }, + "required": [ + "type", + "project_id", + "instance_id", + "database_id", + "change_streams" + ], + "additionalProperties": false, + "examples": [ + { + "spanner-source": { + "type": "spanner", + "project_id": "example-12345", + "instance_id": "example", + "database_id": "example", + "change_streams": { + "change_stream_all": { + "retention_period_hours": 24 + } + } + } + } + ] + }, + "Snowflake": { + "title": "Snowflake", + "description": "Connection configuration for a Snowflake database.", + "type": "object", + "properties": { + "type": { + "title": "Database type", + "description": "Database type identifier. Always `snowflake` for this connection.", + "type": "string", + "const": "snowflake" + }, + "url": { + "title": "JDBC URL", + "description": "Snowflake JDBC connection URL, for example `jdbc:snowflake://account.snowflakecomputing.com/`.", + "type": "string", + "errorMessage": "URL must be a valid Snowflake JDBC connection string" + }, + "user": { + "title": "Snowflake user", + "description": "Username for authentication to Snowflake.", + "type": "string", + "errorMessage": "User must be a valid string" + }, + "password": { + "title": "Snowflake password", + "description": "Password for authentication to Snowflake. For key-pair authentication, omit this field and provide the private key via the `source-db-ssl` secret (`client.key` field).", + "type": "string", + "errorMessage": "Password must be a valid string" + }, + "database": { + "title": "Snowflake database", + "description": "Name of the Snowflake database to connect to.", + "type": "string", + "errorMessage": "Database name must be a valid string" + }, + "warehouse": { + "title": "Snowflake warehouse", + "description": "Name of the Snowflake warehouse used for compute.", + "type": "string", + "errorMessage": "Warehouse name must be a valid string" + }, + "role": { + "title": "Snowflake role", + "description": "Snowflake role used for the connection.", + "type": "string", + "errorMessage": "Role name must be a valid string" + }, + "cdcDatabase": { + "title": "CDC database", + "description": "Database hosting the CDC streams. Defaults to the main `database` if not set.", + "type": "string", + "errorMessage": "CDC database name must be a valid string" + }, + "cdcSchema": { + "title": "CDC schema", + "description": "Schema hosting the CDC streams. Defaults to the main schema if not set.", + "type": "string", + "errorMessage": "CDC schema name must be a valid string" + } + }, + "required": ["type", "url", "user", "warehouse", "database"], + "additionalProperties": false, + "errorMessage": { + "required": "Snowflake connection must include type, url, user, warehouse, and database." + }, + "examples": [ + { + "snowflake": { + "type": "snowflake", + "url": "jdbc:snowflake://myaccount.snowflakecomputing.com/", + "user": "myuser", + "password": "mypassword", + "database": "MYDB", + "warehouse": "COMPUTE_WH" + } + } + ] + } + }, + "type": "object" + }, + "name": { + "title": "Source name", + "description": "Human-readable name for the source collector. Maximum 100 characters.", + "type": "string", + "maxLength": 100, + "errorMessage": { + "type": "Source name must be a string", + "maxLength": "Source name must be at most 100 characters" + } + }, + "type": { + "title": "Collector type", + "description": "Type of the source collector. Use `cdc` (default) for change data capture using [Debezium](https://debezium.io/). Use `flink` for Spanner change streams using the Apache Flink-based collector. Use `riotx` for Snowflake CDC using [RIOT-X](https://redis.github.io/riotx/).", + "type": "string", + "enum": ["cdc", "flink", "riotx"], + "default": "cdc", + "errorMessage": "Source collector type must be one of: cdc, flink, external, riotx" + }, + "active": { + "title": "Collector enabled", + "description": "When `true`, the collector runs; when `false`, the collector is disabled and produces no events.", + "type": "boolean", + "default": true, + "errorMessage": "Active flag must be a boolean value (true/false)" + }, + "logging": { + "title": "Logging configuration", + "description": "Logging settings for this source collector.", + "type": "object", + "properties": { + "level": { + "title": "Logging level", + "description": "Log verbosity for the source collector.", + "type": "string", + "enum": ["trace", "debug", "info", "warn", "error"], + "default": "info", + "errorMessage": "Logging level must be one of: trace, debug, info, warn, error" + } + }, + "additionalProperties": false, + "errorMessage": { + "additionalProperties": "Unknown logging configuration properties. Only 'level' is allowed" + } + }, + "tables": { + "title": "Tables to capture", + "description": "Tables to capture from the source database, keyed by table name. The value configures column selection and key handling for that table.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": ["object", "null"], + "properties": { + "snapshot_sql": { + "title": "Snapshot SQL", + "description": "Custom SQL statement used during the initial snapshot, giving fine-grained control over the data captured.", + "type": "string", + "errorMessage": "Snapshot SQL must be a valid SQL query string" + }, + "columns": { + "title": "Columns to capture", + "description": "List of specific columns to capture for changes. If not specified, all columns will be captured. For RIOTX Snowflake sources, this is rendered as per-table `--table-columns` projection. Note: This property cannot be used for MongoDB connections", + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": "Columns must be an array of column names" + }, + "exclude_columns": { + "title": "Columns to exclude", + "description": "Specific columns to exclude from capture. When omitted, no columns are excluded. Only supported for MongoDB connections.", + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": "Exclude columns must be an array of column names" + }, + "keys": { + "title": "Message keys", + "description": "Optional list of columns to use as a composite unique identifier. For RIOTX Snowflake sources, this is rendered as per-table `--table-keys`. Only required when the table lacks a primary key or unique constraint. Must form a unique combination of fields", + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": "Keys must be an array of column names that form a unique identifier" + } + }, + "additionalProperties": false, + "errorMessage": { + "additionalProperties": "Unknown table properties detected" + } + }, + "errorMessage": { + "minProperties": "Tables configuration cannot be empty" + } + }, + "schemas": { + "title": "Schema names", + "description": "Schema names to capture from the source database. Maps to the underlying connector's `schema.include.list`.", + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": { + "type": "Schemas must be an array of schema names", + "items": "Each schema name must be a valid string" + } + }, + "databases": { + "title": "Database names", + "description": "Database names to capture from the source database. Maps to the underlying connector's `database.include.list`.", + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": { + "type": "Databases must be an array of database names", + "items": "Each database name must be a valid string" + } + }, + "advanced": { + "title": "Advanced configuration", + "description": "Advanced configuration that overrides the underlying engine's defaults. Only required for non-standard tuning.", + "type": "object", + "minProperties": 1, + "properties": { + "sink": { + "title": "RDI Collector stream writer configuration", + "description": "Advanced configuration properties for the RDI Collector stream writer connection and behaviour. **Applies to the `cdc` and `flink` collector types.**

For the `cdc` collector type, see the full list of properties at [Debezium Server — Redis Stream sink](https://debezium.io/documentation/reference/stable/operations/debezium-server.html#_redis_stream). When using a property from that page, omit the `debezium.sink.` prefix.

**The properties listed below only apply to the `flink` collector type.**", + "type": "object", + "minProperties": 1, + "properties": { + "redis.batch.size": { + "title": "Sink batch size", + "description": "Maximum number of records the collector sink writes to Redis in a single batch.", + "type": "integer", + "minimum": 1, + "default": 1000, + "x-ui-expose": ["collector:cdc", "collector:flink"] + }, + "redis.flush.interval.ms": { + "title": "Sink flush interval", + "description": "Maximum time in milliseconds the collector sink waits to fill a batch before flushing it to Redis.", + "type": "integer", + "minimum": 1, + "default": 100, + "x-ui-expose": ["collector:cdc", "collector:flink"] + }, + "redis.connection.timeout.ms": { + "title": "Sink connection timeout", + "description": "Connection timeout in milliseconds for the target Redis client used by the collector sink.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "redis.socket.timeout.ms": { + "title": "Sink socket timeout", + "description": "Socket read/write timeout in milliseconds for the target Redis client used by the collector sink.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "redis.retry.max.attempts": { + "title": "Sink retry max attempts", + "description": "Maximum number of retry attempts for failed Redis operations.", + "type": "integer", + "minimum": 1, + "default": 5 + }, + "redis.retry.initial.delay.ms": { + "title": "Sink retry initial delay", + "description": "Initial delay in milliseconds before the first retry of a failed Redis operation.", + "type": "integer", + "minimum": 1, + "default": 100 + }, + "redis.retry.max.delay.ms": { + "title": "Sink retry max delay", + "description": "Maximum delay in milliseconds between retry attempts for failed Redis operations.", + "type": "integer", + "minimum": 1, + "default": 3000 + }, + "redis.retry.backoff.multiplier": { + "title": "Sink retry backoff multiplier", + "description": "Exponential backoff multiplier between retry attempts for failed Redis operations.", + "type": "number", + "minimum": 1, + "default": 2 + }, + "redis.oom.retry.initial.delay.ms": { + "title": "Sink OOM retry initial delay", + "description": "Initial delay in milliseconds before the first retry after a Redis out-of-memory error.", + "type": "integer", + "minimum": 1, + "default": 1000 + }, + "redis.oom.retry.max.delay.ms": { + "title": "Sink OOM retry max delay", + "description": "Maximum delay in milliseconds between retry attempts after a Redis out-of-memory error.", + "type": "integer", + "minimum": 1, + "default": 10000 + }, + "redis.oom.retry.backoff.multiplier": { + "title": "Sink OOM retry backoff multiplier", + "description": "Exponential backoff multiplier between retry attempts after a Redis out-of-memory error.", + "type": "number", + "minimum": 1, + "default": 2 + }, + "redis.wait.enabled": { + "title": "Sink replica wait enabled", + "description": "When `true`, the collector verifies that each write has been replicated to the configured number of Redis replica shards before acknowledging it.", + "type": "boolean", + "default": false + }, + "redis.wait.write.timeout.ms": { + "title": "Sink replica wait timeout", + "description": "Maximum time in milliseconds to wait for replica write acknowledgements.", + "type": "integer", + "minimum": 1, + "default": 1000 + }, + "redis.wait.retry.enabled": { + "title": "Sink replica wait retry enabled", + "description": "When `true`, the collector keeps retrying a write until replica acknowledgement succeeds; when `false`, it gives up after the first failure.", + "type": "boolean", + "default": false + }, + "redis.wait.retry.delay.ms": { + "title": "Sink replica wait retry delay", + "description": "Delay in milliseconds between replica wait retry attempts.", + "type": "integer", + "minimum": 1, + "default": 1000 + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Sink configuration cannot be empty", + "additionalProperties": "Sink configuration values must be strings, numbers, or booleans" + } + }, + "source": { + "title": "Advanced source settings", + "description": "Advanced configuration properties for the source database connection and CDC behavior. **Applies to the `cdc` and `flink` collector types.**

For the `cdc` collector type, available properties depend on the source database — refer to the relevant Debezium connector documentation: [MySQL](https://debezium.io/documentation/reference/stable/connectors/mysql.html), [MariaDB](https://debezium.io/documentation/reference/stable/connectors/mariadb.html), [PostgreSQL](https://debezium.io/documentation/reference/stable/connectors/postgresql.html), [Oracle](https://debezium.io/documentation/reference/stable/connectors/oracle.html), [SQL Server](https://debezium.io/documentation/reference/stable/connectors/sqlserver.html), [Db2](https://debezium.io/documentation/reference/stable/connectors/db2.html), [MongoDB](https://debezium.io/documentation/reference/stable/connectors/mongodb.html). When using a property from those pages, omit the `debezium.source.` prefix.

**The named properties below cover the most commonly tuned settings: `spanner.*` properties apply to the `flink` collector type, all others apply to the `cdc` collector type. Any other property from the Debezium documentation can still be set as a free-form key-value pair.**", + "type": "object", + "minProperties": 1, + "properties": { + "record.processing.threads": { + "title": "Record processing threads", + "description": "Controls how many worker threads process captured records before they are written downstream.", + "type": "integer", + "minimum": 1, + "x-ui-expose": ["collector:cdc"] + }, + "snapshot.max.threads": { + "title": "Snapshot max threads", + "description": "Sets the maximum number of threads used while taking the initial snapshot.", + "type": "integer", + "minimum": 1, + "default": 1, + "x-ui-expose": ["collector:cdc"] + }, + "poll.interval.ms": { + "title": "Poll interval ms", + "description": "Defines how often the collector polls the source for new changes.", + "type": "integer", + "minimum": 1, + "default": 500, + "x-ui-expose": ["collector:cdc"] + }, + "snapshot.fetch.size": { + "title": "Snapshot fetch size", + "description": "Defines how many rows are fetched per batch during the initial snapshot.", + "type": "integer", + "minimum": 1, + "default": 10000, + "x-ui-expose": ["collector:cdc"] + }, + "max.batch.size": { + "title": "Max batch size", + "description": "Caps how many records are processed together in a single batch.", + "type": "integer", + "minimum": 1, + "default": 2048, + "x-ui-expose": ["collector:cdc"] + }, + "max.queue.size": { + "title": "Max queue size", + "description": "Limits how many records can be buffered in memory before processing catches up.", + "type": "integer", + "minimum": 1, + "default": 8192, + "x-ui-expose": ["collector:cdc"] + }, + "heartbeat.interval.ms": { + "title": "Heartbeat interval ms", + "description": "Sets how often heartbeat events are emitted to keep change tracking active. Use 0 to disable them.", + "type": "integer", + "minimum": 0, + "default": 0, + "x-ui-expose": ["collector:cdc"] + }, + "heartbeat.action.query": { + "title": "Heartbeat action query", + "description": "SQL query executed on the source whenever a heartbeat is emitted.", + "type": "string", + "x-ui-expose": ["collector:cdc"] + }, + "lob.enabled": { + "title": "Lob enabled", + "description": "Determines whether large object columns are included in change capture.", + "type": "boolean", + "default": false, + "x-ui-expose": ["collector:cdc"], + "x-ui-source-types": ["oracle"] + }, + "gtid.source.includes": { + "title": "GTID source includes", + "description": "Restricts MySQL GTID processing to the listed source UUIDs.", + "type": "string", + "x-ui-expose": ["collector:cdc"], + "x-ui-source-types": ["mysql", "mariadb"] + }, + "publication.autocreate.mode": { + "title": "Publication autocreate mode", + "description": "Controls whether and how the PostgreSQL publication is created or updated automatically.", + "type": "string", + "enum": ["all_tables", "filtered", "disabled"], + "default": "all_tables", + "x-ui-expose": ["collector:cdc"], + "x-ui-source-types": ["postgresql"] + }, + "publication.name": { + "title": "Publication name", + "description": "Sets the PostgreSQL logical replication publication name used by the collector.", + "type": "string", + "default": "dbz_publication", + "x-ui-expose": ["collector:cdc"], + "x-ui-source-types": ["postgresql"] + }, + "slot.name": { + "title": "Slot name", + "description": "Sets the PostgreSQL replication slot name the collector reads from.", + "type": "string", + "default": "debezium", + "x-ui-expose": ["collector:cdc"], + "x-ui-source-types": ["postgresql"] + }, + "spanner.version.retention.period.hours": { + "title": "Spanner version retention period", + "description": "Retention period in hours for Spanner change stream versions. Determines how far back the collector can resume after an outage.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "spanner.fetch.timeout.ms": { + "title": "Spanner fetch timeout", + "description": "Timeout in milliseconds for a single change stream fetch request to Spanner.", + "type": "integer", + "minimum": 1, + "default": 500 + }, + "spanner.fetch.heartbeat.ms": { + "title": "Spanner fetch heartbeat interval", + "description": "Interval in milliseconds at which Spanner sends heartbeat records when no data changes are available.", + "type": "integer", + "minimum": 1, + "default": 100 + }, + "spanner.max.rows.per.partition": { + "title": "Spanner max rows per partition", + "description": "Maximum number of rows the collector reads from a single Spanner change stream partition before yielding.", + "type": "integer", + "minimum": 1, + "default": 10000 + }, + "spanner.dialect": { + "title": "Spanner SQL dialect", + "description": "SQL dialect of the Spanner database. Use `GOOGLESQL` for Google Standard SQL or `POSTGRESQL` for the PostgreSQL interface.", + "type": "string", + "enum": ["GOOGLESQL", "POSTGRESQL"], + "default": "GOOGLESQL" + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Source configuration cannot be empty", + "additionalProperties": "Source configuration values must be strings, numbers, or booleans" + } + }, + "quarkus": { + "title": "Quarkus runtime settings", + "description": "Advanced configuration properties for the Quarkus runtime that hosts Debezium Server. **Only applies to the `cdc` collector type.** See the [Debezium Server documentation](https://debezium.io/documentation/reference/stable/operations/debezium-server.html) for runtime configuration options. When using a property from that page, omit the `quarkus.` prefix.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Quarkus configuration cannot be empty", + "additionalProperties": "Quarkus configuration values must be strings, numbers, or booleans" + } + }, + "flink": { + "title": "Advanced Flink settings", + "description": "Advanced configuration properties forwarded to the Flink runtime that hosts the collector. Any property listed in the [Flink configuration documentation](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/deployment/config/) can be set here and will override the RDI default. **Only applies to the `flink` collector type.**

The properties listed below are the ones most likely to require adjustment. **Changing any other Flink property is not recommended unless instructed by Redis support.**", + "type": "object", + "minProperties": 1, + "properties": { + "parallelism.default": { + "title": "Default parallelism", + "description": "Default parallelism for Flink jobs and operators. When unset, Flink uses the number of available task slots across all task managers (`taskManager.replicas × taskmanager.numberOfTaskSlots`). See [parallel execution](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/dev/datastream/execution/parallel/).", + "type": "integer", + "minimum": 1 + }, + "taskmanager.numberOfTaskSlots": { + "title": "Task slots per task manager", + "description": "Number of parallel task slots per task manager pod. Each slot can run one parallel pipeline instance, so this caps the parallelism a single task manager can absorb. See [task slots and resources](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/concepts/flink-architecture/#task-slots-and-resources).", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "taskmanager.memory.process.size": { + "title": "Task manager process memory", + "description": "Total memory budget for each task manager JVM process, expressed with a unit suffix such as `2048m` or `4g`. See [task manager memory configuration](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/deployment/memory/mem_setup_tm/).", + "type": "string" + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Flink configuration cannot be empty", + "additionalProperties": "Flink configuration values must be strings, numbers, or booleans" + } + }, + "resources": { + "title": "Collector resource settings", + "description": "Compute resources allocated to the collector. **Only applies to the `cdc` collector type.**", + "type": "object", + "minProperties": 1, + "properties": { + "cpu": { + "title": "CPU resource value", + "description": "CPU request for the collector container, for example `1` or `500m`.", + "type": "string", + "errorMessage": "CPU resource value must be a valid Kubernetes quantity string" + }, + "memory": { + "title": "Memory resource value", + "description": "Memory request for the collector container, for example `1024Mi` or `2Gi`.", + "type": "string", + "errorMessage": "Memory resource value must be a valid Kubernetes quantity string" + } + }, + "additionalProperties": false, + "errorMessage": { + "minProperties": "Resources configuration cannot be empty", + "additionalProperties": "Unknown resources properties. Only 'cpu' and 'memory' are allowed" + } + }, + "riotx": { + "title": "Advanced RIOT-X settings", + "description": "Advanced configuration properties for the RIOT-X Snowflake collector. **Only applies to the `riotx` collector type.**", + "type": "object", + "minProperties": 1, + "properties": { + "poll": { + "title": "Polling interval", + "description": "Interval between polls for new stream changes, for example `30s` or `PT30S`.", + "type": "string", + "default": "30s", + "errorMessage": "Poll interval must be a valid duration string" + }, + "snapshot": { + "title": "Snapshot mode", + "description": "Initial-load behavior. `INITIAL` performs a one-time snapshot before streaming; `NEVER` skips the snapshot.", + "type": "string", + "enum": ["INITIAL", "NEVER"], + "default": "INITIAL", + "errorMessage": "Snapshot mode must be either 'INITIAL' or 'NEVER'" + }, + "streamPrefix": { + "title": "Redis stream key prefix", + "description": "Prefix used when constructing Redis stream keys, for example `data:`.", + "type": "string", + "default": "data:", + "errorMessage": "Stream prefix must be a valid string" + }, + "streamLimit": { + "title": "Maximum stream length", + "description": "Maximum number of entries kept in each Redis stream before older entries are trimmed.", + "type": "integer", + "minimum": 1, + "errorMessage": "Stream limit must be a positive integer" + }, + "keyColumns": { + "title": "Key columns", + "description": "Deprecated RIOTX global fallback list of columns to use as message keys for every captured table. Prefer `tables..keys`", + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": "Key columns must be an array of column names" + }, + "clearOffset": { + "title": "Clear existing offset", + "description": "When `true`, the stored offset is cleared on startup, forcing a fresh read.", + "type": "boolean", + "default": false, + "errorMessage": "Clear offset must be a boolean value" + }, + "count": { + "title": "Record count limit", + "description": "Maximum number of records to process. Set to `0` for unlimited.", + "type": "integer", + "minimum": 0, + "default": 0, + "errorMessage": "Count must be a non-negative integer" + } + }, + "additionalProperties": false, + "errorMessage": { + "minProperties": "RIOTX configuration cannot be empty", + "additionalProperties": "Unknown RIOTX configuration properties detected" + } + }, + "java_options": { + "title": "Advanced Java options", + "description": "These Java options will be passed to the command line command when launching the source collector. **Only applies to the `cdc` collector type.**", + "type": "string" + } + }, + "additionalProperties": false, + "errorMessage": { + "type": "Advanced configuration must be an object", + "minProperties": "Advanced configuration cannot be empty" + } + } + }, + "required": ["type", "connection"], + "additionalProperties": false, + "errorMessage": { + "required": "Source collector must include 'type' and 'connection' properties", + "additionalProperties": "Unknown source collector properties detected" + }, + "allOf": [ + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "not": { + "enum": [ + "oracle", + "postgresql", + "sqlserver", + "spanner", + "snowflake" + ] + } + } + } + } + } + }, + "then": { + "properties": { + "schemas": false + } + }, + "errorMessage": "Schemas can only be defined for Oracle, PostgreSQL, SQL Server, Spanner, or Snowflake connections" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "not": { + "enum": ["mariadb", "mysql", "mongodb"] + } + } + } + } + } + }, + "then": { + "properties": { + "databases": false + } + }, + "errorMessage": "Databases can only be defined for MariaDB, MySQL, or MongoDB connections" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "const": "snowflake" + } + }, + "required": ["type"] + } + }, + "required": ["connection"] + }, + "then": { + "properties": { + "databases": false + } + }, + "errorMessage": "Snowflake does not support source-level 'databases'; use 'schemas' instead" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "const": "snowflake" + } + }, + "required": ["type"] + } + }, + "required": ["connection"] + }, + "then": { + "required": ["schemas"], + "properties": { + "schemas": { + "minItems": 1 + } + } + }, + "errorMessage": "Snowflake source requires 'schemas' at the source level" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "database": { + "type": "string" + } + }, + "required": ["database"] + } + }, + "required": ["connection", "databases"] + }, + "then": false, + "errorMessage": "Cannot specify both 'database' in connection and 'databases' list - use one or the other" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "enum": ["mongodb"] + } + } + } + } + }, + "then": { + "properties": { + "tables": { + "additionalProperties": { + "properties": { + "columns": false + } + } + } + } + }, + "errorMessage": "'columns' cannot be defined for MongoDB connections" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "not": { + "enum": ["mongodb"] + } + } + } + } + } + }, + "then": { + "properties": { + "tables": { + "additionalProperties": { + "properties": { + "exclude_columns": false + } + } + } + } + }, + "errorMessage": "'exclude_columns' can only be defined for MongoDB connections" + }, + { + "if": { + "properties": { + "type": { + "const": "riotx" + } + }, + "required": ["type"] + }, + "then": { + "properties": { + "connection": { + "properties": { + "type": { + "const": "snowflake" + } + }, + "required": ["type"] + } + } + }, + "errorMessage": "RIOTX collector type requires a Snowflake connection" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "const": "snowflake" + } + }, + "required": ["type"] + } + }, + "required": ["connection"] + }, + "then": { + "properties": { + "type": { + "const": "riotx" + } + }, + "required": ["type"] + }, + "errorMessage": "Snowflake connection type requires RIOTX collector type" + }, + { + "if": { + "properties": { + "type": { + "const": "flink" + } + }, + "required": ["type"] + }, + "then": { + "properties": { + "connection": { + "properties": { + "type": { + "const": "spanner" + } + }, + "required": ["type"] + } + } + }, + "errorMessage": "Flink collector type requires a Spanner connection" + }, + { + "if": { + "properties": { + "connection": { + "properties": { + "type": { + "const": "spanner" + } + }, + "required": ["type"] + } + }, + "required": ["connection"] + }, + "then": { + "properties": { + "type": { + "const": "flink" + } + }, + "required": ["type"] + }, + "errorMessage": "Spanner connection type requires Flink collector type" + } + ] + } + } + }, + "targets": { + "title": "Target connections", + "description": "Target Redis databases where processed records are written. Each key is a target identifier; the value configures the connection.", + "type": "object", + "patternProperties": { + ".*": { + "oneOf": [ + { + "type": "object", + "properties": { + "connection": { + "title": "Database connection", + "description": "Connection configuration for a Redis database.", + "type": "object", + "properties": { + "type": { + "title": "Database type", + "description": "Database type identifier. Always `redis` for this connection.", + "const": "redis", + "errorMessage": "Connection type must be 'redis'" + }, + "host": { + "title": "Database host", + "description": "Hostname or IP address of the Redis server.", + "type": "string", + "errorMessage": "Host must be a valid hostname or IP address" + }, + "port": { + "title": "Database port", + "description": "Network port on which the Redis server is listening.", + "errorMessage": { + "oneOf": "Port must be either a number between 1-65535 or an environment variable reference", + "pattern": "Environment variable reference must be in ${VAR_NAME} format", + "minimum": "Port number must be at least 1", + "maximum": "Port number cannot exceed 65535" + }, + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "user": { + "title": "Database user", + "description": "Username for authentication to the Redis database.", + "type": "string", + "errorMessage": "Database user must be a valid string" + }, + "password": { + "title": "Database password", + "description": "Password for authentication to the Redis database.", + "type": "string", + "errorMessage": "Database password must be a valid string" + }, + "key": { + "title": "Private key file", + "description": "Path to the private key file used for SSL/TLS client authentication.", + "type": "string", + "errorMessage": "Private key file path must be a valid string" + }, + "key_password": { + "title": "Private key password", + "description": "Password used to decrypt the private key file.", + "type": "string", + "errorMessage": "Private key password must be a valid string" + }, + "cert": { + "title": "Client certificate", + "description": "Path to the client certificate file used for SSL/TLS client authentication.", + "type": "string", + "errorMessage": "Client certificate path must be a valid string" + }, + "cacert": { + "title": "CA certificate", + "description": "Path to the Certificate Authority (CA) certificate file used to verify the server's TLS certificate.", + "type": "string", + "errorMessage": "CA certificate path must be a valid string" + } + }, + "required": ["type", "host", "port"], + "minProperties": 3, + "dependentRequired": { + "key": ["cert"], + "cert": ["key"], + "key_password": ["key"] + }, + "errorMessage": { + "minProperties": "Connection must include required properties", + "required": "Connection must include type, host, and port", + "dependentRequired": { + "key": "When using key authentication, both key and cert must be provided", + "cert": "When using certificate authentication, both key and cert must be provided", + "key_password": "Key password can only be specified when a key is provided" + } + }, + "additionalProperties": false + }, + "name": { + "title": "Target name", + "description": "Human-readable name for the target connection. Maximum 100 characters.", + "type": "string", + "maxLength": 100, + "errorMessage": { + "type": "Target name must be a string", + "maxLength": "Target name must be at most 100 characters" + } + } + }, + "required": ["connection"], + "additionalProperties": false, + "errorMessage": { + "required": "Target must include a connection configuration", + "additionalProperties": "Unknown target properties detected" + } + }, + { + "title": "Database connection", + "description": "Connection configuration for a Redis database.", + "type": "object", + "properties": { + "type": { + "title": "Database type", + "description": "Database type identifier. Always `redis` for this connection.", + "const": "redis", + "errorMessage": "Connection type must be 'redis'" + }, + "host": { + "title": "Database host", + "description": "Hostname or IP address of the Redis server.", + "type": "string", + "errorMessage": "Host must be a valid hostname or IP address" + }, + "port": { + "title": "Database port", + "description": "Network port on which the Redis server is listening.", + "errorMessage": { + "oneOf": "Port must be either a number between 1-65535 or an environment variable reference", + "pattern": "Environment variable reference must be in ${VAR_NAME} format", + "minimum": "Port number must be at least 1", + "maximum": "Port number cannot exceed 65535" + }, + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "user": { + "title": "Database user", + "description": "Username for authentication to the Redis database.", + "type": "string", + "errorMessage": "Database user must be a valid string" + }, + "password": { + "title": "Database password", + "description": "Password for authentication to the Redis database.", + "type": "string", + "errorMessage": "Database password must be a valid string" + }, + "key": { + "title": "Private key file", + "description": "Path to the private key file used for SSL/TLS client authentication.", + "type": "string", + "errorMessage": "Private key file path must be a valid string" + }, + "key_password": { + "title": "Private key password", + "description": "Password used to decrypt the private key file.", + "type": "string", + "errorMessage": "Private key password must be a valid string" + }, + "cert": { + "title": "Client certificate", + "description": "Path to the client certificate file used for SSL/TLS client authentication.", + "type": "string", + "errorMessage": "Client certificate path must be a valid string" + }, + "cacert": { + "title": "CA certificate", + "description": "Path to the Certificate Authority (CA) certificate file used to verify the server's TLS certificate.", + "type": "string", + "errorMessage": "CA certificate path must be a valid string" + } + }, + "required": ["type", "host", "port"], + "minProperties": 3, + "dependentRequired": { + "key": ["cert"], + "cert": ["key"], + "key_password": ["key"] + }, + "errorMessage": { + "minProperties": "Connection must include required properties", + "required": "Connection must include type, host, and port", + "dependentRequired": { + "key": "When using key authentication, both key and cert must be provided", + "cert": "When using certificate authentication, both key and cert must be provided", + "key_password": "Key password can only be specified when a key is provided" + } + }, + "additionalProperties": false + } + ], + "errorMessage": "Target must be either a Redis connection object or an object with a connection property" + } + }, + "errorMessage": { + "type": "Targets configuration must be an object mapping target names to their connection settings" + } + }, + "processors": { + "title": "Data processing configuration", + "description": "Settings that control how data is processed, including batch sizes, error handling, and performance tuning.", + "type": ["object", "null"], + "properties": { + "type": { + "title": "Processor type", + "description": "Processor implementation to run. `classic` runs the classic processor; `flink` runs the Apache Flink-based processor (Kubernetes deployments only).", + "type": "string", + "default": "classic", + "enum": ["classic", "flink"], + "errorMessage": { + "enum": "Invalid processor type. The only supported types are 'classic' or 'flink'." + } + }, + "read_batch_size": { + "title": "Read batch size", + "description": "Maximum number of records read from the source streams in a single batch.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$", + "default": 2000, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "Read batch size must be either an integer or an environment variable reference", + "minimum": "Read batch size must be at least 1", + "pattern": "Invalid format in 'processors -> read_batch_size': ${VALUE} does not match the required pattern" + } + }, + "read_batch_timeout_ms": { + "title": "Read batch timeout", + "description": "Maximum time in milliseconds to wait for a batch to fill before processing it.", + "type": ["integer"], + "minimum": 1, + "default": 100, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "Read batch timeout must be an integer", + "minimum": "Read batch timeout must be at least 1 millisecond" + } + }, + "duration": { + "title": "Batch duration limit", + "description": "This property has no effect; use `read_batch_timeout_ms` instead.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$", + "default": 100, + "deprecated": true, + "errorMessage": { + "type": "Duration must be either an integer or an environment variable reference", + "minimum": "Duration must be at least 1 millisecond", + "pattern": "Invalid format in 'processors -> duration': ${VALUE} does not match the required pattern" + } + }, + "write_batch_size": { + "title": "Write batch size", + "description": "Maximum number of records written to the target Redis database in a single batch.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$", + "default": 200, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "Write batch size must be either an integer or an environment variable reference", + "minimum": "Write batch size must be at least 1", + "pattern": "Invalid format in 'processors -> write_batch_size': ${VALUE} does not match the required pattern" + } + }, + "enable_async_processing": { + "title": "Enable async processing", + "description": "When `true`, the processor handles batches asynchronously to improve throughput. **Classic processor only.**", + "type": "boolean", + "default": true, + "x-ui-expose": ["processor:classic"], + "errorMessage": "Enable async processing flag must be a boolean value (true/false)" + }, + "batch_queue_size": { + "title": "Batch queue size", + "description": "Maximum number of batches queued for processing. **Classic processor only.**", + "type": "integer", + "minimum": 1, + "default": 3, + "x-ui-expose": ["processor:classic"], + "errorMessage": { + "type": "Batch queue size must be an integer", + "minimum": "Batch queue size must be at least 1" + } + }, + "ack_queue_size": { + "title": "ACK queue size", + "description": "Maximum number of batches queued for asynchronous acknowledgement. **Classic processor only.**", + "type": "integer", + "minimum": 1, + "default": 10, + "x-ui-expose": ["processor:classic"], + "errorMessage": { + "type": "ACK queue size must be an integer", + "minimum": "ACK queue size must be at least 1" + } + }, + "dedup": { + "title": "Enable deduplication", + "description": "When `true`, the processor deduplicates incoming records. **Classic processor only.**", + "type": "boolean", + "default": false, + "x-ui-expose": ["processor:classic"], + "errorMessage": "Deduplication flag must be a boolean value (true/false)" + }, + "dedup_max_size": { + "title": "Deduplication set size", + "description": "Maximum number of entries kept in the deduplication set. **Classic processor only.**", + "type": "integer", + "minimum": 1, + "default": 1024, + "x-ui-expose": ["processor:classic"], + "errorMessage": { + "type": "Deduplication set size must be an integer", + "minimum": "Deduplication set size must be at least 1" + } + }, + "dedup_strategy": { + "title": "Deduplication strategy", + "description": "This property has no effect — the only supported strategy is `ignore`. Remove it from the configuration. **Classic processor only.**", + "type": "string", + "default": "ignore", + "enum": ["reject", "ignore"], + "deprecated": true, + "errorMessage": { + "enum": "Invalid deduplication strategy. The only supported strategies are 'reject' or 'ignore'." + } + }, + "error_handling": { + "title": "Error handling strategy", + "description": "Strategy for handling failed records. `ignore` silently drops them; `dlq` writes them to the dead-letter queue.", + "default": "dlq", + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": "error_handling must be 'ignore', 'dlq', or an environment variable reference", + "type": "string" + }, + "dlq_max_messages": { + "title": "DLQ message limit", + "description": "Maximum number of messages stored per dead-letter queue stream.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$", + "default": 1000, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "DLQ message limit must be either an integer or an environment variable reference", + "minimum": "DLQ message limit must be at least 1", + "pattern": "Invalid format in 'processors -> dlq_max_messages': ${VALUE} does not match the required pattern" + } + }, + "target_data_type": { + "title": "Target Redis data type", + "description": "Data type used to store target records in Redis. `hash` writes a Redis Hash; `json` writes a RedisJSON document and requires the RedisJSON module.", + "default": "hash", + "errorMessage": "target_data_type must be 'hash', 'json', or an environment variable reference", + "type": "string" + }, + "json_update_strategy": { + "title": "JSON update strategy", + "description": "Strategy for updating existing JSON documents in Redis. `replace` overwrites the entire document; `merge` merges incoming fields into it.", + "default": "replace", + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": "json_update_strategy must be 'replace', 'merge', or an environment variable reference", + "type": "string" + }, + "use_native_json_merge": { + "title": "Use native JSON merge from RedisJSON module", + "description": "Controls whether JSON merge operations use the native `JSON.MERGE` command (when `true`) or Lua scripts (when `false`). Introduced in RDI 1.15.0. The native command provides 2x performance improvement but handles null values differently:

**Previous behavior (Lua merge)**: When merging `{\"field1\": \"value1\", \"field2\": \"value2\"}` with `{\"field2\": null, \"field3\": \"value3\"}`, the result was `{\"field1\": \"value1\", \"field2\": null, \"field3\": \"value3\"}` (null value is preserved).

**New behavior (JSON.MERGE)**: The same merge produces `{\"field1\": \"value1\", \"field3\": \"value3\"}` (null value removes the field, following [RFC 7396](https://datatracker.ietf.org/doc/html/rfc7396)).

**Note**: The native `JSON.MERGE` command requires RedisJSON 2.6.0 or higher. If the target database has an older version of RedisJSON, RDI automatically falls back to Lua-based merge operations regardless of this setting.

**Impact**: If your application logic distinguishes between a field with a `null` value and a missing field, you may need to adjust your data handling. This follows the JSON Merge Patch RFC standard but differs from the previous Lua implementation. Set to `false` to revert to the previous Lua-based merge behavior if needed.

The Flink processor always uses the native `JSON.MERGE` command when the target database supports it. **Classic processor only.**", + "type": "boolean", + "default": true, + "x-ui-expose": ["processor:classic"], + "errorMessage": "Use native JSON merge flag must be a boolean value (true/false)" + }, + "initial_sync_processes": { + "title": "Initial sync processes", + "description": "Number of parallel processes used to perform the initial data synchronization. For the Flink processor, parallelism is controlled by Flink properties instead. **Classic processor only.**", + "type": ["integer", "string"], + "minimum": 1, + "maximum": 32, + "pattern": "^\\${.*}$", + "default": 4, + "x-ui-expose": ["processor:classic"], + "errorMessage": { + "type": "Initial sync processes must be either an integer or an environment variable reference", + "minimum": "Initial sync processes must be at least 1", + "maximum": "Initial sync processes cannot exceed 32", + "pattern": "Invalid format in 'processors -> initial_sync_processes': ${VALUE} does not match the required pattern" + } + }, + "idle_sleep_time_ms": { + "title": "Idle sleep interval", + "description": "Time in milliseconds to sleep between processing batches when idle. **Classic processor only.**", + "type": ["integer", "string"], + "minimum": 1, + "maximum": 999999, + "pattern": "^\\${.*}$", + "default": 200, + "x-ui-expose": ["processor:classic"], + "errorMessage": { + "type": "Idle sleep time must be either an integer or an environment variable reference", + "minimum": "Idle sleep time must be at least 1 millisecond", + "maximum": "Idle sleep time cannot exceed 999999 milliseconds", + "pattern": "Invalid format in 'processors -> idle_sleep_time_ms': ${VALUE} does not match the required pattern" + } + }, + "idle_streams_check_interval_ms": { + "title": "Idle streams check interval", + "description": "Time in milliseconds between checks for new streams when the processor is idle. For the Flink processor, use `processors.advanced.source.discovery.interval.ms` instead to configure a single discovery interval regardless of load. **Classic processor only.**", + "type": ["integer", "string"], + "minimum": 1, + "maximum": 999999, + "pattern": "^\\${.*}$", + "default": 1000, + "x-ui-expose": ["processor:classic"], + "errorMessage": { + "type": "Idle streams check interval must be either an integer or an environment variable reference", + "minimum": "Idle streams check interval must be at least 1 millisecond", + "maximum": "Idle streams check interval cannot exceed 999999 milliseconds", + "pattern": "Invalid format in 'processors -> idle_streams_check_interval_ms': ${VALUE} does not match the required pattern" + } + }, + "busy_streams_check_interval_ms": { + "title": "Busy streams check interval", + "description": "Time in milliseconds between checks for new streams when the processor is busy. For the Flink processor, use `processors.advanced.source.discovery.interval.ms` instead to configure a single discovery interval regardless of load. **Classic processor only.**", + "type": ["integer", "string"], + "minimum": 1, + "maximum": 999999, + "pattern": "^\\${.*}$", + "default": 5000, + "x-ui-expose": ["processor:classic"], + "errorMessage": { + "type": "Busy streams check interval must be either an integer or an environment variable reference", + "minimum": "Busy streams check interval must be at least 1 millisecond", + "maximum": "Busy streams check interval cannot exceed 999999 milliseconds", + "pattern": "Invalid format in 'processors -> busy_streams_check_interval_ms': ${VALUE} does not match the required pattern" + } + }, + "retry_max_attempts": { + "title": "Maximum retry attempts", + "description": "Maximum number of attempts for a failed write to the target Redis database before giving up.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$", + "default": 5, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "Maximum retry attempts must be either an integer or an environment variable reference", + "minimum": "Maximum retry attempts must be at least 1", + "pattern": "Invalid format in 'processors -> retry_max_attempts': ${VALUE} does not match the required pattern" + } + }, + "retry_initial_delay_ms": { + "title": "Initial retry delay", + "description": "Initial delay in milliseconds before the first retry of a failed write.", + "type": ["integer", "string"], + "minimum": 1, + "maximum": 999999, + "pattern": "^\\${.*}$", + "default": 1000, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "Initial retry delay must be either an integer or an environment variable reference", + "minimum": "Initial retry delay must be at least 1 millisecond", + "maximum": "Initial retry delay cannot exceed 999999 milliseconds", + "pattern": "Invalid format in 'processors -> retry_initial_delay_ms': ${VALUE} does not match the required pattern" + } + }, + "retry_max_delay_ms": { + "title": "Maximum retry delay", + "description": "Maximum delay in milliseconds between retry attempts.", + "type": ["integer", "string"], + "minimum": 1, + "maximum": 999999, + "pattern": "^\\${.*}$", + "default": 10000, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "Maximum retry delay must be either an integer or an environment variable reference", + "minimum": "Maximum retry delay must be at least 1 millisecond", + "maximum": "Maximum retry delay cannot exceed 999999 milliseconds", + "pattern": "Invalid format in 'processors -> retry_max_delay_ms': ${VALUE} does not match the required pattern" + } + }, + "wait_enabled": { + "title": "Enable replica wait", + "description": "When `true`, RDI verifies that each write has been replicated to the target database's replica shards before acknowledging it.", + "type": "boolean", + "default": false, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": "Wait enabled flag must be a boolean value (true/false)" + }, + "wait_timeout": { + "title": "Replica wait timeout", + "description": "Maximum time in milliseconds to wait for replica write verification on the target database.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$", + "default": 1000, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": { + "type": "Wait timeout must be either an integer or an environment variable reference", + "minimum": "Wait timeout must be at least 1 millisecond", + "pattern": "Invalid format in 'processors -> wait_timeout': ${VALUE} does not match the required pattern" + } + }, + "retry_on_replica_failure": { + "title": "Retry on replica failure", + "description": "When `true`, RDI keeps retrying a write until replica replication is confirmed; when `false`, it gives up after the first failure.", + "type": "boolean", + "default": true, + "x-ui-expose": ["processor:classic", "processor:flink"], + "errorMessage": "Retry on replica failure flag must be a boolean value (true/false)" + }, + "on_failed_retry_interval": { + "title": "Retry interval on failure", + "description": "This property has no effect; remove it from the configuration.", + "type": ["integer", "string"], + "minimum": 1, + "pattern": "^\\${.*}$", + "default": 5, + "deprecated": true, + "errorMessage": { + "type": "Retry interval must be either an integer or an environment variable reference", + "minimum": "Retry interval must be at least 1 second", + "pattern": "Invalid format in 'processors -> on_failed_retry_interval': ${VALUE} does not match the required pattern" + } + }, + "logging": { + "title": "Logging configuration", + "description": "Logging settings for the processor. **Flink processor only.**", + "type": "object", + "properties": { + "level": { + "title": "Logging level", + "description": "Log verbosity for the processor.", + "type": "string", + "enum": ["trace", "debug", "info", "warn", "error"], + "default": "info", + "x-ui-expose": ["processor:flink"], + "errorMessage": "Logging level must be one of: trace, debug, info, warn, error" + } + }, + "additionalProperties": false, + "errorMessage": { + "additionalProperties": "Unknown logging configuration properties. Only 'level' is allowed" + } + }, + "advanced": { + "title": "Advanced configuration", + "description": "Advanced configuration for fine-tuning the processor. **All properties under `advanced` apply to the Flink processor only and are silently ignored by the classic processor.**", + "type": "object", + "minProperties": 1, + "properties": { + "source": { + "title": "Advanced source settings", + "description": "Advanced configuration properties for the source Redis client and streams reader. **Flink processor only.**", + "type": "object", + "minProperties": 1, + "properties": { + "stream.name.pattern": { + "title": "Source stream name pattern", + "description": "Glob pattern used to discover input streams in the source Redis database, for example `data:*`.", + "type": "string", + "default": "data:*", + "x-ui-expose": ["processor:flink"] + }, + "discovery.interval.ms": { + "title": "Stream discovery interval", + "description": "Time in milliseconds between checks for new input streams. Replaces the classic `processors.idle_streams_check_interval_ms` and `processors.busy_streams_check_interval_ms` properties.", + "type": "integer", + "minimum": 0, + "default": 1000, + "x-ui-expose": ["processor:flink"] + }, + "batch.size": { + "title": "Source batch size", + "description": "Maximum number of records the source operator reads in a single batch. Alias for `processors.read_batch_size`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "batch.timeout.ms": { + "title": "Source batch timeout", + "description": "Maximum time in milliseconds to wait for a source batch to fill before processing. Alias for `processors.read_batch_timeout_ms`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 100 + }, + "connection.timeout.ms": { + "title": "Source connection timeout", + "description": "Connection timeout in milliseconds for the source Redis client.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "socket.timeout.ms": { + "title": "Source socket timeout", + "description": "Socket read/write timeout in milliseconds for the source Redis client.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "retry.max.attempts": { + "title": "Source retry max attempts", + "description": "Maximum number of retry attempts for failed source Redis operations.", + "type": "integer", + "minimum": 1, + "default": 5, + "x-ui-expose": ["processor:flink"] + }, + "retry.initial.delay.ms": { + "title": "Source retry initial delay", + "description": "Initial delay in milliseconds before the first retry of a failed source Redis operation.", + "type": "integer", + "minimum": 1, + "default": 100, + "x-ui-expose": ["processor:flink"] + }, + "retry.max.delay.ms": { + "title": "Source retry max delay", + "description": "Maximum delay in milliseconds between retry attempts for source Redis operations.", + "type": "integer", + "minimum": 1, + "default": 3000, + "x-ui-expose": ["processor:flink"] + }, + "retry.backoff.multiplier": { + "title": "Source retry backoff multiplier", + "description": "Exponential backoff multiplier between retry attempts for source Redis operations.", + "type": "number", + "minimum": 1, + "default": 2, + "x-ui-expose": ["processor:flink"] + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Source configuration cannot be empty", + "additionalProperties": "Source configuration values must be strings, numbers, or booleans" + } + }, + "target": { + "title": "Advanced target settings", + "description": "Advanced configuration properties for the target Redis client and sink. **Flink processor only.**", + "type": "object", + "minProperties": 1, + "properties": { + "batch.size": { + "title": "Target sink batch size", + "description": "Maximum number of records the target sink writes in a single batch. Alias for `processors.write_batch_size`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 200 + }, + "flush.interval.ms": { + "title": "Target sink flush interval", + "description": "Maximum time in milliseconds the target sink waits to fill a batch before flushing it to Redis.", + "type": "integer", + "minimum": 1, + "default": 100, + "x-ui-expose": ["processor:flink"] + }, + "connection.timeout.ms": { + "title": "Target connection timeout", + "description": "Connection timeout in milliseconds for the target Redis client.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "socket.timeout.ms": { + "title": "Target socket timeout", + "description": "Socket read/write timeout in milliseconds for the target Redis client.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "retry.max.attempts": { + "title": "Target retry max attempts", + "description": "Maximum number of retry attempts for failed target Redis operations. Alias for `processors.retry_max_attempts`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 5 + }, + "retry.initial.delay.ms": { + "title": "Target retry initial delay", + "description": "Initial delay in milliseconds before the first retry of a failed target Redis operation. Alias for `processors.retry_initial_delay_ms`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 1000 + }, + "retry.max.delay.ms": { + "title": "Target retry max delay", + "description": "Maximum delay in milliseconds between retry attempts for target Redis operations. Alias for `processors.retry_max_delay_ms`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 10000 + }, + "retry.backoff.multiplier": { + "title": "Target retry backoff multiplier", + "description": "Exponential backoff multiplier between retry attempts for target Redis operations.", + "type": "number", + "minimum": 1, + "default": 2, + "x-ui-expose": ["processor:flink"] + }, + "wait.enabled": { + "title": "Target replica wait enabled", + "description": "When `true`, RDI verifies that each write has been replicated to the target database's replica shards before acknowledging it. Alias for `processors.wait_enabled`; takes priority when both are set.", + "type": "boolean", + "default": false + }, + "wait.write.timeout.ms": { + "title": "Target replica wait timeout", + "description": "Maximum time in milliseconds to wait for target replica write verification. Alias for `processors.wait_timeout`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 1000 + }, + "wait.retry.enabled": { + "title": "Target replica wait retry enabled", + "description": "When `true`, RDI keeps retrying a target write until replica replication is confirmed; when `false`, it gives up after the first failure. Alias for `processors.retry_on_replica_failure`; takes priority when both are set. When enabled, the Flink processor retries indefinitely until the checkpoint timeout, unlike the classic processor which retries once.", + "type": "boolean", + "default": true + }, + "wait.retry.delay.ms": { + "title": "Target replica wait retry delay", + "description": "Delay in milliseconds between target replica wait retry attempts.", + "type": "integer", + "minimum": 1, + "default": 1000 + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Target configuration cannot be empty", + "additionalProperties": "Target configuration values must be strings, numbers, or booleans" + } + }, + "dlq": { + "title": "Advanced DLQ settings", + "description": "Advanced configuration properties for the DLQ Redis client and sink. **Flink processor only.**", + "type": "object", + "minProperties": 1, + "properties": { + "max.len": { + "title": "DLQ sink max length", + "description": "Maximum number of messages stored per dead letter queue stream. Alias for `processors.dlq_max_messages`; takes priority when both are set.", + "type": "integer", + "minimum": 1, + "default": 1000 + }, + "batch.size": { + "title": "DLQ sink batch size", + "description": "Maximum number of records the DLQ sink writes in a single batch.", + "type": "integer", + "minimum": 1, + "default": 100 + }, + "flush.interval.ms": { + "title": "DLQ sink flush interval", + "description": "Maximum time in milliseconds the DLQ sink waits to fill a batch before flushing it to Redis.", + "type": "integer", + "minimum": 1, + "default": 100 + }, + "connection.timeout.ms": { + "title": "DLQ connection timeout", + "description": "Connection timeout in milliseconds for the DLQ Redis client.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "socket.timeout.ms": { + "title": "DLQ socket timeout", + "description": "Socket read/write timeout in milliseconds for the DLQ Redis client.", + "type": "integer", + "minimum": 1, + "default": 2000 + }, + "retry.max.attempts": { + "title": "DLQ retry max attempts", + "description": "Maximum number of retry attempts for failed DLQ Redis operations.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "retry.initial.delay.ms": { + "title": "DLQ retry initial delay", + "description": "Initial delay in milliseconds before the first retry of a failed DLQ Redis operation.", + "type": "integer", + "minimum": 1, + "default": 100 + }, + "retry.max.delay.ms": { + "title": "DLQ retry max delay", + "description": "Maximum delay in milliseconds between retry attempts for DLQ Redis operations.", + "type": "integer", + "minimum": 1, + "default": 3000 + }, + "retry.backoff.multiplier": { + "title": "DLQ retry backoff multiplier", + "description": "Exponential backoff multiplier between retry attempts for DLQ Redis operations.", + "type": "number", + "minimum": 1, + "default": 2 + }, + "wait.enabled": { + "title": "DLQ replica wait enabled", + "description": "When `true`, RDI verifies that each DLQ write has been replicated to the DLQ database's replica shards before acknowledging it.", + "type": "boolean", + "default": false + }, + "wait.write.timeout.ms": { + "title": "DLQ replica wait timeout", + "description": "Maximum time in milliseconds to wait for DLQ replica write verification.", + "type": "integer", + "minimum": 1, + "default": 1000 + }, + "wait.retry.enabled": { + "title": "DLQ replica wait retry enabled", + "description": "When `true`, RDI keeps retrying a DLQ write until replica replication is confirmed; when `false`, it gives up after the first failure.", + "type": "boolean", + "default": false + }, + "wait.retry.delay.ms": { + "title": "DLQ replica wait retry delay", + "description": "Delay in milliseconds between DLQ replica wait retry attempts.", + "type": "integer", + "minimum": 1, + "default": 1000 + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "DLQ configuration cannot be empty", + "additionalProperties": "DLQ configuration values must be strings, numbers, or booleans" + } + }, + "processor": { + "title": "Advanced processor settings", + "description": "Advanced configuration properties for the processor. **Flink processor only.**", + "type": "object", + "minProperties": 1, + "properties": { + "default.data.type": { + "title": "Default target data type", + "description": "Data type to use in Redis when not overridden per job: `hash` for Redis Hash, `json` for RedisJSON. Alias for `processors.target_data_type`; takes priority when both are set.", + "type": "string", + "enum": ["hash", "json"], + "default": "hash" + }, + "default.json.update.strategy": { + "title": "Default JSON update strategy", + "description": "Strategy for updating JSON data in Redis: `replace` to overwrite the entire JSON object, `merge` to merge new data with the existing JSON object. Alias for `processors.json_update_strategy`; takes priority when both are set.", + "type": "string", + "enum": ["replace", "merge"], + "default": "replace" + }, + "dlq.enabled": { + "title": "Enable DLQ", + "description": "When `true`, rejected messages are stored in the dead-letter queue; when `false`, errors are silently skipped. Alias for `processors.error_handling`; takes priority when both are set.", + "type": "boolean", + "default": true + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Processor configuration cannot be empty", + "additionalProperties": "Processor configuration values must be strings, numbers, or booleans" + } + }, + "flink": { + "title": "Advanced Flink settings", + "description": "Advanced configuration properties forwarded to the underlying Flink runtime. Any property listed in the [Flink configuration documentation](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/deployment/config/) can be set here and will override the RDI default. **Flink processor only.**

The properties listed below are the ones most likely to require adjustment. **Changing any other Flink property is not recommended unless instructed by Redis support.**", + "type": "object", + "minProperties": 1, + "properties": { + "parallelism.default": { + "title": "Default parallelism", + "description": "Default parallelism for jobs and operators. When unset, Flink uses the number of available task slots across all task managers (`taskManager.replicas × taskmanager.numberOfTaskSlots`). Increase to fan out work across more task slots; see [parallel execution](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/dev/datastream/execution/parallel/).", + "type": "integer", + "minimum": 1 + }, + "taskmanager.numberOfTaskSlots": { + "title": "Task slots per task manager", + "description": "Number of parallel task slots per task manager pod. Each slot can run one parallel pipeline instance, so this caps the parallelism a single task manager can absorb. See [task slots and resources](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/concepts/flink-architecture/#task-slots-and-resources).", + "type": "integer", + "minimum": 1, + "default": 1, + "x-ui-expose": ["processor:flink"] + }, + "taskmanager.memory.process.size": { + "title": "Task manager process memory", + "description": "Total memory budget for each task manager JVM process (heap + managed + network + metaspace + JVM overhead), expressed with a unit suffix such as `2048m` or `4g`. See [task manager memory configuration](https://nightlies.apache.org/flink/flink-docs-release-2.0/docs/deployment/memory/mem_setup_tm/).", + "type": "string", + "default": "2048m" + } + }, + "additionalProperties": { + "type": ["string", "number", "boolean"] + }, + "errorMessage": { + "minProperties": "Flink configuration cannot be empty", + "additionalProperties": "Flink configuration values must be strings, numbers, or booleans" + } + }, + "resources": { + "title": "Advanced resource settings", + "description": "Compute resources allocated to the Flink job, such as the number of task manager pods. **Flink processor only.**", + "type": "object", + "minProperties": 1, + "properties": { + "taskManager": { + "title": "Task manager resource settings", + "description": "Resource settings for Flink task manager pods.", + "type": "object", + "minProperties": 1, + "properties": { + "replicas": { + "title": "Task manager replicas", + "description": "Number of Flink task manager pods to run.", + "type": "integer", + "minimum": 1, + "errorMessage": { + "type": "Task manager replicas must be an integer", + "minimum": "Task manager replicas must be at least 1" + } + } + }, + "additionalProperties": false, + "errorMessage": { + "minProperties": "Task manager configuration cannot be empty", + "additionalProperties": "Unknown task manager properties detected" + } + } + }, + "additionalProperties": false, + "errorMessage": { + "minProperties": "Resources configuration cannot be empty", + "additionalProperties": "Unknown resources properties detected" + } + } + }, + "additionalProperties": false, + "errorMessage": { + "type": "Advanced configuration must be an object", + "minProperties": "Advanced configuration cannot be empty" + } + } + }, + "additionalProperties": false, + "errorMessage": { + "type": "Processors configuration must be either an object or null", + "additionalProperties": "Unknown processor configuration property detected" + } + }, + "secret-providers": { + "title": "Secret providers", + "description": "External secret providers used to resolve `${...}` references in the configuration.", + "type": "object", + "patternProperties": { + ".*": { + "title": "Secret provider entry", + "type": "object", + "properties": { + "type": { + "title": "Provider type", + "description": "Secret provider backend. `aws` uses AWS Secrets Manager; `vault` uses HashiCorp Vault.", + "type": "string", + "enum": ["aws", "vault"], + "errorMessage": "Secret provider type must be either 'aws' or 'vault'" + }, + "parameters": { + "title": "Provider parameters", + "description": "Configuration parameters for the secret provider.", + "type": "object", + "properties": { + "objects": { + "title": "Secrets objects array", + "description": "Secret objects to fetch from the provider.", + "type": "array", + "items": { + "type": "object", + "title": "Secret object", + "minItems": 1 + }, + "errorMessage": { + "type": "Secrets objects must be an array", + "minItems": "At least one secret object must be specified" + } + } + }, + "required": ["objects"], + "errorMessage": { + "required": "Parameters must include 'objects' array" + } + } + }, + "required": ["type", "parameters"], + "additionalProperties": false, + "errorMessage": { + "required": "Secret provider must include 'type' and 'parameters'", + "additionalProperties": "Unknown secret provider properties detected" + } + } + }, + "errorMessage": { + "type": "Secret providers configuration must be an object" + } + }, + "metadata": { + "title": "Pipeline metadata", + "description": "Optional metadata describing this pipeline, such as a display name and description.", + "type": "object", + "properties": { + "name": { + "title": "Pipeline name", + "description": "Human-readable name for the pipeline. Maximum 100 characters.", + "type": "string", + "maxLength": 100, + "errorMessage": { + "type": "Pipeline name must be a string", + "maxLength": "Pipeline name must be at most 100 characters" + } + }, + "description": { + "title": "Pipeline description", + "description": "Free-form description of what the pipeline does. Maximum 500 characters.", + "type": "string", + "maxLength": 500, + "errorMessage": { + "type": "Pipeline description must be a string", + "maxLength": "Pipeline description must be at most 500 characters" + } + }, + "revision": { + "title": "Pipeline revision", + "description": "Pipeline revision number. Must be a non-negative integer.", + "type": "integer", + "minimum": 0, + "errorMessage": { + "type": "Pipeline revision must be an integer", + "minimum": "Pipeline revision must be greater than or equal to 0" + } + }, + "tags": { + "title": "Pipeline tags", + "description": "Array of pipeline tags. Each tag must be a string of up to 50 characters, containing only alphanumeric characters, dots, dashes, or underscores, and must start and end with an alphanumeric character. Tags must be unique.", + "type": "array", + "items": { + "type": "string", + "maxLength": 50, + "pattern": "^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$", + "errorMessage": { + "type": "Each tag must be a string", + "maxLength": "Each tag must be at most 50 characters", + "pattern": "Each tag must contain only alphanumeric characters, dots, dashes, or underscores, and must start and end with an alphanumeric character" + } + }, + "uniqueItems": true, + "errorMessage": { + "type": "Tags must be an array of strings", + "uniqueItems": "Tags must be unique" + } + } + }, + "additionalProperties": false, + "errorMessage": { + "type": "Metadata configuration must be an object", + "additionalProperties": "Unknown metadata properties detected" + } + } + }, + "additionalProperties": false +} diff --git a/layouts/partials/rdi/render-schema.html b/layouts/partials/rdi/render-schema.html new file mode 100644 index 0000000000..542a3d4450 --- /dev/null +++ b/layouts/partials/rdi/render-schema.html @@ -0,0 +1,274 @@ +{{/* + rdi/render-schema + + Recursive partial that renders a single schema node from the RDI config + JSON-Schema. Headings/prose are emitted as markdown so they pick up + Hugo's standard heading rendering (auto IDs, copy-link icons). The + Properties table is emitted as a raw HTML block, because markdown + table cells inside the percent-form shortcode HTML-escape their + inline HTML (so `
` would otherwise display as literal text). + + Context (passed in as a dict): + node schema node to render + slug accumulated slug for the anchor on this node + path accumulated dotted path for the heading text + name raw name of the current path segment + title human-readable title for the heading (falls back to name) + level markdown heading level (1 for root; 2+ for sub-sections) + isRoot true only for the top-level call from the shortcode +*/}} + +{{- $node := .node -}} +{{- $slug := .slug -}} +{{- $path := .path -}} +{{- $name := .name -}} +{{- $title := .title -}} +{{- $level := .level -}} +{{- $isRoot := .isRoot -}} + +{{/* `oneOf` at the top level (e.g. targets): pick the first branch — same + choice the legacy generator made. */}} +{{- $work := $node -}} +{{- with $node.oneOf -}}{{- $work = index . 0 -}}{{- end -}} + +{{/* Step inside `patternProperties` ({".*": {...}}). */}} +{{- $inner := $work -}} +{{- $patternKey := "" -}} +{{- with $work.patternProperties -}} + {{- range $k, $v := . -}} + {{- $patternKey = $k -}} + {{- $inner = $v -}} + {{- end -}} +{{- end -}} +{{- with $inner.oneOf -}}{{- $inner = index . 0 -}}{{- end -}} + +{{- $props := $inner.properties -}} +{{- $required := $inner.required | default slice -}} +{{- $hasRequiredList := false -}} +{{- with $inner.required -}}{{- $hasRequiredList = true -}}{{- end -}} + +{{/* ---------- Section heading (skipped for the root call) ---------- */}} +{{- if not $isRoot }} + + +{{ strings.Repeat $level "#" }} {{ $path }}: {{ or $title $name }} + +{{ with $node.description }}{{ . | safeHTML }} + +{{ end }} +{{- end }} + +{{/* ---------- Properties table (raw HTML block) ---------- */}} +{{- if $props }} +**Properties**{{ if $patternKey }} (key: {{ $patternKey }}){{ end }} + + + + + + +{{- range $childName, $childNode := $props }} + +{{- /* Anchor slug fragment: lowercase + keep only [a-z0-9_]. */ -}} +{{- $slugFrag := lower $childName -}} +{{- $slugFrag = replaceRE "[^a-z0-9_]" "" $slugFrag -}} +{{- $childSlug := printf "%s%s" $slug $slugFrag -}} + +{{- /* is-complex: does this child warrant its own sub-section? */ -}} +{{- $isComplex := false -}} +{{- $cType := $childNode.type -}} +{{- if eq (printf "%T" $cType) "string" -}} + {{- if or (eq $cType "object") (eq $cType "array") -}}{{- $isComplex = true -}}{{- end -}} +{{- else if eq (printf "%T" $cType) "[]interface {}" -}} + {{- range $cType -}} + {{- if or (eq . "object") (eq . "array") -}}{{- $isComplex = true -}}{{- end -}} + {{- end -}} +{{- end -}} +{{- if $childNode.properties -}}{{- $isComplex = true -}}{{- end -}} +{{- if $childNode.patternProperties -}}{{- $isComplex = true -}}{{- end -}} +{{- if $childNode.items -}}{{- $isComplex = true -}}{{- end -}} +{{- if $childNode.oneOf -}}{{- $isComplex = true -}}{{- end -}} +{{- if $childNode.allOf -}}{{- $isComplex = true -}}{{- end -}} + +{{- /* Cell 1 (HTML): bold name (+ optional anchor link) + optional title */ -}} +{{- $nameInner := printf "%s" (htmlEscape $childName) -}} +{{- $nameCell := $nameInner -}} +{{- if $isComplex -}} + {{- $nameCell = printf `%s` $childSlug $nameInner -}} +{{- end -}} +{{- with $childNode.title -}} + {{- $nameCell = printf "%s
(%s)" $nameCell (htmlEscape .) -}} +{{- end -}} + +{{- /* Cell 2 (HTML): type, comma-joined for type arrays */ -}} +{{- $typeCell := "" -}} +{{- if eq (printf "%T" $cType) "string" -}} + {{- $typeCell = printf "%s" $cType -}} + {{- if eq $cType "array" -}} + {{- $itemType := "any" -}} + {{- with $childNode.items.type -}}{{- $itemType = . -}}{{- end -}} + {{- $typeCell = printf "%s[]" $itemType -}} + {{- end -}} +{{- else if eq (printf "%T" $cType) "[]interface {}" -}} + {{- $parts := slice -}} + {{- range $cType -}}{{- $parts = $parts | append (printf "%s" .) -}}{{- end -}} + {{- $typeCell = delimit $parts ", " -}} +{{- end -}} + +{{- /* Cell 3 (HTML): description (markdownified) + annotations */ -}} +{{- $descHTML := "" -}} +{{- with $childNode.description -}}{{- $descHTML = . | markdownify -}}{{- end -}} + +{{- /* markdownify wraps prose in

...

; strip the outer wrap so the + description sits inline with the annotations. */ -}} +{{- $descHTML = $descHTML | string -}} +{{- $descHTML = replaceRE "^

" "" $descHTML -}} +{{- $descHTML = replaceRE "

\\s*$" "" $descHTML -}} + +{{- $extras := slice -}} +{{- if isset $childNode "default" -}} + {{- $extras = $extras | append (printf "Default: %s" (jsonify (index $childNode "default"))) -}} +{{- end -}} +{{- if isset $childNode "const" -}} + {{- $extras = $extras | append (printf "Constant Value: %s" (jsonify (index $childNode "const"))) -}} +{{- end -}} +{{- with $childNode.enum -}} + {{- $items := slice -}} + {{- range . -}}{{- $items = $items | append (printf "%s" (jsonify .)) -}}{{- end -}} + {{- $extras = $extras | append (printf "Enum: %s" (delimit $items ", ")) -}} +{{- end -}} +{{- with $childNode.minimum -}}{{- $extras = $extras | append (printf "Minimum: %v" .) -}}{{- end -}} +{{- with $childNode.maximum -}}{{- $extras = $extras | append (printf "Maximum: %v" .) -}}{{- end -}} +{{- with $childNode.maxLength -}}{{- $extras = $extras | append (printf "Maximal Length: %v" .) -}}{{- end -}} +{{- with $childNode.pattern -}}{{- $extras = $extras | append (printf "Pattern: %s" (htmlEscape .)) -}}{{- end -}} + +{{- range $extras -}} + {{- $descHTML = printf "%s
%s" $descHTML . -}} +{{- end -}} + +{{- /* Cell 4: required */ -}} +{{- $reqCell := "" -}} +{{- if $hasRequiredList -}} + {{- if in $required $childName -}}{{- $reqCell = "yes" -}}{{- else -}}{{- $reqCell = "no" -}}{{- end -}} +{{- end -}} + +{{ printf "" $nameCell $typeCell $descHTML $reqCell | safeHTML }} +{{- end }} + +
NameTypeDescriptionRequired
%s%s%s%s
+ +{{ end }}{{/* end if $props */}} + +{{/* ---------- Schema-level annotations under the table ---------- */}} +{{- if eq $inner.additionalProperties false }} +**Additional Properties:** not allowed +{{ end -}} + +{{- with $inner.minProperties }} +**Minimal Properties:** {{ . }} +{{ end -}} + +{{- with $inner.dependentRequired }} +{{ range $dep, $needs := . -}} +**If property *{{ $dep }}* is defined**, property/ies *{{ delimit $needs ", " }}* is/are required. + +{{ end -}} +{{ end -}} + +{{- with $inner.examples }} +{{- range . }} +**Example** + +```yaml +{{ . | transform.Remarshal "yaml" -}} +``` +{{ end -}} +{{ end -}} + +{{/* ---------- Recurse into complex child properties ---------- */}} +{{- range $childName, $childNode := $props }} + +{{- $cType2 := $childNode.type -}} +{{- $isComplex2 := false -}} +{{- if eq (printf "%T" $cType2) "string" -}} + {{- if or (eq $cType2 "object") (eq $cType2 "array") -}}{{- $isComplex2 = true -}}{{- end -}} +{{- else if eq (printf "%T" $cType2) "[]interface {}" -}} + {{- range $cType2 -}} + {{- if or (eq . "object") (eq . "array") -}}{{- $isComplex2 = true -}}{{- end -}} + {{- end -}} +{{- end -}} +{{- if $childNode.properties -}}{{- $isComplex2 = true -}}{{- end -}} +{{- if $childNode.patternProperties -}}{{- $isComplex2 = true -}}{{- end -}} +{{- if $childNode.items -}}{{- $isComplex2 = true -}}{{- end -}} +{{- if $childNode.oneOf -}}{{- $isComplex2 = true -}}{{- end -}} +{{- if $childNode.allOf -}}{{- $isComplex2 = true -}}{{- end -}} + +{{- if $isComplex2 }} + +{{- $segDisplay2 := $childName -}} +{{- $segDisplay2 = replace $segDisplay2 "_" "\\_" -}} +{{- $segDisplay2 = replace $segDisplay2 "[" "\\[" -}} +{{- $segDisplay2 = replace $segDisplay2 "]" "\\]" -}} + +{{- $childPath := "" -}} +{{- if $path -}} + {{- $childPath = printf "%s\\.%s" $path $segDisplay2 -}} +{{- else -}} + {{- $childPath = $segDisplay2 -}} +{{- end -}} + +{{- $isArray := false -}} +{{- if eq (printf "%T" $cType2) "string" -}} + {{- if eq $cType2 "array" -}}{{- $isArray = true -}}{{- end -}} +{{- end -}} +{{- if $isArray -}} + {{- $childPath = printf "%s\\[\\]" $childPath -}} +{{- end -}} + +{{- $slugFrag2 := lower $childName -}} +{{- $slugFrag2 = replaceRE "[^a-z0-9_]" "" $slugFrag2 -}} +{{- $childSlugAcc := printf "%s%s" $slug $slugFrag2 -}} + +{{ partial "rdi/render-schema.html" (dict + "node" $childNode + "slug" $childSlugAcc + "path" $childPath + "name" $childName + "title" ($childNode.title | default "") + "level" (add $level 1) + "isRoot" false +) }} +{{- end -}} +{{- end -}} + +{{/* ---------- additionalProperties as named sub-section ---------- */}} +{{- if reflect.IsMap $inner.additionalProperties }} +{{- $apNode := $inner.additionalProperties -}} +{{- $apPath := "" -}} +{{- if $path -}} + {{- $apPath = printf "%s\\.additionalProperties" $path -}} +{{- else -}} + {{- $apPath = "additionalProperties" -}} +{{- end -}} + +{{- $apType := index $apNode "type" -}} +{{- $apTitle := "" -}} +{{- with $apNode.title -}}{{- $apTitle = . -}}{{- end -}} +{{- if not $apTitle -}} + {{- if eq (printf "%T" $apType) "string" -}} + {{- $apTitle = $apType -}} + {{- else if eq (printf "%T" $apType) "[]interface {}" -}} + {{- $apTitle = delimit $apType "," -}} + {{- end -}} +{{- end -}} + +{{ partial "rdi/render-schema.html" (dict + "node" $apNode + "slug" (printf "%sadditionalproperties" $slug) + "path" $apPath + "name" "additionalProperties" + "title" $apTitle + "level" (add $level 1) + "isRoot" false +) }} +{{- end -}} diff --git a/layouts/shortcodes/rdi-config-reference.html b/layouts/shortcodes/rdi-config-reference.html new file mode 100644 index 0000000000..9a1adb15aa --- /dev/null +++ b/layouts/shortcodes/rdi-config-reference.html @@ -0,0 +1,29 @@ +{{/* + rdi-config-reference + + Renders the RDI config-file reference from data/rdi-reference/config.json. + + Use with the `%` shortcode form so the emitted markdown is processed: + + {{% rdi-config-reference %}} + + The shortcode emits: + 1. The root schema's description (the page intro paragraph). + 2. The Properties table for the root schema. + 3. Recursive sub-sections for every complex child property. +*/}} +{{- $root := index site.Data "rdi-reference" "config" -}} +{{- with $root }} +{{ .description }} + + +{{ partial "rdi/render-schema.html" (dict + "node" . + "slug" "" + "path" "" + "name" "" + "title" "" + "level" 1 + "isRoot" true +) }} +{{- end }} From d973c045846d1effa8d2d7d9141d57c8089dcdb6 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Wed, 3 Jun 2026 16:48:58 +0100 Subject: [PATCH 2/5] DOC-6711 initial generation of CLI commands from JSON --- .../reference/cli-v2/_content.gotmpl | 59 ++++++++++++ .../reference/cli-v2/_index.md | 18 ++++ layouts/partials/rdi/cli-page-body.html | 91 +++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 content/integrate/redis-data-integration/reference/cli-v2/_content.gotmpl create mode 100644 content/integrate/redis-data-integration/reference/cli-v2/_index.md create mode 100644 layouts/partials/rdi/cli-page-body.html diff --git a/content/integrate/redis-data-integration/reference/cli-v2/_content.gotmpl b/content/integrate/redis-data-integration/reference/cli-v2/_content.gotmpl new file mode 100644 index 0000000000..8acc046daa --- /dev/null +++ b/content/integrate/redis-data-integration/reference/cli-v2/_content.gotmpl @@ -0,0 +1,59 @@ +{{/* + RDI CLI reference content adapter. + + Reads the Click-introspection schema in data/rdi-reference/cli.json and + emits one page per command (the root `redis-di` plus each subcommand). + + Each emitted page mirrors the frontmatter of the legacy hand-authored + pages in ../cli/ (Title, linkTitle, description, weight, categories). + The body is rendered by `rdi/cli-page-body.html`. +*/}} + +{{- $cli := index .Site.Data "rdi-reference" "cli" -}} + +{{/* Root page: `redis-di` itself. */}} +{{- $rootBody := partial "rdi/cli-page-body.html" (dict + "cmd" $cli + "name" "redis-di" +) -}} +{{- $.AddPage (dict + "kind" "page" + "path" "redis-di" + "title" "redis-di" + "params" (dict + "linkTitle" "redis-di" + "description" $cli.help + "weight" 10 + "alwaysopen" false + "categories" (slice "redis-di") + ) + "content" (dict + "mediaType" "text/markdown" + "value" $rootBody + ) +) -}} + +{{/* One page per subcommand. */}} +{{- range $name, $cmd := $cli.commands -}} + {{- $fullName := printf "redis-di %s" $name -}} + {{- $body := partial "rdi/cli-page-body.html" (dict + "cmd" $cmd + "name" $fullName + ) -}} + {{- $.AddPage (dict + "kind" "page" + "path" (printf "redis-di-%s" $name) + "title" $fullName + "params" (dict + "linkTitle" $fullName + "description" $cmd.help + "weight" 10 + "alwaysopen" false + "categories" (slice "redis-di") + ) + "content" (dict + "mediaType" "text/markdown" + "value" $body + ) + ) -}} +{{- end -}} diff --git a/content/integrate/redis-data-integration/reference/cli-v2/_index.md b/content/integrate/redis-data-integration/reference/cli-v2/_index.md new file mode 100644 index 0000000000..7b93560ab9 --- /dev/null +++ b/content/integrate/redis-data-integration/reference/cli-v2/_index.md @@ -0,0 +1,18 @@ +--- +Title: CLI reference (rebuilt from data) +alwaysopen: false +categories: + - docs + - integrate + - rs + - rdi +description: Reference for the RDI CLI commands, rendered from data/rdi-reference/cli.json +group: di +hideListLinks: false +linkTitle: CLI commands (v2) +summary: + Redis Data Integration keeps Redis in sync with the primary database in near + real time. +type: integration +weight: 61 +--- diff --git a/layouts/partials/rdi/cli-page-body.html b/layouts/partials/rdi/cli-page-body.html new file mode 100644 index 0000000000..19bcee1918 --- /dev/null +++ b/layouts/partials/rdi/cli-page-body.html @@ -0,0 +1,91 @@ +{{/* + rdi/cli-page-body + + Renders the markdown body for one RDI CLI command page from the + Click-introspection schema in data/rdi-reference/cli.json. + + Emits a `## Usage` block and a `## Options` bullet list, in the same + shape as the legacy generator. The legacy "## CLI help" section is + deliberately omitted — it duplicates the Options data. + + Context (dict): + cmd one entry from cli.json (the root or a subcommand) + name fully-qualified command name for the Usage line + (e.g. "redis-di" or "redis-di add-context") +*/}} + +{{- $cmd := .cmd -}} +{{- $name := .name -}} + +{{/* Split out positional arguments. They contribute to the Usage line + but are still listed (along with options) in the Options block, + matching the legacy layout. */}} +{{- $args := slice -}} +{{- range $p := $cmd.params -}} + {{- if eq $p.param_type_name "argument" -}}{{- $args = $args | append $p -}}{{- end -}} +{{- end -}} + +{{/* Build the Usage line. */}} +{{- $usage := printf "Usage: %s [OPTIONS]" $name -}} +{{- range $a := $args -}} + {{- $usage = printf "%s %s" $usage (upper $a.name) -}} +{{- end -}} +{{- if $cmd.commands -}} + {{- $usage = printf "%s COMMAND [ARGS]..." $usage -}} +{{- end -}} + +{{/* Build each option block as a discrete string, then join with blank + lines. This gives precise control over the spacing the legacy + generator produced. */}} +{{- $blocks := slice -}} +{{- range $p := $cmd.params -}} + + {{- $reqSuffix := "" -}} + {{- if $p.required -}}{{- $reqSuffix = " (REQUIRED)" -}}{{- end -}} + + {{- /* Type string */ -}} + {{- $t := $p.type -}} + {{- $typeStr := "" -}} + {{- if eq $t.param_type "Bool" -}}{{- $typeStr = "BOOL" -}} + {{- else if eq $t.param_type "String" -}}{{- $typeStr = "STRING" -}} + {{- else if eq $t.param_type "Int" -}}{{- $typeStr = "INT" -}} + {{- else if eq $t.param_type "IntRange" -}} + {{- $lo := "<=" -}}{{- if $t.min_open -}}{{- $lo = "<" -}}{{- end -}} + {{- $hi := "<=" -}}{{- if $t.max_open -}}{{- $hi = "<" -}}{{- end -}} + {{- $typeStr = printf "" $t.min $lo $hi $t.max -}} + {{- else if eq $t.param_type "Choice" -}} + {{- $quoted := slice -}} + {{- range $c := $t.choices -}}{{- $quoted = $quoted | append (printf "'%s'" $c) -}}{{- end -}} + {{- $typeStr = printf "Choice([%s])" (delimit $quoted ", ") -}} + {{- else if eq $t.param_type "Path" -}}{{- $typeStr = "" -}} + {{- else -}}{{- $typeStr = $t.param_type -}} + {{- end -}} + + {{- /* Default string: `none` for null/missing, else lowercased str repr. */ -}} + {{- $defaultStr := "none" -}} + {{- if isset $p "default" -}} + {{- $dv := index $p "default" -}} + {{- if ne $dv nil -}}{{- $defaultStr = lower (printf "%v" $dv) -}}{{- end -}} + {{- end -}} + + {{- /* Opts joined with a literal newline (matches the legacy quirk). */ -}} + {{- $optsStr := delimit $p.opts "\n" -}} + + {{- $block := printf "- `%s`%s:\n - Type: %s\n - Default: `%s`\n - Usage: `%s`" $p.name $reqSuffix $typeStr $defaultStr $optsStr -}} + + {{- with $p.help -}} + {{- $block = printf "%s\n\n %s" $block . -}} + {{- end -}} + + {{- $blocks = $blocks | append $block -}} +{{- end -}} + +## Usage + +``` +{{ $usage }} +``` + +## Options + +{{ delimit $blocks "\n\n" }} From 826024a77255d5b757229086dada21bf2008d61e Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 4 Jun 2026 15:50:34 +0100 Subject: [PATCH 3/5] DOC-6711 updated layout for props lists --- layouts/partials/rdi/render-property.html | 207 ++++++++++++++ layouts/partials/rdi/render-schema.html | 274 ------------------- layouts/partials/rdi/render-tree.html | 142 ++++++++++ layouts/shortcodes/rdi-config-reference.html | 242 ++++++++++++++-- 4 files changed, 573 insertions(+), 292 deletions(-) create mode 100644 layouts/partials/rdi/render-property.html delete mode 100644 layouts/partials/rdi/render-schema.html create mode 100644 layouts/partials/rdi/render-tree.html diff --git a/layouts/partials/rdi/render-property.html b/layouts/partials/rdi/render-property.html new file mode 100644 index 0000000000..c966044363 --- /dev/null +++ b/layouts/partials/rdi/render-property.html @@ -0,0 +1,207 @@ +{{/* + rdi/render-property + + Renders one schema property as a stacked block in the flat-tree + layout (driven by `rdi/render-tree.html`). + + Layout: + Line 1: full dot path [required] [deprecated] [object|array] + Line 2: short label (title) · type · constraints · N properties + Line 3: long description (only if it adds info beyond the title) + + Visual variants: + - leaf — primitives and arrays-of-primitives. Minimal style + (left-border accent). + - container — objects / arrays-of-objects with nested structure. + Bordered card with a light tint to flag that the + entries that follow are its children. + + Context (dict): + displayPath full dotted path shown as the name (e.g. + "sources.connection.Spanner.type") + slug accumulated slug for this property's anchor + node schema node for this property + isInRequiredList whether the parent's `required` array lists + this property + parentHasReqList whether the parent has a `required` field + at all (controls whether `required: no` is + even meaningful) +*/}} + +{{- $displayPath := .displayPath -}} +{{- $slug := .slug -}} +{{- $node := .node -}} +{{- $isRequired := and .parentHasReqList .isInRequiredList -}} + +{{/* Resolve oneOf at this level. */}} +{{- $work := $node -}} +{{- with $node.oneOf -}}{{- $work = index . 0 -}}{{- end -}} + +{{/* Container check — same predicate render-tree uses, so the visual + variant matches whether children are about to follow. */}} +{{- $cType := $node.type -}} +{{- $isContainer := false -}} +{{- $isArrayOfObjects := false -}} +{{- if eq (printf "%T" $cType) "string" -}} + {{- if eq $cType "object" -}} + {{- if or $node.properties $node.patternProperties $node.oneOf $node.allOf (reflect.IsMap $node.additionalProperties) -}} + {{- $isContainer = true -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- if and (eq (printf "%T" $cType) "string") (eq $cType "array") -}} + {{- with $node.items -}} + {{- if or .properties .patternProperties -}} + {{- $isContainer = true -}} + {{- $isArrayOfObjects = true -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- $isDeprecated := false -}} +{{- if $node.deprecated -}}{{- $isDeprecated = true -}}{{- end -}} + +{{/* Type display string. */}} +{{- $typeDisplay := "" -}} +{{- if eq (printf "%T" $cType) "string" -}} + {{- if eq $cType "array" -}} + {{- $itemType := "any" -}} + {{- with $node.items.type -}}{{- $itemType = . -}}{{- end -}} + {{- $typeDisplay = printf "%s[]" $itemType -}} + {{- else -}} + {{- $typeDisplay = $cType -}} + {{- end -}} +{{- else if eq (printf "%T" $cType) "[]interface {}" -}} + {{- $typeDisplay = delimit $cType " | " -}} +{{- else if isset $node "const" -}} + {{- $typeDisplay = "constant" -}} +{{- end -}} + +{{/* Property count for container cards. */}} +{{- $childCount := 0 -}} +{{- $work2 := $node -}} +{{- with $node.oneOf -}}{{- $work2 = index . 0 -}}{{- end -}} +{{- with $work2.patternProperties -}} + {{- range $k, $v := . -}}{{- $work2 = $v -}}{{- end -}} +{{- end -}} +{{- /* Step through additionalProperties wrapper the same way render-tree does. */ -}} +{{- if and (not $work2.properties) (not $work2.patternProperties) (reflect.IsMap $work2.additionalProperties) -}} + {{- $work2 = $work2.additionalProperties -}} +{{- end -}} +{{- with $work2.oneOf -}}{{- $work2 = index . 0 -}}{{- end -}} +{{- with $work2.properties -}}{{- $childCount = len . -}}{{- end -}} +{{- if and (eq $childCount 0) $isArrayOfObjects -}} + {{- with $node.items.properties -}}{{- $childCount = len . -}}{{- end -}} +{{- end -}} + +{{/* The HTML id, used for #-permalinks and autocomplete jumps. */}} +{{- $propId := printf "prop-%s" $slug -}} + +{{/* For backwards compatibility with deep links into the old schema + section anchors (#sources, #sourcesconnection, ...), emit a bare + next to each *container* entry. */}} +{{- if $isContainer }} + +{{- end }} + +{{/* Build per-segment links for the dot path: clicking any non-leaf + segment jumps to that subtree's section anchor. The final segment + stays as plain text (you're already on it). */}} +{{- $segments := split $displayPath "." -}} +{{- $lastIdx := sub (len $segments) 1 -}} + +{{/* x-ui-expose / x-ui-source-types: applicability tags supplied by + the RDI team. Surfaced as space-separated data attributes; the + filter UI reads them to drive the processor / collector / database + dropdowns. */}} +{{- $expose := "" -}} +{{- with index $node "x-ui-expose" -}} + {{- $expose = delimit . " " -}} +{{- end -}} +{{- $sourceTypes := "" -}} +{{- with index $node "x-ui-source-types" -}} + {{- $sourceTypes = delimit . " " -}} +{{- end -}} + +
+ +
+ + {{- $accSlug := "" -}} + {{- range $i, $seg := $segments -}} + {{- $segFrag := lower $seg -}} + {{- $segFrag = replaceRE "[^a-z0-9_]" "" $segFrag -}} + {{- $accSlug = printf "%s%s" $accSlug $segFrag -}} + {{- if $i }}.{{ end -}} + {{- if lt $i $lastIdx -}} + {{ $seg }} + {{- else -}} + {{ $seg }} + {{- end -}} + {{- end -}} + + # + {{- if $isRequired }} + required + {{- end }} + {{- if $isDeprecated }} + deprecated + {{- end }} +
+ +
+ {{- with $node.title }} + {{ . }} + {{- end }} + {{- if $typeDisplay }} + {{ $typeDisplay }} + {{- end }} + {{- if isset $node "default" }} + {{- $dv := index $node "default" -}} + {{- if ne $dv nil }} + default: {{ jsonify $dv }} + {{- end }} + {{- end }} + {{- with $node.const }} + constant: {{ jsonify . }} + {{- end }} + {{- with $node.minimum }} + min: {{ . }} + {{- end }} + {{- with $node.maximum }} + max: {{ . }} + {{- end }} + {{- with $node.maxLength }} + max length: {{ . }} + {{- end }} + {{- with $node.enum }} + enum: + {{- range $i, $v := . -}} + {{ if $i }}, {{ end }}{{ jsonify $v }} + {{- end -}} + + {{- end }} + {{- with $node.pattern }} + pattern: {{ . }} + {{- end }} + {{- if and $isContainer (gt $childCount 0) }} + {{ $childCount }} {{ if eq $childCount 1 }}property{{ else }}properties{{ end }} + {{- end }} +
+ + {{- with $node.description }} +
{{ . | markdownify }}
+ {{- end }} +
diff --git a/layouts/partials/rdi/render-schema.html b/layouts/partials/rdi/render-schema.html deleted file mode 100644 index 542a3d4450..0000000000 --- a/layouts/partials/rdi/render-schema.html +++ /dev/null @@ -1,274 +0,0 @@ -{{/* - rdi/render-schema - - Recursive partial that renders a single schema node from the RDI config - JSON-Schema. Headings/prose are emitted as markdown so they pick up - Hugo's standard heading rendering (auto IDs, copy-link icons). The - Properties table is emitted as a raw HTML block, because markdown - table cells inside the percent-form shortcode HTML-escape their - inline HTML (so `
` would otherwise display as literal text). - - Context (passed in as a dict): - node schema node to render - slug accumulated slug for the anchor on this node - path accumulated dotted path for the heading text - name raw name of the current path segment - title human-readable title for the heading (falls back to name) - level markdown heading level (1 for root; 2+ for sub-sections) - isRoot true only for the top-level call from the shortcode -*/}} - -{{- $node := .node -}} -{{- $slug := .slug -}} -{{- $path := .path -}} -{{- $name := .name -}} -{{- $title := .title -}} -{{- $level := .level -}} -{{- $isRoot := .isRoot -}} - -{{/* `oneOf` at the top level (e.g. targets): pick the first branch — same - choice the legacy generator made. */}} -{{- $work := $node -}} -{{- with $node.oneOf -}}{{- $work = index . 0 -}}{{- end -}} - -{{/* Step inside `patternProperties` ({".*": {...}}). */}} -{{- $inner := $work -}} -{{- $patternKey := "" -}} -{{- with $work.patternProperties -}} - {{- range $k, $v := . -}} - {{- $patternKey = $k -}} - {{- $inner = $v -}} - {{- end -}} -{{- end -}} -{{- with $inner.oneOf -}}{{- $inner = index . 0 -}}{{- end -}} - -{{- $props := $inner.properties -}} -{{- $required := $inner.required | default slice -}} -{{- $hasRequiredList := false -}} -{{- with $inner.required -}}{{- $hasRequiredList = true -}}{{- end -}} - -{{/* ---------- Section heading (skipped for the root call) ---------- */}} -{{- if not $isRoot }} - - -{{ strings.Repeat $level "#" }} {{ $path }}: {{ or $title $name }} - -{{ with $node.description }}{{ . | safeHTML }} - -{{ end }} -{{- end }} - -{{/* ---------- Properties table (raw HTML block) ---------- */}} -{{- if $props }} -**Properties**{{ if $patternKey }} (key: {{ $patternKey }}){{ end }} - - - - - - -{{- range $childName, $childNode := $props }} - -{{- /* Anchor slug fragment: lowercase + keep only [a-z0-9_]. */ -}} -{{- $slugFrag := lower $childName -}} -{{- $slugFrag = replaceRE "[^a-z0-9_]" "" $slugFrag -}} -{{- $childSlug := printf "%s%s" $slug $slugFrag -}} - -{{- /* is-complex: does this child warrant its own sub-section? */ -}} -{{- $isComplex := false -}} -{{- $cType := $childNode.type -}} -{{- if eq (printf "%T" $cType) "string" -}} - {{- if or (eq $cType "object") (eq $cType "array") -}}{{- $isComplex = true -}}{{- end -}} -{{- else if eq (printf "%T" $cType) "[]interface {}" -}} - {{- range $cType -}} - {{- if or (eq . "object") (eq . "array") -}}{{- $isComplex = true -}}{{- end -}} - {{- end -}} -{{- end -}} -{{- if $childNode.properties -}}{{- $isComplex = true -}}{{- end -}} -{{- if $childNode.patternProperties -}}{{- $isComplex = true -}}{{- end -}} -{{- if $childNode.items -}}{{- $isComplex = true -}}{{- end -}} -{{- if $childNode.oneOf -}}{{- $isComplex = true -}}{{- end -}} -{{- if $childNode.allOf -}}{{- $isComplex = true -}}{{- end -}} - -{{- /* Cell 1 (HTML): bold name (+ optional anchor link) + optional title */ -}} -{{- $nameInner := printf "%s" (htmlEscape $childName) -}} -{{- $nameCell := $nameInner -}} -{{- if $isComplex -}} - {{- $nameCell = printf `%s` $childSlug $nameInner -}} -{{- end -}} -{{- with $childNode.title -}} - {{- $nameCell = printf "%s
(%s)" $nameCell (htmlEscape .) -}} -{{- end -}} - -{{- /* Cell 2 (HTML): type, comma-joined for type arrays */ -}} -{{- $typeCell := "" -}} -{{- if eq (printf "%T" $cType) "string" -}} - {{- $typeCell = printf "%s" $cType -}} - {{- if eq $cType "array" -}} - {{- $itemType := "any" -}} - {{- with $childNode.items.type -}}{{- $itemType = . -}}{{- end -}} - {{- $typeCell = printf "%s[]" $itemType -}} - {{- end -}} -{{- else if eq (printf "%T" $cType) "[]interface {}" -}} - {{- $parts := slice -}} - {{- range $cType -}}{{- $parts = $parts | append (printf "%s" .) -}}{{- end -}} - {{- $typeCell = delimit $parts ", " -}} -{{- end -}} - -{{- /* Cell 3 (HTML): description (markdownified) + annotations */ -}} -{{- $descHTML := "" -}} -{{- with $childNode.description -}}{{- $descHTML = . | markdownify -}}{{- end -}} - -{{- /* markdownify wraps prose in

...

; strip the outer wrap so the - description sits inline with the annotations. */ -}} -{{- $descHTML = $descHTML | string -}} -{{- $descHTML = replaceRE "^

" "" $descHTML -}} -{{- $descHTML = replaceRE "

\\s*$" "" $descHTML -}} - -{{- $extras := slice -}} -{{- if isset $childNode "default" -}} - {{- $extras = $extras | append (printf "Default: %s" (jsonify (index $childNode "default"))) -}} -{{- end -}} -{{- if isset $childNode "const" -}} - {{- $extras = $extras | append (printf "Constant Value: %s" (jsonify (index $childNode "const"))) -}} -{{- end -}} -{{- with $childNode.enum -}} - {{- $items := slice -}} - {{- range . -}}{{- $items = $items | append (printf "%s" (jsonify .)) -}}{{- end -}} - {{- $extras = $extras | append (printf "Enum: %s" (delimit $items ", ")) -}} -{{- end -}} -{{- with $childNode.minimum -}}{{- $extras = $extras | append (printf "Minimum: %v" .) -}}{{- end -}} -{{- with $childNode.maximum -}}{{- $extras = $extras | append (printf "Maximum: %v" .) -}}{{- end -}} -{{- with $childNode.maxLength -}}{{- $extras = $extras | append (printf "Maximal Length: %v" .) -}}{{- end -}} -{{- with $childNode.pattern -}}{{- $extras = $extras | append (printf "Pattern: %s" (htmlEscape .)) -}}{{- end -}} - -{{- range $extras -}} - {{- $descHTML = printf "%s
%s" $descHTML . -}} -{{- end -}} - -{{- /* Cell 4: required */ -}} -{{- $reqCell := "" -}} -{{- if $hasRequiredList -}} - {{- if in $required $childName -}}{{- $reqCell = "yes" -}}{{- else -}}{{- $reqCell = "no" -}}{{- end -}} -{{- end -}} - -{{ printf "" $nameCell $typeCell $descHTML $reqCell | safeHTML }} -{{- end }} - -
NameTypeDescriptionRequired
%s%s%s%s
- -{{ end }}{{/* end if $props */}} - -{{/* ---------- Schema-level annotations under the table ---------- */}} -{{- if eq $inner.additionalProperties false }} -**Additional Properties:** not allowed -{{ end -}} - -{{- with $inner.minProperties }} -**Minimal Properties:** {{ . }} -{{ end -}} - -{{- with $inner.dependentRequired }} -{{ range $dep, $needs := . -}} -**If property *{{ $dep }}* is defined**, property/ies *{{ delimit $needs ", " }}* is/are required. - -{{ end -}} -{{ end -}} - -{{- with $inner.examples }} -{{- range . }} -**Example** - -```yaml -{{ . | transform.Remarshal "yaml" -}} -``` -{{ end -}} -{{ end -}} - -{{/* ---------- Recurse into complex child properties ---------- */}} -{{- range $childName, $childNode := $props }} - -{{- $cType2 := $childNode.type -}} -{{- $isComplex2 := false -}} -{{- if eq (printf "%T" $cType2) "string" -}} - {{- if or (eq $cType2 "object") (eq $cType2 "array") -}}{{- $isComplex2 = true -}}{{- end -}} -{{- else if eq (printf "%T" $cType2) "[]interface {}" -}} - {{- range $cType2 -}} - {{- if or (eq . "object") (eq . "array") -}}{{- $isComplex2 = true -}}{{- end -}} - {{- end -}} -{{- end -}} -{{- if $childNode.properties -}}{{- $isComplex2 = true -}}{{- end -}} -{{- if $childNode.patternProperties -}}{{- $isComplex2 = true -}}{{- end -}} -{{- if $childNode.items -}}{{- $isComplex2 = true -}}{{- end -}} -{{- if $childNode.oneOf -}}{{- $isComplex2 = true -}}{{- end -}} -{{- if $childNode.allOf -}}{{- $isComplex2 = true -}}{{- end -}} - -{{- if $isComplex2 }} - -{{- $segDisplay2 := $childName -}} -{{- $segDisplay2 = replace $segDisplay2 "_" "\\_" -}} -{{- $segDisplay2 = replace $segDisplay2 "[" "\\[" -}} -{{- $segDisplay2 = replace $segDisplay2 "]" "\\]" -}} - -{{- $childPath := "" -}} -{{- if $path -}} - {{- $childPath = printf "%s\\.%s" $path $segDisplay2 -}} -{{- else -}} - {{- $childPath = $segDisplay2 -}} -{{- end -}} - -{{- $isArray := false -}} -{{- if eq (printf "%T" $cType2) "string" -}} - {{- if eq $cType2 "array" -}}{{- $isArray = true -}}{{- end -}} -{{- end -}} -{{- if $isArray -}} - {{- $childPath = printf "%s\\[\\]" $childPath -}} -{{- end -}} - -{{- $slugFrag2 := lower $childName -}} -{{- $slugFrag2 = replaceRE "[^a-z0-9_]" "" $slugFrag2 -}} -{{- $childSlugAcc := printf "%s%s" $slug $slugFrag2 -}} - -{{ partial "rdi/render-schema.html" (dict - "node" $childNode - "slug" $childSlugAcc - "path" $childPath - "name" $childName - "title" ($childNode.title | default "") - "level" (add $level 1) - "isRoot" false -) }} -{{- end -}} -{{- end -}} - -{{/* ---------- additionalProperties as named sub-section ---------- */}} -{{- if reflect.IsMap $inner.additionalProperties }} -{{- $apNode := $inner.additionalProperties -}} -{{- $apPath := "" -}} -{{- if $path -}} - {{- $apPath = printf "%s\\.additionalProperties" $path -}} -{{- else -}} - {{- $apPath = "additionalProperties" -}} -{{- end -}} - -{{- $apType := index $apNode "type" -}} -{{- $apTitle := "" -}} -{{- with $apNode.title -}}{{- $apTitle = . -}}{{- end -}} -{{- if not $apTitle -}} - {{- if eq (printf "%T" $apType) "string" -}} - {{- $apTitle = $apType -}} - {{- else if eq (printf "%T" $apType) "[]interface {}" -}} - {{- $apTitle = delimit $apType "," -}} - {{- end -}} -{{- end -}} - -{{ partial "rdi/render-schema.html" (dict - "node" $apNode - "slug" (printf "%sadditionalproperties" $slug) - "path" $apPath - "name" "additionalProperties" - "title" $apTitle - "level" (add $level 1) - "isRoot" false -) }} -{{- end -}} diff --git a/layouts/partials/rdi/render-tree.html b/layouts/partials/rdi/render-tree.html new file mode 100644 index 0000000000..526daa3e10 --- /dev/null +++ b/layouts/partials/rdi/render-tree.html @@ -0,0 +1,142 @@ +{{/* + rdi/render-tree + + Recursive partial that renders the property subtree under one node as + a single flat list, depth-first: + + - leaf properties first, alphabetical + - then each container property, alphabetical, followed inline by + its own subtree (recursion) + + No section sub-headings are emitted — the dot-path name on each + property entry shows the hierarchy. The top-level

heading is + emitted by the shortcode, not here. + + Context (dict): + node schema node to descend into (an object schema) + path dotted display path of this node + (e.g. "sources.connection.Spanner") + slug accumulated slug used for anchors / IDs + isTopLevel true on the first call from the shortcode for one + of the five top-level sections; suppresses + rendering this node itself, only its descendants + parentRequired slice of names that the *enclosing* schema marks + required (used to compute the `required` pill on + this node's own entry, never on this call's leaves) + parentHasReqList true iff the enclosing schema has `required` +*/}} + +{{- $node := .node -}} +{{- $path := .path -}} +{{- $slug := .slug -}} +{{- $isTopLevel := .isTopLevel -}} +{{- $parentRequired := .parentRequired | default slice -}} +{{- $parentHasReqList := .parentHasReqList -}} + +{{/* Render *this* node as an entry, unless it's a top-level section + (handled by the shortcode). */}} +{{- if not $isTopLevel -}} +{{ partial "rdi/render-property.html" (dict + "displayPath" $path + "slug" $slug + "node" $node + "isInRequiredList" (in $parentRequired (index (last 1 (split $path ".")) 0)) + "parentHasReqList" $parentHasReqList +) }} +{{- end -}} + +{{/* Step inside the node to find its own properties. + Apply the same flattening rules used by render-schema: + - oneOf at this level → pick first branch + - patternProperties wrapper → step into the .* value + - inner oneOf → pick first branch + This lets us reach the actual properties dict regardless of how + the schema wraps it. */}} +{{- $work := $node -}} +{{- with $node.oneOf -}}{{- $work = index . 0 -}}{{- end -}} +{{- $inner := $work -}} +{{- with $work.patternProperties -}} + {{- range $k, $v := . -}}{{- $inner = $v -}}{{- end -}} +{{- end -}} +{{- /* When the inner schema's nested fields live under + `additionalProperties` (as an object schema) — e.g. `sources.tables` + — step into it too. The "additionalProperties" segment stays + invisible in the display path, matching how patternProperties keys + are hidden. */ -}} +{{- if and (not $inner.properties) (not $inner.patternProperties) (reflect.IsMap $inner.additionalProperties) -}} + {{- $inner = $inner.additionalProperties -}} +{{- end -}} +{{- with $inner.oneOf -}}{{- $inner = index . 0 -}}{{- end -}} + +{{- $props := $inner.properties -}} +{{- $childRequired := $inner.required | default slice -}} +{{- $childHasReqList := false -}} +{{- with $inner.required -}}{{- $childHasReqList = true -}}{{- end -}} + +{{/* Split children into leaves and containers. */}} +{{- $leaves := slice -}} +{{- $containers := slice -}} +{{- range $childName, $childNode := $props -}} + {{- $cType := $childNode.type -}} + + {{/* Object with structure → container */}} + {{- $isContainer := false -}} + {{- if eq (printf "%T" $cType) "string" -}} + {{- if eq $cType "object" -}} + {{- if or $childNode.properties $childNode.patternProperties $childNode.oneOf $childNode.allOf (reflect.IsMap $childNode.additionalProperties) -}} + {{- $isContainer = true -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- /* Array of objects → container */ -}} + {{- if and (eq (printf "%T" $cType) "string") (eq $cType "array") -}} + {{- with $childNode.items -}} + {{- if or .properties .patternProperties -}}{{- $isContainer = true -}}{{- end -}} + {{- end -}} + {{- end -}} + + {{- if $isContainer -}} + {{- $containers = $containers | append (dict "name" $childName "node" $childNode) -}} + {{- else -}} + {{- $leaves = $leaves | append (dict "name" $childName "node" $childNode) -}} + {{- end -}} +{{- end -}} + +{{/* Alphabetise both buckets. */}} +{{- $leaves := sort $leaves "name" -}} +{{- $containers := sort $containers "name" -}} + +{{/* Render leaves first. */}} +{{- range $leaf := $leaves -}} + {{- $childPath := printf "%s.%s" $path $leaf.name -}} + {{- $slugFrag := lower $leaf.name -}} + {{- $slugFrag = replaceRE "[^a-z0-9_]" "" $slugFrag -}} + {{- $childSlug := printf "%s%s" $slug $slugFrag -}} +{{ partial "rdi/render-property.html" (dict + "displayPath" $childPath + "slug" $childSlug + "node" $leaf.node + "isInRequiredList" (in $childRequired $leaf.name) + "parentHasReqList" $childHasReqList +) }} +{{- end -}} + +{{/* Then each container, followed inline by its own subtree. */}} +{{- range $container := $containers -}} + {{- $childPath := printf "%s.%s" $path $container.name -}} + {{- $slugFrag := lower $container.name -}} + {{- $slugFrag = replaceRE "[^a-z0-9_]" "" $slugFrag -}} + {{- $childSlug := printf "%s%s" $slug $slugFrag -}} +{{ partial "rdi/render-tree.html" (dict + "node" $container.node + "path" $childPath + "slug" $childSlug + "isTopLevel" false + "parentRequired" $childRequired + "parentHasReqList" $childHasReqList +) }} +{{- end -}} + +{{/* `additionalProperties` is absorbed into the inner-stepping above + when this node uses it instead of `properties`/`patternProperties`, + so no separate recursion needed here. */}} diff --git a/layouts/shortcodes/rdi-config-reference.html b/layouts/shortcodes/rdi-config-reference.html index 9a1adb15aa..4823e2f114 100644 --- a/layouts/shortcodes/rdi-config-reference.html +++ b/layouts/shortcodes/rdi-config-reference.html @@ -3,27 +3,233 @@ Renders the RDI config-file reference from data/rdi-reference/config.json. - Use with the `%` shortcode form so the emitted markdown is processed: + Use the percent shortcode form so the emitted markdown is processed: {{% rdi-config-reference %}} - The shortcode emits: - 1. The root schema's description (the page intro paragraph). - 2. The Properties table for the root schema. - 3. Recursive sub-sections for every complex child property. + Page layout: + 1. Sticky filter/search bar — free-text filter, jump-to autocomplete, + toggles for required-only / hide-deprecated. + 2. Quick-nav list linking the five top-level sections in domain + order (sources → processors → targets → secret-providers → + metadata). + 3. Per top-level section:

heading + section description, then + a single flat list of every property at every depth below, + rendered by `rdi/render-tree.html` (leaves first, alphabetical; + then containers, alphabetical, each followed inline by its own + descendants). */}} + {{- $root := index site.Data "rdi-reference" "config" -}} -{{- with $root }} -{{ .description }} - - -{{ partial "rdi/render-schema.html" (dict - "node" . - "slug" "" - "path" "" - "name" "" - "title" "" - "level" 1 - "isRoot" true +{{- $order := slice "sources" "processors" "targets" "secret-providers" "metadata" -}} + +
+
+
+ + +
+ + + + + + +
+
+ +{{ $root.description | safeHTML }} + +**Jump to a section:** + +{{ range $sectionName := $order -}} +{{- $sectionNode := index $root.properties $sectionName -}} +- [`{{ $sectionName }}`](#{{ $sectionName }}) — {{ $sectionNode.title }} +{{ end }} + +{{ range $sectionName := $order }} +{{- $sectionNode := index $root.properties $sectionName -}} +{{- $sectionTitle := $sectionNode.title -}} + + + +## {{ $sectionName }}: {{ $sectionTitle }} + +{{ with $sectionNode.description }}{{ . | safeHTML }}{{ end }} + +{{ partial "rdi/render-tree.html" (dict + "node" $sectionNode + "path" $sectionName + "slug" $sectionName + "isTopLevel" true + "parentRequired" slice + "parentHasReqList" false ) }} -{{- end }} + +{{ end }} + + From 323eed7b3f3b0d91946d1a7f0b748212da0c0c5b Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 4 Jun 2026 16:14:20 +0100 Subject: [PATCH 4/5] DOC-6711 further layout updates --- layouts/shortcodes/rdi-config-reference.html | 61 ++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/layouts/shortcodes/rdi-config-reference.html b/layouts/shortcodes/rdi-config-reference.html index 4823e2f114..493b61c4fd 100644 --- a/layouts/shortcodes/rdi-config-reference.html +++ b/layouts/shortcodes/rdi-config-reference.html @@ -116,6 +116,52 @@ if (!props.length || !searchEl) return; + /* ---------- section index ---------- + Identify the five top-level sections by the bare tags + emitted next to each H2. For each section, capture: + headerEls — the elements between this anchor and the first + `.rdi-prop` (the

-wrapped anchor itself, the

, + and the section description). Hidden when the + section has no visible properties under the current + filter, so empty sections don't leave dangling + headings on the page. + props — the property entries that belong to this section + (everything up to the next section's anchor). + tocLink — the corresponding "Jump to a section" markdown link + at the top of the page; dimmed when empty. */ + const sectionNames = ['sources', 'processors', 'targets', 'secret-providers', 'metadata']; + const sections = []; + for (let i = 0; i < sectionNames.length; i++) { + const name = sectionNames[i]; + const anchor = document.querySelector(`a[name="${name}"]`); + if (!anchor) continue; + const startNode = anchor.closest('p') || anchor; + const nextAnchor = i + 1 < sectionNames.length + ? document.querySelector(`a[name="${sectionNames[i + 1]}"]`) + : null; + const endNode = nextAnchor ? (nextAnchor.closest('p') || nextAnchor) : null; + + const headerEls = []; + const sectionProps = []; + let node = startNode; + let seenProp = false; + while (node && node !== endNode) { + if (node.matches && node.matches('[data-rdi-prop="true"]')) { + seenProp = true; + sectionProps.push(node); + } else if (!seenProp) { + headerEls.push(node); + } + node = node.nextElementSibling; + } + sections.push({ + name, + headerEls, + props: sectionProps, + tocLink: document.querySelector(`a[href="#${name}"]`) + }); + } + /* ---------- filter ---------- */ /* x-ui-expose mixes two dimensions ("processor:*" and "collector:*"). @@ -176,6 +222,21 @@ counterEl.textContent = filtering ? `${visible} of ${props.length} shown` : `${props.length} properties`; + + /* Hide section headings + descriptions for sections that have no + visible properties under the current filter. Also dim the TOC + entry so the "Jump to" list reflects what's available. */ + for (const section of sections) { + const anyVisible = section.props.some(p => p.style.display !== 'none'); + const hide = filtering && !anyVisible; + for (const el of section.headerEls) { + el.style.display = hide ? 'none' : ''; + } + if (section.tocLink) { + section.tocLink.style.opacity = hide ? '0.4' : ''; + section.tocLink.style.pointerEvents = hide ? 'none' : ''; + } + } } applyFilter(); From 4d03e85db159f84ef70476b5c7bf92e133308692 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 5 Jun 2026 10:29:21 +0100 Subject: [PATCH 5/5] DOC-6711 added collections feature --- Makefile | 9 +- build/validate_rdi_collections.py | 86 ++++++++++++++++++++ data/rdi-reference/collections.json | 61 ++++++++++++++ layouts/shortcodes/rdi-config-reference.html | 67 ++++++++++++++- 4 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 build/validate_rdi_collections.py create mode 100644 data/rdi-reference/collections.json diff --git a/Makefile b/Makefile index de7aedbe56..94c8bfc46f 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,12 @@ components: components_local: @python3 build/make.py --stack ./data/components_local/index.json -hugo: +# Validate that paths in data/rdi-reference/collections.json all resolve +# to nodes in config.json. Catches drift when the upstream schema changes. +validate_rdi: + @python3 build/validate_rdi_collections.py + +hugo: validate_rdi @hugo $(HUGO_DEBUG) $(HUGO_BUILD) # json_transform requires hugo to have populated public/ with index.json files @@ -33,7 +38,7 @@ ndjson: json_transform @echo "Compressing NDJSON feed..." @gzip -kf public/docs.ndjson -serve_hugo: +serve_hugo: validate_rdi @hugo serve clean: diff --git a/build/validate_rdi_collections.py b/build/validate_rdi_collections.py new file mode 100644 index 0000000000..679e388905 --- /dev/null +++ b/build/validate_rdi_collections.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Validate data/rdi-reference/collections.json against config.json. + +Every property path listed in a collection must resolve to a node in the +schema. Use this as a CI check (or run by hand) to catch broken +collections when the schema changes. + +Exit code is non-zero if any path is unresolvable, so the script slots +straight into a Makefile target or pre-commit hook. + +Usage: + python3 build/validate_rdi_collections.py +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCHEMA = REPO_ROOT / "data" / "rdi-reference" / "config.json" +COLLECTIONS = REPO_ROOT / "data" / "rdi-reference" / "collections.json" + + +def collect_paths(node: dict, path: str = "") -> list[str]: + """Walk the schema the same way the Hugo render-tree partial does. + + Steps through `oneOf` (first branch), `patternProperties` (single + wildcard key), and `additionalProperties` (treated as an inline + wrapper) so the path strings match what the rendered page emits. + """ + out: list[str] = [] + if not isinstance(node, dict): + return out + + inner = node + if "oneOf" in inner: + inner = inner["oneOf"][0] + if "patternProperties" in inner: + inner = next(iter(inner["patternProperties"].values())) + if ( + "additionalProperties" in inner + and isinstance(inner["additionalProperties"], dict) + and "properties" not in inner + and "patternProperties" not in inner + ): + inner = inner["additionalProperties"] + if "oneOf" in inner: + inner = inner["oneOf"][0] + + for name, child in (inner.get("properties") or {}).items(): + child_path = f"{path}.{name}" if path else name + out.append(child_path) + out.extend(collect_paths(child, child_path)) + return out + + +def main() -> int: + with SCHEMA.open() as f: + schema = json.load(f) + with COLLECTIONS.open() as f: + collections = json.load(f) + + valid = set(collect_paths(schema)) + + errors: list[str] = [] + for entry in collections: + cid = entry.get("id", "") + for path in entry.get("properties", []): + if path not in valid: + errors.append(f" [{cid}] unknown path: {path}") + + if errors: + print("collections.json references paths that do not exist in config.json:") + print("\n".join(errors)) + print(f"\n{len(errors)} broken path(s) across {len(collections)} collection(s).") + return 1 + + total = sum(len(c.get("properties", [])) for c in collections) + print(f"OK: {total} paths across {len(collections)} collection(s) all resolve.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/data/rdi-reference/collections.json b/data/rdi-reference/collections.json new file mode 100644 index 0000000000..2df44db18f --- /dev/null +++ b/data/rdi-reference/collections.json @@ -0,0 +1,61 @@ +[ + { + "id": "getting-started", + "name": "Getting started", + "description": "A minimum set of properties to define a working RDI pipeline against a SQL source and a Redis target. Treat this as a starting point — most real pipelines will need a few more options.", + "properties": [ + "metadata.name", + "sources.type", + "sources.name", + "sources.connection.SQL database.type", + "sources.connection.SQL database.host", + "sources.connection.SQL database.port", + "sources.connection.SQL database.database", + "sources.connection.SQL database.user", + "sources.connection.SQL database.password", + "sources.tables", + "targets.connection.type", + "targets.connection.host", + "targets.connection.port", + "targets.connection.password", + "processors.type", + "processors.error_handling" + ] + }, + { + "id": "classic-optimization", + "name": "Classic processor tuning", + "description": "Properties most relevant when tuning the classic processor (batch sizing, deduplication, retry behaviour, replica wait).", + "properties": [ + "processors.type", + "processors.read_batch_size", + "processors.read_batch_timeout_ms", + "processors.write_batch_size", + "processors.enable_async_processing", + "processors.batch_queue_size", + "processors.ack_queue_size", + "processors.dedup", + "processors.dedup_max_size", + "processors.initial_sync_processes", + "processors.idle_sleep_time_ms", + "processors.idle_streams_check_interval_ms", + "processors.busy_streams_check_interval_ms", + "processors.retry_max_attempts", + "processors.retry_initial_delay_ms", + "processors.retry_max_delay_ms", + "processors.wait_enabled", + "processors.wait_timeout", + "processors.retry_on_replica_failure", + "processors.dlq_max_messages" + ] + }, + { + "id": "flink-optimization", + "name": "Flink processor tuning", + "description": "Properties most relevant when tuning the Apache Flink processor (Kubernetes deployments only). Includes everything under `processors.advanced`, which only applies to Flink.", + "properties": [ + "processors.type", + "processors.advanced" + ] + } +] diff --git a/layouts/shortcodes/rdi-config-reference.html b/layouts/shortcodes/rdi-config-reference.html index 493b61c4fd..d61c9863cb 100644 --- a/layouts/shortcodes/rdi-config-reference.html +++ b/layouts/shortcodes/rdi-config-reference.html @@ -22,6 +22,7 @@ {{- $root := index site.Data "rdi-reference" "config" -}} {{- $order := slice "sources" "processors" "targets" "secret-providers" "metadata" -}} +{{- $collections := index site.Data "rdi-reference" "collections" -}}
@@ -34,6 +35,15 @@
+