From 0daddd5b2a186a1122876def3b14dd3ad0812233 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Thu, 7 Aug 2025 10:27:40 -0600 Subject: [PATCH 01/15] Add queue for exposing telemetry from subsystems The subsystems are (mostly) self-encapsulated and communicate via queues. In order to expose internal system metrics and state a queue will be used to export telemetry data. This will decouple the metrics capture / exposure from the internal subsystems. --- src/controller/custom_workflow.py | 3 +++ src/controller/events.py | 2 ++ src/controller/scheduler.py | 2 ++ src/koreo_controller.py | 3 ++- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/controller/custom_workflow.py b/src/controller/custom_workflow.py index d387311..57d2f8b 100644 --- a/src/controller/custom_workflow.py +++ b/src/controller/custom_workflow.py @@ -46,11 +46,13 @@ async def workflow_controller_system( api: kr8s.asyncio.Api, namespace: str, workflow_updates_queue: events.WatchQueue, + telemetry_sink: asyncio.Queue | None = None, ): event_handler, request_queue = reconcile.get_event_handler(namespace=namespace) event_config = events.Configuration( event_handler=event_handler, + telemetry_sink=telemetry_sink, namespace=namespace, max_unknown_errors=10, retry_delay_base=30, @@ -105,6 +107,7 @@ async def workflow_controller_system( retry_delay_base=30, retry_delay_max=900, work_processor=_configure_reconciler(api=managed_resource_api), + telemetry_sink=telemetry_sink, ) async with asyncio.TaskGroup() as tg: diff --git a/src/controller/events.py b/src/controller/events.py index 602cb8d..a2378d6 100644 --- a/src/controller/events.py +++ b/src/controller/events.py @@ -50,6 +50,8 @@ class Configuration(NamedTuple): retry_delay_jitter: int = 30 retry_delay_max: int = 900 + telemetry_sink: asyncio.Queue | None = None + def _watch_key(api_version: str, kind: str): return f"{kind}.{api_version}" diff --git a/src/controller/scheduler.py b/src/controller/scheduler.py index 42c050b..cdcf847 100644 --- a/src/controller/scheduler.py +++ b/src/controller/scheduler.py @@ -34,6 +34,8 @@ class Configuration[T](NamedTuple): retry_delay_max: int = 300 retry_delay_jitter: int = 30 + telemetry_sink: asyncio.Queue | None = None + class Request[T](NamedTuple): at: float # Timestamp to, approximately, run at diff --git a/src/koreo_controller.py b/src/koreo_controller.py index 420273d..f909b3e 100644 --- a/src/koreo_controller.py +++ b/src/koreo_controller.py @@ -126,7 +126,7 @@ async def _controller_engine_wrapper( shutdown_trigger.set() -async def main(): +async def main(telemetry_sink: asyncio.Queue | None = None): logger.info("Koreo Controller Starting") api = await kr8s.asyncio.api() @@ -213,6 +213,7 @@ async def main(): api=api, namespace=RESOURCE_NAMESPACE, workflow_updates_queue=workflow_updates_queue, + telemetry_sink=telemetry_sink, ), ), name="workflow-controller", From 9862b0b8e42a23703fbe2f1b69dc7f5e80aab79e Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Thu, 7 Aug 2025 15:23:47 -0600 Subject: [PATCH 02/15] Add optional health/telemetry capability Adding telemetry in order to add health checks and aid in debugging. This is disabled by default and the structures returned are alpha quality so that we can better understand the practical usage of them. --- src/controller/__init__.py | 219 ++++++++++++++++++++++++++++++++++++ src/controller/status.py | 109 ++++++++++++++++++ src/koreo_controller.py | 221 +++---------------------------------- 3 files changed, 346 insertions(+), 203 deletions(-) create mode 100644 src/controller/__init__.py create mode 100644 src/controller/status.py diff --git a/src/controller/__init__.py b/src/controller/__init__.py new file mode 100644 index 0000000..64d4b7f --- /dev/null +++ b/src/controller/__init__.py @@ -0,0 +1,219 @@ +from typing import Awaitable +import asyncio +import logging +import os + + +import kr8s.asyncio + +from koreo.constants import API_GROUP, DEFAULT_API_VERSION +from koreo.resource_function.prepare import prepare_resource_function +from koreo.resource_function.structure import ResourceFunction +from koreo.resource_template.prepare import prepare_resource_template +from koreo.resource_template.structure import ResourceTemplate +from koreo.value_function.prepare import prepare_value_function +from koreo.value_function.structure import ValueFunction +from koreo.workflow.structure import Workflow + +from controller import koreo_cache +from controller import load_schemas +from controller.custom_workflow import workflow_controller_system +from controller.workflow_prepare_shim import get_workflow_preparer + +RECONNECT_TIMEOUT = 900 + +API_VERSION = f"{API_GROUP}/{DEFAULT_API_VERSION}" + +HOT_LOADING = True + +KOREO_NAMESPACE = os.environ.get("KOREO_NAMESPACE", "koreo-testing") + +TEMPLATE_NAMESPACE = os.environ.get("TEMPLATE_NAMESPACE", "koreo-testing") + +RESOURCE_NAMESPACE = os.environ.get("RESOURCE_NAMESPACE", "koreo-testing") + + +# NOTE: These are ordered so that each group's dependencies will already be +# loaded when initially loaded into cache. +KOREO_RESOURCES = [ + ( + TEMPLATE_NAMESPACE, + "ResourceTemplate", + ResourceTemplate, + prepare_resource_template, + ), + (KOREO_NAMESPACE, "ValueFunction", ValueFunction, prepare_value_function), + (KOREO_NAMESPACE, "ResourceFunction", ResourceFunction, prepare_resource_function), + # NOTE: Workflow is appended within `main` to integrate updates queue. +] + +logger = logging.getLogger("controller") + + +async def _koreo_resource_cache_manager( + api: kr8s.asyncio.Api, + namespace: str, + kind_title: str, + resource_class: type, + preparer, + shutdown_trigger: asyncio.Event, +): + """ + These are long-term (infinite) cache maintainers that will run in the + background to watch for updates to Koreo Resources. + """ + try: + await koreo_cache.maintain_cache( + api=api, + namespace=namespace, + api_version=API_VERSION, + plural_kind=f"{kind_title.lower()}s", + kind_title=kind_title, + resource_class=resource_class, + preparer=preparer, + reconnect_timeout=RECONNECT_TIMEOUT, + ) + finally: + shutdown_trigger.set() + + +async def _controller_engine_wrapper( + shutdown_trigger: asyncio.Event, controller: Awaitable +): + try: + await controller + + except KeyboardInterrupt: + logger.debug(f"Controller engine quit due to user quit.") + raise + + except asyncio.CancelledError: + logger.info(f"Controller engine quit due to cancel.") + raise + + except BaseException as err: + logger.error(f"Controller engine quit due to error: {err}.") + raise + + finally: + shutdown_trigger.set() + + +async def controller_main(telemetry_sink: asyncio.Queue | None = None): + logger.info("Koreo Controller Starting") + + api = await kr8s.asyncio.api() + api.timeout = RECONNECT_TIMEOUT + + # The schemas must be loaded before Koreo resources can be prepared. + await load_schemas.load_koreo_resource_schemas(api) + + # This is so the resources can be re-reconciled if their Workflows are + # updated. + prepare_workflow, workflow_updates_queue = get_workflow_preparer() + + KOREO_RESOURCES.append( + (KOREO_NAMESPACE, "Workflow", Workflow, prepare_workflow), + ) + + for namespace, kind_title, resource_class, preparer in KOREO_RESOURCES: + try: + # Load the Koreo resources sequentially, for efficiency purposes. + await koreo_cache.load_cache( + api=api, + namespace=namespace, + api_version=API_VERSION, + plural_kind=f"{kind_title.lower()}s", + kind_title=kind_title, + resource_class=resource_class, + preparer=preparer, + ) + + # There is a trailing return + continue + + except KeyboardInterrupt: + logger.info( + f"Initiating shutdown due to user-request. (Koreo {kind_title} Resource Load)" + ) + + except asyncio.CancelledError: + logger.info( + f"Initiating shutdown due to cancel. (Koreo {kind_title} Resource Load)" + ) + + except BaseException as err: + logger.error( + f"Initiating shutdown due to error {err}. (Koreo {kind_title} Resource Load)" + ) + + except: + logger.critical( + f"Initiating shutdown due to non-error exception. (Koreo {kind_title} Resource Load)" + ) + + # This means the continue was not hit + return + + async with asyncio.TaskGroup() as controller_tasks: + shutdown_trigger = asyncio.Event() + + tasks: list[asyncio.Task] = [] + + if HOT_LOADING: + logger.info("Hot-loading Koreo Resource enabled") + for namespace, kind_title, resource_class, preparer in KOREO_RESOURCES: + tasks.append( + controller_tasks.create_task( + _koreo_resource_cache_manager( + api=api, + namespace=namespace, + kind_title=kind_title, + resource_class=resource_class, + preparer=preparer, + shutdown_trigger=shutdown_trigger, + ), + name=f"cache-maintainer-{kind_title.lower()}", + ) + ) + + # This is the schedule watcher / dispatcher for workflow crdRefs. + tasks.append( + asyncio.create_task( + _controller_engine_wrapper( + shutdown_trigger=shutdown_trigger, + controller=workflow_controller_system( + api=api, + namespace=RESOURCE_NAMESPACE, + workflow_updates_queue=workflow_updates_queue, + telemetry_sink=telemetry_sink, + ), + ), + name="workflow-controller", + ) + ) + + try: + await shutdown_trigger.wait() + + except KeyboardInterrupt: + logger.info("Initiating shutdown due to user-request.") + + except asyncio.CancelledError: + logger.info("Initiating shutdown due to cancel.") + + except BaseException as err: + logger.error(f"Initiating shutdown due to error {err}.") + + except: + logger.critical(f"Initiating shutdown due to non-error exception.") + + logger.info("Shutting down workers") + for task in tasks: + task_name = task.get_name() + + if not (task.done() or task.cancelling()): + task.cancel("System shutdown") + logger.info(f"Stopping {task_name}") + + logger.info("Koreo controller is shutdown") diff --git a/src/controller/status.py b/src/controller/status.py new file mode 100644 index 0000000..8344da1 --- /dev/null +++ b/src/controller/status.py @@ -0,0 +1,109 @@ +import asyncio +import copy +import logging +import os +import time + +import uvicorn + +from starlette.applications import Starlette +from starlette.responses import JSONResponse +from starlette.routing import Route + +from koreo import status + +logger = logging.getLogger("controller.status") + +PORT = int(os.environ.get("PORT", 5000)) + + +async def koreo_cache_status(_): + return JSONResponse(status.list_resources()) + + +def workflow_events_status(telemetry: dict): + async def _handler(_): + if not telemetry: + return JSONResponse({"error": "No telemetry data"}) + + return JSONResponse(telemetry.get("event_watcher")) + + return _handler + + +def resource_events_status(telemetry: dict): + async def _handler(_): + if not telemetry: + return JSONResponse({"error": "No telemetry data"}) + + status_request_time = time.monotonic() + + scheduler_telemetry = copy.copy(telemetry.get("scheduler")) + if not scheduler_telemetry: + return JSONResponse({}) + + schedule = scheduler_telemetry.get("schedule") + if schedule: + scheduler_telemetry["schedule"] = [ + [scheduled_for - status_request_time, *rest] + for scheduled_for, *rest in schedule + ] + + return JSONResponse(scheduler_telemetry) + + return _handler + + +async def aggregator( + telemetry_sink: asyncio.Queue | None = asyncio.Queue(), + telemetry_data: dict | None = None, +): + if not telemetry_sink or telemetry_data is None: + return + + while True: + telemetry_update = await telemetry_sink.get() + telemetry_data["__last_update__"] = time.monotonic() + + update_source = telemetry_update.get("source") + + if not update_source: + continue + + telemetry_data[update_source] = telemetry_update.get("telemetry") + + +async def status_main(telemetry_sink: asyncio.Queue | None = None): + logger.info("Koreo Status Server Starting") + + telemetry_data = {"__system_start__": time.monotonic()} + + app = Starlette( + debug=True, + routes=[ + Route("/koreo/cache", koreo_cache_status), + Route("/controller/events", workflow_events_status(telemetry_data)), + Route("/controller/scheduler", resource_events_status(telemetry_data)), + ], + ) + + config = uvicorn.Config(app, port=PORT, log_level="info") + server = uvicorn.Server(config) + + try: + async with asyncio.TaskGroup() as main_tg: + main_tg.create_task( + aggregator( + telemetry_sink=telemetry_sink, telemetry_data=telemetry_data + ), + name="telemetry-aggregator", + ) + main_tg.create_task(server.serve(), name="status-server") + except KeyboardInterrupt: + logger.info("Initiating shutdown due to user stop.") + exit(0) + + except asyncio.CancelledError: + logger.info("Initiating shutdown due to cancel.") + + exit(1) diff --git a/src/koreo_controller.py b/src/koreo_controller.py index f909b3e..4373383 100644 --- a/src/koreo_controller.py +++ b/src/koreo_controller.py @@ -1,9 +1,10 @@ -from typing import Awaitable import asyncio import json import logging import os +import uvloop + class JsonFormatter(logging.Formatter): def format(self, record): @@ -33,217 +34,27 @@ def format(self, record): import uvloop -import kr8s.asyncio - -from koreo.constants import API_GROUP, DEFAULT_API_VERSION -from koreo.resource_function.prepare import prepare_resource_function -from koreo.resource_function.structure import ResourceFunction -from koreo.resource_template.prepare import prepare_resource_template -from koreo.resource_template.structure import ResourceTemplate -from koreo.value_function.prepare import prepare_value_function -from koreo.value_function.structure import ValueFunction -from koreo.workflow.structure import Workflow - -from controller import koreo_cache -from controller import load_schemas -from controller.workflow_prepare_shim import get_workflow_preparer -from controller.custom_workflow import workflow_controller_system - -RECONNECT_TIMEOUT = 900 - -API_VERSION = f"{API_GROUP}/{DEFAULT_API_VERSION}" +from controller import controller_main +from controller.status import status_main -HOT_LOADING = True +DIAGNOSTICS = os.environ.get("DIAGNOSTICS", "disabled") -KOREO_NAMESPACE = os.environ.get("KOREO_NAMESPACE", "koreo-testing") -TEMPLATE_NAMESPACE = os.environ.get("TEMPLATE_NAMESPACE", "koreo-testing") +async def main(): + if DIAGNOSTICS.lower().strip() != "enabled": + return await controller_main() -RESOURCE_NAMESPACE = os.environ.get("RESOURCE_NAMESPACE", "koreo-testing") + telemetry_sink: asyncio.Queue | None = asyncio.Queue() - -# NOTE: These are ordered so that each group's dependencies will already be -# loaded when initially loaded into cache. -KOREO_RESOURCES = [ - ( - TEMPLATE_NAMESPACE, - "ResourceTemplate", - ResourceTemplate, - prepare_resource_template, - ), - (KOREO_NAMESPACE, "ValueFunction", ValueFunction, prepare_value_function), - (KOREO_NAMESPACE, "ResourceFunction", ResourceFunction, prepare_resource_function), - # NOTE: Workflow is appended within `main` to integrate updates queue. -] - - -async def _koreo_resource_cache_manager( - api: kr8s.asyncio.Api, - namespace: str, - kind_title: str, - resource_class: type, - preparer, - shutdown_trigger: asyncio.Event, -): - """ - These are long-term (infinite) cache maintainers that will run in the - background to watch for updates to Koreo Resources. - """ try: - await koreo_cache.maintain_cache( - api=api, - namespace=namespace, - api_version=API_VERSION, - plural_kind=f"{kind_title.lower()}s", - kind_title=kind_title, - resource_class=resource_class, - preparer=preparer, - reconnect_timeout=RECONNECT_TIMEOUT, - ) - finally: - shutdown_trigger.set() - - -async def _controller_engine_wrapper( - shutdown_trigger: asyncio.Event, controller: Awaitable -): - try: - await controller - + async with asyncio.TaskGroup() as main_tg: + main_tg.create_task(controller_main(telemetry_sink=telemetry_sink)) + main_tg.create_task(status_main(telemetry_sink=telemetry_sink)) except KeyboardInterrupt: - logger.debug(f"Controller engine quit due to user quit.") - raise + logger.info("Initiating shutdown due to user-request.") except asyncio.CancelledError: - logger.info(f"Controller engine quit due to cancel.") - raise - - except BaseException as err: - logger.error(f"Controller engine quit due to error: {err}.") - raise - - finally: - shutdown_trigger.set() - - -async def main(telemetry_sink: asyncio.Queue | None = None): - logger.info("Koreo Controller Starting") - - api = await kr8s.asyncio.api() - api.timeout = RECONNECT_TIMEOUT - - # The schemas must be loaded before Koreo resources can be prepared. - await load_schemas.load_koreo_resource_schemas(api) - - # This is so the resources can be re-reconciled if their Workflows are - # updated. - prepare_workflow, workflow_updates_queue = get_workflow_preparer() - - KOREO_RESOURCES.append( - (KOREO_NAMESPACE, "Workflow", Workflow, prepare_workflow), - ) - - for namespace, kind_title, resource_class, preparer in KOREO_RESOURCES: - try: - # Load the Koreo resources sequentially, for efficiency purposes. - await koreo_cache.load_cache( - api=api, - namespace=namespace, - api_version=API_VERSION, - plural_kind=f"{kind_title.lower()}s", - kind_title=kind_title, - resource_class=resource_class, - preparer=preparer, - ) - - # There is a trailing return - continue - - except KeyboardInterrupt: - logger.info( - f"Initiating shutdown due to user-request. (Koreo {kind_title} Resource Load)" - ) - - except asyncio.CancelledError: - logger.info( - f"Initiating shutdown due to cancel. (Koreo {kind_title} Resource Load)" - ) - - except BaseException as err: - logger.error( - f"Initiating shutdown due to error {err}. (Koreo {kind_title} Resource Load)" - ) - - except: - logger.critical( - f"Initiating shutdown due to non-error exception. (Koreo {kind_title} Resource Load)" - ) - - # This means the continue was not hit - return - - async with asyncio.TaskGroup() as main_tg: - shutdown_trigger = asyncio.Event() - - tasks: list[asyncio.Task] = [] - - if HOT_LOADING: - logger.info("Hot-loading Koreo Resource enabled") - for namespace, kind_title, resource_class, preparer in KOREO_RESOURCES: - tasks.append( - main_tg.create_task( - _koreo_resource_cache_manager( - api=api, - namespace=namespace, - kind_title=kind_title, - resource_class=resource_class, - preparer=preparer, - shutdown_trigger=shutdown_trigger, - ), - name=f"cache-maintainer-{kind_title.lower()}", - ) - ) - - # This is the schedule watcher / dispatcher for workflow crdRefs. - tasks.append( - asyncio.create_task( - _controller_engine_wrapper( - shutdown_trigger=shutdown_trigger, - controller=workflow_controller_system( - api=api, - namespace=RESOURCE_NAMESPACE, - workflow_updates_queue=workflow_updates_queue, - telemetry_sink=telemetry_sink, - ), - ), - name="workflow-controller", - ) - ) - - try: - await shutdown_trigger.wait() - - except KeyboardInterrupt: - logger.info("Initiating shutdown due to user-request.") - - except asyncio.CancelledError: - logger.info("Initiating shutdown due to cancel.") - - except BaseException as err: - logger.error(f"Initiating shutdown due to error {err}.") - - except: - logger.critical(f"Initiating shutdown due to non-error exception.") - - logger.info("Shutting down workers") - for task in tasks: - task_name = task.get_name() - - if not (task.done() or task.cancelling()): - task.cancel("System shutdown") - logger.info(f"Stopping {task_name}") - - logger.info("Shutdown") + logger.info("Initiating shutdown due to cancel.") if __name__ == "__main__": @@ -251,6 +62,10 @@ async def main(telemetry_sink: asyncio.Queue | None = None): try: uvloop.run(main()) except KeyboardInterrupt: + logger.info("Initiating shutdown due to user stop.") exit(0) + except asyncio.CancelledError: + logger.info("Initiating shutdown due to cancel.") + exit(1) From 1efc311804ccaa7ff6f23b01df9777f3d365f54b Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Thu, 7 Aug 2025 15:41:09 -0600 Subject: [PATCH 03/15] Return seconds-to-next-reconcile-attempt as formatted string --- src/controller/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controller/status.py b/src/controller/status.py index 8344da1..5e9364e 100644 --- a/src/controller/status.py +++ b/src/controller/status.py @@ -45,7 +45,7 @@ async def _handler(_): schedule = scheduler_telemetry.get("schedule") if schedule: scheduler_telemetry["schedule"] = [ - [scheduled_for - status_request_time, *rest] + [f"{scheduled_for - status_request_time:.4f}", *rest] for scheduled_for, *rest in schedule ] From ba4655a7e3db664e43a62e3d8cdfb9736f00d935 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Thu, 7 Aug 2025 16:15:35 -0600 Subject: [PATCH 04/15] Emit telemetry from the scheduler and event watcher For the initial telemetry capture only the reconciliation scheduler and event watcher subsystems emit data. This is rough and expected to evolve after further testing and design. --- src/controller/events.py | 47 ++++++++++++++++++++++++++++++++++++ src/controller/scheduler.py | 48 +++++++++++++++++++++++++++++++++++++ src/controller/status.py | 26 +++++++++++++------- src/koreo_controller.py | 3 ++- 4 files changed, 114 insertions(+), 10 deletions(-) diff --git a/src/controller/events.py b/src/controller/events.py index a2378d6..24a2689 100644 --- a/src/controller/events.py +++ b/src/controller/events.py @@ -1,5 +1,6 @@ from typing import Literal, NamedTuple, Protocol import asyncio +import copy import logging import random @@ -129,6 +130,52 @@ async def chief_of_the_watch( finally: watch_requests.task_done() + try: + _emit_telemetry( + configuration=configuration, + watch_requests=watch_requests, + watchstanders=watchstanders, + workflow_watches=workflow_watches, + resource_watchers=resource_watchers, + ) + except: + logger.exception("Error emitting event-watcher telemetry") + + +def _emit_telemetry( + configuration: Configuration, + watch_requests: WatchQueue, + watchstanders: dict[str, tuple[asyncio.Task, asyncio.Queue]], + workflow_watches: dict[str, str], + resource_watchers: dict[str, set[str]], +): + if not configuration.telemetry_sink: + return + + try: + configuration.telemetry_sink.put_nowait( + { + "source": "event_watcher", + "telemetry": { + "pending_watch_requests": watch_requests.qsize(), + "watchstander_status": { + key: { + "done": watchstander.done(), + "cancelled": watchstander.cancelled(), + } + for key, (watchstander, _) in watchstanders.items() + }, + "workflow_watches": copy.copy(workflow_watches), + "resource_watchers": { + key: list(watchers) + for key, watchers in resource_watchers.items() + }, + }, + } + ) + except asyncio.QueueFull: + logger.info("Telemetry queue full, skipping event-watcher telemetry") + async def _cancel_watch( workflow: str, diff --git a/src/controller/scheduler.py b/src/controller/scheduler.py index cdcf847..3be73c7 100644 --- a/src/controller/scheduler.py +++ b/src/controller/scheduler.py @@ -68,6 +68,16 @@ async def orchestrator[T]( work: asyncio.Queue[Request[T] | Shutdown] = asyncio.Queue() while True: + try: + _emit_telemetry( + configuration=configuration, + request_schedule=request_schedule, + workers=workers, + work=work, + ) + except: + logger.exception("Error emitting scheduler telemetry") + # Ensure reconcilers are running if len(workers) < max(configuration.concurrency, 1): worker_task = tg.create_task( @@ -130,6 +140,44 @@ async def orchestrator[T]( continue +def _emit_telemetry( + configuration: Configuration, + request_schedule: list[Request], + workers: set[asyncio.Task], + work: asyncio.Queue[Request | Shutdown], +): + if not configuration.telemetry_sink: + return + + try: + configuration.telemetry_sink.put_nowait( + { + "source": "scheduler", + "telemetry": { + "schedule": [ + ( + request.at, + request.payload, + request.user_retries, + request.sys_error_retries, + ) + for request in request_schedule + ], + "workers": [ + { + "done": worker.done(), + "cancelled": worker.cancelled(), + } + for worker in workers + ], + "unscheduled_work": work.qsize(), + }, + } + ) + except asyncio.QueueFull: + logger.info("Telemetry queue full, skipping scheduler telemetry") + + async def _worker_task[T]( work: asyncio.Queue[Request[T] | Shutdown], requests: asyncio.Queue[Request[T] | Shutdown], diff --git a/src/controller/status.py b/src/controller/status.py index 5e9364e..f7a1e94 100644 --- a/src/controller/status.py +++ b/src/controller/status.py @@ -62,15 +62,23 @@ async def aggregator( return while True: - telemetry_update = await telemetry_sink.get() - telemetry_data["__last_update__"] = time.monotonic() - - update_source = telemetry_update.get("source") - - if not update_source: - continue - - telemetry_data[update_source] = telemetry_update.get("telemetry") + try: + telemetry_update = await telemetry_sink.get() + # We don't want to retry if these fail to process. + telemetry_sink.task_done() + + telemetry_data["__last_update__"] = time.monotonic() + + update_source = telemetry_update.get("source") + if not update_source: + logging.debug( + f"Malformed controller telemetry data ({telemetry_update})" + ) + continue + + telemetry_data[update_source] = telemetry_update.get("telemetry") + except (KeyboardInterrupt, asyncio.CancelledError): + raise async def status_main(telemetry_sink: asyncio.Queue | None = None): diff --git a/src/koreo_controller.py b/src/koreo_controller.py index 4373383..f6bde5b 100644 --- a/src/koreo_controller.py +++ b/src/koreo_controller.py @@ -44,7 +44,8 @@ async def main(): if DIAGNOSTICS.lower().strip() != "enabled": return await controller_main() - telemetry_sink: asyncio.Queue | None = asyncio.Queue() + # Not 100% certain what the limit should be, perhaps higher. + telemetry_sink: asyncio.Queue | None = asyncio.Queue(100) try: async with asyncio.TaskGroup() as main_tg: From 6c79587fde865101a7344d992740f9950ab1b915 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Sat, 9 Aug 2025 14:07:50 -0600 Subject: [PATCH 05/15] Bump Koreo Core version --- pdm.lock | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pdm.lock b/pdm.lock index 2078780..49fae65 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "all", "test", "tooling"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:be022792e65393a95bd40e758ddd53bdf5b905b56c5d629f438c15be3d950277" +content_hash = "sha256:3f9b0d2234d6f6eebbff1ee81045a881aead42f577d4ed1f5a307f3b301dc1b0" [[metadata.targets]] requires_python = "==3.13.*" @@ -315,7 +315,7 @@ files = [ [[package]] name = "koreo-core" -version = "0.1.15" +version = "0.1.17" requires_python = ">=3.13" summary = "Type-safe and testable KRM Templates and Workflows." groups = ["default", "all"] @@ -326,23 +326,23 @@ dependencies = [ "kr8s==0.20.7", ] files = [ - {file = "koreo_core-0.1.15-py3-none-any.whl", hash = "sha256:72c40463b95826d7c3bbb89bf6ce86b32cc172630962cec79e94c252b094f1d2"}, - {file = "koreo_core-0.1.15.tar.gz", hash = "sha256:249f7626b5eabd32b74d108fcacad69834002c56b059603ba28ab4d0af529d21"}, + {file = "koreo_core-0.1.17-py3-none-any.whl", hash = "sha256:270a8f6b857a13c907b4e08147914a9c8125194ef979a93610c3ce97da3fc8de"}, + {file = "koreo_core-0.1.17.tar.gz", hash = "sha256:0bfc111cb9f9d51a180626f7c225a678e0f79c71d8d3002b9b245bd370f95cea"}, ] [[package]] name = "koreo-core" -version = "0.1.15" +version = "0.1.17" extras = ["test", "tooling"] requires_python = ">=3.13" summary = "Type-safe and testable KRM Templates and Workflows." groups = ["all"] dependencies = [ - "koreo-core==0.1.15", + "koreo-core==0.1.17", ] files = [ - {file = "koreo_core-0.1.15-py3-none-any.whl", hash = "sha256:72c40463b95826d7c3bbb89bf6ce86b32cc172630962cec79e94c252b094f1d2"}, - {file = "koreo_core-0.1.15.tar.gz", hash = "sha256:249f7626b5eabd32b74d108fcacad69834002c56b059603ba28ab4d0af529d21"}, + {file = "koreo_core-0.1.17-py3-none-any.whl", hash = "sha256:270a8f6b857a13c907b4e08147914a9c8125194ef979a93610c3ce97da3fc8de"}, + {file = "koreo_core-0.1.17.tar.gz", hash = "sha256:0bfc111cb9f9d51a180626f7c225a678e0f79c71d8d3002b9b245bd370f95cea"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index b1abe99..fd529fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ authors = [ {name = "Robert Kluin", email = "robert.kluin@realkinetic.com"}, ] dependencies = [ - "koreo-core==0.1.15", + "koreo-core==0.1.17", "cel-python==0.3.0", "kr8s==0.20.7", "uvloop==0.21.0", From 2d1193541c129fd005192024b44562c5c14c2e36 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Sat, 9 Aug 2025 14:08:58 -0600 Subject: [PATCH 06/15] Add Starlette and uvicorn libs --- pdm.lock | 51 +++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 2 ++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/pdm.lock b/pdm.lock index 49fae65..3f7ec0a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "all", "test", "tooling"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:3f9b0d2234d6f6eebbff1ee81045a881aead42f577d4ed1f5a307f3b301dc1b0" +content_hash = "sha256:75dc0de06a6f5a674c576fd63d539db553c1ba7d995221a0245333f691db36e7" [[metadata.targets]] requires_python = "==3.13.*" @@ -105,13 +105,27 @@ files = [ {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] +[[package]] +name = "click" +version = "8.2.1" +requires_python = ">=3.10" +summary = "Composable command line interface toolkit" +groups = ["default"] +dependencies = [ + "colorama; platform_system == \"Windows\"", +] +files = [ + {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, + {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, +] + [[package]] name = "colorama" version = "0.4.6" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Cross-platform colored terminal text." -groups = ["test"] -marker = "sys_platform == \"win32\"" +groups = ["default", "test"] +marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -616,6 +630,21 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "starlette" +version = "0.47.2" +requires_python = ">=3.9" +summary = "The little ASGI library that shines." +groups = ["default"] +dependencies = [ + "anyio<5,>=3.6.2", + "typing-extensions>=4.10.0; python_version < \"3.13\"", +] +files = [ + {file = "starlette-0.47.2-py3-none-any.whl", hash = "sha256:c5847e96134e5c5371ee9fac6fdf1a67336d5815e09eb2a01fdb57a351ef915b"}, + {file = "starlette-0.47.2.tar.gz", hash = "sha256:6ae9aa5db235e4846decc1e7b79c4f346adf41e9777aebeb49dfd09bbd7023d8"}, +] + [[package]] name = "types-pyyaml" version = "6.0.12.20250516" @@ -649,6 +678,22 @@ files = [ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] +[[package]] +name = "uvicorn" +version = "0.35.0" +requires_python = ">=3.9" +summary = "The lightning-fast ASGI server." +groups = ["default"] +dependencies = [ + "click>=7.0", + "h11>=0.8", + "typing-extensions>=4.0; python_version < \"3.11\"", +] +files = [ + {file = "uvicorn-0.35.0-py3-none-any.whl", hash = "sha256:197535216b25ff9b785e29a0b79199f55222193d47f820816e7da751e9bc8d4a"}, + {file = "uvicorn-0.35.0.tar.gz", hash = "sha256:bc662f087f7cf2ce11a1d7fd70b90c9f98ef2e2831556dd078d131b96cc94a01"}, +] + [[package]] name = "uvloop" version = "0.21.0" diff --git a/pyproject.toml b/pyproject.toml index fd529fd..6e7a9e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ dependencies = [ "cel-python==0.3.0", "kr8s==0.20.7", "uvloop==0.21.0", + "starlette==0.47.2", + "uvicorn==0.35.0", ] requires-python = "==3.13.*" readme = "README.md" From c5a7b6f6b08a43540913fe17aa5669f160bcf304 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Wed, 13 Aug 2025 09:19:28 -0600 Subject: [PATCH 07/15] Update lockfile --- pdm.lock | 208 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 106 insertions(+), 102 deletions(-) diff --git a/pdm.lock b/pdm.lock index 3f7ec0a..1401987 100644 --- a/pdm.lock +++ b/pdm.lock @@ -12,9 +12,9 @@ requires_python = "==3.13.*" [[package]] name = "anyio" -version = "4.9.0" +version = "4.10.0" requires_python = ">=3.9" -summary = "High level compatibility layer for multiple asynchronous event loop implementations" +summary = "High-level concurrency and networking framework on top of asyncio or Trio" groups = ["default", "all"] dependencies = [ "exceptiongroup>=1.0.2; python_version < \"3.11\"", @@ -23,8 +23,8 @@ dependencies = [ "typing-extensions>=4.5; python_version < \"3.13\"", ] files = [ - {file = "anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c"}, - {file = "anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028"}, + {file = "anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1"}, + {file = "anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6"}, ] [[package]] @@ -71,13 +71,13 @@ files = [ [[package]] name = "certifi" -version = "2025.4.26" -requires_python = ">=3.6" +version = "2025.8.3" +requires_python = ">=3.7" summary = "Python package for providing Mozilla's CA Bundle." groups = ["default", "all"] files = [ - {file = "certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3"}, - {file = "certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6"}, + {file = "certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5"}, + {file = "certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407"}, ] [[package]] @@ -133,106 +133,110 @@ files = [ [[package]] name = "coverage" -version = "7.8.0" +version = "7.10.3" requires_python = ">=3.9" summary = "Code coverage measurement for Python" groups = ["test"] files = [ - {file = "coverage-7.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd"}, - {file = "coverage-7.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00"}, - {file = "coverage-7.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64"}, - {file = "coverage-7.8.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067"}, - {file = "coverage-7.8.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008"}, - {file = "coverage-7.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733"}, - {file = "coverage-7.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323"}, - {file = "coverage-7.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3"}, - {file = "coverage-7.8.0-cp313-cp313-win32.whl", hash = "sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d"}, - {file = "coverage-7.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487"}, - {file = "coverage-7.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25"}, - {file = "coverage-7.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42"}, - {file = "coverage-7.8.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502"}, - {file = "coverage-7.8.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1"}, - {file = "coverage-7.8.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4"}, - {file = "coverage-7.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73"}, - {file = "coverage-7.8.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a"}, - {file = "coverage-7.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883"}, - {file = "coverage-7.8.0-cp313-cp313t-win32.whl", hash = "sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada"}, - {file = "coverage-7.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257"}, - {file = "coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7"}, - {file = "coverage-7.8.0.tar.gz", hash = "sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501"}, + {file = "coverage-7.10.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b99e87304ffe0eb97c5308447328a584258951853807afdc58b16143a530518a"}, + {file = "coverage-7.10.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4af09c7574d09afbc1ea7da9dcea23665c01f3bc1b1feb061dac135f98ffc53a"}, + {file = "coverage-7.10.3-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:488e9b50dc5d2aa9521053cfa706209e5acf5289e81edc28291a24f4e4488f46"}, + {file = "coverage-7.10.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:913ceddb4289cbba3a310704a424e3fb7aac2bc0c3a23ea473193cb290cf17d4"}, + {file = "coverage-7.10.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b1f91cbc78c7112ab84ed2a8defbccd90f888fcae40a97ddd6466b0bec6ae8a"}, + {file = "coverage-7.10.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0bac054d45af7cd938834b43a9878b36ea92781bcb009eab040a5b09e9927e3"}, + {file = "coverage-7.10.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:fe72cbdd12d9e0f4aca873fa6d755e103888a7f9085e4a62d282d9d5b9f7928c"}, + {file = "coverage-7.10.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1e2e927ab3eadd7c244023927d646e4c15c65bb2ac7ae3c3e9537c013700d21"}, + {file = "coverage-7.10.3-cp313-cp313-win32.whl", hash = "sha256:24d0c13de473b04920ddd6e5da3c08831b1170b8f3b17461d7429b61cad59ae0"}, + {file = "coverage-7.10.3-cp313-cp313-win_amd64.whl", hash = "sha256:3564aae76bce4b96e2345cf53b4c87e938c4985424a9be6a66ee902626edec4c"}, + {file = "coverage-7.10.3-cp313-cp313-win_arm64.whl", hash = "sha256:f35580f19f297455f44afcd773c9c7a058e52eb6eb170aa31222e635f2e38b87"}, + {file = "coverage-7.10.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07009152f497a0464ffdf2634586787aea0e69ddd023eafb23fc38267db94b84"}, + {file = "coverage-7.10.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8dd2ba5f0c7e7e8cc418be2f0c14c4d9e3f08b8fb8e4c0f83c2fe87d03eb655e"}, + {file = "coverage-7.10.3-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1ae22b97003c74186e034a93e4f946c75fad8c0ce8d92fbbc168b5e15ee2841f"}, + {file = "coverage-7.10.3-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb329f1046888a36b1dc35504d3029e1dd5afe2196d94315d18c45ee380f67d5"}, + {file = "coverage-7.10.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce01048199a91f07f96ca3074b0c14021f4fe7ffd29a3e6a188ac60a5c3a4af8"}, + {file = "coverage-7.10.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:08b989a06eb9dfacf96d42b7fb4c9a22bafa370d245dc22fa839f2168c6f9fa1"}, + {file = "coverage-7.10.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:669fe0d4e69c575c52148511029b722ba8d26e8a3129840c2ce0522e1452b256"}, + {file = "coverage-7.10.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3262d19092771c83f3413831d9904b1ccc5f98da5de4ffa4ad67f5b20c7aaf7b"}, + {file = "coverage-7.10.3-cp313-cp313t-win32.whl", hash = "sha256:cc0ee4b2ccd42cab7ee6be46d8a67d230cb33a0a7cd47a58b587a7063b6c6b0e"}, + {file = "coverage-7.10.3-cp313-cp313t-win_amd64.whl", hash = "sha256:03db599f213341e2960430984e04cf35fb179724e052a3ee627a068653cf4a7c"}, + {file = "coverage-7.10.3-cp313-cp313t-win_arm64.whl", hash = "sha256:46eae7893ba65f53c71284585a262f083ef71594f05ec5c85baf79c402369098"}, + {file = "coverage-7.10.3-py3-none-any.whl", hash = "sha256:416a8d74dc0adfd33944ba2f405897bab87b7e9e84a391e09d241956bd953ce1"}, + {file = "coverage-7.10.3.tar.gz", hash = "sha256:812ba9250532e4a823b070b0420a36499859542335af3dca8f47fc6aa1a05619"}, ] [[package]] name = "coverage" -version = "7.8.0" +version = "7.10.3" extras = ["toml"] requires_python = ">=3.9" summary = "Code coverage measurement for Python" groups = ["test"] dependencies = [ - "coverage==7.8.0", + "coverage==7.10.3", "tomli; python_full_version <= \"3.11.0a6\"", ] files = [ - {file = "coverage-7.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd"}, - {file = "coverage-7.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00"}, - {file = "coverage-7.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64"}, - {file = "coverage-7.8.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067"}, - {file = "coverage-7.8.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008"}, - {file = "coverage-7.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733"}, - {file = "coverage-7.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323"}, - {file = "coverage-7.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3"}, - {file = "coverage-7.8.0-cp313-cp313-win32.whl", hash = "sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d"}, - {file = "coverage-7.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487"}, - {file = "coverage-7.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25"}, - {file = "coverage-7.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42"}, - {file = "coverage-7.8.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502"}, - {file = "coverage-7.8.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1"}, - {file = "coverage-7.8.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4"}, - {file = "coverage-7.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73"}, - {file = "coverage-7.8.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a"}, - {file = "coverage-7.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883"}, - {file = "coverage-7.8.0-cp313-cp313t-win32.whl", hash = "sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada"}, - {file = "coverage-7.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257"}, - {file = "coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7"}, - {file = "coverage-7.8.0.tar.gz", hash = "sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501"}, + {file = "coverage-7.10.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b99e87304ffe0eb97c5308447328a584258951853807afdc58b16143a530518a"}, + {file = "coverage-7.10.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4af09c7574d09afbc1ea7da9dcea23665c01f3bc1b1feb061dac135f98ffc53a"}, + {file = "coverage-7.10.3-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:488e9b50dc5d2aa9521053cfa706209e5acf5289e81edc28291a24f4e4488f46"}, + {file = "coverage-7.10.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:913ceddb4289cbba3a310704a424e3fb7aac2bc0c3a23ea473193cb290cf17d4"}, + {file = "coverage-7.10.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b1f91cbc78c7112ab84ed2a8defbccd90f888fcae40a97ddd6466b0bec6ae8a"}, + {file = "coverage-7.10.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0bac054d45af7cd938834b43a9878b36ea92781bcb009eab040a5b09e9927e3"}, + {file = "coverage-7.10.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:fe72cbdd12d9e0f4aca873fa6d755e103888a7f9085e4a62d282d9d5b9f7928c"}, + {file = "coverage-7.10.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1e2e927ab3eadd7c244023927d646e4c15c65bb2ac7ae3c3e9537c013700d21"}, + {file = "coverage-7.10.3-cp313-cp313-win32.whl", hash = "sha256:24d0c13de473b04920ddd6e5da3c08831b1170b8f3b17461d7429b61cad59ae0"}, + {file = "coverage-7.10.3-cp313-cp313-win_amd64.whl", hash = "sha256:3564aae76bce4b96e2345cf53b4c87e938c4985424a9be6a66ee902626edec4c"}, + {file = "coverage-7.10.3-cp313-cp313-win_arm64.whl", hash = "sha256:f35580f19f297455f44afcd773c9c7a058e52eb6eb170aa31222e635f2e38b87"}, + {file = "coverage-7.10.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07009152f497a0464ffdf2634586787aea0e69ddd023eafb23fc38267db94b84"}, + {file = "coverage-7.10.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8dd2ba5f0c7e7e8cc418be2f0c14c4d9e3f08b8fb8e4c0f83c2fe87d03eb655e"}, + {file = "coverage-7.10.3-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1ae22b97003c74186e034a93e4f946c75fad8c0ce8d92fbbc168b5e15ee2841f"}, + {file = "coverage-7.10.3-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb329f1046888a36b1dc35504d3029e1dd5afe2196d94315d18c45ee380f67d5"}, + {file = "coverage-7.10.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce01048199a91f07f96ca3074b0c14021f4fe7ffd29a3e6a188ac60a5c3a4af8"}, + {file = "coverage-7.10.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:08b989a06eb9dfacf96d42b7fb4c9a22bafa370d245dc22fa839f2168c6f9fa1"}, + {file = "coverage-7.10.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:669fe0d4e69c575c52148511029b722ba8d26e8a3129840c2ce0522e1452b256"}, + {file = "coverage-7.10.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3262d19092771c83f3413831d9904b1ccc5f98da5de4ffa4ad67f5b20c7aaf7b"}, + {file = "coverage-7.10.3-cp313-cp313t-win32.whl", hash = "sha256:cc0ee4b2ccd42cab7ee6be46d8a67d230cb33a0a7cd47a58b587a7063b6c6b0e"}, + {file = "coverage-7.10.3-cp313-cp313t-win_amd64.whl", hash = "sha256:03db599f213341e2960430984e04cf35fb179724e052a3ee627a068653cf4a7c"}, + {file = "coverage-7.10.3-cp313-cp313t-win_arm64.whl", hash = "sha256:46eae7893ba65f53c71284585a262f083ef71594f05ec5c85baf79c402369098"}, + {file = "coverage-7.10.3-py3-none-any.whl", hash = "sha256:416a8d74dc0adfd33944ba2f405897bab87b7e9e84a391e09d241956bd953ce1"}, + {file = "coverage-7.10.3.tar.gz", hash = "sha256:812ba9250532e4a823b070b0420a36499859542335af3dca8f47fc6aa1a05619"}, ] [[package]] name = "cryptography" -version = "44.0.3" +version = "45.0.6" requires_python = "!=3.9.0,!=3.9.1,>=3.7" summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." groups = ["default", "all"] dependencies = [ - "cffi>=1.12; platform_python_implementation != \"PyPy\"", -] -files = [ - {file = "cryptography-44.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01"}, - {file = "cryptography-44.0.3-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d"}, - {file = "cryptography-44.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904"}, - {file = "cryptography-44.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44"}, - {file = "cryptography-44.0.3-cp37-abi3-win32.whl", hash = "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d"}, - {file = "cryptography-44.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d"}, - {file = "cryptography-44.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c"}, - {file = "cryptography-44.0.3-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f"}, - {file = "cryptography-44.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5"}, - {file = "cryptography-44.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b"}, - {file = "cryptography-44.0.3-cp39-abi3-win32.whl", hash = "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028"}, - {file = "cryptography-44.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334"}, - {file = "cryptography-44.0.3.tar.gz", hash = "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053"}, + "cffi>=1.14; platform_python_implementation != \"PyPy\"", +] +files = [ + {file = "cryptography-45.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:048e7ad9e08cf4c0ab07ff7f36cc3115924e22e2266e034450a890d9e312dd74"}, + {file = "cryptography-45.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:44647c5d796f5fc042bbc6d61307d04bf29bccb74d188f18051b635f20a9c75f"}, + {file = "cryptography-45.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e40b80ecf35ec265c452eea0ba94c9587ca763e739b8e559c128d23bff7ebbbf"}, + {file = "cryptography-45.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:00e8724bdad672d75e6f069b27970883179bd472cd24a63f6e620ca7e41cc0c5"}, + {file = "cryptography-45.0.6-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a3085d1b319d35296176af31c90338eeb2ddac8104661df79f80e1d9787b8b2"}, + {file = "cryptography-45.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1b7fa6a1c1188c7ee32e47590d16a5a0646270921f8020efc9a511648e1b2e08"}, + {file = "cryptography-45.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:275ba5cc0d9e320cd70f8e7b96d9e59903c815ca579ab96c1e37278d231fc402"}, + {file = "cryptography-45.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f4028f29a9f38a2025abedb2e409973709c660d44319c61762202206ed577c42"}, + {file = "cryptography-45.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ee411a1b977f40bd075392c80c10b58025ee5c6b47a822a33c1198598a7a5f05"}, + {file = "cryptography-45.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2a21a8eda2d86bb604934b6b37691585bd095c1f788530c1fcefc53a82b3453"}, + {file = "cryptography-45.0.6-cp311-abi3-win32.whl", hash = "sha256:d063341378d7ee9c91f9d23b431a3502fc8bfacd54ef0a27baa72a0843b29159"}, + {file = "cryptography-45.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:833dc32dfc1e39b7376a87b9a6a4288a10aae234631268486558920029b086ec"}, + {file = "cryptography-45.0.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:3436128a60a5e5490603ab2adbabc8763613f638513ffa7d311c900a8349a2a0"}, + {file = "cryptography-45.0.6-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0d9ef57b6768d9fa58e92f4947cea96ade1233c0e236db22ba44748ffedca394"}, + {file = "cryptography-45.0.6-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea3c42f2016a5bbf71825537c2ad753f2870191134933196bee408aac397b3d9"}, + {file = "cryptography-45.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:20ae4906a13716139d6d762ceb3e0e7e110f7955f3bc3876e3a07f5daadec5f3"}, + {file = "cryptography-45.0.6-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dac5ec199038b8e131365e2324c03d20e97fe214af051d20c49db129844e8b3"}, + {file = "cryptography-45.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:18f878a34b90d688982e43f4b700408b478102dd58b3e39de21b5ebf6509c301"}, + {file = "cryptography-45.0.6-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5bd6020c80c5b2b2242d6c48487d7b85700f5e0038e67b29d706f98440d66eb5"}, + {file = "cryptography-45.0.6-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:eccddbd986e43014263eda489abbddfbc287af5cddfd690477993dbb31e31016"}, + {file = "cryptography-45.0.6-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:550ae02148206beb722cfe4ef0933f9352bab26b087af00e48fdfb9ade35c5b3"}, + {file = "cryptography-45.0.6-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b64e668fc3528e77efa51ca70fadcd6610e8ab231e3e06ae2bab3b31c2b8ed9"}, + {file = "cryptography-45.0.6-cp37-abi3-win32.whl", hash = "sha256:780c40fb751c7d2b0c6786ceee6b6f871e86e8718a8ff4bc35073ac353c7cd02"}, + {file = "cryptography-45.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:20d15aed3ee522faac1a39fbfdfee25d17b1284bafd808e1640a74846d7c4d1b"}, + {file = "cryptography-45.0.6.tar.gz", hash = "sha256:5c966c732cf6e4a276ce83b6e4c729edda2df6929083a952cc7da973c539c719"}, ] [[package]] @@ -441,13 +445,13 @@ files = [ [[package]] name = "pluggy" -version = "1.5.0" -requires_python = ">=3.8" +version = "1.6.0" +requires_python = ">=3.9" summary = "plugin and hook calling mechanisms for python" groups = ["test"] files = [ - {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, - {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, ] [[package]] @@ -464,13 +468,13 @@ files = [ [[package]] name = "pygments" -version = "2.19.1" +version = "2.19.2" requires_python = ">=3.8" summary = "Pygments is a syntax highlighting package written in Python." groups = ["test"] files = [ - {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, - {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, ] [[package]] @@ -553,13 +557,13 @@ files = [ [[package]] name = "python-jsonpath" -version = "1.3.0" +version = "1.3.1" requires_python = ">=3.8" summary = "JSONPath, JSON Pointer and JSON Patch for Python." groups = ["default", "all"] files = [ - {file = "python_jsonpath-1.3.0-py3-none-any.whl", hash = "sha256:ce586ec5bd934ce97bc2f06600b00437d9684138b77273ced5b70694a8ef3a76"}, - {file = "python_jsonpath-1.3.0.tar.gz", hash = "sha256:ea5eb4d9b1296c8c19cc53538eb0f20fc54128f84571559ee63539e57875fefe"}, + {file = "python_jsonpath-1.3.1-py3-none-any.whl", hash = "sha256:df33fcd56ed6f3eef2eae990b74719c4111f709247ed73cbe1b088063a1aac0c"}, + {file = "python_jsonpath-1.3.1.tar.gz", hash = "sha256:90d348be9c6f0ee070cb6419d4b2b08fcbb07f5a2a34060b8833bef024099315"}, ] [[package]] @@ -647,24 +651,24 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.20250516" +version = "6.0.12.20250809" requires_python = ">=3.9" summary = "Typing stubs for PyYAML" groups = ["default", "all"] files = [ - {file = "types_pyyaml-6.0.12.20250516-py3-none-any.whl", hash = "sha256:8478208feaeb53a34cb5d970c56a7cd76b72659442e733e268a94dc72b2d0530"}, - {file = "types_pyyaml-6.0.12.20250516.tar.gz", hash = "sha256:9f21a70216fc0fa1b216a8176db5f9e0af6eb35d2f2932acb87689d03a5bf6ba"}, + {file = "types_pyyaml-6.0.12.20250809-py3-none-any.whl", hash = "sha256:032b6003b798e7de1a1ddfeefee32fac6486bdfe4845e0ae0e7fb3ee4512b52f"}, + {file = "types_pyyaml-6.0.12.20250809.tar.gz", hash = "sha256:af4a1aca028f18e75297da2ee0da465f799627370d74073e96fee876524f61b5"}, ] [[package]] name = "typing-extensions" -version = "4.13.2" -requires_python = ">=3.8" -summary = "Backported and Experimental Type Hints for Python 3.8+" +version = "4.14.1" +requires_python = ">=3.9" +summary = "Backported and Experimental Type Hints for Python 3.9+" groups = ["default", "all", "tooling"] files = [ - {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, - {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, + {file = "typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76"}, + {file = "typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36"}, ] [[package]] From 75d75e8b36381ea8c62e65aa4251031ab452e04a Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Thu, 14 Aug 2025 09:56:54 -0600 Subject: [PATCH 08/15] Add jitter to scheduler's periodic checks Without jitter, reconciliation of all resources a controller is monitoring can become aligned. This is meant to help scatter that in order to more effectively spread load. --- src/controller/scheduler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/controller/scheduler.py b/src/controller/scheduler.py index 3be73c7..e183877 100644 --- a/src/controller/scheduler.py +++ b/src/controller/scheduler.py @@ -27,6 +27,7 @@ class Configuration[T](NamedTuple): concurrency: int = 5 frequency_seconds: int = 1200 + schedule_jitter: int = 90 timeout_seconds: int = 30 retry_max_retries: int = 10 @@ -259,7 +260,9 @@ async def _worker[T]( # Ok, Skip, or DepSkip. Recheck at scheduled frequency. await requests.put( Request( - at=time.monotonic() + configuration.frequency_seconds, + at=time.monotonic() + + configuration.frequency_seconds + + random.randint(0, configuration.schedule_jitter), payload=request.payload, sys_error_retries=0, user_retries=0, @@ -273,6 +276,7 @@ async def _worker[T]( return False # User-requested Retry case + # Should this have some jitter as well? reconcile_at = time.monotonic() + result.delay await requests.put( From e5fb3f61b77f207b85276d005bd2c0ac47c67dad Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Thu, 14 Aug 2025 09:58:16 -0600 Subject: [PATCH 09/15] Ensure uvicorn starts before the controller Starting uvicorn prior to the controller helps uvicorn start successfully. The cause is not yet clear. --- src/koreo_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/koreo_controller.py b/src/koreo_controller.py index f6bde5b..17eaafd 100644 --- a/src/koreo_controller.py +++ b/src/koreo_controller.py @@ -49,8 +49,8 @@ async def main(): try: async with asyncio.TaskGroup() as main_tg: - main_tg.create_task(controller_main(telemetry_sink=telemetry_sink)) main_tg.create_task(status_main(telemetry_sink=telemetry_sink)) + main_tg.create_task(controller_main(telemetry_sink=telemetry_sink)) except KeyboardInterrupt: logger.info("Initiating shutdown due to user-request.") From 6025c88a9ca5c33febd7aa31cea6157583783fc9 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Fri, 15 Aug 2025 16:58:28 -0600 Subject: [PATCH 10/15] Remove gratuitous try/excepts Removing try/excepts in order to debug an issue with unclean uvicorn shutdowns. --- src/controller/status.py | 46 +++++++++++++--------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/src/controller/status.py b/src/controller/status.py index f7a1e94..e20c8ec 100644 --- a/src/controller/status.py +++ b/src/controller/status.py @@ -62,23 +62,18 @@ async def aggregator( return while True: - try: - telemetry_update = await telemetry_sink.get() - # We don't want to retry if these fail to process. - telemetry_sink.task_done() + telemetry_update = await telemetry_sink.get() + # We don't want to retry if these fail to process. + telemetry_sink.task_done() - telemetry_data["__last_update__"] = time.monotonic() + telemetry_data["__last_update__"] = time.monotonic() - update_source = telemetry_update.get("source") - if not update_source: - logging.debug( - f"Malformed controller telemetry data ({telemetry_update})" - ) - continue + update_source = telemetry_update.get("source") + if not update_source: + logging.debug(f"Malformed controller telemetry data ({telemetry_update})") + continue - telemetry_data[update_source] = telemetry_update.get("telemetry") - except (KeyboardInterrupt, asyncio.CancelledError): - raise + telemetry_data[update_source] = telemetry_update.get("telemetry") async def status_main(telemetry_sink: asyncio.Queue | None = None): @@ -98,20 +93,9 @@ async def status_main(telemetry_sink: asyncio.Queue | None = None): config = uvicorn.Config(app, port=PORT, log_level="info") server = uvicorn.Server(config) - try: - async with asyncio.TaskGroup() as main_tg: - main_tg.create_task( - aggregator( - telemetry_sink=telemetry_sink, telemetry_data=telemetry_data - ), - name="telemetry-aggregator", - ) - main_tg.create_task(server.serve(), name="status-server") - except KeyboardInterrupt: - logger.info("Initiating shutdown due to user stop.") - exit(0) - - except asyncio.CancelledError: - logger.info("Initiating shutdown due to cancel.") - - exit(1) + async with asyncio.TaskGroup() as main_tg: + main_tg.create_task( + aggregator(telemetry_sink=telemetry_sink, telemetry_data=telemetry_data), + name="telemetry-aggregator", + ) + main_tg.create_task(server.serve(), name="status-server") From ab664699beff4db1dcd59d0766be94302fd5ce55 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Fri, 15 Aug 2025 17:24:02 -0600 Subject: [PATCH 11/15] Add logging for SystemExit --- src/controller/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/controller/__init__.py b/src/controller/__init__.py index 64d4b7f..90bd31e 100644 --- a/src/controller/__init__.py +++ b/src/controller/__init__.py @@ -198,9 +198,15 @@ async def controller_main(telemetry_sink: asyncio.Queue | None = None): except KeyboardInterrupt: logger.info("Initiating shutdown due to user-request.") + raise + + except SystemExit: + logger.info("Initiating shutdown due to system exit.") + raise except asyncio.CancelledError: logger.info("Initiating shutdown due to cancel.") + raise except BaseException as err: logger.error(f"Initiating shutdown due to error {err}.") From f84ab6fa5367253fab3e5ab8d6b21152de05dd15 Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Tue, 19 Aug 2025 15:54:41 -0600 Subject: [PATCH 12/15] Add more robust logging around core task groups Python's TaskGroup has several special case errors. If these aren't handled specially, then very verbose errors are dumped as the system exits. --- src/controller/__init__.py | 145 +++++++++++++----------------- src/controller/custom_workflow.py | 96 ++++++++++++++++---- src/controller/status.py | 79 ++++++++++++++-- src/koreo_controller.py | 23 ++++- 4 files changed, 230 insertions(+), 113 deletions(-) diff --git a/src/controller/__init__.py b/src/controller/__init__.py index 90bd31e..8600084 100644 --- a/src/controller/__init__.py +++ b/src/controller/__init__.py @@ -50,53 +50,19 @@ logger = logging.getLogger("controller") -async def _koreo_resource_cache_manager( - api: kr8s.asyncio.Api, - namespace: str, - kind_title: str, - resource_class: type, - preparer, - shutdown_trigger: asyncio.Event, -): - """ - These are long-term (infinite) cache maintainers that will run in the - background to watch for updates to Koreo Resources. - """ +async def _done_watcher(guard: asyncio.Event, task: Awaitable): + """This is to ensure that critical tasks exiting cause a crash.""" try: - await koreo_cache.maintain_cache( - api=api, - namespace=namespace, - api_version=API_VERSION, - plural_kind=f"{kind_title.lower()}s", - kind_title=kind_title, - resource_class=resource_class, - preparer=preparer, - reconnect_timeout=RECONNECT_TIMEOUT, - ) + return await task + finally: - shutdown_trigger.set() + guard.set() -async def _controller_engine_wrapper( - shutdown_trigger: asyncio.Event, controller: Awaitable -): - try: - await controller +class ControllerSystemFailure(Exception): + """Controller system process exited unexpectedly.""" - except KeyboardInterrupt: - logger.debug(f"Controller engine quit due to user quit.") - raise - - except asyncio.CancelledError: - logger.info(f"Controller engine quit due to cancel.") - raise - - except BaseException as err: - logger.error(f"Controller engine quit due to error: {err}.") - raise - - finally: - shutdown_trigger.set() + pass async def controller_main(telemetry_sink: asyncio.Queue | None = None): @@ -155,34 +121,35 @@ async def controller_main(telemetry_sink: asyncio.Queue | None = None): # This means the continue was not hit return - async with asyncio.TaskGroup() as controller_tasks: - shutdown_trigger = asyncio.Event() - - tasks: list[asyncio.Task] = [] + try: + async with asyncio.TaskGroup() as controller_tasks: + shutdown_trigger = asyncio.Event() - if HOT_LOADING: - logger.info("Hot-loading Koreo Resource enabled") - for namespace, kind_title, resource_class, preparer in KOREO_RESOURCES: - tasks.append( + if HOT_LOADING: + logger.info("Hot-loading Koreo Resource enabled") + for namespace, kind_title, resource_class, preparer in KOREO_RESOURCES: controller_tasks.create_task( - _koreo_resource_cache_manager( - api=api, - namespace=namespace, - kind_title=kind_title, - resource_class=resource_class, - preparer=preparer, - shutdown_trigger=shutdown_trigger, + _done_watcher( + guard=shutdown_trigger, + task=koreo_cache.maintain_cache( + api=api, + namespace=namespace, + api_version=API_VERSION, + plural_kind=f"{kind_title.lower()}s", + kind_title=kind_title, + resource_class=resource_class, + preparer=preparer, + reconnect_timeout=RECONNECT_TIMEOUT, + ), ), name=f"cache-maintainer-{kind_title.lower()}", ) - ) - # This is the schedule watcher / dispatcher for workflow crdRefs. - tasks.append( + # This is the schedule watcher / dispatcher for workflow crdRefs. asyncio.create_task( - _controller_engine_wrapper( - shutdown_trigger=shutdown_trigger, - controller=workflow_controller_system( + _done_watcher( + guard=shutdown_trigger, + task=workflow_controller_system( api=api, namespace=RESOURCE_NAMESPACE, workflow_updates_queue=workflow_updates_queue, @@ -191,35 +158,43 @@ async def controller_main(telemetry_sink: asyncio.Queue | None = None): ), name="workflow-controller", ) - ) - try: await shutdown_trigger.wait() - except KeyboardInterrupt: - logger.info("Initiating shutdown due to user-request.") - raise + await shutdown_trigger.wait() + logger.info("Controller system task exited unexpectedly.") - except SystemExit: - logger.info("Initiating shutdown due to system exit.") - raise + _task_cancelled = False + for task in controller_tasks._tasks: + if not task.done(): + continue - except asyncio.CancelledError: - logger.info("Initiating shutdown due to cancel.") - raise + if task.cancelled(): + _task_cancelled = True + continue - except BaseException as err: - logger.error(f"Initiating shutdown due to error {err}.") + if task.exception() is not None: + return - except: - logger.critical(f"Initiating shutdown due to non-error exception.") + if _task_cancelled: + raise asyncio.CancelledError( + "Controller system task cancelled unexpectedly." + ) - logger.info("Shutting down workers") - for task in tasks: - task_name = task.get_name() + raise ControllerSystemFailure( + "Controller system task returned unexpectedly." + ) + + except KeyboardInterrupt: + logger.info("Initiating shutdown due to user-request.") + return - if not (task.done() or task.cancelling()): - task.cancel("System shutdown") - logger.info(f"Stopping {task_name}") + except SystemExit: + logger.info("Initiating shutdown due to system exit.") + return - logger.info("Koreo controller is shutdown") + except (BaseExceptionGroup, ExceptionGroup) as errs: + logger.error("Unhandled exception in controller system main.") + for idx, err in enumerate(errs.exceptions): + logger.error(f"Error[{idx}]: {type(err)}({err})") + raise diff --git a/src/controller/custom_workflow.py b/src/controller/custom_workflow.py index 57d2f8b..fdca957 100644 --- a/src/controller/custom_workflow.py +++ b/src/controller/custom_workflow.py @@ -1,3 +1,4 @@ +from typing import Awaitable import asyncio import logging import os @@ -42,6 +43,20 @@ async def wrapped( return wrapped +async def _done_watcher(guard: asyncio.Event, task: Awaitable): + try: + return await task + + finally: + guard.set() + + +class WorkflowControllerFailure(Exception): + """Workflow controller process exited unexpectedly.""" + + pass + + async def workflow_controller_system( api: kr8s.asyncio.Api, namespace: str, @@ -110,20 +125,67 @@ async def workflow_controller_system( telemetry_sink=telemetry_sink, ) - async with asyncio.TaskGroup() as tg: - tg.create_task( - events.chief_of_the_watch( - api=api, - tg=tg, - watch_requests=workflow_updates_queue, - configuration=event_config, - ), - name="workflow-chief-of-the-watch", - ) - - tg.create_task( - scheduler.orchestrator( - tg=tg, requests=request_queue, configuration=scheduler_config - ), - name="workflow-reconcile-scheduler", - ) + try: + async with asyncio.TaskGroup() as tg: + shutdown_trigger = asyncio.Event() + + tg.create_task( + _done_watcher( + guard=shutdown_trigger, + task=events.chief_of_the_watch( + api=api, + tg=tg, + watch_requests=workflow_updates_queue, + configuration=event_config, + ), + ), + name="workflow-chief-of-the-watch", + ) + + tg.create_task( + _done_watcher( + guard=shutdown_trigger, + task=scheduler.orchestrator( + tg=tg, requests=request_queue, configuration=scheduler_config + ), + ), + name="workflow-reconcile-scheduler", + ) + + await shutdown_trigger.wait() + logger.info("Workflow controller task exited unexpectedly.") + + _task_cancelled = False + for task in tg._tasks: + if not task.done(): + continue + + if task.cancelled(): + _task_cancelled = True + continue + + if task.exception() is not None: + return + + if _task_cancelled: + raise asyncio.CancelledError( + "Workflow controller task cancelled unexpectedly." + ) + + raise WorkflowControllerFailure( + "Workflow controller task exited unexpectedly." + ) + + except KeyboardInterrupt: + logger.info("Workflow controller shutdown due to user-request.") + return + + except SystemExit: + logger.info("Workflow controller shutdown due to system exit.") + return + + except (BaseExceptionGroup, ExceptionGroup) as errs: + logger.error("Unhandled exception in Workflow controller main.") + for idx, err in enumerate(errs.exceptions): + logger.error(f"Error[{idx}]: {type(err)}({err})") + raise diff --git a/src/controller/status.py b/src/controller/status.py index e20c8ec..dfbda5c 100644 --- a/src/controller/status.py +++ b/src/controller/status.py @@ -1,3 +1,4 @@ +from typing import Awaitable import asyncio import copy import logging @@ -76,6 +77,23 @@ async def aggregator( telemetry_data[update_source] = telemetry_update.get("telemetry") +async def _done_watcher(guard: asyncio.Event, task: Awaitable): + try: + return await task + + except KeyboardInterrupt: + pass + + finally: + guard.set() + + +class StatusServiceFailure(Exception): + """Status service process exited unexpectedly.""" + + pass + + async def status_main(telemetry_sink: asyncio.Queue | None = None): logger.info("Koreo Status Server Starting") @@ -90,12 +108,59 @@ async def status_main(telemetry_sink: asyncio.Queue | None = None): ], ) - config = uvicorn.Config(app, port=PORT, log_level="info") + config = uvicorn.Config(app, port=PORT, log_level="info", reload=False, workers=1) server = uvicorn.Server(config) - async with asyncio.TaskGroup() as main_tg: - main_tg.create_task( - aggregator(telemetry_sink=telemetry_sink, telemetry_data=telemetry_data), - name="telemetry-aggregator", - ) - main_tg.create_task(server.serve(), name="status-server") + shutdown_trigger = asyncio.Event() + + try: + async with asyncio.TaskGroup() as main_tg: + main_tg.create_task( + _done_watcher( + guard=shutdown_trigger, + task=aggregator( + telemetry_sink=telemetry_sink, telemetry_data=telemetry_data + ), + ), + name="telemetry-aggregator", + ) + main_tg.create_task( + _done_watcher(guard=shutdown_trigger, task=server.serve()), + name="status-server", + ) + + await shutdown_trigger.wait() + logger.info("Status service task exited unexpectedly.") + + _task_cancelled = False + for task in main_tg._tasks: + if not task.done(): + continue + + if task.cancelled(): + _task_cancelled = True + continue + + if task.exception() is not None: + return + + if _task_cancelled: + raise asyncio.CancelledError( + "Status service task cancelled unexpectedly." + ) + + raise StatusServiceFailure("Status service task exited unexpectedly.") + + except KeyboardInterrupt: + logger.info("Status service shutdown due to user-request.") + return + + except SystemExit: + logger.info("Status service shutdown due to system exit.") + return + + except (BaseExceptionGroup, ExceptionGroup) as errs: + logger.error("Unhandled exception in status process main.") + for idx, err in enumerate(errs.exceptions): + logger.error(f"Error[{idx}]: {type(err)}({err})") + raise diff --git a/src/koreo_controller.py b/src/koreo_controller.py index 17eaafd..a24a2c4 100644 --- a/src/koreo_controller.py +++ b/src/koreo_controller.py @@ -54,19 +54,34 @@ async def main(): except KeyboardInterrupt: logger.info("Initiating shutdown due to user-request.") + except SystemExit: + logger.info("Initiating shutdown due to system exit.") + except asyncio.CancelledError: - logger.info("Initiating shutdown due to cancel.") + logger.debug("Initiating shutdown due to cancellations.") + + except (BaseExceptionGroup, ExceptionGroup) as errs: + logger.error("Unhandled exception in system main.") + for idx, err in enumerate(errs.exceptions): + logger.error(f"Error[{idx}]: {type(err)}({err})") + raise if __name__ == "__main__": asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) try: uvloop.run(main()) + logger.info("System Shutdown.") + except KeyboardInterrupt: - logger.info("Initiating shutdown due to user stop.") + logger.info("Shutdown due to user request.") exit(0) - except asyncio.CancelledError: - logger.info("Initiating shutdown due to cancel.") + except SystemExit: + logger.info("Shutdown due to system exit.") + exit(0) + + except BaseException as err: + logger.critical("System crash", exc_info=True) exit(1) From 910c3706d8cebc11c8fc9a261196adb9840b9ebc Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Tue, 19 Aug 2025 16:18:56 -0600 Subject: [PATCH 13/15] Default to None, not an object --- src/controller/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controller/status.py b/src/controller/status.py index dfbda5c..048b0c5 100644 --- a/src/controller/status.py +++ b/src/controller/status.py @@ -56,10 +56,10 @@ async def _handler(_): async def aggregator( - telemetry_sink: asyncio.Queue | None = asyncio.Queue(), + telemetry_sink: asyncio.Queue | None = None, telemetry_data: dict | None = None, ): - if not telemetry_sink or telemetry_data is None: + if telemetry_sink is None or telemetry_data is None: return while True: From 75299fe03bb1819afbeb681b0bbc251620bec80e Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Wed, 20 Aug 2025 13:49:18 -0600 Subject: [PATCH 14/15] Return on SystemExit or KeyboardInterrupt, raise CancelledError --- src/controller/koreo_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/controller/koreo_cache.py b/src/controller/koreo_cache.py index 967b8f4..4fc0257 100644 --- a/src/controller/koreo_cache.py +++ b/src/controller/koreo_cache.py @@ -111,14 +111,24 @@ async def maintain_cache( metadata=resource.metadata, spec=resource.raw.get("spec"), ) - except (asyncio.CancelledError, KeyboardInterrupt): + + except (SystemExit, KeyboardInterrupt): + logger.debug( + f"Stopping {plural_kind}.{api_version} cache maintainer watch " + "due to system shutdown." + ) + return + + except asyncio.CancelledError: raise + except asyncio.TimeoutError: logger.debug( f"Restarting {plural_kind}.{api_version} cache maintainer watch " - "due to normal reconnect timeout." + "due to reconnect timeout." ) error_retries = 0 + except BaseException as err: error_retries += 1 From 1ec85c25d120fea2eb7cdd081cc3063785ddbdde Mon Sep 17 00:00:00 2001 From: Robert Kluin Date: Wed, 20 Aug 2025 14:04:31 -0600 Subject: [PATCH 15/15] Make hot loading capable of toggling --- src/controller/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controller/__init__.py b/src/controller/__init__.py index 8600084..1f7322a 100644 --- a/src/controller/__init__.py +++ b/src/controller/__init__.py @@ -24,7 +24,7 @@ API_VERSION = f"{API_GROUP}/{DEFAULT_API_VERSION}" -HOT_LOADING = True +HOT_LOADING = int(os.environ.get("HOT_LOADING", "1")) KOREO_NAMESPACE = os.environ.get("KOREO_NAMESPACE", "koreo-testing")