From 18437c3926d2d23e47c88bdaff2e75031d2eaa70 Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Mon, 4 May 2026 06:46:39 +0200 Subject: [PATCH 1/5] Support extra nodes in config and better flags for datasource config --- docs/mapget-api.md | 45 ++++- docs/mapget-config.md | 12 +- docs/mapget-dev-guide.md | 8 +- docs/mapget-user-guide.md | 2 +- libs/http-service/src/cli.cpp | 9 +- libs/http-service/src/config-handler.cpp | 105 +++++++++--- libs/http-service/src/tiles-ws-controller.cpp | 27 +++ libs/service/include/mapget/service/config.h | 29 ++++ libs/service/include/mapget/service/service.h | 10 ++ libs/service/src/config.cpp | 126 ++++++++++++++ libs/service/src/service.cpp | 52 +++++- test/unit/test-config.cpp | 47 ++++++ test/unit/test-http-datasource.cpp | 155 ++++++++++++++---- 13 files changed, 553 insertions(+), 74 deletions(-) diff --git a/docs/mapget-api.md b/docs/mapget-api.md index 3c8479ce..7f91ec53 100644 --- a/docs/mapget-api.md +++ b/docs/mapget-api.md @@ -80,6 +80,14 @@ To cancel an in-flight HTTP stream, close the HTTP connection. - `Status` frames contain UTF-8 JSON describing per-request `RequestStatus` transitions and a human-readable message. The final status frame has `"allDone": true`. - `LoadStateChange` exists in the protocol but is currently not emitted by the HTTP service. +Each entry in a status frame's `requests` array contains `index`, `mapId`, `layerId`, numeric `status`, and `statusText`. For `NoDataSource` statuses, servers may also include `noDataSourceReason`: + +- `emptySources` +- `allSourcesDisabled` +- `datasourceInitializationFailed` +- `missingMapOrLayer` +- `noConfig` + To cancel, either send a new request message on the same connection (which replaces the current one) or close the WebSocket connection. ## `/tiles/next` – pull binary tile frames @@ -238,10 +246,17 @@ The page shows the number of active datasources and worker threads, cache statis The response contains: - `timestampMs` -- `service`: service statistics, datasource info, cache occupancy, and optional tile-size-distribution data +- `service`: service statistics, datasource info, cache occupancy, datasource-config counts, and optional tile-size-distribution data - `cache`: cache hit/miss counters and cache sizes - `tilesWebsocket`: control-channel metrics such as active sessions, pending queued frames for `/tiles/next`, blocked pull requests, and total forwarded bytes / frames +`service.datasource-config` reports datasource YAML load diagnostics: + +- `configured`: number of entries under `sources`. +- `enabled`: number of entries not disabled by `enabled: false`. +- `disabled`: number of entries skipped because `enabled: false`. +- `construction-failed`: number of enabled entries whose datasource construction failed. + ## `/locate` – resolve external feature IDs `POST /locate` resolves external feature references to the tile IDs and feature IDs that contain them. This is commonly used together with feature search results or external databases that store map references. @@ -262,7 +277,7 @@ Datasources are free to implement more advanced resolution schemes (for example ## `/config` – inspect and update configuration -The `/config` endpoint family exposes the YAML configuration used by `mapget` for datasource wiring and HTTP settings. It is optional and can be enabled or disabled from the command line. +The `/config` endpoint family exposes the YAML configuration used by `mapget` for datasource wiring and HTTP settings. Command-line flags control whether datasource config is exposed and whether updates are accepted. @@ -271,21 +286,37 @@ The `/config` endpoint family exposes the YAML configuration used by `mapget` fo - **Method:** `GET` - **Request body:** none - **Response:** `application/json` object with the keys: - - `model`: JSON representation of the current YAML config, limited to the `sources` and `http-settings` top‑level keys. - - `schema`: JSON Schema used to validate incoming configurations. + - `schema`: JSON Schema used to validate datasource-model configurations. + - `model`: JSON representation of the current YAML config, limited to datasource-model top-level keys such as `sources` and `http-settings`. - `readOnly`: boolean flag indicating whether `POST /config` is enabled. + - `datasourceConfigUnavailable`: boolean flag indicating that datasource config could not or must not be exposed. + - `datasourceConfigUnavailableReason`: `null` on success, otherwise a stable reason string. + - Additional public sections registered by the embedding application, returned as top-level siblings of `model`. + +When the endpoint handler is reached, `GET /config` returns HTTP `200`. If datasource configuration is unavailable, `schema` and `model` are empty objects, `readOnly` is `true`, and `datasourceConfigUnavailable` is `true`. + +Unavailable reason values are: + +- `getConfigDisabled` +- `configPathUnset` +- `configFileMissing` +- `configFileOpenFailed` +- `configParseFailed` +- `configValidationFailed` + +On a successful datasource-config response, `datasourceConfigUnavailable` is `false` and `datasourceConfigUnavailableReason` is `null`. The returned model masks sensitive fields: any `password` or `api-key` values are replaced with stable masked tokens. -This call is disabled if the server is started with `--no-get-config`. When enabled, it provides a safe way for tools and UIs to read the active configuration without exposing secrets in plain text: any `password` or `api-key` fields are replaced with stable masked tokens. +Registered public sections are read-only. They are included as top-level siblings of `model` on successful responses. On datasource-config unavailable responses, registered public sections are still present but empty. ### `POST /config` - **Method:** `POST` - **Request body:** `application/json` matching the schema returned by `GET /config`. - - Must contain both `sources` and `http-settings` keys at the top level. + - Must contain the datasource-model keys required by the schema. - **Response:** - `text/plain` success message when the configuration was validated, written to disk and successfully applied. - `text/plain` error description and a 4xx/5xx status code if validation or application failed. -This call is only accepted if the server is started with `--allow-post-config`. When a valid configuration is posted, mapget rewrites the underlying YAML file, preserving real secret values where masked tokens were supplied, and then reloads the datasource configuration. Clients should be prepared for temporary 5xx errors if reloading fails. +This call is only accepted if the server is started with `--allow-post-config`. When a valid configuration is posted, mapget rewrites the datasource-model fields in the underlying YAML file, preserving real secret values where masked tokens were supplied, and then reloads the datasource configuration. Unknown top-level YAML sections, including registered public sections, are preserved but not edited through this endpoint. Clients should be prepared for temporary 5xx errors if reloading fails. diff --git a/docs/mapget-config.md b/docs/mapget-config.md index 9fa54eb8..fb1c44ab 100644 --- a/docs/mapget-config.md +++ b/docs/mapget-config.md @@ -24,6 +24,8 @@ For integration with configuration UIs there is an additional top‑level key: - `http-settings` (optional) stores HTTP‑related settings used by frontends or tooling. Mapget itself does not interpret its contents, but exposes it via `/config` and ensures that sensitive fields such as `password` and `api-key` are masked in responses. +Embedded applications can also register additional public top-level sections for `GET /config`. These sections are returned as siblings of `model`, not inside `model`, and are outside mapget's datasource schema. For example, MapViewer registers an `erdblick` section for frontend defaults. `POST /config` remains scoped to datasource-model keys and preserves unknown public sections in the YAML file. + Changes to the `sources` section take effect while the server is running. Changes to options under `mapget` only apply after the server is restarted. ## The `sources` section @@ -39,6 +41,14 @@ By default, the HTTP service registers the following datasource types: Additional datasource types can be registered from C++ code using `DataSourceConfigService`, but those are outside the scope of this guide. +Every datasource entry accepts these generic fields in addition to type-specific fields: + +- `enabled` (optional, default `true`): when set to `false`, the entry is skipped before its type-specific constructor is called. +- `ttl` (optional): cache time-to-live override in seconds. `0` means infinite. +- `auth-header` (optional): header-to-regular-expression map as described below. + +Disabled entries are counted separately in service diagnostics and are not treated as construction failures. + ### Restricting access with `auth-header` @@ -313,7 +323,7 @@ mapget: cache-max-tiles: 20000 # --cache-max-tiles (0 disables the limit) clear-cache: false # --clear-cache allow-post-config: true # --allow-post-config (enables POST /config) - no-get-config: false # --no-get-config (set to true to disable GET /config) + no-get-config: false # --no-get-config (set to true to hide datasource config in GET /config) memory-trim-binary-interval: 100 # --memory-trim-binary-interval memory-trim-json-interval: 0 # --memory-trim-json-interval diff --git a/docs/mapget-dev-guide.md b/docs/mapget-dev-guide.md index 2a66f424..cbe747ed 100644 --- a/docs/mapget-dev-guide.md +++ b/docs/mapget-dev-guide.md @@ -249,10 +249,12 @@ This means `priorityTileIds` affects both backend scheduling and already-produce The HTTP service also implements `/config`: -- `GET /config` reads the YAML configuration file, extracts the `sources` and `http-settings` keys, masks any `password` or `api-key` values and returns the result alongside the JSON Schema configured via `--config-schema`. -- `POST /config` validates an incoming JSON document against that schema, merges it back into the YAML file (unmasking secrets) and relies on `DataSourceConfigService` to apply the updated datasource configuration. +- `GET /config` reads the YAML configuration file, extracts datasource-model keys such as `sources` and `http-settings`, masks any `password` or `api-key` values and returns the result alongside the JSON Schema configured via `--config-schema`. +- `GET /config` also merges public read-only sections registered through `DataSourceConfigService::registerPublicConfigSection(...)` as top-level siblings of `model`. Mapget does not interpret those sections itself. +- If datasource configuration cannot be exposed, `GET /config` still returns HTTP `200` from the handler and reports the problem through `datasourceConfigUnavailable` plus `datasourceConfigUnavailableReason`. +- `POST /config` validates an incoming datasource-model JSON document against the schema, merges it back into the YAML file (unmasking secrets), preserves unknown top-level sections, and relies on `DataSourceConfigService` to apply the updated datasource configuration. -These endpoints are guarded by command‑line flags: `--no-get-config` disables the GET endpoint, and `--allow-post-config` enables the POST endpoint. They are primarily intended for configuration editors and admin tools. +These endpoints are guarded by command-line flags: `--no-get-config` makes `GET /config` return the unavailable response shape, and `--allow-post-config` enables the POST endpoint. They are primarily intended for configuration editors and admin tools. ## Binary streaming and simfil integration diff --git a/docs/mapget-user-guide.md b/docs/mapget-user-guide.md index 5385df53..5294f24e 100644 --- a/docs/mapget-user-guide.md +++ b/docs/mapget-user-guide.md @@ -9,7 +9,7 @@ Most readers will only need the setup and API chapters. The model and configurat The guide is split into several focused documents: - [**Setup Guide**](mapget-setup.md) explains how to install mapget via `pip`, how to build the native executable from source, and how to start a server or use the built‑in `fetch` client for quick experiments. -- [**Configuration Guide**](mapget-config.md) documents the YAML configuration file used with `--config`, the supported datasource types (`DataSourceHost`, `DataSourceProcess`, `GridDataSource`, `GeoJsonFolder) and the optional `http-settings` section used by tools and UIs. +- [**Configuration Guide**](mapget-config.md) documents the YAML configuration file used with `--config`, the supported datasource types (`DataSourceHost`, `DataSourceProcess`, `GridDataSource`, `GeoJsonFolder`), generic datasource fields such as `enabled`, and optional UI-facing sections such as `http-settings`. - [**HTTP / WebSocket API Guide**](mapget-api.md) describes the endpoints exposed by `mapget serve`, including `/sources`, `/tiles`, `/tiles/next`, `/status`, `/status-data`, `/locate` and `/config`, along with their request and response formats and example calls. - [**Caching Guide**](mapget-cache.md) covers the available cache modes (`memory`, `persistent`, `none`), explains how to configure cache size and location, and shows how to inspect cache statistics via the status endpoint. - [**Simfil Language Extensions**](mapget-simfil-extensions.md) introduces the feature model, tiling scheme, geometry and validity concepts, and the binary tile stream format. This chapter is especially relevant if you are writing datasources or low‑level clients. diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp index 90e92e76..e3b4b10d 100644 --- a/libs/http-service/src/cli.cpp +++ b/libs/http-service/src/cli.cpp @@ -72,7 +72,6 @@ nlohmann::json gridDataSourceSchema() return { {"type", "object"}, {"properties", { - {"enabled", {{"type", "boolean"}, {"title", "Enabled"}}}, {"mapId", {{"type", "string"}, {"title", "Map ID"}}}, {"spatialCoherence", {{"type", "boolean"}}}, {"collisionGridSize", {{"type", "number"}}}, @@ -228,13 +227,7 @@ void registerDefaultDatasourceTypes() { dataSourceProcessSchema()); service.registerDataSourceType( "GridDataSource", - [](YAML::Node const& config) -> DataSource::Ptr { - // Check if enabled flag is present and set to false - if (config["enabled"].IsDefined() && !config["enabled"].as()) { - return nullptr; // Skip this datasource - } - return std::make_shared(config); - }, + [](YAML::Node const& config) -> DataSource::Ptr { return std::make_shared(config); }, gridDataSourceSchema()); service.registerDataSourceType( "GeoJsonFolder", diff --git a/libs/http-service/src/config-handler.cpp b/libs/http-service/src/config-handler.cpp index 82ac86e6..b9b13829 100644 --- a/libs/http-service/src/config-handler.cpp +++ b/libs/http-service/src/config-handler.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,43 @@ namespace mapget { +namespace +{ + +constexpr std::string_view kUnavailableReasonGetConfigDisabled = "getConfigDisabled"; +constexpr std::string_view kUnavailableReasonConfigPathUnset = "configPathUnset"; +constexpr std::string_view kUnavailableReasonConfigFileMissing = "configFileMissing"; +constexpr std::string_view kUnavailableReasonConfigFileOpenFailed = "configFileOpenFailed"; +constexpr std::string_view kUnavailableReasonConfigParseFailed = "configParseFailed"; +constexpr std::string_view kUnavailableReasonConfigValidationFailed = "configValidationFailed"; + +[[nodiscard]] nlohmann::json buildUnavailableConfigResponse(std::string_view reason) +{ + auto& configService = DataSourceConfigService::get(); + nlohmann::json response = { + {"schema", nlohmann::json::object()}, + {"model", nlohmann::json::object()}, + {"readOnly", true}, + {"datasourceConfigUnavailable", true}, + {"datasourceConfigUnavailableReason", reason}, + }; + auto publicSections = configService.getPublicConfigSections(YAML::Node{}); + for (auto& [name, value] : publicSections.items()) { + response[name] = std::move(value); + } + return response; +} + +[[nodiscard]] drogon::HttpResponsePtr jsonResponse(nlohmann::json payload) +{ + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k200OK); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + resp->setBody(payload.dump(2)); + return resp; +} + +} // namespace drogon::HttpResponsePtr HttpService::Impl::openConfigFile(std::ifstream& configFile) { @@ -55,49 +93,67 @@ void HttpService::Impl::handleGetConfigRequest( const drogon::HttpRequestPtr& /*req*/, std::function&& callback) { + auto& configService = DataSourceConfigService::get(); + if (!isGetConfigEndpointEnabled()) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k403Forbidden); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody("The GET /config endpoint is disabled by the server administrator."); - callback(resp); + callback(jsonResponse(buildUnavailableConfigResponse(kUnavailableReasonGetConfigDisabled))); return; } - std::ifstream configFile; - if (auto errorResp = openConfigFile(configFile)) { - callback(errorResp); + auto configFilePath = configService.getConfigFilePath(); + if (!configFilePath.has_value()) { + callback(jsonResponse(buildUnavailableConfigResponse(kUnavailableReasonConfigPathUnset))); return; } - nlohmann::json jsonSchema = DataSourceConfigService::get().getDataSourceConfigSchema(); + std::filesystem::path path = *configFilePath; + if (!std::filesystem::exists(path)) { + callback(jsonResponse(buildUnavailableConfigResponse(kUnavailableReasonConfigFileMissing))); + return; + } + + std::ifstream configFile(*configFilePath); + if (!configFile) { + callback(jsonResponse(buildUnavailableConfigResponse(kUnavailableReasonConfigFileOpenFailed))); + return; + } try { YAML::Node configYaml = YAML::Load(configFile); + configService.validateDataSourceConfig(configYaml); + nlohmann::json jsonConfig; std::unordered_map maskedSecretMap; - for (const auto& key : DataSourceConfigService::get().topLevelDataSourceConfigKeys()) { + for (const auto& key : configService.topLevelDataSourceConfigKeys()) { if (auto configYamlEntry = configYaml[key]) jsonConfig[key] = yamlToJson(configYaml[key], true, &maskedSecretMap); } - nlohmann::json combinedJson; - combinedJson["schema"] = jsonSchema; - combinedJson["model"] = jsonConfig; - combinedJson["readOnly"] = !isPostConfigEndpointEnabled(); + nlohmann::json combinedJson = { + {"schema", configService.getDataSourceConfigSchema()}, + {"model", std::move(jsonConfig)}, + {"readOnly", !isPostConfigEndpointEnabled()}, + {"datasourceConfigUnavailable", false}, + {"datasourceConfigUnavailableReason", nullptr}, + }; + auto publicSections = configService.getPublicConfigSections(configYaml); + for (auto& [name, value] : publicSections.items()) { + combinedJson[name] = std::move(value); + } - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k200OK); - resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); - resp->setBody(combinedJson.dump(2)); - callback(resp); + callback(jsonResponse(std::move(combinedJson))); + } + catch (const std::invalid_argument& validationError) { + log().warn("GET /config validation failed: {}", validationError.what()); + callback(jsonResponse(buildUnavailableConfigResponse(kUnavailableReasonConfigValidationFailed))); + } + catch (const YAML::Exception& yamlError) { + log().warn("GET /config parse failed: {}", yamlError.what()); + callback(jsonResponse(buildUnavailableConfigResponse(kUnavailableReasonConfigParseFailed))); } catch (const std::exception& e) { - auto resp = drogon::HttpResponse::newHttpResponse(); - resp->setStatusCode(drogon::k500InternalServerError); - resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); - resp->setBody(std::string("Error processing config file: ") + e.what()); - callback(resp); + log().warn("GET /config failed: {}", e.what()); + callback(jsonResponse(buildUnavailableConfigResponse(kUnavailableReasonConfigParseFailed))); } } @@ -226,4 +282,3 @@ void HttpService::Impl::handlePostConfigRequest( } } // namespace mapget - diff --git a/libs/http-service/src/tiles-ws-controller.cpp b/libs/http-service/src/tiles-ws-controller.cpp index 2632e632..2fe756d6 100644 --- a/libs/http-service/src/tiles-ws-controller.cpp +++ b/libs/http-service/src/tiles-ws-controller.cpp @@ -122,6 +122,25 @@ enum class ClientRequestUpdateMode return "Unknown"; } +[[nodiscard]] std::string_view noDataSourceReasonToString(NoDataSourceReason reason) +{ + switch (reason) { + case NoDataSourceReason::EmptySources: + return "emptySources"; + case NoDataSourceReason::AllSourcesDisabled: + return "allSourcesDisabled"; + case NoDataSourceReason::DatasourceInitializationFailed: + return "datasourceInitializationFailed"; + case NoDataSourceReason::MissingMapOrLayer: + return "missingMapOrLayer"; + case NoDataSourceReason::NoConfig: + return "noConfig"; + case NoDataSourceReason::None: + break; + } + return ""; +} + /// Convert tile load-state enum values to stable UI-facing strings. [[nodiscard]] std::string_view loadStateToString(TileLayer::LoadState s) { @@ -648,6 +667,7 @@ class TilesWsSession : public std::enable_shared_from_this nextRequestInfos.push_back(RequestInfo{ .mapId = parsed.request.mapId, .layerId = parsed.request.layerId, + .noDataSourceReason = parsed.context.noDataSourceReason_, }); std::vector> tileIdsByNextStageToFetch; @@ -837,6 +857,7 @@ class TilesWsSession : public std::enable_shared_from_this { std::string mapId; std::string layerId; + NoDataSourceReason noDataSourceReason = NoDataSourceReason::None; }; /// One queued websocket frame plus metadata used for bookkeeping. @@ -1455,6 +1476,12 @@ class TilesWsSession : public std::enable_shared_from_this reqJson["layerId"] = requestInfos_[i].layerId; reqJson["status"] = static_cast>(status); reqJson["statusText"] = std::string(requestStatusToString(status)); + if (status == RequestStatus::NoDataSource) { + auto reason = noDataSourceReasonToString(requestInfos_[i].noDataSourceReason); + if (!reason.empty()) { + reqJson["noDataSourceReason"] = std::string(reason); + } + } requestsJson.push_back(std::move(reqJson)); } } diff --git a/libs/service/include/mapget/service/config.h b/libs/service/include/mapget/service/config.h index fbc27f05..608e904b 100644 --- a/libs/service/include/mapget/service/config.h +++ b/libs/service/include/mapget/service/config.h @@ -18,6 +18,13 @@ namespace mapget { +struct DataSourceConfigStats { + size_t configured = 0; + size_t enabled = 0; + size_t disabled = 0; + size_t constructionFailed = 0; +}; + /** * Singleton class that watches a particular YAML config file path. @@ -32,6 +39,9 @@ namespace mapget class DataSourceConfigService { public: + using PublicConfigSectionSerializer = + std::function; + /** * Gets the singleton instance of the DataSourceConfig class. * @return Reference to the singleton instance. @@ -124,6 +134,23 @@ class DataSourceConfigService /** Top-level JSON keys allowed by current schema (properties keys). */ [[nodiscard]] std::vector topLevelDataSourceConfigKeys() const; + /** Latest datasource config statistics from the most recent load. */ + [[nodiscard]] DataSourceConfigStats getDataSourceConfigStats() const; + + /** + * Register an additional public top-level config section for GET /config. + * The serializer receives the full YAML config document. + */ + void registerPublicConfigSection( + std::string name, + PublicConfigSectionSerializer serializer); + + /** + * Serialize registered public config sections. + * Every registered key is present in the result. + */ + [[nodiscard]] nlohmann::json getPublicConfigSections(YAML::Node const& fullConfig) const; + /** * Call this to stop the config file watching thread. */ @@ -185,6 +212,8 @@ class DataSourceConfigService std::optional schemaPatch_; mutable std::optional schema_; mutable std::unique_ptr validator_; + std::map publicConfigSectionSerializers_; + DataSourceConfigStats dataSourceConfigStats_; // Atomic flag to control the file watching thread. std::atomic watching_ = false; diff --git a/libs/service/include/mapget/service/service.h b/libs/service/include/mapget/service/service.h index 6d1e1563..e1a7d9ea 100644 --- a/libs/service/include/mapget/service/service.h +++ b/libs/service/include/mapget/service/service.h @@ -24,8 +24,18 @@ enum class RequestStatus { Aborted = 0x4 /** Canceled, e.g. because a bundled request cannot be fulfilled. */ }; +enum class NoDataSourceReason { + None = 0x0, + EmptySources = 0x1, + AllSourcesDisabled = 0x2, + DatasourceInitializationFailed = 0x3, + MissingMapOrLayer = 0x4, + NoConfig = 0x5, +}; + struct LayerRequestContext { RequestStatus status_ = RequestStatus::NoDataSource; + NoDataSourceReason noDataSourceReason_ = NoDataSourceReason::MissingMapOrLayer; LayerType layerType_ = LayerType::Features; uint32_t stages_ = 1; }; diff --git a/libs/service/src/config.cpp b/libs/service/src/config.cpp index 378f745b..26f6d812 100644 --- a/libs/service/src/config.cpp +++ b/libs/service/src/config.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -37,6 +38,36 @@ nlohmann::json authHeaderSchema() }; } +nlohmann::json enabledSchema() +{ + return { + {"type", "boolean"}, + {"title", "Enabled"}, + {"description", "If false, this datasource entry is skipped."} + }; +} + +bool isYamlNodeMeaningful(YAML::Node const& node) +{ + if (!node || node.IsNull()) { + return false; + } + if (node.IsScalar()) { + auto const scalar = node.Scalar(); + if (scalar.empty()) { + return false; + } + return std::any_of( + scalar.begin(), + scalar.end(), + [](unsigned char c) { return !std::isspace(c); }); + } + if (node.IsSequence() || node.IsMap()) { + return node.size() > 0; + } + return true; +} + } // namespace DataSourceConfigService& DataSourceConfigService::get() @@ -130,9 +161,20 @@ void DataSourceConfigService::loadConfig() std::lock_guard memberAccessLock(memberAccessMutex_); validateDataSourceConfig(config); currentConfig_.clear(); + dataSourceConfigStats_ = {}; if (auto sourcesNode = config["sources"]) { for (auto const& node : sourcesNode) + { + ++dataSourceConfigStats_.configured; + const bool enabled = !node["enabled"].IsDefined() || node["enabled"].as(true); + if (enabled) { + ++dataSourceConfigStats_.enabled; + } + else { + ++dataSourceConfigStats_.disabled; + } currentConfig_.push_back(node); + } } else { log().debug(fmt::format("The config file {} does not have a sources node.", configFilePath_)); @@ -165,6 +207,18 @@ void DataSourceConfigService::loadConfig() DataSource::Ptr DataSourceConfigService::makeDataSource(YAML::Node const& descriptor) { + try { + if (auto enabledNode = descriptor["enabled"]; + enabledNode.IsDefined() && !enabledNode.as(true)) + { + return nullptr; + } + } + catch (std::exception const& e) { + log().error("Invalid datasource `enabled` value: {}", e.what()); + return nullptr; + } + if (auto typeNode = descriptor["type"]) { std::lock_guard memberAccessLock(memberAccessMutex_); auto type = typeNode.as(); @@ -365,6 +419,8 @@ nlohmann::json DataSourceConfigService::getDataSourceConfigSchema() const {"enum", nlohmann::json::array({typeName})} }; + if (!properties.contains("enabled")) + properties["enabled"] = enabledSchema(); if (!properties.contains("ttl")) properties["ttl"] = ttlSchema(); if (!properties.contains("auth-header")) @@ -388,6 +444,7 @@ nlohmann::json DataSourceConfigService::getDataSourceConfigSchema() const {"type", "object"}, {"properties", { {"type", typeProperty}, + {"enabled", enabledSchema()}, {"ttl", ttlSchema()}, {"auth-header", authHeaderSchema()} }}, @@ -443,6 +500,73 @@ std::vector DataSourceConfigService::topLevelDataSourceConfigKeys() return keys; } +DataSourceConfigStats DataSourceConfigService::getDataSourceConfigStats() const +{ + std::lock_guard memberAccessLock(memberAccessMutex_); + return dataSourceConfigStats_; +} + +void DataSourceConfigService::registerPublicConfigSection( + std::string name, + PublicConfigSectionSerializer serializer) +{ + if (name.empty()) { + log().warn("Refusing to register public config section with empty name."); + return; + } + if (!serializer) { + log().warn("Refusing to register public config section {} with NULL serializer.", name); + return; + } + + std::lock_guard memberAccessLock(memberAccessMutex_); + publicConfigSectionSerializers_[std::move(name)] = std::move(serializer); +} + +nlohmann::json DataSourceConfigService::getPublicConfigSections(YAML::Node const& fullConfig) const +{ + std::lock_guard memberAccessLock(memberAccessMutex_); + auto result = nlohmann::json::object(); + for (auto const& [name, serializer] : publicConfigSectionSerializers_) { + nlohmann::json section = nlohmann::json::object(); + if (!serializer) { + result[name] = std::move(section); + continue; + } + + try { + auto sectionNode = fullConfig[name]; + if (!isYamlNodeMeaningful(sectionNode)) { + result[name] = std::move(section); + continue; + } + + auto serialized = serializer(fullConfig); + if (serialized.is_object()) { + section = std::move(serialized); + } else { + log().warn( + "Public config section {} serializer returned non-object payload. Replacing with empty object.", + name); + } + } + catch (std::exception const& e) { + log().warn( + "Public config section {} serializer failed: {}. Replacing with empty object.", + name, + e.what()); + } + catch (...) { + log().warn( + "Public config section {} serializer failed with unknown error. Replacing with empty object.", + name); + } + + result[name] = std::move(section); + } + return result; +} + void DataSourceConfigService::validateDataSourceConfig(nlohmann::json json) const { std::lock_guard memberAccessLock(memberAccessMutex_); @@ -573,6 +697,8 @@ void DataSourceConfigService::reset() { configFilePath_.clear(); schema_.reset(); validator_.reset(); + publicConfigSectionSerializers_.clear(); + dataSourceConfigStats_ = {}; end(); } diff --git a/libs/service/src/service.cpp b/libs/service/src/service.cpp index d4101499..d7e1e994 100644 --- a/libs/service/src/service.cpp +++ b/libs/service/src/service.cpp @@ -45,6 +45,18 @@ std::vector> normalizeTileBuckets(std::vector(true); + } catch (...) { + return true; + } +} + } // namespace LayerTilesRequest::LayerTilesRequest( @@ -603,6 +615,7 @@ struct Service::Impl : public Service::Controller mutable std::shared_mutex dataSourcesMutex_; std::unique_ptr configSubscription_; std::vector dataSourcesFromConfig_; + size_t dataSourceConstructionFailed_ = 0; explicit Impl( Cache::Ptr cache, @@ -630,12 +643,18 @@ struct Service::Impl : public Service::Controller // Add datasources present in the new configuration. auto index = 0; std::vector configuredDataSources; + size_t constructionFailures = 0; for (const auto& configNode : dataSourceConfigNodes) { + if (!isDataSourceDescriptorEnabled(configNode)) { + ++index; + continue; + } if (auto dataSource = DataSourceConfigService::get().makeDataSource(configNode)) { addDataSource(dataSource); configuredDataSources.push_back(dataSource); } else { + ++constructionFailures; log().error( "Failed to make datasource at index {}.", index); } @@ -644,6 +663,7 @@ struct Service::Impl : public Service::Controller std::unique_lock lock(dataSourcesMutex_); dataSourcesFromConfig_ = std::move(configuredDataSources); + dataSourceConstructionFailed_ = constructionFailures; }); } @@ -665,6 +685,7 @@ struct Service::Impl : public Service::Controller dataSourceInfo_.clear(); addOnDataSources_.clear(); dataSourcesFromConfig_.clear(); + dataSourceConstructionFailed_ = 0; } // Wake up all workers to check shouldTerminate_. jobsAvailable_.notify_all(); @@ -1038,9 +1059,26 @@ LayerRequestContext Service::resolveLayerRequest( if (layerExists && unauthorized) { result.status_ = RequestStatus::Unauthorized; + result.noDataSourceReason_ = NoDataSourceReason::None; } else { result.status_ = RequestStatus::NoDataSource; + result.noDataSourceReason_ = NoDataSourceReason::MissingMapOrLayer; + + auto const configPath = DataSourceConfigService::get().getConfigFilePath(); + auto const configStats = DataSourceConfigService::get().getDataSourceConfigStats(); + if (!configPath.has_value()) { + result.noDataSourceReason_ = NoDataSourceReason::NoConfig; + } + else if (configStats.configured == 0) { + result.noDataSourceReason_ = NoDataSourceReason::EmptySources; + } + else if (configStats.enabled == 0) { + result.noDataSourceReason_ = NoDataSourceReason::AllSourcesDisabled; + } + else if (impl_->dataSourceInfo_.empty() && impl_->dataSourceConstructionFailed_ > 0) { + result.noDataSourceReason_ = NoDataSourceReason::DatasourceInitializationFailed; + } } return result; } @@ -1145,13 +1183,25 @@ nlohmann::json Service::getStatistics(bool includeCachedFeatureTreeBytes, bool i } size_t activeRequests = 0; + size_t constructionFailures = 0; { std::unique_lock lock(impl_->jobsMutex_); activeRequests = impl_->requests_.size(); } + { + std::shared_lock lock(impl_->dataSourcesMutex_); + constructionFailures = impl_->dataSourceConstructionFailed_; + } + auto configStats = DataSourceConfigService::get().getDataSourceConfigStats(); auto result = nlohmann::json{ {"datasources", datasources}, - {"active-requests", activeRequests} + {"active-requests", activeRequests}, + {"datasource-config", nlohmann::json{ + {"configured", configStats.configured}, + {"enabled", configStats.enabled}, + {"disabled", configStats.disabled}, + {"construction-failed", constructionFailures} + }} }; if (!includeCachedFeatureTreeBytes && !includeTileSizeDistribution) { diff --git a/test/unit/test-config.cpp b/test/unit/test-config.cpp index 5d77cc9c..5f7ef679 100644 --- a/test/unit/test-config.cpp +++ b/test/unit/test-config.cpp @@ -214,3 +214,50 @@ TEST_CASE("Datasource Config", "[DataSourceConfig]") fs::remove_all(tempDir); DataSourceConfigService::get().end(); } + +TEST_CASE("Datasource enabled flag is handled generically", "[DataSourceConfig]") +{ + auto tempDir = fs::temp_directory_path() / test::generateTimestampedDirectoryName("mapget_test_ds_enabled"); + fs::create_directory(tempDir); + auto tempConfigPath = tempDir / "temp_config.yaml"; + + DataSourceConfigService::get().reset(); + DataSourceConfigService::get().registerDataSourceType( + "TestDataSource", + [](const YAML::Node&) -> DataSource::Ptr + { return std::make_shared(); }); + + auto cache = std::make_shared(); + Service service(cache, true); + + { + std::ofstream out(tempConfigPath, std::ios_base::trunc); + out << R"( +sources: + - type: TestDataSource + enabled: false + - type: TestDataSource + enabled: true +)"; + out.flush(); + out.close(); + syncFile(tempConfigPath); + } + + DataSourceConfigService::get().loadConfig(tempConfigPath.string()); + waitForCondition([&service]() { return service.info().size() == 1; }); + + auto stats = DataSourceConfigService::get().getDataSourceConfigStats(); + REQUIRE(stats.configured == 2); + REQUIRE(stats.enabled == 1); + REQUIRE(stats.disabled == 1); + + auto serviceStats = service.getStatistics(); + REQUIRE(serviceStats["datasource-config"]["configured"].get() == 2); + REQUIRE(serviceStats["datasource-config"]["enabled"].get() == 1); + REQUIRE(serviceStats["datasource-config"]["disabled"].get() == 1); + REQUIRE(serviceStats["datasource-config"]["construction-failed"].get() == 0); + + fs::remove_all(tempDir); + DataSourceConfigService::get().end(); +} diff --git a/test/unit/test-http-datasource.cpp b/test/unit/test-http-datasource.cpp index 54948884..c0b84352 100644 --- a/test/unit/test-http-datasource.cpp +++ b/test/unit/test-http-datasource.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -692,6 +693,24 @@ TEST_CASE("HttpDataSource", "[HttpDataSource]") static_cast(RequestStatus::Unauthorized)); } + // WebSocket tiles: include noDataSourceReason in status payload when available. + { + auto req = nlohmann::json::object({ + {"requests", nlohmann::json::array({nlohmann::json::object({ + {"mapId", "UnknownMap"}, + {"layerId", "WayLayer"}, + {"tileIds", nlohmann::json::array({1234})}, + })})}, + }).dump(); + + auto [status, wsTileCount] = runWsTilesRequest(true, req); + REQUIRE(wsTileCount == 0); + REQUIRE(status["requests"].size() == 1); + REQUIRE(status["requests"][0]["status"].get() == + static_cast(RequestStatus::NoDataSource)); + REQUIRE(status["requests"][0]["noDataSourceReason"].get() == "missingMapOrLayer"); + } + // WebSocket tiles: invalid request stays on the same connection, then succeeds. { WsTilesClient wsClient(service.port(), layerInfo); @@ -781,8 +800,8 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") auto tempDir = fs::temp_directory_path() / test::generateTimestampedDirectoryName("mapget_test_http_config"); fs::create_directory(tempDir); auto tempConfigPath = tempDir / "temp_config.yaml"; + auto tempMissingConfigPath = tempDir / "missing_config.yaml"; - // Set up the config file. DataSourceConfigService::get().reset(); struct ConfigWatchGuard { ~ConfigWatchGuard() { DataSourceConfigService::get().end(); } @@ -790,6 +809,13 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") struct SchemaPatchGuard { ~SchemaPatchGuard() { DataSourceConfigService::get().setDataSourceConfigSchemaPatch(nlohmann::json::object()); } } schemaPatchGuard; + struct EndpointToggleGuard { + ~EndpointToggleGuard() + { + setGetConfigEndpointEnabled(true); + setPostConfigEndpointEnabled(false); + } + } endpointToggleGuard; auto schemaPatch = nlohmann::json::parse(R"( { @@ -803,23 +829,21 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") )"); DataSourceConfigService::get().setDataSourceConfigSchemaPatch(schemaPatch); - SECTION("Get Configuration - Config File Not Found") - { - DataSourceConfigService::get().loadConfig(tempConfigPath.string()); - auto [result, res] = cli.get("/config"); - REQUIRE(result == drogon::ReqResult::Ok); - REQUIRE(res != nullptr); - REQUIRE(res->statusCode() == drogon::k404NotFound); - REQUIRE(std::string(res->body()) == "The server does not have a config file."); - } + DataSourceConfigService::get().registerPublicConfigSection( + "publicConfig", + [](YAML::Node const& fullConfig) -> nlohmann::json { + auto section = fullConfig["publicConfig"]; + if (!section || !section.IsMap() || section.size() == 0) { + return nlohmann::json::object(); + } + return yamlToJson(section, false); + }); - // Create config file for tests that need it - { + auto writeConfigFile = [&](std::string const& contents) { std::ofstream configFile(tempConfigPath); - configFile << "sources: []\nhttp-settings: [{'password': 'hunter2'}]"; + configFile << contents; configFile.flush(); configFile.close(); - #ifndef _WIN32 int fd = open(tempConfigPath.c_str(), O_RDONLY); if (fd != -1) { @@ -828,49 +852,122 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") } #endif std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - + }; + + writeConfigFile( + "sources: []\n" + "http-settings: [{'password': 'hunter2'}]\n" + "publicConfig:\n" + " featureFlag: true\n" + "erdblick:\n" + " keepMe: true\n"); DataSourceConfigService::get().loadConfig(tempConfigPath.string()); std::this_thread::sleep_for(std::chrono::milliseconds(500)); - SECTION("Get Configuration - Not allowed") + SECTION("Get Configuration - Config File Missing") + { + DataSourceConfigService::get().loadConfig(tempMissingConfigPath.string()); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(res != nullptr); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto payload = nlohmann::json::parse(std::string(res->body())); + REQUIRE(payload["datasourceConfigUnavailable"].get() == true); + REQUIRE(payload["datasourceConfigUnavailableReason"].get() == "configFileMissing"); + REQUIRE(payload["model"] == nlohmann::json::object()); + REQUIRE(payload["schema"] == nlohmann::json::object()); + REQUIRE(payload["publicConfig"] == nlohmann::json::object()); + } + + SECTION("Get Configuration - Endpoint disabled returns flagged 200") { setGetConfigEndpointEnabled(false); auto [result, res] = cli.get("/config"); REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->statusCode() == drogon::k403Forbidden); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto payload = nlohmann::json::parse(std::string(res->body())); + REQUIRE(payload["datasourceConfigUnavailable"].get() == true); + REQUIRE(payload["datasourceConfigUnavailableReason"].get() == "getConfigDisabled"); + REQUIRE(payload["model"] == nlohmann::json::object()); + REQUIRE(payload["schema"] == nlohmann::json::object()); + REQUIRE(payload["publicConfig"] == nlohmann::json::object()); } SECTION("Get Configuration - No Config File Path Set") { - setGetConfigEndpointEnabled(true); DataSourceConfigService::get().loadConfig(""); auto [result, res] = cli.get("/config"); REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); - REQUIRE(res->statusCode() == drogon::k404NotFound); - REQUIRE(std::string(res->body()) == - "The config file path is not set. Check the server configuration."); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto payload = nlohmann::json::parse(std::string(res->body())); + REQUIRE(payload["datasourceConfigUnavailable"].get() == true); + REQUIRE(payload["datasourceConfigUnavailableReason"].get() == "configPathUnset"); + REQUIRE(payload["model"] == nlohmann::json::object()); + REQUIRE(payload["schema"] == nlohmann::json::object()); + REQUIRE(payload["publicConfig"] == nlohmann::json::object()); + } + + SECTION("Get Configuration - Validation failure is flagged") + { + writeConfigFile("sources: []\n"); + DataSourceConfigService::get().loadConfig(tempConfigPath.string()); + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(res != nullptr); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto payload = nlohmann::json::parse(std::string(res->body())); + REQUIRE(payload["datasourceConfigUnavailable"].get() == true); + REQUIRE(payload["datasourceConfigUnavailableReason"].get() == "configValidationFailed"); + REQUIRE(payload["model"] == nlohmann::json::object()); + REQUIRE(payload["schema"] == nlohmann::json::object()); + REQUIRE(payload["publicConfig"] == nlohmann::json::object()); } SECTION("Get Configuration - Success") { - setGetConfigEndpointEnabled(true); auto [result, res] = cli.get("/config"); REQUIRE(result == drogon::ReqResult::Ok); REQUIRE(res != nullptr); REQUIRE(res->statusCode() == drogon::k200OK); - auto body = std::string(res->body()); - REQUIRE(body.find("sources") != std::string::npos); - REQUIRE(body.find("http-settings") != std::string::npos); + auto payload = nlohmann::json::parse(std::string(res->body())); + REQUIRE(payload["datasourceConfigUnavailable"].get() == false); + REQUIRE(payload["datasourceConfigUnavailableReason"].is_null()); + REQUIRE(payload["readOnly"].get() == true); + REQUIRE(payload["model"].contains("sources")); + REQUIRE(payload["model"].contains("http-settings")); + REQUIRE(payload["model"].contains("publicConfig") == false); + REQUIRE(payload["publicConfig"]["featureFlag"].get() == true); + + auto body = payload.dump(); REQUIRE(body.find("hunter2") == std::string::npos); REQUIRE( body.find("MASKED:0:f52fbd32b2b3b86ff88ef6c490628285f482af15ddcb29541f94bcf526a3f6c7") != std::string::npos); } + SECTION("Get Configuration - Public section serializer exceptions are tolerated") + { + DataSourceConfigService::get().registerPublicConfigSection( + "throwingSection", + [](YAML::Node const&) -> nlohmann::json { throw std::runtime_error("boom"); }); + + auto [result, res] = cli.get("/config"); + REQUIRE(result == drogon::ReqResult::Ok); + REQUIRE(res != nullptr); + REQUIRE(res->statusCode() == drogon::k200OK); + + auto payload = nlohmann::json::parse(std::string(res->body())); + REQUIRE(payload["datasourceConfigUnavailable"].get() == false); + REQUIRE(payload["throwingSection"] == nlohmann::json::object()); + } + SECTION("Post Configuration - Not Enabled") { setPostConfigEndpointEnabled(false); @@ -910,7 +1007,7 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") REQUIRE(std::string(res->body()).starts_with("Validation failed")); } - SECTION("Post Configuration - Valid JSON Config") + SECTION("Post Configuration - Valid JSON Config preserves top-level erdblick") { setPostConfigEndpointEnabled(true); std::string newConfig = R"({ @@ -929,8 +1026,10 @@ TEST_CASE("Configuration Endpoint Tests", "[Configuration]") configContentStream << config.rdbuf(); auto configContent = configContentStream.str(); REQUIRE(configContent.find("hunter2") != std::string::npos); + REQUIRE(configContent.find("erdblick") != std::string::npos); + REQUIRE(configContent.find("keepMe") != std::string::npos); } DataSourceConfigService::get().end(); - fs::remove(tempConfigPath); + fs::remove_all(tempDir); } From f8e87dd466ccd044ce6784f64d4d2cb4e7a697ae Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Tue, 5 May 2026 08:37:45 +0200 Subject: [PATCH 2/5] Add static mount command --- libs/http-service/src/cli.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libs/http-service/src/cli.cpp b/libs/http-service/src/cli.cpp index e3b4b10d..61151bd1 100644 --- a/libs/http-service/src/cli.cpp +++ b/libs/http-service/src/cli.cpp @@ -276,6 +276,7 @@ struct ServeCommand int64_t cacheMaxTiles_ = 1024; bool clearCache_ = false; std::string webapp_; + std::vector staticMounts_; int64_t ttlSeconds_ = 0; uint64_t memoryTrimIntervalBinary_ = HttpServiceConfig{}.memoryTrimIntervalBinary; // Use default from config uint64_t memoryTrimIntervalJson_ = HttpServiceConfig{}.memoryTrimIntervalJson; // Use default from config @@ -325,6 +326,10 @@ struct ServeCommand "-w,--webapp", webapp_, "Serve a static web application, in the format [:]."); + serveCmd->add_option( + "--static-mount", + staticMounts_, + "Serve an additional static filesystem mount, in the format [:]. Can be specified multiple times."); serveCmd->add_flag( "--allow-post-config", isPostConfigEndpointEnabled_, @@ -453,6 +458,16 @@ struct ServeCommand } } + if (!staticMounts_.empty()) { + for (auto const& staticMount : staticMounts_) { + log().info("Static mount: {}", staticMount); + if (!srv.mountFileSystem(staticMount)) { + log().error(" ...failed to mount!"); + raise("Failed to mount static filesystem path."); + } + } + } + srv.go("0.0.0.0", port_); srv.waitForSignal(); } From e065891eef7ece3575b5a36c57b83189de8c7071 Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Tue, 5 May 2026 08:51:37 +0200 Subject: [PATCH 3/5] Update docs --- docs/mapget-config.md | 5 ++++- docs/mapget-dev-guide.md | 1 + docs/mapget-setup.md | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/mapget-config.md b/docs/mapget-config.md index fb1c44ab..561197b4 100644 --- a/docs/mapget-config.md +++ b/docs/mapget-config.md @@ -324,6 +324,9 @@ mapget: clear-cache: false # --clear-cache allow-post-config: true # --allow-post-config (enables POST /config) no-get-config: false # --no-get-config (set to true to hide datasource config in GET /config) + webapp: /srv/my-ui # --webapp, one application document root + static-mount: # --static-mount, additional static aliases + - /assets:/srv/assets memory-trim-binary-interval: 100 # --memory-trim-binary-interval memory-trim-json-interval: 0 # --memory-trim-json-interval @@ -331,6 +334,6 @@ http-settings: ... sources: ... ``` -Adjust or omit fields as needed; unspecified options fall back to the same defaults as the CLI flags (for example, in-memory cache, port 0, GET `/config` enabled, POST `/config` disabled). +Adjust or omit fields as needed; unspecified options fall back to the same defaults as the CLI flags (for example, in-memory cache, port 0, GET `/config` enabled, POST `/config` disabled). Static mount entries use `[:]` syntax and are served as plain files; mapget does not attach application-specific meaning to those files. diff --git a/docs/mapget-dev-guide.md b/docs/mapget-dev-guide.md index cbe747ed..4bf42832 100644 --- a/docs/mapget-dev-guide.md +++ b/docs/mapget-dev-guide.md @@ -221,6 +221,7 @@ For interactive clients, tile streaming uses WebSocket `GET /tiles` as a control - map HTTP/WebSocket endpoints to service calls (`/sources`, `/tiles`, `/tiles/next`, `/status`, `/status-data`, `/locate`, `/config`), - parse JSON requests and build `LayerTilesRequest` objects, - serialize tile responses as JSONL or binary streams, +- mount static filesystem roots configured through `--webapp` and `--static-mount`, - provide `/config` as a JSON view on the YAML config file. ### Tile streaming diff --git a/docs/mapget-setup.md b/docs/mapget-setup.md index b26b1c29..4e3dd811 100644 --- a/docs/mapget-setup.md +++ b/docs/mapget-setup.md @@ -53,6 +53,21 @@ mapget --config examples/config/sample-service.yaml serve When `--config` is used, the HTTP service subscribes to the referenced YAML file. Any changes to its `sources` section are picked up while the server is running. Details of that file format are described in `mapget-config.md`. +## Serving static files + +`mapget serve` can expose static filesystem directories in addition to datasource endpoints: + +```bash +mapget serve \ + --port 8080 \ + --webapp /srv/my-ui \ + --static-mount /assets:/srv/assets +``` + +`--webapp` is the single application document root, usually mounted at `/`. `--static-mount` is for additional static aliases and can be specified multiple times. Both options use `[:]` syntax; if the URL scope is omitted, the directory is mounted at `/`. + +Static mounts are generic HTTP serving functionality. Mapget does not interpret or validate the files beyond requiring the filesystem path to exist and be a directory. Higher-level applications such as MapViewer may use static mounts for their own assets, but application-specific configuration semantics live in those applications, not in mapget. + ## Using the fetch client The `fetch` subcommand is a simple HTTP client for `/tiles`. It requests one map layer for a set of tiles and prints each tile as a JSON feature collection. @@ -78,4 +93,3 @@ For local experimentation it is often useful to increase log verbosity. You can - `MAPGET_LOG_FILE_MAXSIZE` limits the size of a log file before it is rotated. These settings apply to both the Python entry point and the native executable and are especially handy when debugging datasources or cache behaviour. - From 3d2142d97b65b0af8da6c4ee3f087f6afacc4774 Mon Sep 17 00:00:00 2001 From: Wagram Airiian Date: Thu, 7 May 2026 12:12:52 +0200 Subject: [PATCH 4/5] Introduce API for dynamic static mounts --- docs/mapget-dev-guide.md | 1 + docs/mapget-setup.md | 2 +- .../include/mapget/detail/http-server.h | 27 ++ libs/http-datasource/src/http-server.cpp | 234 ++++++++++++++++-- libs/service/src/config.cpp | 29 --- 5 files changed, 237 insertions(+), 56 deletions(-) diff --git a/docs/mapget-dev-guide.md b/docs/mapget-dev-guide.md index 4bf42832..f106c40b 100644 --- a/docs/mapget-dev-guide.md +++ b/docs/mapget-dev-guide.md @@ -222,6 +222,7 @@ For interactive clients, tile streaming uses WebSocket `GET /tiles` as a control - parse JSON requests and build `LayerTilesRequest` objects, - serialize tile responses as JSONL or binary streams, - mount static filesystem roots configured through `--webapp` and `--static-mount`, +- expose `mapget::ensureStaticMount()` and `mapget::removeStaticMount()` for embedding applications that discover additional static roots after startup, - provide `/config` as a JSON view on the YAML config file. ### Tile streaming diff --git a/docs/mapget-setup.md b/docs/mapget-setup.md index 4e3dd811..ece31930 100644 --- a/docs/mapget-setup.md +++ b/docs/mapget-setup.md @@ -66,7 +66,7 @@ mapget serve \ `--webapp` is the single application document root, usually mounted at `/`. `--static-mount` is for additional static aliases and can be specified multiple times. Both options use `[:]` syntax; if the URL scope is omitted, the directory is mounted at `/`. -Static mounts are generic HTTP serving functionality. Mapget does not interpret or validate the files beyond requiring the filesystem path to exist and be a directory. Higher-level applications such as MapViewer may use static mounts for their own assets, but application-specific configuration semantics live in those applications, not in mapget. +Static mounts are generic HTTP serving functionality. Mapget does not interpret or validate the files beyond requiring the filesystem path to exist and be a directory. Higher-level applications such as MapViewer may use static mounts for their own assets, but application-specific configuration semantics live in those applications, not in mapget. Embedded applications that discover static roots after startup can call `mapget::ensureStaticMount()` instead of synthesizing command-line options. ## Using the fetch client diff --git a/libs/http-datasource/include/mapget/detail/http-server.h b/libs/http-datasource/include/mapget/detail/http-server.h index cc6db8cb..28871318 100644 --- a/libs/http-datasource/include/mapget/detail/http-server.h +++ b/libs/http-datasource/include/mapget/detail/http-server.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -90,4 +91,30 @@ class HttpServer std::unique_ptr impl_; }; +/** + * Ensure that a static filesystem mount is available to the running HTTP server. + * This is intended for embedding applications that discover additional static + * assets after the server has already started. The mount is process-global + * because Drogon exposes a process-global application instance. + * + * @param urlPrefix URL path prefix, e.g. `/assets`. + * @param filesystemRoot Directory to serve under the prefix. + * @return true when the mount exists or was added successfully. + */ +bool ensureStaticMount( + std::string const& urlPrefix, + std::filesystem::path const& filesystemRoot); + +/** + * Ensure a static mount using the same `[:]` syntax + * as HttpServer::mountFileSystem(). + */ +bool ensureStaticMount(std::string const& pathFromTo); + +/** + * Remove a runtime static mount previously added through ensureStaticMount(). + * Startup mounts added through HttpServer::mountFileSystem() are not affected. + */ +bool removeStaticMount(std::string const& urlPrefix); + } // namespace mapget diff --git a/libs/http-datasource/src/http-server.cpp b/libs/http-datasource/src/http-server.cpp index e39cb847..73d35884 100644 --- a/libs/http-datasource/src/http-server.cpp +++ b/libs/http-datasource/src/http-server.cpp @@ -2,6 +2,8 @@ #include "mapget/log.h" #include +#include +#include #include #include @@ -12,9 +14,11 @@ #include #include #include +#include #include #include #include +#include #include #include "fmt/format.h" @@ -38,6 +42,12 @@ struct MountPoint std::filesystem::path fsRoot; }; +struct RuntimeStaticMountRegistry +{ + std::mutex mutex; + std::vector mounts; +}; + [[nodiscard]] bool looksLikeWindowsDrivePath(std::string_view s) { if (s.size() < 3) @@ -57,6 +67,133 @@ struct MountPoint return prefix; } +[[nodiscard]] RuntimeStaticMountRegistry& runtimeStaticMountRegistry() +{ + static RuntimeStaticMountRegistry registry; + return registry; +} + +[[nodiscard]] std::optional normalizeMountPoint( + std::string urlPrefix, + std::filesystem::path fsRoot) +{ + urlPrefix = normalizeUrlPrefix(std::move(urlPrefix)); + + std::error_code ec; + fsRoot = std::filesystem::absolute(fsRoot, ec); + if (ec) + return std::nullopt; + fsRoot = std::filesystem::weakly_canonical(fsRoot, ec); + if (ec) + return std::nullopt; + + auto exists = std::filesystem::exists(fsRoot, ec); + if (!exists || ec) + return std::nullopt; + auto isDirectory = std::filesystem::is_directory(fsRoot, ec); + if (!isDirectory || ec) + return std::nullopt; + + return MountPoint{std::move(urlPrefix), std::move(fsRoot)}; +} + +[[nodiscard]] std::optional parseMountPoint(std::string const& pathFromTo) +{ + std::string urlPrefix; + std::string fsRootStr; + + const auto firstColon = pathFromTo.find(':'); + if (firstColon == std::string::npos || looksLikeWindowsDrivePath(pathFromTo)) { + urlPrefix = "/"; + fsRootStr = pathFromTo; + } else { + urlPrefix = pathFromTo.substr(0, firstColon); + fsRootStr = pathFromTo.substr(firstColon + 1); + if (fsRootStr.empty()) + return std::nullopt; + } + + return normalizeMountPoint(std::move(urlPrefix), std::filesystem::path(fsRootStr)); +} + +[[nodiscard]] bool pathIsWithin(std::filesystem::path const& path, std::filesystem::path const& root) +{ + auto pathIt = path.begin(); + for (auto rootIt = root.begin(); rootIt != root.end(); ++rootIt, ++pathIt) { + if (pathIt == path.end() || *pathIt != *rootIt) { + return false; + } + } + return true; +} + +[[nodiscard]] bool requestPathMatchesPrefix(std::string_view requestPath, std::string_view urlPrefix) +{ + if (requestPath == urlPrefix) + return true; + return requestPath.size() > urlPrefix.size() + && requestPath.substr(0, urlPrefix.size()) == urlPrefix + && requestPath[urlPrefix.size()] == '/'; +} + +[[nodiscard]] std::optional resolveMountedRequestPath( + MountPoint const& mount, + std::string const& requestPath) +{ + if (!requestPathMatchesPrefix(requestPath, mount.urlPrefix)) + return std::nullopt; + + auto relativePath = requestPath == mount.urlPrefix + ? std::string() + : requestPath.substr(mount.urlPrefix.size() + 1); + if (relativePath.empty()) + return std::nullopt; + + std::error_code ec; + auto candidate = std::filesystem::weakly_canonical(mount.fsRoot / std::filesystem::path(relativePath), ec); + if (ec || !pathIsWithin(candidate, mount.fsRoot)) + return std::nullopt; + + auto exists = std::filesystem::exists(candidate, ec); + if (!exists || ec) + return std::nullopt; + auto isRegularFile = std::filesystem::is_regular_file(candidate, ec); + if (!isRegularFile || ec) + return std::nullopt; + + return candidate; +} + +[[nodiscard]] std::optional dynamicStaticMountResponse( + drogon::HttpRequestPtr const& req) +{ + if (req->method() != drogon::Get && req->method() != drogon::Head) { + return std::nullopt; + } + + std::vector mounts; + { + auto& registry = runtimeStaticMountRegistry(); + std::lock_guard lock(registry.mutex); + mounts = registry.mounts; + } + if (mounts.empty()) + return std::nullopt; + + std::sort( + mounts.begin(), + mounts.end(), + [](MountPoint const& a, MountPoint const& b) { return a.urlPrefix.size() > b.urlPrefix.size(); }); + + for (auto const& mount : mounts) { + if (auto localPath = resolveMountedRequestPath(mount, req->path())) { + return drogon::HttpResponse::newFileResponse(localPath->string(), "", drogon::CT_NONE, "", req); + } + } + + return std::nullopt; +} + } // namespace struct HttpServer::Impl @@ -150,6 +287,17 @@ void HttpServer::go(std::string const& interfaceAddr, uint16_t port, uint32_t wa } } + app.registerPreRoutingAdvice( + [](drogon::HttpRequestPtr const& req, + drogon::AdviceCallback&& callback, + drogon::AdviceChainCallback&& chainCallback) { + if (auto response = dynamicStaticMountResponse(req)) { + callback(*response); + return; + } + chainCallback(); + }); + // Allow derived class to set up the server. setup(app); @@ -245,43 +393,77 @@ void HttpServer::waitForSignal() bool HttpServer::mountFileSystem(std::string const& pathFromTo) { - std::string urlPrefix; - std::string fsRootStr; + auto mount = parseMountPoint(pathFromTo); + if (!mount) + return false; - const auto firstColon = pathFromTo.find(':'); - if (firstColon == std::string::npos || looksLikeWindowsDrivePath(pathFromTo)) { - urlPrefix = "/"; - fsRootStr = pathFromTo; - } else { - urlPrefix = pathFromTo.substr(0, firstColon); - fsRootStr = pathFromTo.substr(firstColon + 1); - if (fsRootStr.empty()) - return false; - } + std::scoped_lock lock(impl_->mountsMutex_); + impl_->mounts_.emplace_back(std::move(*mount)); + return true; +} - urlPrefix = normalizeUrlPrefix(std::move(urlPrefix)); +void HttpServer::printPortToStdOut(bool enabled) +{ + impl_->printPortToStdout_ = enabled; +} - std::filesystem::path fsRoot(fsRootStr); - std::error_code ec; - fsRoot = std::filesystem::absolute(fsRoot, ec); - if (ec) +bool ensureStaticMount(std::string const& urlPrefix, std::filesystem::path const& filesystemRoot) +{ + auto mount = normalizeMountPoint(urlPrefix, filesystemRoot); + if (!mount) return false; - auto exists = std::filesystem::exists(fsRoot, ec); - if (!exists || ec) - return false; - auto isDirectory = std::filesystem::is_directory(fsRoot, ec); - if (!isDirectory || ec) + auto& registry = runtimeStaticMountRegistry(); + std::lock_guard lock(registry.mutex); + for (auto const& existing : registry.mounts) { + if (existing.urlPrefix != mount->urlPrefix) + continue; + if (existing.fsRoot == mount->fsRoot) + return true; + log().warn( + "Refusing to remount static URL prefix {} from {} to {}.", + mount->urlPrefix, + existing.fsRoot.generic_string(), + mount->fsRoot.generic_string()); return false; + } - std::scoped_lock lock(impl_->mountsMutex_); - impl_->mounts_.emplace_back(MountPoint{std::move(urlPrefix), std::move(fsRoot)}); + log().info( + "Static mount: {}:{}", + mount->urlPrefix, + mount->fsRoot.generic_string()); + registry.mounts.emplace_back(std::move(*mount)); return true; } -void HttpServer::printPortToStdOut(bool enabled) +bool ensureStaticMount(std::string const& pathFromTo) { - impl_->printPortToStdout_ = enabled; + auto mount = parseMountPoint(pathFromTo); + if (!mount) + return false; + return ensureStaticMount(mount->urlPrefix, mount->fsRoot); +} + +bool removeStaticMount(std::string const& urlPrefix) +{ + auto normalizedUrlPrefix = normalizeUrlPrefix(urlPrefix); + auto& registry = runtimeStaticMountRegistry(); + std::lock_guard lock(registry.mutex); + auto previousSize = registry.mounts.size(); + registry.mounts.erase( + std::remove_if( + registry.mounts.begin(), + registry.mounts.end(), + [&normalizedUrlPrefix](MountPoint const& mount) { + return mount.urlPrefix == normalizedUrlPrefix; + }), + registry.mounts.end()); + + if (registry.mounts.size() == previousSize) + return false; + + log().info("Removed static mount: {}", normalizedUrlPrefix); + return true; } } // namespace mapget diff --git a/libs/service/src/config.cpp b/libs/service/src/config.cpp index 26f6d812..1b1d69b8 100644 --- a/libs/service/src/config.cpp +++ b/libs/service/src/config.cpp @@ -1,13 +1,11 @@ #include #include -#include #include #include #include #include #include -#include #include #include @@ -47,27 +45,6 @@ nlohmann::json enabledSchema() }; } -bool isYamlNodeMeaningful(YAML::Node const& node) -{ - if (!node || node.IsNull()) { - return false; - } - if (node.IsScalar()) { - auto const scalar = node.Scalar(); - if (scalar.empty()) { - return false; - } - return std::any_of( - scalar.begin(), - scalar.end(), - [](unsigned char c) { return !std::isspace(c); }); - } - if (node.IsSequence() || node.IsMap()) { - return node.size() > 0; - } - return true; -} - } // namespace DataSourceConfigService& DataSourceConfigService::get() @@ -535,12 +512,6 @@ nlohmann::json DataSourceConfigService::getPublicConfigSections(YAML::Node const } try { - auto sectionNode = fullConfig[name]; - if (!isYamlNodeMeaningful(sectionNode)) { - result[name] = std::move(section); - continue; - } - auto serialized = serializer(fullConfig); if (serialized.is_object()) { section = std::move(serialized); From a6bd408bdc41eb2e509d85e2cf54005e7bb09c44 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 7 May 2026 14:26:12 +0200 Subject: [PATCH 5/5] Use simfil 0.7.2 --- cmake/deps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/deps.cmake b/cmake/deps.cmake index e3abef3e..e2684812 100644 --- a/cmake/deps.cmake +++ b/cmake/deps.cmake @@ -34,7 +34,7 @@ if (NOT "${_mapget_simfil_source_dir}" STREQUAL "") "SIMFIL_SHARED OFF") else() CPMAddPackage( - URI "gh:Klebert-Engineering/simfil@0.7.2#modelnode-data-accessor" + URI "gh:Klebert-Engineering/simfil@0.7.2" OPTIONS "SIMFIL_WITH_MODEL_JSON ON" "SIMFIL_SHARED OFF")