diff --git a/examples/python/datasource.py b/examples/python/datasource.py index 185f87cb..214baef5 100644 --- a/examples/python/datasource.py +++ b/examples/python/datasource.py @@ -48,13 +48,47 @@ def handle_tile_request(tile: mapget.TileFeatureLayer): # Add an attribute layer attr_layer: mapget.Object = feature.attribute_layers().new_layer("rules") attr: mapget.Attribute = attr_layer.new_attribute("SPEED_LIMIT") - # TODO: Add Python bindings for validities. - # attr.set_direction(mapget.Direction.POSITIVE) + attr.validity().new_offset_range( + mapget.ValidityGeometryOffsetType.RELATIVE_LENGTH, + 0.0, + 1.0, + direction=mapget.Direction.POSITIVE) attr.add_field("speedLimit", 50) - # Add a child feature ID - # TODO: Add Python bindings for relations. - # feature.children().append(tile.new_feature_id("Way", [("wayId", 10)])) + # Add a feature relation + target = tile.new_feature_id("Way", [("wayId", 10)]) + relation = feature.add_relation("successor", target) + relation.source_validity().new_complete() + + # Attach source-data provenance. + refs = tile.new_source_data_references([ + ("RawWayLayer", "primary", mapget.SourceDataAddress(0, 128)) + ]) + feature.set_source_data_references(refs) + + +def handle_source_data_request(tile: mapget.TileSourceDataLayer): + compound = tile.new_compound(2) + compound.set_schema_name("example.RawWay") + compound.set_source_data_address(mapget.SourceDataAddress(0, 128)) + compound.add_field("wayId", 0) + compound.add_field("name", "Main St.") + tile.add_root(compound) + + +def handle_locate_request(request: mapget.LocateRequest): + response = mapget.LocateResponse(request) + response.tile_key = mapget.MapTileKey( + mapget.LayerType.FEATURES, + request.map_id, + "WayLayer", + mapget.TileId(12345), + 0) + return [response] + + +def handle_cache_expired(tile_key: mapget.MapTileKey, expired_at_us: int): + print(f"Cached tile expired: {tile_key} at {expired_at_us}us.") # Instantiate a data source with a minimal mandatory set @@ -71,12 +105,18 @@ def handle_tile_request(tile: mapget.TileFeatureLayer): }]] } ] + }, + "RawWayLayer": { + "type": "SourceData" } }, "mapId": "TestMap" }) # Set the callback which is invoked when a tile is requested. ds.on_tile_feature_request(handle_tile_request) +ds.on_tile_sourcedata_request(handle_source_data_request) +ds.on_locate_request(handle_locate_request) +ds.on_cache_expired(handle_cache_expired) # Parse port as optional first argument port = 0 # Pick random free port diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h index 5a00d000..aba55bf9 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-client.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-client.h @@ -53,6 +53,9 @@ class RemoteDataSource : public DataSource DataSourceInfo const& info, TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; + void onCacheExpired( + MapTileKey const& tileKey, + std::chrono::system_clock::time_point expiredAt) override; private: // DataSourceInfo is fetched in the constructor @@ -97,6 +100,9 @@ class RemoteDataSourceProcess : public DataSource DataSourceInfo const& info, TileLayer::LoadStateCallback loadStateCallback = {}) override; std::vector locate(const mapget::LocateRequest &req) override; + void onCacheExpired( + MapTileKey const& tileKey, + std::chrono::system_clock::time_point expiredAt) override; private: std::unique_ptr remoteSource_; diff --git a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h index a0deda49..86c1a40f 100644 --- a/libs/http-datasource/include/mapget/http-datasource/datasource-server.h +++ b/libs/http-datasource/include/mapget/http-datasource/datasource-server.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "mapget/model/sourcedatalayer.h" #include "mapget/model/featurelayer.h" @@ -44,6 +45,13 @@ class DataSourceServer : public HttpServer DataSourceServer& onLocateRequest(std::function(LocateRequest const&)> const&); + /** + * Set the callback which will be invoked when a service reports that a + * cached tile for this remote datasource expired and is being refreshed. + */ + DataSourceServer& onCacheExpired( + std::function const&); + /** * Get the DataSourceInfo metadata which this instance was constructed with. */ diff --git a/libs/http-datasource/src/datasource-client.cpp b/libs/http-datasource/src/datasource-client.cpp index 9fc99f22..6665cc1c 100644 --- a/libs/http-datasource/src/datasource-client.cpp +++ b/libs/http-datasource/src/datasource-client.cpp @@ -158,6 +158,33 @@ std::vector RemoteDataSource::locate(const LocateRequest& req) return responseVector; } +void RemoteDataSource::onCacheExpired( + MapTileKey const& tileKey, + std::chrono::system_clock::time_point expiredAt) +{ + auto& client = httpClients_[(nextClient_++) % httpClients_.size()]; + + auto cacheExpiredReq = drogon::HttpRequest::newHttpRequest(); + cacheExpiredReq->setMethod(drogon::Post); + cacheExpiredReq->setPath("/cache-expired"); + cacheExpiredReq->setContentTypeCode(drogon::CT_APPLICATION_JSON); + cacheExpiredReq->setBody(nlohmann::json{ + {"tileKey", tileKey.toString()}, + {"expiredAt", std::chrono::duration_cast( + expiredAt.time_since_epoch()).count()}, + }.dump()); + + auto [resultCode, response] = client->sendRequest(cacheExpiredReq); + if (resultCode != drogon::ReqResult::Ok || !response || (int)response->statusCode() >= 300) { + log().warn( + "Failed to notify remote data source about cache expiry for {}: {}", + tileKey.toString(), + resultCode != drogon::ReqResult::Ok + ? drogon::to_string(resultCode) + : response ? fmt::format("HTTP {}", (int)response->statusCode()) : "No remote response"); + } +} + std::shared_ptr RemoteDataSource::fromHostPort(const std::string& hostPort) { auto delimiterPos = hostPort.find(':'); @@ -269,4 +296,13 @@ std::vector RemoteDataSourceProcess::locate(const LocateRequest& return remoteSource_->locate(req); } +void RemoteDataSourceProcess::onCacheExpired( + MapTileKey const& tileKey, + std::chrono::system_clock::time_point expiredAt) +{ + if (!remoteSource_) + raise("Remote data source is not initialized."); + remoteSource_->onCacheExpired(tileKey, expiredAt); +} + } diff --git a/libs/http-datasource/src/datasource-server.cpp b/libs/http-datasource/src/datasource-server.cpp index 1d93813c..38940840 100644 --- a/libs/http-datasource/src/datasource-server.cpp +++ b/libs/http-datasource/src/datasource-server.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,7 @@ struct DataSourceServer::Impl throw std::runtime_error("TileSourceDataLayer callback is unset!"); }; std::function(const LocateRequest&)> locateCallback_; + std::function cacheExpiredCallback_; std::shared_ptr strings_; explicit Impl(DataSourceInfo info) : info_(std::move(info)), strings_(std::make_shared(info_.nodeId_)) @@ -59,6 +61,13 @@ DataSourceServer& DataSourceServer::onLocateRequest( return *this; } +DataSourceServer& DataSourceServer::onCacheExpired( + const std::function& callback) +{ + impl_->cacheExpiredCallback_ = callback; + return *this; +} + DataSourceInfo const& DataSourceServer::info() { return impl_->info_; } void DataSourceServer::setup(drogon::HttpAppFramework& app) @@ -194,6 +203,34 @@ void DataSourceServer::setup(drogon::HttpAppFramework& app) } }, {drogon::Post}); + + app.registerHandler( + "/cache-expired", + [this](const drogon::HttpRequestPtr& req, std::function&& callback) + { + try { + if (impl_->cacheExpiredCallback_) { + auto const body = nlohmann::json::parse(std::string(req->body())); + auto const tileKey = MapTileKey(body.at("tileKey").get()); + auto const expiredAtUs = body.at("expiredAt").get(); + auto const expiredAt = std::chrono::system_clock::time_point{ + std::chrono::microseconds{expiredAtUs}}; + impl_->cacheExpiredCallback_(tileKey, expiredAt); + } + + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k204NoContent); + callback(resp); + } + catch (std::exception const& e) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setStatusCode(drogon::k400BadRequest); + resp->setContentTypeCode(drogon::CT_TEXT_PLAIN); + resp->setBody(std::string("Invalid request: ") + e.what()); + callback(resp); + } + }, + {drogon::Post}); } } // namespace mapget diff --git a/libs/pymapget/binding/py-datasource.h b/libs/pymapget/binding/py-datasource.h index 111ed926..29f6acb9 100644 --- a/libs/pymapget/binding/py-datasource.h +++ b/libs/pymapget/binding/py-datasource.h @@ -2,6 +2,7 @@ #include "mapget/http-datasource/datasource-server.h" +#include #include #include #include @@ -9,11 +10,140 @@ namespace py = pybind11; using namespace py::literals; +namespace +{ +py::object datasourceJsonToPython(nlohmann::json const& j) +{ + switch (j.type()) { + case nlohmann::json::value_t::null: + return py::none(); + case nlohmann::json::value_t::boolean: + return py::bool_(j.get()); + case nlohmann::json::value_t::number_integer: + return py::int_(j.get()); + case nlohmann::json::value_t::number_unsigned: + return py::int_(j.get()); + case nlohmann::json::value_t::number_float: + return py::float_(j.get()); + case nlohmann::json::value_t::string: + return py::str(j.get()); + case nlohmann::json::value_t::array: { + py::list result; + for (auto const& item : j) + result.append(datasourceJsonToPython(item)); + return result; + } + case nlohmann::json::value_t::object: { + py::dict result; + for (auto const& [key, value] : j.items()) + result[py::str(key)] = datasourceJsonToPython(value); + return result; + } + default: + mapget::raise("Unsupported JSON value type."); + } +} +} + void bindDataSourceServer(py::module_& m) { using namespace mapget; using namespace simfil; + py::enum_(m, "LayerType") + .value("FEATURES", LayerType::Features) + .value("HEIGHTMAP", LayerType::Heightmap) + .value("ORTHO_IMAGE", LayerType::OrthoImage) + .value("GLTF", LayerType::GLTF) + .value("SOURCE_DATA", LayerType::SourceData); + + py::class_(m, "MapTileKey") + .def(py::init<>(), "Construct an empty map tile key.") + .def(py::init(), py::arg("value"), "Parse a map tile key from its string form.") + .def(py::init(), + py::arg("layer_type"), + py::arg("map_id"), + py::arg("layer_id"), + py::arg("tile_id"), + py::arg("stage") = 0, + "Construct a map tile key from individual components.") + .def_readwrite("layer_type", &MapTileKey::layer_) + .def_readwrite("map_id", &MapTileKey::mapId_) + .def_readwrite("layer_id", &MapTileKey::layerId_) + .def_readwrite("tile_id", &MapTileKey::tileId_) + .def_readwrite("stage", &MapTileKey::stage_) + .def("to_string", &MapTileKey::toString, "Convert this key to its stable string form.") + .def("__str__", &MapTileKey::toString); + + py::class_(m, "LocateRequest") + .def(py::init([](std::string mapId, std::string typeId, KeyValuePairVec const& featureIdParts) { + return LocateRequest(std::move(mapId), std::move(typeId), castToKeyValue(castToKeyValueView(featureIdParts))); + }), + py::arg("map_id"), + py::arg("type_id"), + py::arg("feature_id_parts"), + "Construct a locate request for a feature id.") + .def(py::init([](py::dict const& dict) { + py::module jsonModule = py::module::import("json"); + auto jsonString = jsonModule.attr("dumps")(dict).cast(); + return LocateRequest(nlohmann::json::parse(jsonString)); + }), + py::arg("dict"), + "Construct a locate request from a Python dictionary.") + .def_readwrite("map_id", &LocateRequest::mapId_) + .def_readwrite("type_id", &LocateRequest::typeId_) + .def_property("feature_id_parts", + [](LocateRequest const& self) { + KeyValuePairVec result; + for (auto const& [key, value] : self.featureId_) { + std::visit( + [&result, &key](auto&& vv) { + result.emplace_back(key, vv); + }, + value); + } + return result; + }, + [](LocateRequest& self, KeyValuePairVec const& parts) { + self.setFeatureId(castToKeyValueView(parts)); + }) + .def("set_feature_id", [](LocateRequest& self, KeyValuePairVec const& parts) { + self.setFeatureId(castToKeyValueView(parts)); + }, + py::arg("feature_id_parts"), + "Replace the feature-id parts.") + .def("get_int_id_part", &LocateRequest::getIntIdPart, + py::arg("part_id"), + "Get an integer id part by name.") + .def("get_str_id_part", &LocateRequest::getStrIdPart, + py::arg("part_id"), + "Get a string id part by name.") + .def("to_dict", [](LocateRequest const& self) { + return datasourceJsonToPython(self.serialize()); + }, + "Serialize the request to a Python dictionary.") + .def("to_json", [](LocateRequest const& self) { return self.serialize().dump(); }, + "Serialize the request to a JSON string."); + + py::class_(m, "LocateResponse") + .def(py::init(), + py::arg("request"), + "Construct a response initialized from a request.") + .def(py::init([](py::dict const& dict) { + py::module jsonModule = py::module::import("json"); + auto jsonString = jsonModule.attr("dumps")(dict).cast(); + return LocateResponse(nlohmann::json::parse(jsonString)); + }), + py::arg("dict"), + "Construct a locate response from a Python dictionary.") + .def_readwrite("tile_key", &LocateResponse::tileKey_) + .def("to_dict", [](LocateResponse const& self) { + return datasourceJsonToPython(self.serialize()); + }, + "Serialize the response to a Python dictionary.") + .def("to_json", [](LocateResponse const& self) { return self.serialize().dump(); }, + "Serialize the response to a JSON string."); + py::class_>(m, "DataSourceServer") .def( py::init( @@ -34,13 +164,18 @@ void bindDataSourceServer(py::module_& m) }), R"pbdoc( Construct a DataSource with a DataSourceInfo metadata instance. - )pbdoc", + )pbdoc", py::arg("info_dict")) .def( "on_tile_feature_request", - &DataSourceServer::onTileFeatureRequest, + [](DataSourceServer& self, py::function callback) -> DataSourceServer& { + return self.onTileFeatureRequest( + [callback = std::move(callback)](TileFeatureLayer::Ptr tile) { + py::gil_scoped_acquire gil; + callback(std::move(tile)); + }); + }, py::arg("callback"), - py::call_guard(), R"pbdoc( Set the Callback which will be invoked when a `/tile`-request for a feature layer is received. @@ -51,16 +186,54 @@ void bindDataSourceServer(py::module_& m) )pbdoc") .def( "on_tile_sourcedata_request", - &DataSourceServer::onTileSourceDataRequest, + [](DataSourceServer& self, py::function callback) -> DataSourceServer& { + return self.onTileSourceDataRequest( + [callback = std::move(callback)](TileSourceDataLayer::Ptr tile) { + py::gil_scoped_acquire gil; + callback(std::move(tile)); + }); + }, py::arg("callback"), - py::call_guard(), R"pbdoc( Set the Callback which will be invoked when a `/tile`-request for a source-data layer is received. - The callback argument is a fresh TileFeatureLayer, which the callback must - fill according to the set TileFeatureLayer's layer info and tile id. If an + The callback argument is a fresh TileSourceDataLayer, which the callback must + fill according to the set TileSourceDataLayer's layer info and tile id. If an error occurs while filling the tile, the callback can use - TileFeatureLayer::setError(...) to signal the error downstream. + TileSourceDataLayer::setError(...) to signal the error downstream. + )pbdoc") + .def( + "on_locate_request", + [](DataSourceServer& self, py::function callback) -> DataSourceServer& { + return self.onLocateRequest( + [callback = std::move(callback)](LocateRequest const& request) { + py::gil_scoped_acquire gil; + return callback(request).cast>(); + }); + }, + py::arg("callback"), + R"pbdoc( + Set the Callback which will be invoked when a `/locate` request is received. + The callback receives a LocateRequest and must return a list of LocateResponse objects. + )pbdoc") + .def( + "on_cache_expired", + [](DataSourceServer& self, py::function callback) -> DataSourceServer& { + return self.onCacheExpired( + [callback = std::move(callback)]( + MapTileKey const& tileKey, + std::chrono::system_clock::time_point expiredAt) { + py::gil_scoped_acquire gil; + auto expiredAtUs = std::chrono::duration_cast( + expiredAt.time_since_epoch()).count(); + callback(tileKey, expiredAtUs); + }); + }, + py::arg("callback"), + R"pbdoc( + Set the callback invoked when a service reports that an expired cached + tile for this datasource is being refreshed. The callback receives + (MapTileKey, expired_at_unix_microseconds). )pbdoc") .def( "go", @@ -103,7 +276,7 @@ void bindDataSourceServer(py::module_& m) )pbdoc") .def( "info", - &DataSourceServer::info, + [](DataSourceServer& self) { return datasourceJsonToPython(self.info().toJson()); }, R"pbdoc( Get the DataSourceInfo metadata which this instance was constructed with. )pbdoc"); diff --git a/libs/pymapget/binding/py-layer.h b/libs/pymapget/binding/py-layer.h index 8ec52dc6..6b798f15 100644 --- a/libs/pymapget/binding/py-layer.h +++ b/libs/pymapget/binding/py-layer.h @@ -1,6 +1,8 @@ #pragma once #include "mapget/model/featurelayer.h" +#include "mapget/model/sourcedata.h" +#include "mapget/model/sourcedatalayer.h" #include #include @@ -11,12 +13,173 @@ namespace py = pybind11; using namespace py::literals; +namespace mapget +{ + +struct BoundSourceDataCompound : public BoundModelNode +{ + model_ptr& ptr() { return *modelNodePtr_; } + model_ptr const& ptr() const { return *modelNodePtr_; } + + static void bind(py::module_& m) + { + py::class_(m, "SourceDataCompoundNode") + .def("schema_name", [](BoundSourceDataCompound& self) { + return self.ptr()->schemaName(); + }, + "Get the schema/type name associated with this source-data node.") + .def("set_schema_name", [](BoundSourceDataCompound& self, std::string_view const& name) { + self.ptr()->setSchemaName(name); + }, + py::arg("name"), + "Set the schema/type name associated with this source-data node.") + .def("source_data_address", [](BoundSourceDataCompound& self) { + return self.ptr()->sourceDataAddress(); + }, + "Get the source-data address associated with this node.") + .def("set_source_data_address", [](BoundSourceDataCompound& self, SourceDataAddress const& address) { + self.ptr()->setSourceDataAddress(address); + }, + py::arg("address"), + "Set the source-data address associated with this node.") + .def("add_field", [](BoundSourceDataCompound& self, + std::string_view const& name, + py::object const& pyValue) { + auto cppValue = pyValueToModel(pyValue, self.ptr()->model()); + auto object = self.ptr()->object(); + std::visit( + [&object, &name](auto&& value) { + if constexpr (std::is_same_v, bool>) + object->addBool(name, value); + else + object->addField(name, value); + }, + cppValue); + }, + py::arg("name"), + py::arg("value"), + "Add a field to this source-data compound object.") + .def("to_dict", [](BoundSourceDataCompound& self) { + return nodeToPython(self.node(), self.ptr()->model()); + }, + "Convert this source-data compound object to Python values."); + } + + ModelNode::Ptr node() override { return ModelNode::Ptr(ptr()); } + + explicit BoundSourceDataCompound(model_ptr ptr) + : modelNodePtr_(std::make_shared>(std::move(ptr))) + { + } + + std::shared_ptr> modelNodePtr_; +}; + +inline SourceDataAddress sourceDataAddressFromPython(py::handle value) +{ + if (py::isinstance(value)) { + return value.cast(); + } + return SourceDataAddress(value.cast()); +} + +inline model_ptr makeSourceDataReferences( + TileFeatureLayer& self, + py::iterable const& entries) +{ + std::vector refs; + for (auto const& item : entries) { + std::string layerId; + std::string qualifier; + SourceDataAddress address; + + if (py::isinstance(item)) { + auto dict = py::reinterpret_borrow(item); + py::object layerIdObject; + if (dict.contains("layer_id")) { + layerIdObject = dict["layer_id"]; + } else { + layerIdObject = dict["layerId"]; + } + layerId = py::str(layerIdObject); + qualifier = py::str(dict["qualifier"]); + address = sourceDataAddressFromPython(dict["address"]); + } else { + auto tuple = py::reinterpret_borrow(item); + if (tuple.size() != 3) { + throw py::value_error("source-data references must be dicts or (layer_id, qualifier, address) tuples"); + } + layerId = py::str(tuple[0]); + qualifier = py::str(tuple[1]); + address = sourceDataAddressFromPython(tuple[2]); + } + + auto layerStringId = self.strings()->emplace(layerId); + if (!layerStringId) + raise(layerStringId.error().message); + auto qualifierStringId = self.strings()->emplace(qualifier); + if (!qualifierStringId) + raise(qualifierStringId.error().message); + + refs.push_back(QualifiedSourceDataReference{ + address, + *layerStringId, + *qualifierStringId, + }); + } + + return self.newSourceDataReferenceCollection(std::span{refs}); +} + +} // namespace mapget + void bindTileLayer(py::module_& m) { using namespace mapget; using namespace simfil; - py::class_( + py::class_(m, "TileLayer") + .def("tile_id", &TileLayer::tileId, "Get the layer tile id.") + .def("map_id", &TileLayer::mapId, "Get the map id.") + .def("layer_id", [](TileLayer const& self) { return self.layerInfo()->layerId_; }, + "Get the layer id.") + .def("error", &TileLayer::error, "Get the tile error if one was set.") + .def("set_error", [](TileLayer& self, std::string const& e) { self.setError(e); }, + py::arg("err"), + "Set the tile error.") + .def("error_code", &TileLayer::errorCode, "Get the tile error code if one was set.") + .def("set_error_code", [](TileLayer& self, int code) { self.setErrorCode(code); }, + py::arg("code"), + "Set the tile error code.") + .def("ttl", [](TileLayer const& self) { return self.ttl() ? self.ttl()->count() : -1; }, + "Get the tile TTL in milliseconds, or -1 if unset.") + .def("set_ttl", [](TileLayer& self, int64_t ms) { + if (ms >= 0) + self.setTtl(std::chrono::milliseconds(ms)); + else + self.setTtl(std::nullopt); + }, + py::arg("time_to_live_in_ms"), + "Set the tile TTL in milliseconds, or -1 to clear it.") + .def("set_info", [](TileLayer& self, std::string const& key, simfil::ScalarValueType const& value) { + std::visit( + [&](auto&& vv) { + using V = std::decay_t; + if constexpr (std::is_same_v) { + return; + } else if constexpr (std::is_same_v) { + self.setInfo(key, vv.toHex()); + } else { + self.setInfo(key, vv); + } + }, + value); + }, + py::arg("key"), + py::arg("value"), + "Set a JSON metadata field on this tile."); + + py::class_( m, "TileFeatureLayer") .def( @@ -232,6 +395,33 @@ void bindTileLayer(py::module_& m) R"pbdoc( Create a new geometry of the given type. )pbdoc") + .def( + "new_relation", + [](TileFeatureLayer& self, std::string_view const& name, BoundFeatureId const& target) + { return BoundRelation(self.newRelation(name, target.modelNodePtr_)); }, + py::arg("name"), + py::arg("target"), + R"pbdoc( + Create a new relation object. Attach it to a feature with + Feature.add_relation(relation) or Feature.add_relation(name, target). + )pbdoc") + .def( + "new_validity_collection", + [](TileFeatureLayer& self, size_t initialCapacity) + { return BoundMultiValidity(self.newValidityCollection(initialCapacity)); }, + py::arg("initial_capacity") = 2, + R"pbdoc( + Create a new validity collection which can be attached to attributes or relations. + )pbdoc") + .def( + "new_source_data_references", + [](TileFeatureLayer& self, py::iterable const& entries) + { return BoundSourceDataReferenceCollection(makeSourceDataReferences(self, entries)); }, + py::arg("entries"), + R"pbdoc( + Create qualified source-data references from dictionaries or + (layer_id, qualifier, address) tuples. + )pbdoc") .def( "geojson", [](TileFeatureLayer& self) @@ -246,4 +436,29 @@ void bindTileLayer(py::module_& m) if (i < 0 || i >= sz) throw py::index_error(); return BoundFeature(self.at((size_t)i)); }); + + py::enum_(m, "SourceDataAddressFormat") + .value("UNKNOWN", TileSourceDataLayer::SourceDataAddressFormat::Unknown) + .value("BIT_RANGE", TileSourceDataLayer::SourceDataAddressFormat::BitRange); + + BoundSourceDataCompound::bind(m); + + py::class_(m, "TileSourceDataLayer") + .def("new_compound", [](TileSourceDataLayer& self, size_t initialSize) { + return BoundSourceDataCompound(self.newCompound(initialSize)); + }, + py::arg("initial_size") = 2, + "Create a new source-data compound node.") + .def("add_root", [](TileSourceDataLayer& self, BoundSourceDataCompound const& node) { + self.addRoot(ModelNode::Ptr(node.ptr())); + }, + py::arg("node"), + "Add a source-data compound node as a root of this layer.") + .def("source_data_address_format", &TileSourceDataLayer::sourceDataAddressFormat, + "Get the source-data address format.") + .def("set_source_data_address_format", &TileSourceDataLayer::setSourceDataAddressFormat, + py::arg("format"), + "Set the source-data address format.") + .def("to_json", [](TileSourceDataLayer& self) { return self.toJson().dump(); }, + "Convert this source-data layer to JSON."); } diff --git a/libs/pymapget/binding/py-model.h b/libs/pymapget/binding/py-model.h index d1be095b..3beb064f 100644 --- a/libs/pymapget/binding/py-model.h +++ b/libs/pymapget/binding/py-model.h @@ -4,6 +4,7 @@ #include "mapget/model/feature.h" #include "mapget/model/featurelayer.h" #include "simfil/value.h" +#include "mapget/model/sourcedatareference.h" #include #include @@ -16,7 +17,7 @@ using namespace simfil; namespace mapget { -py::object nodeToPython(model_ptr const& n, TileFeatureLayer& fl, bool checkMultimap = false); +py::object nodeToPython(model_ptr const& n, simfil::ModelPool& model, bool checkMultimap = false); struct BoundModelNode { @@ -152,7 +153,7 @@ struct BoundModelNodeBase : public BoundModelNode using ModelVariant = std::variant; -ModelVariant pyValueToModel(py::object const& pyValue, TileFeatureLayer& model) +ModelVariant pyValueToModel(py::object const& pyValue, simfil::ModelPool& model) { if (py::isinstance(pyValue)) { return pyValue.cast(); @@ -421,27 +422,82 @@ struct BoundGeometryCollection : public BoundModelNode model_ptr modelNodePtr_; }; +struct BoundValidity : public BoundModelNode +{ + static void bind(py::module_& m); + ModelNode::Ptr node() override; + explicit BoundValidity(model_ptr const& ptr); + model_ptr modelNodePtr_; +}; + +struct BoundMultiValidity : public BoundModelNode +{ + static void bind(py::module_& m); + ModelNode::Ptr node() override; + explicit BoundMultiValidity(model_ptr const& ptr); + model_ptr modelNodePtr_; +}; + +struct BoundSourceDataReferenceCollection : public BoundModelNode +{ + static void bind(py::module_& m); + ModelNode::Ptr node() override; + explicit BoundSourceDataReferenceCollection(model_ptr const& ptr); + model_ptr modelNodePtr_; +}; + +struct BoundRelation : public BoundModelNode +{ + static void bind(py::module_& m); + ModelNode::Ptr node() override; + explicit BoundRelation(model_ptr const& ptr); + model_ptr modelNodePtr_; +}; + struct BoundAttribute : public BoundObject { static void bind(py::module_& m) { - py::enum_(m, "Direction") - .value("EMPTY", Validity::Direction::Empty) - .value("POSITIVE", Validity::Direction::Positive) - .value("NEGATIVE", Validity::Direction::Negative) - .value("COMPLETE", Validity::Direction::Both) - .value("NONE", Validity::Direction::None); - auto boundClass = py::class_(m, "Attribute") .def( "validity", - [](BoundAttribute& self) { return self.modelNodePtr_->validityOrNull(); }, - "Get the validity of the attribute.") + [](BoundAttribute& self) { return BoundMultiValidity(self.modelNodePtr_->validity()); }, + "Get or create the attribute validity collection.") + .def( + "validity_or_none", + [](BoundAttribute& self) -> py::object { + if (auto validity = self.modelNodePtr_->validityOrNull()) + return py::cast(BoundMultiValidity(validity)); + return py::none(); + }, + "Get the attribute validity collection if present.") + .def( + "set_validity", + [](BoundAttribute& self, BoundMultiValidity const& validity) { + self.modelNodePtr_->setValidity(validity.modelNodePtr_); + }, + py::arg("validity"), + "Assign an existing validity collection to this attribute.") .def( "name", [](BoundAttribute& self) { return self.modelNodePtr_->name(); }, - "Get the name of the attribute."); + "Get the name of the attribute.") + .def( + "source_data_references", + [](BoundAttribute& self) -> py::object { + if (auto refs = self.modelNodePtr_->sourceDataReferences()) + return py::cast(BoundSourceDataReferenceCollection(refs)); + return py::none(); + }, + "Get source-data references attached to this attribute.") + .def( + "set_source_data_references", + [](BoundAttribute& self, BoundSourceDataReferenceCollection const& refs) { + self.modelNodePtr_->setSourceDataReferences(refs.modelNodePtr_); + }, + py::arg("refs"), + "Attach source-data references to this attribute."); bindObjectMethods(boundClass); boundClass.def("__iter__", [](BoundAttribute& self) { @@ -573,7 +629,25 @@ struct BoundFeatureId : public BoundModelNode .def( "external_map_id", [](BoundFeatureId& self) { return self.modelNodePtr_->externalMapId(); }, - "Get the explicitly stored external map ID, or None for local references."); + "Get the explicitly stored external map ID, or None for local references.") + .def( + "key_value_pairs", + [](BoundFeatureId& self) { + KeyValuePairVec result; + for (auto const& [key, value] : self.modelNodePtr_->keyValuePairs()) { + std::visit( + [&result, &key](auto&& vv) { + using V = std::decay_t; + if constexpr (std::is_same_v) + result.emplace_back(std::string(key), std::string(vv)); + else + result.emplace_back(std::string(key), vv); + }, + value); + } + return result; + }, + "Get feature-id parts as name/value pairs."); } ModelNode::Ptr node() override { return modelNodePtr_; } @@ -616,8 +690,75 @@ struct BoundFeature : public BoundModelNode "Access this feature's attribute layer collection.") .def( "relations", - [](BoundFeature& self) { return BoundArray(self.modelNodePtr_->relations()); }, - "Access this feature's relation list.") + [](BoundFeature& self) { + py::list result; + self.modelNodePtr_->forEachRelation( + [&result](model_ptr const& relation) { + result.append(BoundRelation(relation)); + return true; + }); + return result; + }, + "Get this feature's relations as typed Relation objects.") + .def( + "num_relations", + [](BoundFeature& self) { return self.modelNodePtr_->numRelations(); }, + "Get the number of relations attached to this feature.") + .def( + "relation_at", + [](BoundFeature& self, int64_t i) { + auto sz = (int64_t)self.modelNodePtr_->numRelations(); + if (i < 0) i += sz; + if (i < 0 || i >= sz) throw py::index_error(); + return BoundRelation(self.modelNodePtr_->getRelation((uint32_t)i)); + }, + py::arg("index"), + "Get a typed Relation at the given index.") + .def( + "add_relation", + [](BoundFeature& self, std::string_view const& name, BoundFeatureId const& target) { + return BoundRelation(self.modelNodePtr_->addRelation(name, target.modelNodePtr_)); + }, + py::arg("name"), + py::arg("target"), + "Create and attach a named relation to an existing target FeatureId.") + .def( + "add_relation", + [](BoundFeature& self, BoundRelation const& relation) { + return BoundRelation(self.modelNodePtr_->addRelation(relation.modelNodePtr_)); + }, + py::arg("relation"), + "Attach an existing Relation object to this feature.") + .def( + "add_relation", + [](BoundFeature& self, + std::string_view const& name, + std::string_view const& targetType, + KeyValuePairVec const& targetIdParts) { + return BoundRelation(self.modelNodePtr_->addRelation( + name, + targetType, + castToKeyValueView(targetIdParts))); + }, + py::arg("name"), + py::arg("target_type"), + py::arg("target_id_parts"), + "Create and attach a named relation by target type and id parts.") + .def( + "source_data_references", + [](BoundFeature& self) -> py::object { + if (auto refs = self.modelNodePtr_->sourceDataReferences()) + return py::cast(BoundSourceDataReferenceCollection(refs)); + return py::none(); + }, + "Get source-data references attached to this feature.") + .def( + "set_source_data_references", + [](BoundFeature& self, BoundSourceDataReferenceCollection const& refs) { + self.modelNodePtr_->setSourceDataReferences(refs.modelNodePtr_); + }, + py::arg("refs"), + "Attach source-data references to this feature.") .def( "add_point", [](BoundFeature& self, Point const& p) { @@ -662,6 +803,345 @@ struct BoundFeature : public BoundModelNode model_ptr modelNodePtr_; }; +inline BoundValidity::BoundValidity(model_ptr const& ptr) : modelNodePtr_(ptr) {} + +inline ModelNode::Ptr BoundValidity::node() { return modelNodePtr_; } + +inline void BoundValidity::bind(py::module_& m) +{ + py::enum_(m, "Direction") + .value("EMPTY", Validity::Direction::Empty) + .value("POSITIVE", Validity::Direction::Positive) + .value("NEGATIVE", Validity::Direction::Negative) + .value("COMPLETE", Validity::Direction::Both) + .value("NONE", Validity::Direction::None); + + py::enum_(m, "ValidityGeometryDescriptionType") + .value("NO_GEOMETRY", Validity::NoGeometry) + .value("SIMPLE_GEOMETRY", Validity::SimpleGeometry) + .value("OFFSET_POINT", Validity::OffsetPointValidity) + .value("OFFSET_RANGE", Validity::OffsetRangeValidity) + .value("FEATURE_TRANSITION", Validity::FeatureTransition); + + py::enum_(m, "ValidityGeometryOffsetType") + .value("INVALID", Validity::InvalidOffsetType) + .value("GEO_POSITION", Validity::GeoPosOffset) + .value("BUFFER", Validity::BufferOffset) + .value("RELATIVE_LENGTH", Validity::RelativeLengthOffset) + .value("METRIC_LENGTH", Validity::MetricLengthOffset); + + py::enum_(m, "TransitionEnd") + .value("START", Validity::Start) + .value("END", Validity::End); + + py::class_(m, "Validity") + .def("direction", [](BoundValidity& self) { return self.modelNodePtr_->direction(); }, + "Get the direction in which this validity applies.") + .def("set_direction", [](BoundValidity& self, Validity::Direction direction) { + self.modelNodePtr_->setDirection(direction); + }, + py::arg("direction"), + "Set the direction in which this validity applies.") + .def("geometry_description_type", [](BoundValidity& self) { + return self.modelNodePtr_->geometryDescriptionType(); + }, + "Get the kind of geometry restriction stored by this validity.") + .def("geometry_offset_type", [](BoundValidity& self) { + return self.modelNodePtr_->geometryOffsetType(); + }, + "Get the offset interpretation used by point/range validities.") + .def("geometry_stage", [](BoundValidity& self) { + return self.modelNodePtr_->geometryStage(); + }, + "Get the referenced geometry stage if one is set.") + .def("set_geometry_stage", [](BoundValidity& self, std::optional stage) { + self.modelNodePtr_->setGeometryStage(stage); + }, + py::arg("stage"), + "Set or clear the referenced geometry stage.") + .def("feature_id", [](BoundValidity& self) -> py::object { + if (auto featureId = self.modelNodePtr_->featureId()) + return py::cast(BoundFeatureId(featureId)); + return py::none(); + }, + "Get the feature-id referenced by this validity, if present.") + .def("set_feature_id", [](BoundValidity& self, BoundFeatureId const& featureId) { + self.modelNodePtr_->setFeatureId(featureId.modelNodePtr_); + }, + py::arg("feature_id"), + "Reference another feature's geometry for this validity.") + .def("offset_point", [](BoundValidity& self) { + return self.modelNodePtr_->offsetPoint(); + }, + "Get the stored offset point if this is a point validity.") + .def("set_offset_point", [](BoundValidity& self, Point const& point) { + self.modelNodePtr_->setOffsetPoint(point); + }, + py::arg("point"), + "Set this validity to a geographic point restriction.") + .def("set_offset_point", [](BoundValidity& self, Validity::GeometryOffsetType offsetType, double point) { + self.modelNodePtr_->setOffsetPoint(offsetType, point); + }, + py::arg("offset_type"), + py::arg("point"), + "Set this validity to a one-dimensional point offset restriction.") + .def("offset_range", [](BoundValidity& self) { + return self.modelNodePtr_->offsetRange(); + }, + "Get the stored offset range if this is a range validity.") + .def("set_offset_range", [](BoundValidity& self, Point const& start, Point const& end) { + self.modelNodePtr_->setOffsetRange(start, end); + }, + py::arg("start"), + py::arg("end"), + "Set this validity to a geographic range restriction.") + .def("set_offset_range", [](BoundValidity& self, + Validity::GeometryOffsetType offsetType, + double start, + double end) { + self.modelNodePtr_->setOffsetRange(offsetType, start, end); + }, + py::arg("offset_type"), + py::arg("start"), + py::arg("end"), + "Set this validity to a one-dimensional range offset restriction.") + .def("simple_geometry", [](BoundValidity& self) -> py::object { + if (auto geom = self.modelNodePtr_->simpleGeometry()) + return py::cast(BoundGeometry(geom)); + return py::none(); + }, + "Get the explicit geometry stored by this validity, if present.") + .def("set_simple_geometry", [](BoundValidity& self, BoundGeometry const& geometry) { + self.modelNodePtr_->setSimpleGeometry(geometry.modelNodePtr_); + }, + py::arg("geometry"), + "Store an explicit geometry as this validity's restriction.") + .def("transition_number", [](BoundValidity& self) { + return self.modelNodePtr_->transitionNumber(); + }, + "Get the semantic transition number, if this validity is a transition.") + .def("transition_from_connected_end", [](BoundValidity& self) { + return self.modelNodePtr_->transitionFromConnectedEnd(); + }, + "Get the connected endpoint of the transition source feature.") + .def("transition_to_connected_end", [](BoundValidity& self) { + return self.modelNodePtr_->transitionToConnectedEnd(); + }, + "Get the connected endpoint of the transition target feature."); +} + +inline BoundMultiValidity::BoundMultiValidity(model_ptr const& ptr) : modelNodePtr_(ptr) {} + +inline ModelNode::Ptr BoundMultiValidity::node() { return modelNodePtr_; } + +inline void BoundMultiValidity::bind(py::module_& m) +{ + py::class_(m, "MultiValidity") + .def("new_point", [](BoundMultiValidity& self, + Point const& point, + std::optional geometryStage, + Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newPoint(point, geometryStage, direction)); + }, + py::arg("point"), + py::arg("geometry_stage") = std::nullopt, + py::arg("direction") = Validity::Empty, + "Append a geographic point validity.") + .def("new_offset_point", [](BoundMultiValidity& self, + Validity::GeometryOffsetType offsetType, + double point, + std::optional geometryStage, + Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newPoint(offsetType, point, geometryStage, direction)); + }, + py::arg("offset_type"), + py::arg("point"), + py::arg("geometry_stage") = std::nullopt, + py::arg("direction") = Validity::Empty, + "Append a one-dimensional point-offset validity.") + .def("new_range", [](BoundMultiValidity& self, + Point const& start, + Point const& end, + std::optional geometryStage, + Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newRange(start, end, geometryStage, direction)); + }, + py::arg("start"), + py::arg("end"), + py::arg("geometry_stage") = std::nullopt, + py::arg("direction") = Validity::Empty, + "Append a geographic range validity.") + .def("new_offset_range", [](BoundMultiValidity& self, + Validity::GeometryOffsetType offsetType, + double start, + double end, + std::optional geometryStage, + Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newRange(offsetType, start, end, geometryStage, direction)); + }, + py::arg("offset_type"), + py::arg("start"), + py::arg("end"), + py::arg("geometry_stage") = std::nullopt, + py::arg("direction") = Validity::Empty, + "Append a one-dimensional range-offset validity.") + .def("new_geometry", [](BoundMultiValidity& self, + BoundGeometry const& geometry, + Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newGeometry(geometry.modelNodePtr_, direction)); + }, + py::arg("geometry"), + py::arg("direction") = Validity::Empty, + "Append a validity that stores an explicit geometry.") + .def("new_feature_id", [](BoundMultiValidity& self, + BoundFeatureId const& featureId, + Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newFeatureId(featureId.modelNodePtr_, direction)); + }, + py::arg("feature_id"), + py::arg("direction") = Validity::Empty, + "Append a validity that references another feature's full geometry.") + .def("new_geom_stage", [](BoundMultiValidity& self, + uint32_t geometryStage, + Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newGeomStage(geometryStage, direction)); + }, + py::arg("geometry_stage"), + py::arg("direction") = Validity::Empty, + "Append a validity that references one staged geometry.") + .def("new_complete", [](BoundMultiValidity& self, Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newComplete(direction)); + }, + py::arg("direction") = Validity::Empty, + "Append a validity covering the complete referenced geometry.") + .def("new_direction", [](BoundMultiValidity& self, Validity::Direction direction) { + return BoundValidity(self.modelNodePtr_->newDirection(direction)); + }, + py::arg("direction") = Validity::Empty, + "Append a direction-only validity.") + .def("__len__", [](BoundMultiValidity& self) { return self.modelNodePtr_->size(); }) + .def("__getitem__", [](BoundMultiValidity& self, int64_t i) { + auto sz = (int64_t)self.modelNodePtr_->size(); + if (i < 0) i += sz; + if (i < 0 || i >= sz) throw py::index_error(); + auto node = self.modelNodePtr_->at(i); + BoundModelNodeBase base; + base.modelNodePtr_ = node; + return BoundValidity(base.featureLayer().resolve(*node)); + }) + .def("__iter__", [](BoundMultiValidity& self) { + py::list result; + for (auto i = 0u; i < self.modelNodePtr_->size(); ++i) { + auto node = self.modelNodePtr_->at(i); + BoundModelNodeBase base; + base.modelNodePtr_ = node; + result.append(BoundValidity(base.featureLayer().resolve(*node))); + } + return py::iter(result); + }); +} + +inline BoundSourceDataReferenceCollection::BoundSourceDataReferenceCollection( + model_ptr const& ptr) + : modelNodePtr_(ptr) +{ +} + +inline ModelNode::Ptr BoundSourceDataReferenceCollection::node() { return modelNodePtr_; } + +inline void BoundSourceDataReferenceCollection::bind(py::module_& m) +{ + py::class_(m, "SourceDataAddress") + .def(py::init<>(), "Construct an empty source-data address.") + .def(py::init(), + py::arg("bit_offset"), + py::arg("bit_size"), + "Construct a source-data address from bit offset and bit size.") + .def(py::init(), py::arg("value"), "Construct a source-data address from its packed integer.") + .def("value", &SourceDataAddress::u64, "Get the packed 64-bit address value.") + .def("bit_offset", &SourceDataAddress::bitOffset, "Get the bit offset.") + .def("bit_size", &SourceDataAddress::bitSize, "Get the bit size."); + + py::class_(m, "SourceDataReferenceCollection") + .def("__len__", [](BoundSourceDataReferenceCollection& self) { return self.modelNodePtr_->size(); }) + .def("__iter__", [](BoundSourceDataReferenceCollection& self) { + py::list result; + self.modelNodePtr_->forEachReference([&result](SourceDataReferenceItem const& item) { + py::dict entry; + entry["layer_id"] = py::str(std::string(item.layerId())); + entry["qualifier"] = py::str(std::string(item.qualifier())); + entry["address"] = item.address(); + result.append(entry); + }); + return py::iter(result); + }) + .def("to_list", [](BoundSourceDataReferenceCollection& self) { + py::list result; + self.modelNodePtr_->forEachReference([&result](SourceDataReferenceItem const& item) { + py::dict entry; + entry["layer_id"] = py::str(std::string(item.layerId())); + entry["qualifier"] = py::str(std::string(item.qualifier())); + entry["address"] = py::int_(item.address().u64()); + result.append(entry); + }); + return result; + }, "Convert references to Python dictionaries with packed integer addresses."); +} + +inline BoundRelation::BoundRelation(model_ptr const& ptr) : modelNodePtr_(ptr) {} + +inline ModelNode::Ptr BoundRelation::node() { return modelNodePtr_; } + +inline void BoundRelation::bind(py::module_& m) +{ + py::class_(m, "Relation") + .def("name", [](BoundRelation& self) { return self.modelNodePtr_->name(); }, + "Get the relation name.") + .def("target", [](BoundRelation& self) { return BoundFeatureId(self.modelNodePtr_->target()); }, + "Get the target feature id.") + .def("source_validity", [](BoundRelation& self) { + return BoundMultiValidity(self.modelNodePtr_->sourceValidity()); + }, + "Get or create the source-side validity collection.") + .def("source_validity_or_none", [](BoundRelation& self) -> py::object { + if (auto validity = self.modelNodePtr_->sourceValidityOrNull()) + return py::cast(BoundMultiValidity(validity)); + return py::none(); + }, + "Get the source-side validity collection if present.") + .def("set_source_validity", [](BoundRelation& self, BoundMultiValidity const& validity) { + self.modelNodePtr_->setSourceValidity(validity.modelNodePtr_); + }, + py::arg("validity"), + "Assign an existing source-side validity collection.") + .def("target_validity", [](BoundRelation& self) { + return BoundMultiValidity(self.modelNodePtr_->targetValidity()); + }, + "Get or create the target-side validity collection.") + .def("target_validity_or_none", [](BoundRelation& self) -> py::object { + if (auto validity = self.modelNodePtr_->targetValidityOrNull()) + return py::cast(BoundMultiValidity(validity)); + return py::none(); + }, + "Get the target-side validity collection if present.") + .def("set_target_validity", [](BoundRelation& self, BoundMultiValidity const& validity) { + self.modelNodePtr_->setTargetValidity(validity.modelNodePtr_); + }, + py::arg("validity"), + "Assign an existing target-side validity collection.") + .def("source_data_references", [](BoundRelation& self) -> py::object { + if (auto refs = self.modelNodePtr_->sourceDataReferences()) + return py::cast(BoundSourceDataReferenceCollection(refs)); + return py::none(); + }, + "Get source-data references attached to this relation.") + .def("set_source_data_references", [](BoundRelation& self, BoundSourceDataReferenceCollection const& refs) { + self.modelNodePtr_->setSourceDataReferences(refs.modelNodePtr_); + }, + py::arg("refs"), + "Attach source-data references to this relation."); +} + /// Recursively convert a ModelNode tree to native Python objects. /// Mirrors simfil's ModelNode::toJson() (nodes.cpp) but builds /// py::dict/py::list/scalars directly, avoiding JSON serialization. @@ -672,7 +1152,7 @@ struct BoundFeature : public BoundModelNode /// marker. Only AttributeLayer-level objects can have duplicates, /// so callers should only pass true at that level to avoid overhead. /// - ByteArray scalars: converted to {"_bytes": True, "hex": ..., "number": ...}. -py::object nodeToPython(model_ptr const& n, TileFeatureLayer& fl, bool checkMultimap) +py::object nodeToPython(model_ptr const& n, simfil::ModelPool& model, bool checkMultimap) { auto type = n->type(); if (type == ValueType::Object) { @@ -699,12 +1179,12 @@ py::object nodeToPython(model_ptr const& n, TileFeatureLayer& fl, boo } if (isMultiMap) { for (auto const& [fieldId, child] : n->fields()) { - if (auto key = fl.lookupStringId(fieldId)) { + if (auto key = model.lookupStringId(fieldId)) { auto pyKey = py::str(std::string(*key)); if (d.contains(pyKey)) - d[pyKey].cast().append(nodeToPython(child, fl)); + d[pyKey].cast().append(nodeToPython(child, model)); else - d[pyKey] = py::list(py::make_tuple(nodeToPython(child, fl))); + d[pyKey] = py::list(py::make_tuple(nodeToPython(child, model))); } } d[py::str("_multimap")] = py::bool_(true); @@ -712,15 +1192,15 @@ py::object nodeToPython(model_ptr const& n, TileFeatureLayer& fl, boo } } for (auto const& [fieldId, child] : n->fields()) { - if (auto key = fl.lookupStringId(fieldId)) - d[py::str(std::string(*key))] = nodeToPython(child, fl); + if (auto key = model.lookupStringId(fieldId)) + d[py::str(std::string(*key))] = nodeToPython(child, model); } return d; } if (type == ValueType::Array) { py::list l; for (uint32_t i = 0; i < n->size(); ++i) - l.append(nodeToPython(n->at(i), fl)); + l.append(nodeToPython(n->at(i), model)); return l; } auto v = n->value(); @@ -760,9 +1240,13 @@ void bindModel(py::module& m) mapget::BoundArray::bind(m); mapget::BoundGeometry::bind(m); mapget::BoundGeometryCollection::bind(m); + mapget::BoundFeatureId::bind(m); + mapget::BoundValidity::bind(m); + mapget::BoundMultiValidity::bind(m); + mapget::BoundSourceDataReferenceCollection::bind(m); mapget::BoundAttribute::bind(m); mapget::BoundAttributeLayer::bind(m); mapget::BoundAttributeLayerList::bind(m); - mapget::BoundFeatureId::bind(m); + mapget::BoundRelation::bind(m); mapget::BoundFeature::bind(m); } diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 1af0caf9..15182888 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -11,6 +11,12 @@ add_wheel_test(test-local-example # that does not require networking -f "./cpp-sample-local-service") +add_wheel_test(test-python-bindings-smoke + WORKING_DIRECTORY + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" + COMMANDS + -f "python ${CMAKE_CURRENT_LIST_DIR}/python-bindings-smoke.py") + if (NOT WITH_COVERAGE) # Python wheels currently don't play well with gcov. message(STATUS "WITH_COVERAGE: ${WITH_COVERAGE}: Enabling wheel-based integration test.") diff --git a/test/integration/detect-ports-and-prepare-config-yaml.py b/test/integration/detect-ports-and-prepare-config-yaml.py index df8e652a..03654fc8 100644 --- a/test/integration/detect-ports-and-prepare-config-yaml.py +++ b/test/integration/detect-ports-and-prepare-config-yaml.py @@ -129,7 +129,10 @@ def main() -> int: examples_config = repo_root / "examples" / "config" sample_service = (examples_config / "sample-service.yaml").read_text(encoding="utf-8") - cache_path = str((out_dir / "mapget-cache.db").resolve()) + cache_file = out_dir / "mapget-cache.db" + for stale_cache_file in (cache_file, out_dir / "mapget-cache.db-shm", out_dir / "mapget-cache.db-wal"): + stale_cache_file.unlink(missing_ok=True) + cache_path = str(cache_file.resolve()) (out_dir / "sample-service.yaml").write_text( _patch_cache_dir( _patch_sample_service_yaml(sample_service, mapget_port, datasource_cpp_port, datasource_py_port), diff --git a/test/integration/python-bindings-smoke.py b/test/integration/python-bindings-smoke.py new file mode 100644 index 00000000..5d9357c9 --- /dev/null +++ b/test/integration/python-bindings-smoke.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import urllib.request + +import mapget + + +def _get_json(url: str): + with urllib.request.urlopen(url, timeout=10) as response: + return json.loads(response.read().decode("utf-8")) + + +def _post_json(url: str, body: dict): + request = urllib.request.Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={"content-type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=10) as response: + payload = response.read().decode("utf-8") + return json.loads(payload) if payload else None + + +def main() -> int: + point = mapget.Point + cache_expired_calls: list[tuple[str, int]] = [] + + def fill_feature_tile(tile: mapget.TileFeatureLayer) -> None: + feature = tile.new_feature("Way", [("wayId", 1)]) + feature.add_line([point(1.0, 2.0), point(2.0, 3.0)]) + + attr = feature.attribute_layers().new_layer("rules").new_attribute("speed") + attr.validity().new_offset_range( + mapget.ValidityGeometryOffsetType.RELATIVE_LENGTH, + 0.1, + 0.5, + direction=mapget.Direction.POSITIVE, + ) + attr.add_field("value", 42) + + target = tile.new_feature_id("Way", [("wayId", 2)]) + relation = feature.add_relation("next", target) + relation.source_validity().new_complete() + + source_refs = tile.new_source_data_references( + [("RawLayer", "primary", mapget.SourceDataAddress(10, 20))] + ) + feature.set_source_data_references(source_refs) + + def fill_source_data_tile(tile: mapget.TileSourceDataLayer) -> None: + compound = tile.new_compound(2) + compound.set_schema_name("example.Type") + compound.set_source_data_address(mapget.SourceDataAddress(1, 8)) + compound.add_field("answer", 42) + tile.add_root(compound) + + def locate(request: mapget.LocateRequest) -> list[mapget.LocateResponse]: + response = mapget.LocateResponse(request) + response.tile_key = mapget.MapTileKey( + mapget.LayerType.FEATURES, + request.map_id, + "WayLayer", + mapget.TileId(1), + 0, + ) + return [response] + + def on_cache_expired(tile_key: mapget.MapTileKey, expired_at_us: int) -> None: + cache_expired_calls.append((tile_key.to_string(), expired_at_us)) + + datasource = mapget.DataSourceServer( + { + "nodeId": "python-bindings-smoke", + "mapId": "Map", + "layers": { + "WayLayer": { + "featureTypes": [ + { + "name": "Way", + "uniqueIdCompositions": [[{"partId": "wayId", "datatype": "I64"}]], + } + ] + }, + "RawLayer": {"type": "SourceData"}, + }, + } + ) + datasource.on_tile_feature_request(fill_feature_tile) + datasource.on_tile_sourcedata_request(fill_source_data_tile) + datasource.on_locate_request(locate) + datasource.on_cache_expired(on_cache_expired) + + datasource.go("127.0.0.1", 0, 1000) + try: + base_url = f"http://127.0.0.1:{datasource.port()}" + + feature_tile = _get_json(f"{base_url}/tile?layer=WayLayer&tileId=1&responseType=json") + feature = feature_tile["features"][0] + assert feature["relations"][0]["name"] == "next" + assert feature["relations"][0]["sourceValidity"]["direction"] == "COMPLETE" + assert feature["properties"]["layer"]["rules"]["speed"]["validity"]["offsetType"] == "RelativeLengthOffset" + assert feature["_sourceData"][0]["qualifier"] == "primary" + + source_tile = _get_json(f"{base_url}/tile?layer=RawLayer&tileId=1&responseType=json") + assert source_tile == [{"answer": 42}] + + locate_response = _post_json( + f"{base_url}/locate", + {"mapId": "Map", "typeId": "Way", "featureId": ["wayId", 1]}, + ) + assert locate_response[0]["tileId"] == "Features:Map:WayLayer:1:0" + + _post_json( + f"{base_url}/cache-expired", + {"tileKey": "Features:Map:WayLayer:1:0", "expiredAt": 123456}, + ) + assert cache_expired_calls == [("Features:Map:WayLayer:1:0", 123456)] + finally: + datasource.stop() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())