diff --git a/.env.example b/.env.example index 7b97864..3c25e9b 100644 --- a/.env.example +++ b/.env.example @@ -18,7 +18,7 @@ ADMIN_NODES='!aae8900d' # The root URL of the Meshflow API STORAGE_API_ROOT='http://localhost:8000' STORAGE_API_TOKEN=... -STORAGE_API_VERSION=2 +STORAGE_API_VERSION=3 # Use these if you want to upload to a second API (usually used during testing) # STORAGE_API_2_ROOT=... diff --git a/docs/MESHTASTIC.md b/docs/MESHTASTIC.md index e3d8fd4..b0d0b24 100644 --- a/docs/MESHTASTIC.md +++ b/docs/MESHTASTIC.md @@ -15,14 +15,15 @@ The concrete radio implementation is `MeshtasticRadio` in [`src/meshtastic/radio When `STORAGE_API_ROOT` is set and `RADIO_PROTOCOL=meshtastic`: -- **v2:** `POST /api/packets/{my_nodenum}/ingest/` for raw packets. +- **v3 (default):** `POST /api/v3/packets/{my_nodenum}/ingest/` for raw packets; node upsert uses strict `meshtastic_*` field names. +- **v2:** `POST /api/packets/{my_nodenum}/ingest/` (legacy URL prefix; server accepts legacy nested field aliases on node upsert). - **v1:** `POST /api/raw-packet/`. | Variable | Description | |----------|-------------| | `STORAGE_API_ROOT` | Base URL of meshflow-api | | `STORAGE_API_TOKEN` | Bearer / API token | -| `STORAGE_API_VERSION` | `1` or `2` | +| `STORAGE_API_VERSION` | `1`, `2`, or `3` (default `3` for new installs; use `2` only with older API builds) | | `STORAGE_API_2_*` | Optional second destination | Failed uploads can be retained under `data/failed_packets/` when configured. diff --git a/src/api/StorageAPI.py b/src/api/StorageAPI.py index 330d38f..7991cce 100644 --- a/src/api/StorageAPI.py +++ b/src/api/StorageAPI.py @@ -94,17 +94,21 @@ def _get_url(self, path: str, args: Optional[dict] = None) -> str: } else: local_nodenum = self._local_meshtastic_nodenum_provider() + if self.api_version == 3: + prefix = f"/api/v3/packets/{local_nodenum}" + else: + prefix = f"/api/packets/{local_nodenum}" api_paths = { - "raw_packet": f"/api/packets/{local_nodenum}/ingest/", - "nodes": f"/api/packets/{local_nodenum}/nodes/", - "bot_version": f"/api/packets/{local_nodenum}/bot-version/", + "raw_packet": f"{prefix}/ingest/", + "nodes": f"{prefix}/nodes/", + "bot_version": f"{prefix}/bot-version/", "node_by_id": f"/api/nodes/{args.get('node_id', '')}", } return api_paths[path] def report_bot_version(self) -> bool: - """Report meshflow-bot version to the API (v2 only). Returns True on success.""" - if self.api_version != 2: + """Report meshflow-bot version to the API (v2/v3). Returns True on success.""" + if self.api_version not in (2, 3): logger.debug( "Skipping bot version report (api_version=%s)", self.api_version ) @@ -258,7 +262,8 @@ def store_node(self, node: MeshNode) -> Optional[dict]: return response.json() except HTTPError as exc: self._error_counter.increment("storage.store_node.http") - logger.error("HTTP error storing node: %s", exc.response.text) + node_id = getattr(getattr(node, "user", None), "id", None) + logger.error("HTTP error storing node %s: %s", node_id, exc.response.text) except RequestException as exc: self._error_counter.increment("storage.store_node.network") logger.error("Network error storing node: %s", exc) diff --git a/src/meshtastic/serializers.py b/src/meshtastic/serializers.py index 338007c..891ba8a 100644 --- a/src/meshtastic/serializers.py +++ b/src/meshtastic/serializers.py @@ -19,6 +19,15 @@ from src.data_classes import MeshNode +def _meshtastic_location_source_for_api(source: str | None) -> str: + """Map bot/Meshtastic location source strings to meshflow-api labels.""" + if not source: + return "UNSET" + if source in ("LOC_UNSET", "LOC_UNKNOWN"): + return "UNSET" + return source + + class AbstractModelSerializer(ABC): @classmethod def to_api_dict(cls, model) -> dict: @@ -41,23 +50,32 @@ class PositionSerializer(AbstractModelSerializer): @classmethod def to_api_dict(cls, position: MeshNode.Position) -> dict: return { - "logged_time": cls.date_to_api(position.logged_time), # api v1 compatibility - "reported_time": cls.date_to_api(position.reported_time), # api v2 compatibility + "logged_time": cls.date_to_api( + position.logged_time + ), # api v1 compatibility + "reported_time": cls.date_to_api( + position.reported_time + ), # api v2 compatibility "latitude": position.latitude, "longitude": position.longitude, "altitude": position.altitude, - "location_source": position.location_source or "LOC_UNKNOWN", + "meshtastic_location_source": _meshtastic_location_source_for_api( + position.location_source + ), } @classmethod def from_api_dict(cls, position_data: dict) -> MeshNode.Position: return MeshNode.Position( - logged_time=cls.date_from_api(position_data['logged_time']), - reported_time=cls.date_from_api(position_data['reported_time']), - latitude=position_data['latitude'], - longitude=position_data['longitude'], - altitude=position_data['altitude'], - location_source=position_data['location_source'] + logged_time=cls.date_from_api(position_data["logged_time"]), + reported_time=cls.date_from_api(position_data["reported_time"]), + latitude=position_data["latitude"], + longitude=position_data["longitude"], + altitude=position_data["altitude"], + location_source=position_data.get( + "meshtastic_location_source", + position_data.get("location_source", ""), + ), ) @@ -65,24 +83,34 @@ class DeviceMetricsSerializer(AbstractModelSerializer): @classmethod def to_api_dict(cls, device_metrics: MeshNode.DeviceMetrics) -> dict: return { - "logged_time": cls.date_to_api(device_metrics.logged_time), # api v1 compatibility - "reported_time": cls.date_to_api(device_metrics.logged_time), # api v2 compatibility + "logged_time": cls.date_to_api( + device_metrics.logged_time + ), # api v1 compatibility + "reported_time": cls.date_to_api( + device_metrics.logged_time + ), # api v2 compatibility "battery_level": device_metrics.battery_level, "voltage": device_metrics.voltage, - "channel_utilization": device_metrics.channel_utilization, - "air_util_tx": device_metrics.air_util_tx, - "uptime_seconds": device_metrics.uptime_seconds + "meshtastic_channel_utilization": device_metrics.channel_utilization or 0.0, + "meshtastic_air_util_tx": device_metrics.air_util_tx or 0.0, + "uptime_seconds": device_metrics.uptime_seconds, } @classmethod def from_api_dict(cls, device_metrics_data: dict) -> MeshNode.DeviceMetrics: return MeshNode.DeviceMetrics( - logged_time=cls.date_from_api(device_metrics_data['logged_time']), - battery_level=device_metrics_data['battery_level'], - voltage=device_metrics_data['voltage'], - channel_utilization=device_metrics_data['channel_utilization'], - air_util_tx=device_metrics_data['air_util_tx'], - uptime_seconds=device_metrics_data['uptime_seconds'] + logged_time=cls.date_from_api(device_metrics_data["logged_time"]), + battery_level=device_metrics_data["battery_level"], + voltage=device_metrics_data["voltage"], + channel_utilization=device_metrics_data.get( + "meshtastic_channel_utilization", + device_metrics_data.get("channel_utilization", 0.0), + ), + air_util_tx=device_metrics_data.get( + "meshtastic_air_util_tx", + device_metrics_data.get("air_util_tx", 0.0), + ), + uptime_seconds=device_metrics_data["uptime_seconds"], ) @@ -96,40 +124,45 @@ def to_api_dict(cls, node: MeshNode) -> dict: "macaddr": node.user.macaddr, "hw_model": node.user.hw_model, "public_key": node.user.public_key, - 'user': { + "user": { "long_name": node.user.long_name, - "short_name": node.user.short_name - } + "short_name": node.user.short_name, + }, } # only log a position if it's actually set - if node.position and not \ - (node.position.latitude == 0 and node.position.longitude == 0 and node.position.altitude == 0): - node_data['position'] = PositionSerializer.to_api_dict(node.position) + if node.position and not ( + node.position.latitude == 0 + and node.position.longitude == 0 + and node.position.altitude == 0 + ): + node_data["position"] = PositionSerializer.to_api_dict(node.position) if node.device_metrics: - node_data['device_metrics'] = DeviceMetricsSerializer.to_api_dict(node.device_metrics) + node_data["device_metrics"] = DeviceMetricsSerializer.to_api_dict( + node.device_metrics + ) return node_data @classmethod def from_api_dict(cls, node_data: dict) -> MeshNode: - user_data = node_data['user'] + user_data = node_data["user"] user = MeshNode.User( - node_id=node_data['id'], - macaddr=node_data['macaddr'], - hw_model=node_data['hw_model'], - public_key=node_data['public_key'], - long_name=user_data['long_name'], - short_name=user_data['short_name'] + node_id=node_data["id"], + macaddr=node_data["macaddr"], + hw_model=node_data["hw_model"], + public_key=node_data["public_key"], + long_name=user_data["long_name"], + short_name=user_data["short_name"], ) - position_data = node_data.get('position') + position_data = node_data.get("position") position = None if position_data: position = PositionSerializer.from_api_dict(position_data) - device_metrics_data = node_data.get('device_metrics') + device_metrics_data = node_data.get("device_metrics") device_metrics = None if device_metrics_data: device_metrics = DeviceMetricsSerializer.from_api_dict(device_metrics_data) diff --git a/test/meshtastic/test_serializers.py b/test/meshtastic/test_serializers.py index 7459563..3b3000b 100644 --- a/test/meshtastic/test_serializers.py +++ b/test/meshtastic/test_serializers.py @@ -45,6 +45,38 @@ def test_serialise_node_round_trip(self): self.assertEqual(node2.user.id, "!12345678") self.assertEqual(node2.user.long_name, "Alice") + def test_serialise_node_uses_meshtastic_api_field_names(self): + import datetime + + node = MeshNode() + node.user = MeshNode.User(node_id="!abcdef12", long_name="Bob", short_name="B") + node.position = MeshNode.Position( + logged_time=datetime.datetime( + 2026, 5, 22, 7, 0, 0, tzinfo=datetime.timezone.utc + ), + reported_time=datetime.datetime( + 2026, 5, 22, 7, 0, 0, tzinfo=datetime.timezone.utc + ), + latitude=51.5, + longitude=-0.1, + altitude=10, + location_source="", + ) + node.device_metrics = MeshNode.DeviceMetrics( + logged_time=datetime.datetime( + 2026, 5, 22, 7, 0, 0, tzinfo=datetime.timezone.utc + ), + channel_utilization=None, + air_util_tx=None, + ) + out = self.serializer.serialise_node(node) + self.assertEqual(out["position"]["meshtastic_location_source"], "UNSET") + self.assertNotIn("location_source", out["position"]) + self.assertEqual(out["device_metrics"]["meshtastic_channel_utilization"], 0.0) + self.assertEqual(out["device_metrics"]["meshtastic_air_util_tx"], 0.0) + self.assertNotIn("channel_utilization", out["device_metrics"]) + self.assertNotIn("air_util_tx", out["device_metrics"]) + if __name__ == "__main__": unittest.main() diff --git a/test/test_bot_version_report.py b/test/test_bot_version_report.py index 0d375c4..3836fba 100644 --- a/test/test_bot_version_report.py +++ b/test/test_bot_version_report.py @@ -50,6 +50,32 @@ def test_report_bot_version_skipped_when_meshcore_prefix_missing() -> None: put.assert_not_called() +def test_report_bot_version_puts_v3_path() -> None: + wrapper = StorageAPIWrapper( + "http://api.test", + token="secret", + api_version=3, + serializer=MeshtasticPacketSerializer(), + local_meshtastic_nodenum_provider=lambda: 42424242, + ) + mock_response = MagicMock() + with patch.object(wrapper, "_put", return_value=mock_response) as put: + assert wrapper.report_bot_version() is True + put.assert_called_once() + assert put.call_args[0][0] == "/api/v3/packets/42424242/bot-version/" + + +def test_get_url_v3_ingest_and_nodes() -> None: + wrapper = StorageAPIWrapper( + "http://api.test", + api_version=3, + serializer=MeshtasticPacketSerializer(), + local_meshtastic_nodenum_provider=lambda: 12345, + ) + assert wrapper._get_url("raw_packet") == "/api/v3/packets/12345/ingest/" + assert wrapper._get_url("nodes") == "/api/v3/packets/12345/nodes/" + + def test_report_bot_version_skipped_for_v1() -> None: wrapper = StorageAPIWrapper( "http://api.test",