From a66da572c90d99954ba9a5984fa661d364224129 Mon Sep 17 00:00:00 2001 From: Patrick Skillen Date: Fri, 22 May 2026 11:14:50 +0100 Subject: [PATCH] feat(packets): feeder v3 routes and v2 node-upsert legacy aliases Accept location_source/channel_utilization/air_util_tx on POST /api/packets/{id}/nodes/ for deployed bots. Add /api/v3/packets/... with strict meshtastic_* node upsert wire. Closes #339 Related: pskillen/meshflow-bot#110 --- Meshflow/Meshflow/urls.py | 1 + Meshflow/packets/serializers.py | 120 +++++++++++++--- Meshflow/packets/tests/test_feeder_v3_urls.py | 11 ++ .../packets/tests/test_packet_serializers.py | 76 ++++++++++ Meshflow/packets/urls_v3.py | 18 +++ Meshflow/packets/views.py | 22 ++- docs/features/meshcore/phase-2-outstanding.md | 2 +- openapi.yaml | 132 +++++++++++++++++- 8 files changed, 349 insertions(+), 33 deletions(-) create mode 100644 Meshflow/packets/tests/test_feeder_v3_urls.py create mode 100644 Meshflow/packets/urls_v3.py diff --git a/Meshflow/Meshflow/urls.py b/Meshflow/Meshflow/urls.py index 7326cc3b..df23d7c7 100644 --- a/Meshflow/Meshflow/urls.py +++ b/Meshflow/Meshflow/urls.py @@ -33,6 +33,7 @@ [ path("status/", StatusView.as_view(), name="status"), path("packets/", include("packets.urls")), + path("v3/packets/", include("packets.urls_v3")), path("meshcore/", include("meshcore_packets.urls")), path("constellations/", include("constellations.urls")), path("nodes/", include("nodes.urls")), diff --git a/Meshflow/packets/serializers.py b/Meshflow/packets/serializers.py index 2adcadfe..16267e06 100644 --- a/Meshflow/packets/serializers.py +++ b/Meshflow/packets/serializers.py @@ -1266,13 +1266,87 @@ def create(self, validated_data): return packet +def _normalize_position_block(position_data, *, allow_legacy_fields): + """Map feeder position payload to nested PositionSerializer input.""" + if not allow_legacy_fields: + if "location_source" in position_data and position_data.get("meshtastic_location_source") is None: + raise serializers.ValidationError( + { + "position": { + "location_source": ["Legacy field not accepted on API v3; use meshtastic_location_source."] + } + } + ) + + location_source = position_data.get("meshtastic_location_source") + if allow_legacy_fields and location_source is None: + location_source = position_data.get("location_source") + if location_source in (None, ""): + location_source = "UNSET" + + block = { + "reported_time": position_data.get("reported_time"), + "latitude": position_data.get("latitude"), + "longitude": position_data.get("longitude"), + "altitude": position_data.get("altitude"), + "meshtastic_location_source": location_source, + } + if position_data.get("heading") is not None: + block["heading"] = position_data.get("heading") + return block + + +def _normalize_device_metrics_block(metrics_data, *, allow_legacy_fields): + """Map feeder device_metrics payload to nested DeviceMetricsSerializer input.""" + if not allow_legacy_fields: + if "channel_utilization" in metrics_data and metrics_data.get("meshtastic_channel_utilization") is None: + raise serializers.ValidationError( + { + "device_metrics": { + "channel_utilization": [ + "Legacy field not accepted on API v3; use meshtastic_channel_utilization." + ] + } + } + ) + if "air_util_tx" in metrics_data and metrics_data.get("meshtastic_air_util_tx") is None: + raise serializers.ValidationError( + { + "device_metrics": { + "air_util_tx": ["Legacy field not accepted on API v3; use meshtastic_air_util_tx."] + } + } + ) + + channel_utilization = metrics_data.get("meshtastic_channel_utilization") + if channel_utilization is None and allow_legacy_fields: + channel_utilization = metrics_data.get("channel_utilization") + if channel_utilization is None: + channel_utilization = 0.0 + + air_util_tx = metrics_data.get("meshtastic_air_util_tx") + if air_util_tx is None and allow_legacy_fields: + air_util_tx = metrics_data.get("air_util_tx") + if air_util_tx is None: + air_util_tx = 0.0 + + return { + "reported_time": metrics_data.get("reported_time"), + "battery_level": metrics_data.get("battery_level"), + "voltage": metrics_data.get("voltage"), + "meshtastic_channel_utilization": channel_utilization, + "meshtastic_air_util_tx": air_util_tx, + "uptime_seconds": metrics_data.get("uptime_seconds"), + } + + class PositionSerializer(serializers.Serializer): logged_time = serializers.DateTimeField(read_only=True, default=django_timezone.now) reported_time = serializers.DateTimeField(required=True) latitude = serializers.FloatField() longitude = serializers.FloatField() altitude = serializers.FloatField() - heading = serializers.FloatField() + heading = serializers.FloatField(required=False, allow_null=True) meshtastic_location_source = serializers.CharField() def to_internal_value(self, data): @@ -1281,10 +1355,9 @@ def to_internal_value(self, data): validated_data = super().to_internal_value(data) # Convert meshtastic_location_source from string to integer using LocationSource - if "meshtastic_location_source" in validated_data and validated_data["meshtastic_location_source"]: - validated_data["meshtastic_location_source"] = convert_location_source( - validated_data["meshtastic_location_source"] - ) + validated_data["meshtastic_location_source"] = convert_location_source( + validated_data.get("meshtastic_location_source") + ) return validated_data @@ -1299,8 +1372,10 @@ class DeviceMetricsSerializer(serializers.Serializer): uptime_seconds = serializers.IntegerField() -class NodeSerializer(serializers.ModelSerializer): - """Serializer for node information updates.""" +class BaseNodeSerializer(serializers.ModelSerializer): + """Shared node upsert serializer; subclasses set allow_legacy_fields.""" + + allow_legacy_fields = True class UserSerializer(serializers.Serializer): long_name = serializers.CharField(required=False, allow_null=True) @@ -1374,25 +1449,14 @@ def to_internal_value(self, data): # Handle position data if "position" in data: position_data = data.pop("position") - data["position"] = { - "reported_time": position_data.get("reported_time"), - "latitude": position_data.get("latitude"), - "longitude": position_data.get("longitude"), - "altitude": position_data.get("altitude"), - "meshtastic_location_source": position_data.get("meshtastic_location_source"), - } + data["position"] = _normalize_position_block(position_data, allow_legacy_fields=self.allow_legacy_fields) # Handle device metrics if "device_metrics" in data: metrics_data = data.pop("device_metrics") - data["device_metrics"] = { - "reported_time": metrics_data.get("reported_time"), - "battery_level": metrics_data.get("battery_level"), - "voltage": metrics_data.get("voltage"), - "meshtastic_channel_utilization": metrics_data.get("meshtastic_channel_utilization"), - "meshtastic_air_util_tx": metrics_data.get("meshtastic_air_util_tx"), - "uptime_seconds": metrics_data.get("uptime_seconds"), - } + data["device_metrics"] = _normalize_device_metrics_block( + metrics_data, allow_legacy_fields=self.allow_legacy_fields + ) return super().to_internal_value(data) @@ -1479,6 +1543,18 @@ def update(self, instance, validated_data): return instance +class NodeSerializer(BaseNodeSerializer): + """Feeder node upsert (API v2 paths): legacy field aliases + meshtastic_*.""" + + allow_legacy_fields = True + + +class NodeSerializerV3(BaseNodeSerializer): + """Feeder node upsert (API v3 paths): meshtastic_* wire only.""" + + allow_legacy_fields = False + + class PrefetchedPacketObservationSerializer(serializers.ModelSerializer): """Serializer for packet observations.""" diff --git a/Meshflow/packets/tests/test_feeder_v3_urls.py b/Meshflow/packets/tests/test_feeder_v3_urls.py new file mode 100644 index 00000000..92e9c294 --- /dev/null +++ b/Meshflow/packets/tests/test_feeder_v3_urls.py @@ -0,0 +1,11 @@ +"""URL routing for feeder API v3.""" + +from django.urls import reverse + + +def test_v3_node_upsert_url_resolves(): + url = reverse( + "meshtastic-node-upsert-v3", + kwargs={"node_id": 2997338904}, + ) + assert url == "/api/v3/packets/2997338904/nodes/" diff --git a/Meshflow/packets/tests/test_packet_serializers.py b/Meshflow/packets/tests/test_packet_serializers.py index 1cedc657..0f918610 100644 --- a/Meshflow/packets/tests/test_packet_serializers.py +++ b/Meshflow/packets/tests/test_packet_serializers.py @@ -782,6 +782,82 @@ def test_node_upsert_with_device_metrics_updates_nodelateststatus(self): self.assertEqual(latest_status.uptime_seconds, 10000) self.assertIsNotNone(latest_status.metrics_reported_time) + def test_node_upsert_v2_compat_legacy_position_and_metrics_fields(self): + """Legacy bot wire (location_source, channel_utilization) validates on v2 serializer.""" + from django.utils import timezone + + data = { + "id": self.from_node.meshtastic_node_id, + "position": { + "reported_time": timezone.now().isoformat(), + "latitude": 51.5, + "longitude": -0.12, + "altitude": 5.0, + "location_source": "", + }, + "device_metrics": { + "reported_time": timezone.now().isoformat(), + "battery_level": 90.0, + "voltage": 4.0, + "channel_utilization": 0.15, + "air_util_tx": 0.05, + "uptime_seconds": 5000, + }, + } + + serializer = NodeSerializer(instance=self.from_node, data=data, partial=True) + assert_serializer_valid(serializer) + serializer.save() + + latest_status = NodeLatestStatus.objects.get(node=self.from_node) + self.assertEqual(latest_status.meshtastic_location_source, LocationSource.UNSET) + self.assertEqual(latest_status.meshtastic_channel_utilization, 0.15) + self.assertEqual(latest_status.meshtastic_air_util_tx, 0.05) + + def test_node_upsert_v3_rejects_legacy_only_position_field(self): + from django.utils import timezone + + from packets.serializers import NodeSerializerV3 + + data = { + "id": self.from_node.meshtastic_node_id, + "position": { + "reported_time": timezone.now().isoformat(), + "latitude": 51.5, + "longitude": -0.12, + "altitude": 5.0, + "location_source": "LOC_INTERNAL", + }, + } + + serializer = NodeSerializerV3(instance=self.from_node, data=data, partial=True) + self.assertFalse(serializer.is_valid()) + self.assertIn("position", serializer.errors) + + def test_node_upsert_v3_accepts_meshtastic_fields(self): + from django.utils import timezone + + from packets.serializers import NodeSerializerV3 + + data = { + "id": self.from_node.meshtastic_node_id, + "device_metrics": { + "reported_time": timezone.now().isoformat(), + "battery_level": 80.0, + "voltage": 3.8, + "meshtastic_channel_utilization": 0.1, + "meshtastic_air_util_tx": 0.2, + "uptime_seconds": 100, + }, + } + + serializer = NodeSerializerV3(instance=self.from_node, data=data, partial=True) + assert_serializer_valid(serializer) + serializer.save() + + latest_status = NodeLatestStatus.objects.get(node=self.from_node) + self.assertEqual(latest_status.meshtastic_channel_utilization, 0.1) + @override_settings(PACKET_DEDUP_WINDOW_MINUTES=10) class PacketDeduplicationTest(BasePacketSerializerTestCase): diff --git a/Meshflow/packets/urls_v3.py b/Meshflow/packets/urls_v3.py new file mode 100644 index 00000000..ef802b53 --- /dev/null +++ b/Meshflow/packets/urls_v3.py @@ -0,0 +1,18 @@ +"""URL configuration for feeder API v3 (strict meshtastic_* node upsert wire).""" + +from django.urls import include, path + +from .views import ManagedNodeBotVersionView, NodeUpsertViewV3, PacketIngestView + +urlpatterns = [ + path( + "/", + include( + [ + path("ingest/", PacketIngestView.as_view(), name="meshtastic-packet-ingest-v3"), + path("nodes/", NodeUpsertViewV3.as_view(), name="meshtastic-node-upsert-v3"), + path("bot-version/", ManagedNodeBotVersionView.as_view(), name="meshtastic-bot-version-v3"), + ] + ), + ), +] diff --git a/Meshflow/packets/views.py b/Meshflow/packets/views.py index 46f8b54c..d70115ef 100644 --- a/Meshflow/packets/views.py +++ b/Meshflow/packets/views.py @@ -24,7 +24,7 @@ TrafficManagementStatsPacket, ) -from .serializers import NodeSerializer, PacketIngestSerializer +from .serializers import NodeSerializer, NodeSerializerV3, PacketIngestSerializer from .signals import ( air_quality_metrics_packet_received, device_metrics_packet_received, @@ -160,16 +160,16 @@ def post(self, request, node_id, format=None): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) -class NodeUpsertView(APIView): +class BaseNodeUpsertView(APIView): """ Meshtastic observed-node upsert for feeders (Node API key). Creates or updates an ``ObservedNode`` row for Meshtastic ``node_id`` / ``node_id_str``. - MeshCore will use separate endpoints when implemented. """ authentication_classes = [NodeAPIKeyAuthentication] permission_classes = [NodeAuthorizationPermission] + serializer_class = NodeSerializer def post(self, request, node_id, format=None): """ @@ -233,10 +233,10 @@ def post(self, request, node_id, format=None): q = ObservedNode.objects.filter(meshtastic_node_id=observed_node_id, protocol=Protocol.MESHTASTIC) if q.exists(): node = q.first() - serializer = NodeSerializer(instance=node, data=request.data, partial=True) + serializer = self.serializer_class(instance=node, data=request.data, partial=True) else: node = None - serializer = NodeSerializer(data=request.data, partial=True) + serializer = self.serializer_class(data=request.data, partial=True) if serializer.is_valid(): try: @@ -270,6 +270,18 @@ def post(self, request, node_id, format=None): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) +class NodeUpsertView(BaseNodeUpsertView): + """Node upsert on ``/api/packets/...`` (v2 wire: legacy field aliases allowed).""" + + serializer_class = NodeSerializer + + +class NodeUpsertViewV3(BaseNodeUpsertView): + """Node upsert on ``/api/v3/packets/...`` (strict ``meshtastic_*`` wire).""" + + serializer_class = NodeSerializerV3 + + class ManagedNodeBotVersionSerializer(serializers.Serializer): """Body for feeder-reported meshflow-bot version.""" diff --git a/docs/features/meshcore/phase-2-outstanding.md b/docs/features/meshcore/phase-2-outstanding.md index 62577764..2e6ac809 100644 --- a/docs/features/meshcore/phase-2-outstanding.md +++ b/docs/features/meshcore/phase-2-outstanding.md @@ -21,7 +21,7 @@ Items **skipped**, **incomplete**, or **discovered during Phase 2 / rename execu - [ ] **Deploy order:** api #324 → ui #271 (bot #98 anytime). Migrations through `nodes.0041`, `constellations/0008`, `text_messages/0007`, `packets/0018`. - [ ] **`tests/integration/`** full run after deploy (`pytest tests/integration/ -v`). - [ ] **Release notes** — numeric observed-node bookmarks → UI redirect + api `by-meshtastic-id`. -- [ ] **Feeder upsert gap** — bot may send `location_source` / `channel_utilization`; api maps metrics from `meshtastic_*` only; optional `NodeSerializer` aliases. +- [x] **Feeder upsert gap** — v2 paths accept legacy `location_source` / `channel_utilization` / `air_util_tx` aliases; v3 at `/api/v3/packets/.../nodes/` (meshflow-api #339). - [ ] **OpenAPI path param naming** — ingest/stats still use parameter name `node_id` (Meshtastic nodenum); cosmetic rename to `meshtastic_node_id` deferred. - [ ] **Cursor rename index** — mark SP-05 `skipped` in `meshcore-rename-index.plan.md` when convenient (plan repo). diff --git a/openapi.yaml b/openapi.yaml index 9f01dc6d..359696f9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -3931,12 +3931,13 @@ paths: /packets/{meshtastic_node_id}/nodes/: post: - summary: Upsert observed node (feeder) + summary: Upsert observed node (feeder, API v2 wire) description: > - Meshtastic feeder bots upsert observed-node rows via this endpoint. The path segment is - the observer's Meshtastic numeric node id (``meshtastic_node_id``) and must match the - authenticated ManagedNode. Similar to ``/nodes/observed-nodes/`` but scoped to Node API - key auth for bots. + Meshtastic feeder bots upsert observed-node rows via this endpoint (client ``STORAGE_API_VERSION=2``). + The path segment is the observer's Meshtastic numeric node id (``meshtastic_node_id``) and must + match the authenticated ManagedNode. Nested ``position`` / ``device_metrics`` accept legacy + unprefixed fields (``location_source``, ``channel_utilization``, ``air_util_tx``) as aliases for + ``meshtastic_*`` fields. New installs should use ``POST /v3/packets/{meshtastic_node_id}/nodes/``. tags: [Meshtastic packets] security: - NodeApiKeyAuth: [] @@ -3961,6 +3962,127 @@ paths: schema: $ref: '#/components/schemas/ObservedNode' + /v3/packets/{meshtastic_node_id}/ingest/: + post: + summary: Ingest a Meshtastic packet (feeder API v3) + description: > + Same behaviour as ``POST /packets/{meshtastic_node_id}/ingest/``. Use with client + ``STORAGE_API_VERSION=3``. + tags: [Meshtastic packets] + security: + - NodeApiKeyAuth: [] + parameters: + - name: meshtastic_node_id + in: path + required: true + schema: + type: integer + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/IncomingPacket' + responses: + '201': + description: Packet ingested successfully + content: + application/json: + schema: + $ref: '#/components/schemas/Success' + '400': + description: Invalid packet data + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /v3/packets/{meshtastic_node_id}/bot-version/: + put: + summary: Report meshflow-bot version (feeder API v3) + description: Same as ``PUT /packets/{meshtastic_node_id}/bot-version/`` for ``STORAGE_API_VERSION=3``. + tags: [Meshtastic packets] + security: + - NodeApiKeyAuth: [] + parameters: + - name: meshtastic_node_id + in: path + required: true + schema: + type: integer + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [bot_version] + properties: + bot_version: + type: string + maxLength: 128 + responses: + '200': + description: Version recorded + '400': + description: Invalid or empty bot_version + '403': + description: API key not linked to this managed node + post: + summary: Report meshflow-bot version (feeder API v3, alias) + tags: [Meshtastic packets] + security: + - NodeApiKeyAuth: [] + parameters: + - name: meshtastic_node_id + in: path + required: true + schema: + type: integer + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [bot_version] + properties: + bot_version: + type: string + maxLength: 128 + responses: + '200': + description: Version recorded + + /v3/packets/{meshtastic_node_id}/nodes/: + post: + summary: Upsert observed node (feeder API v3) + description: > + Strict feeder node upsert for ``STORAGE_API_VERSION=3``. Requires ``meshtastic_*`` nested + fields only (no legacy ``location_source`` / ``channel_utilization`` / ``air_util_tx``). + tags: [Meshtastic packets] + security: + - NodeApiKeyAuth: [] + parameters: + - name: meshtastic_node_id + in: path + required: true + schema: + type: integer + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ObservedNodeUpdate' + responses: + '200': + description: Node information updated successfully + content: + application/json: + schema: + $ref: '#/components/schemas/ObservedNode' + /constellations/: get: summary: List constellations