diff --git a/Meshflow/meshcore_packets/serializers.py b/Meshflow/meshcore_packets/serializers.py index e8a1396..f5ad844 100644 --- a/Meshflow/meshcore_packets/serializers.py +++ b/Meshflow/meshcore_packets/serializers.py @@ -13,6 +13,8 @@ ) from meshcore_packets.services.channel import resolve_mc_channel from meshcore_packets.services.dedup import find_existing_packet +from meshcore_packets.services.path_hashes import enrich_validated_data_paths +from meshcore_packets.services.path_twin import sync_path_from_rx_log_twin, sync_path_to_channel_text_twin def _parse_rx_time(value) -> datetime: @@ -63,8 +65,21 @@ def validate(self, attrs): pass # rx_log_data ADVERT may supply adv fields without from_pubkey in envelope return attrs + def _run_path_twin_sync(self, packet, observer): + if self.observation is None: + return + if packet.payload_type == MeshCorePayloadType.RAW: + sync_path_to_channel_text_twin( + packet=packet, + observer=observer, + observation=self.observation, + ) + elif packet.payload_type == MeshCorePayloadType.CHANNEL_TEXT and isinstance(packet, MeshCoreTextPacket): + sync_path_from_rx_log_twin(packet=packet, observer=observer) + def create(self, validated_data): observer = self.context["observer"] + enrich_validated_data_paths(validated_data) rx_time = _parse_rx_time(validated_data["rx_time"]) from_pubkey = validated_data.get("from_pubkey") or None @@ -85,6 +100,7 @@ def create(self, validated_data): if existing: self.instance = existing self._ensure_observation(existing, observer, validated_data, rx_time) + self._run_path_twin_sync(existing, observer) return existing payload_type_map = { @@ -126,6 +142,7 @@ def create(self, validated_data): self.instance = packet self._ensure_observation(packet, observer, validated_data, rx_time) + self._run_path_twin_sync(packet, observer) return packet def _ensure_observation(self, packet, observer, validated_data, rx_time): diff --git a/Meshflow/meshcore_packets/services/path_hashes.py b/Meshflow/meshcore_packets/services/path_hashes.py new file mode 100644 index 0000000..d90abbc --- /dev/null +++ b/Meshflow/meshcore_packets/services/path_hashes.py @@ -0,0 +1,62 @@ +"""Split MeshCore wire ``path`` hex into hop hash segments (server-side).""" + +from __future__ import annotations + +from typing import Any + + +def split_path_hex(path: str, path_hash_size: int) -> list[str]: + """Split concatenated path hex into per-hop segments (``path_hash_size`` bytes each).""" + if not path or not isinstance(path, str): + return [] + size = int(path_hash_size or 2) + if size < 1: + size = 2 + width = size * 2 + return [path[i : i + width] for i in range(0, len(path), width) if path[i : i + width]] + + +def _path_from_payload(payload: dict) -> tuple[list[str] | None, int | None, int | None]: + if not isinstance(payload, dict): + return None, None, None + existing = payload.get("path_hashes") + if isinstance(existing, list) and existing: + return [str(p) for p in existing], payload.get("path_hash_size"), payload.get("path_hash_mode") + path = payload.get("path") + size = payload.get("path_hash_size") + if size is None: + size = 2 + if isinstance(path, list) and path: + return [str(p) for p in path], int(size), payload.get("path_hash_mode") + if isinstance(path, str) and path: + segments = split_path_hex(path, int(size)) + return segments or None, int(size), payload.get("path_hash_mode") + return None, payload.get("path_hash_size"), payload.get("path_hash_mode") + + +def path_hashes_from_ingest(validated_data: dict[str, Any]) -> list[str] | None: + """Resolve path_hashes from ingest body or nested capture envelope.""" + segments, _, _ = _path_from_payload(validated_data) + if segments: + return segments + + raw = validated_data.get("raw") + if not isinstance(raw, dict): + return None + + envelope = raw + if isinstance(raw.get("raw"), dict): + envelope = raw["raw"] + elif raw.get("protocol") != "meshcore" and isinstance(raw.get("payload"), dict): + envelope = raw + + payload = envelope.get("payload") if isinstance(envelope, dict) else None + segments, _, _ = _path_from_payload(payload or {}) + return segments + + +def enrich_validated_data_paths(validated_data: dict[str, Any]) -> None: + """Populate ``path_hashes`` on validated_data when only wire ``path`` is present.""" + segments = path_hashes_from_ingest(validated_data) + if segments and not validated_data.get("path_hashes"): + validated_data["path_hashes"] = segments diff --git a/Meshflow/meshcore_packets/services/path_twin.py b/Meshflow/meshcore_packets/services/path_twin.py new file mode 100644 index 0000000..3eb106b --- /dev/null +++ b/Meshflow/meshcore_packets/services/path_twin.py @@ -0,0 +1,165 @@ +"""Copy path_hashes from rx_log TEXT_MSG/PATH rows onto channel_text packet observations.""" + +from __future__ import annotations + +import logging + +from meshcore_packets.models import ( + MeshCorePacketObservation, + MeshCorePayloadType, + MeshCoreRawPacket, + MeshCoreTextPacket, +) +from meshcore_packets.services.channel import resolve_mc_channel +from meshcore_packets.services.dedup import decoded_twin_window + +logger = logging.getLogger(__name__) + +PATH_RX_TYPENAMES = frozenset({"TEXT_MSG", "PATH"}) + + +def rx_log_payload_typename(packet: MeshCoreRawPacket) -> str: + raw_json = packet.raw_json or {} + envelope = raw_json + nested = raw_json.get("raw") + if isinstance(nested, dict): + envelope = nested + if not isinstance(envelope, dict): + return "" + payload = envelope.get("payload") or {} + return str(payload.get("payload_typename", "")).upper() + + +def channel_idx_from_packet_raw_json(packet: MeshCoreRawPacket) -> int | None: + raw_json = packet.raw_json or {} + if raw_json.get("channel_idx") is not None: + return int(raw_json["channel_idx"]) + envelope = raw_json.get("raw") + if isinstance(envelope, dict): + payload = envelope.get("payload") or {} + if payload.get("channel_idx") is not None: + return int(payload["channel_idx"]) + return None + + +def _path_fields_from_observation(observation: MeshCorePacketObservation) -> dict | None: + if not observation.path_hashes: + return None + return { + "path_hashes": observation.path_hashes, + "path_hash_size": observation.path_hash_size, + "path_hash_mode": observation.path_hash_mode, + } + + +def _prefer_path_fields(existing: list[str] | None, incoming: list[str]) -> list[str]: + if not existing: + return incoming + if len(incoming) > len(existing): + return incoming + return existing + + +def apply_path_to_text_observation( + *, + text_packet: MeshCoreTextPacket, + observer, + path_hashes: list[str], + path_hash_size: int | None = None, + path_hash_mode: int | None = None, +) -> bool: + """Merge path onto the text packet observation; prefer longer hop lists.""" + if not path_hashes: + return False + obs, _ = MeshCorePacketObservation.objects.get_or_create( + packet=text_packet, + observer=observer, + ) + merged = _prefer_path_fields(obs.path_hashes, path_hashes) + changed = ( + obs.path_hashes != merged + or (path_hash_size is not None and obs.path_hash_size != path_hash_size) + or (path_hash_mode is not None and obs.path_hash_mode != path_hash_mode) + ) + if not changed: + return False + obs.path_hashes = merged + if path_hash_size is not None: + obs.path_hash_size = path_hash_size + if path_hash_mode is not None: + obs.path_hash_mode = path_hash_mode + obs.save( + update_fields=["path_hashes", "path_hash_size", "path_hash_mode"], + ) + return True + + +def _pick_channel_text_twin(*, observer, anchor_time, channel_idx: int | None): + window = decoded_twin_window() + candidates = MeshCoreTextPacket.objects.filter( + observer=observer, + payload_type=MeshCorePayloadType.CHANNEL_TEXT, + rx_time__gte=anchor_time - window, + rx_time__lte=anchor_time + window, + ).order_by("-rx_time") + count = candidates.count() + if count == 0: + return None + if count == 1: + return candidates.first() + if channel_idx is not None: + channel = resolve_mc_channel(observer, channel_idx) + if channel: + narrowed = candidates.filter(channel=channel) + if narrowed.count() == 1: + return narrowed.first() + logger.debug("path_twin: %s channel_text candidates in window; skip merge", count) + return None + + +def sync_path_to_channel_text_twin( + *, packet: MeshCoreRawPacket, observer, observation: MeshCorePacketObservation +) -> bool: + """After ingesting rx_log raw TEXT_MSG/PATH, copy path onto a nearby channel_text packet.""" + if packet.payload_type != MeshCorePayloadType.RAW: + return False + if packet.event_type != "rx_log_data": + return False + if rx_log_payload_typename(packet) not in PATH_RX_TYPENAMES: + return False + fields = _path_fields_from_observation(observation) + if not fields: + return False + twin = _pick_channel_text_twin( + observer=observer, + anchor_time=packet.rx_time, + channel_idx=channel_idx_from_packet_raw_json(packet), + ) + if not twin: + return False + return apply_path_to_text_observation(text_packet=twin, observer=observer, **fields) + + +def sync_path_from_rx_log_twin(*, packet: MeshCoreTextPacket, observer) -> bool: + """After ingesting channel_text, copy path from a nearby rx_log TEXT_MSG/PATH observation.""" + if packet.payload_type != MeshCorePayloadType.CHANNEL_TEXT: + return False + window = decoded_twin_window() + raw_packets = MeshCoreRawPacket.objects.filter( + observer=observer, + payload_type=MeshCorePayloadType.RAW, + event_type="rx_log_data", + rx_time__gte=packet.rx_time - window, + rx_time__lte=packet.rx_time + window, + ).order_by("-rx_time") + for raw in raw_packets: + if rx_log_payload_typename(raw) not in PATH_RX_TYPENAMES: + continue + obs = MeshCorePacketObservation.objects.filter(packet=raw, observer=observer).first() + if not obs: + continue + fields = _path_fields_from_observation(obs) + if not fields: + continue + return apply_path_to_text_observation(text_packet=packet, observer=observer, **fields) + return False diff --git a/Meshflow/meshcore_packets/tests/test_path_hashes.py b/Meshflow/meshcore_packets/tests/test_path_hashes.py new file mode 100644 index 0000000..797e875 --- /dev/null +++ b/Meshflow/meshcore_packets/tests/test_path_hashes.py @@ -0,0 +1,36 @@ +"""Tests for meshcore_packets.services.path_hashes.""" + +from meshcore_packets.services.path_hashes import enrich_validated_data_paths, path_hashes_from_ingest, split_path_hex + + +def test_split_path_hex_two_byte_hops(): + assert split_path_hex("aabbccdd", 2) == ["aabb", "ccdd"] + + +def test_split_path_hex_three_byte_hop(): + assert split_path_hex("f3bcf1", 3) == ["f3bcf1"] + + +def test_path_hashes_from_ingest_top_level_path(): + data = {"path": "aabb", "path_hash_size": 2} + assert path_hashes_from_ingest(data) == ["aabb"] + + +def test_path_hashes_from_nested_envelope(): + data = { + "raw": { + "protocol": "meshcore", + "event_type": "rx_log_data", + "payload": { + "path": "f3bcf1", + "path_hash_size": 3, + }, + }, + } + assert path_hashes_from_ingest(data) == ["f3bcf1"] + + +def test_enrich_validated_data_paths(): + data = {"path": "aabbcc", "path_hash_size": 2} + enrich_validated_data_paths(data) + assert data["path_hashes"] == ["aabb", "cc"] diff --git a/Meshflow/meshcore_packets/tests/test_path_twin.py b/Meshflow/meshcore_packets/tests/test_path_twin.py new file mode 100644 index 0000000..0086edd --- /dev/null +++ b/Meshflow/meshcore_packets/tests/test_path_twin.py @@ -0,0 +1,153 @@ +"""Tests for rx_log → channel_text path twin merge.""" + +import json +from pathlib import Path + +from django.utils import timezone + +import pytest + +from meshcore_packets.models import MeshCorePacketObservation, MeshCorePayloadType, MeshCoreTextPacket +from meshcore_packets.services.channel_sync import reconcile_mc_channels +from meshcore_packets.tests.conftest import FEEDER_MC_PUBKEY_PREFIX, feeder_url +from text_messages.models import TextMessage + +DOCS = Path(__file__).resolve().parents[3] / "docs" / "packets" / "meshcore" +CHANNEL_MSG = DOCS / "channel_message" / "20260507_094921_075978.json" + + +def _load(path: Path) -> dict: + return json.loads(path.read_text()) + + +@pytest.fixture +def ingest_client(meshcore_feeder): + from rest_framework.test import APIClient + + client = APIClient() + client.credentials(HTTP_X_API_KEY=meshcore_feeder["api_key"].key) + return client + + +@pytest.mark.django_db +def test_channel_text_then_raw_path_populates_text_observation(meshcore_feeder, ingest_client): + reconcile_mc_channels( + meshcore_feeder["node"], + [{"mc_channel_idx": 0, "name": "Public", "mc_channel_type": "PUBLIC"}], + ) + now = timezone.now() + url = feeder_url("meshcore-feeder-packet-ingest", FEEDER_MC_PUBKEY_PREFIX) + channel_dump = _load(CHANNEL_MSG) + text = channel_dump["payload"]["text"] + rx_time = now.timestamp() + + r1 = ingest_client.post( + url, + { + "event_type": "channel_message", + "payload_type": "channel_text", + "channel_idx": 0, + "rx_time": rx_time, + "text": text, + "path_hash_mode": 2, + "path_hash_size": 2, + "raw": channel_dump, + }, + format="json", + ) + assert r1.status_code == 201 + text_packet = MeshCoreTextPacket.objects.get(text=text) + text_obs = MeshCorePacketObservation.objects.get(packet=text_packet) + assert not text_obs.path_hashes + + path_dump = json.loads((DOCS / "rx_log_data_path.json").read_text()) + r2 = ingest_client.post( + url, + { + "event_type": "rx_log_data", + "payload_type": "raw", + "pkt_hash": path_dump["payload"]["pkt_hash"], + "rx_time": rx_time, + "path_hashes": ["f3bcf1"], + "path_hash_size": 3, + "path_hash_mode": path_dump["payload"].get("path_hash_mode"), + "raw": path_dump, + }, + format="json", + ) + assert r2.status_code == 201 + text_obs.refresh_from_db() + assert text_obs.path_hashes == ["f3bcf1"] + assert text_obs.path_hash_size == 3 + + tm = TextMessage.objects.get(message_text=text) + assert tm.original_mc_packet_id == text_packet.id + + +@pytest.mark.django_db +def test_raw_path_then_channel_text_populates_text_observation(meshcore_feeder, ingest_client): + reconcile_mc_channels( + meshcore_feeder["node"], + [{"mc_channel_idx": 0, "name": "Public", "mc_channel_type": "PUBLIC"}], + ) + now = timezone.now() + url = feeder_url("meshcore-feeder-packet-ingest", FEEDER_MC_PUBKEY_PREFIX) + path_dump = json.loads((DOCS / "rx_log_data_path.json").read_text()) + rx_time = now.timestamp() + + ingest_client.post( + url, + { + "event_type": "rx_log_data", + "payload_type": "raw", + "pkt_hash": path_dump["payload"]["pkt_hash"], + "rx_time": rx_time, + "path_hashes": ["f3bcf1"], + "path_hash_size": 3, + "raw": path_dump, + }, + format="json", + ) + + channel_dump = _load(CHANNEL_MSG) + text = "twin order test message" + channel_dump = dict(channel_dump) + channel_dump["payload"] = dict(channel_dump["payload"]) + channel_dump["payload"]["text"] = text + + ingest_client.post( + url, + { + "event_type": "channel_message", + "payload_type": "channel_text", + "channel_idx": 0, + "rx_time": rx_time, + "text": text, + "raw": channel_dump, + }, + format="json", + ) + text_packet = MeshCoreTextPacket.objects.get(text=text) + text_obs = MeshCorePacketObservation.objects.get(packet=text_packet) + assert text_obs.path_hashes == ["f3bcf1"] + + +@pytest.mark.django_db +def test_raw_path_without_channel_text_twin_leaves_no_text_path(meshcore_feeder, ingest_client): + now = timezone.now() + url = feeder_url("meshcore-feeder-packet-ingest", FEEDER_MC_PUBKEY_PREFIX) + path_dump = json.loads((DOCS / "rx_log_data_path.json").read_text()) + response = ingest_client.post( + url, + { + "event_type": "rx_log_data", + "payload_type": "raw", + "pkt_hash": 999888777, + "rx_time": now.timestamp(), + "path_hashes": ["aa"], + "raw": path_dump, + }, + format="json", + ) + assert response.status_code == 201 + assert MeshCoreTextPacket.objects.filter(payload_type=MeshCorePayloadType.CHANNEL_TEXT).count() == 0 diff --git a/Meshflow/text_messages/tests/test_heard_api.py b/Meshflow/text_messages/tests/test_heard_api.py index 10084c1..4f69135 100644 --- a/Meshflow/text_messages/tests/test_heard_api.py +++ b/Meshflow/text_messages/tests/test_heard_api.py @@ -91,3 +91,55 @@ def test_message_list_query_count_bounded(meshcore_feeder, ingest_client): assert response.status_code == 200 assert len(response.data["results"]) >= 3 assert len(ctx) <= 25 + + +@pytest.mark.django_db +def test_mc_message_heard_path_via_rx_log_twin(meshcore_feeder, ingest_client): + """Tier 1: channel_text without path + raw PATH twin → heard[] shows path_hashes.""" + reconcile_mc_channels( + meshcore_feeder["node"], + [{"mc_channel_idx": 0, "name": "Public", "mc_channel_type": "PUBLIC"}], + ) + now = timezone.now() + url = feeder_url("meshcore-feeder-packet-ingest", FEEDER_MC_PUBKEY_PREFIX) + rx_time = now.timestamp() + message_text = "tier1 heard twin path" + + ingest_client.post( + url, + { + "event_type": "channel_message", + "payload_type": "channel_text", + "channel_idx": 0, + "rx_time": rx_time, + "text": message_text, + "path_hash_size": 2, + "path_hash_mode": 2, + "raw": {}, + }, + format="json", + ) + ingest_client.post( + url, + { + "event_type": "rx_log_data", + "payload_type": "raw", + "pkt_hash": 3138934464, + "rx_time": rx_time, + "path_hashes": ["ab", "cd"], + "path_hash_size": 2, + "raw": { + "protocol": "meshcore", + "event_type": "rx_log_data", + "payload": {"payload_typename": "PATH", "path": "abcd"}, + }, + }, + format="json", + ) + + tm = TextMessage.objects.get(message_text=message_text) + client = APIClient() + list_url = reverse("textmessage-list") + response = client.get(list_url, {"channel_id": tm.channel_id}) + row = next(item for item in response.data["results"] if item["id"] == str(tm.id)) + assert row["heard"][0]["path_hashes"] == ["ab", "cd"] diff --git a/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md b/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md index 66e4290..7589820 100644 --- a/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md +++ b/docs/features/meshcore/packet-path-tracing/packet-path-tracing-outstanding.md @@ -105,7 +105,7 @@ Implications for closing this gap (direction only — not scheduled here): ### Follow-up (tracking) -- [ ] **Server-led ingest design** — extend API (and optionally bot upload surface) so channel-text messages get `path_hashes` on the observation tied to `original_mc_packet`, without bot-side path logic. Likely depends on [#266](https://github.com/pskillen/meshflow-api/issues/266) / `rx_log_data` TEXT_MSG ingest and/or dedup correlation spike. +- [ ] **Tier 1 — server-led ingest (ship)** — [#385](https://github.com/pskillen/meshflow-api/issues/385): `path_hashes` on observation tied to `original_mc_packet` for channel `TextMessage` traffic; thin bot upload of TEXT_MSG/PATH `rx_log_data`; API twin-merge. Design: [tier-1-message-path-twin.md](./tier-1-message-path-twin.md). - [ ] **Confirm with M2 spike** — whether `path_hash_mode` changes segment identity when we do get text paths. - [ ] **Optional:** re-run pre-prod queries after deploy (`Meshflow/ai-env` + Django shell; local skill `MeshFlow/.cursor/skills/preprod-database/`) — breakdown by `payload_type` + `event_type`. diff --git a/docs/features/meshcore/packet-path-tracing/tier-1-message-path-twin.md b/docs/features/meshcore/packet-path-tracing/tier-1-message-path-twin.md new file mode 100644 index 0000000..a0b2aaa --- /dev/null +++ b/docs/features/meshcore/packet-path-tracing/tier-1-message-path-twin.md @@ -0,0 +1,43 @@ +# Tier 1 — message path via rx_log decoded twin + +**Tracking:** [meshflow-api#385](https://github.com/pskillen/meshflow-api/issues/385) + +## Problem + +Decoded `channel_message` ingest (`channel_text`) usually has `path_hash_mode` / `path_hash_size` but **no `path` hex**. Companion `rx_log_data` **TEXT_MSG** / **PATH** frames carry `pkt_hash` and often `path`. Heard reads `path_hashes` on observations for `TextMessage.original_mc_packet` only. + +## Approach (MVP) + +**Thin bot:** upload TEXT_MSG and PATH `rx_log_data` as `payload_type: raw` (forward envelope + `path_hashes` when present). + +**Fat API:** after ingest, **bidirectional twin merge** within `MESHCORE_DECODED_TWIN_WINDOW_SECONDS` (default 30s, see `meshcore_packets.services.dedup.decoded_twin_window`): + +| Order | Action | +| --- | --- | +| `channel_text` then `raw` TEXT_MSG/PATH | On `raw` ingest, copy path fields onto the matching `MeshCoreTextPacket` observation (same feeder) | +| `raw` then `channel_text` | On `channel_text` ingest, copy path from recent matching `raw` observation | + +Matching rules (MVP): + +- Same `observer` (feeder `ManagedNode`). +- `rx_time` within decoded-twin window. +- Target packet: `CHANNEL_TEXT` only (not DMs). +- If multiple candidates: narrow by `channel_idx` in `raw_json` vs text packet / observation `channel` when available; if still ambiguous, skip merge (debug log). + +Raw rows are still stored (M1 rollups / debugging). Heard uses the **text** packet observation after merge. + +## Failure modes + +- No twin in window → `path_hashes` stay empty on text observation; heard schematic empty. +- `channel_message` without companion `rx_log_data` on that feeder → no path (ops / library gap). +- Full `raw_packet_fk` linkage → deferred to [ADR-0004](../../packet-ingestion/adr/0004-mc-dedup-key.md) / [#276](https://github.com/pskillen/meshflow-api/issues/276). + +## Implementation + +- `meshcore_packets.services.path_hashes` — server-side `path` hex split when bot omits `path_hashes`. +- `meshcore_packets.services.path_twin` — `sync_path_to_channel_text_twin`, `sync_path_from_rx_log_twin`. +- Wired from `MeshCorePacketIngestSerializer.create` after `_ensure_observation`. + +## Verification + +See [#385](https://github.com/pskillen/meshflow-api/issues/385) acceptance criteria and [phase-3-outstanding.md](../phase-3-outstanding.md). diff --git a/docs/features/meshcore/phase-3-outstanding.md b/docs/features/meshcore/phase-3-outstanding.md index 199f240..3769054 100644 --- a/docs/features/meshcore/phase-3-outstanding.md +++ b/docs/features/meshcore/phase-3-outstanding.md @@ -7,20 +7,41 @@ Items **skipped**, **incomplete**, or **discovered during Phase 3** ([#267](http --- +## MVP tiers (bot → message heard map) + +Gap analysis (Jun 2026): channel **Heard** is the user-facing “map view” for MeshCore today — geo map shows sender + feeders; hop detail is **schematic** per feeder until hashes resolve to positions. + +| Tier | User outcome | Tracking | +| --- | --- | --- | +| **Tier 1** (ship next) | Real `#channel` messages show **non-empty `path_hashes`** in Heard (unknown hop labels OK) | **[#385](https://github.com/pskillen/meshflow-api/issues/385)** | +| **Tier 2** | Hop **polylines on Leaflet** when hashes map to node positions | M2/M3 matcher ([#373](https://github.com/pskillen/meshflow-api/issues/373), [#374](https://github.com/pskillen/meshflow-api/issues/374)); wire `heard[]` to M1 `MeshCorePathSegmentResolution`; UI draw MC waypoints with `position` | +| **Tier 3** | Full path tracing product (Neo4j, realtime WS, M7 topology) | [#372](https://github.com/pskillen/meshflow-api/issues/372) milestones M4–M7, [meshflow-ui#309](https://github.com/pskillen/meshflow-ui/issues/309) | + +**Tier 1 blocker (pre-prod):** `channel_text` linked to `TextMessage` has empty `path_hashes`; PATH/TEXT_MSG `rx_log_data` with `path` is not uploaded. Precursor + UI ([#369](https://github.com/pskillen/meshflow-api/issues/369), [#304](https://github.com/pskillen/meshflow-ui/issues/304), [#311](https://github.com/pskillen/meshflow-ui/issues/311)) already work when data exists — **no new UI for Tier 1**. + +**Recommendations** + +- **Thin bot / fat server** — bot uploads additional `rx_log_data` typenames; API splits `path` and correlates to `original_mc_packet` (see [#385](https://github.com/pskillen/meshflow-api/issues/385)). +- **Deploy** precursor + M1 ([#372](https://github.com/pskillen/meshflow-api/issues/372)) in parallel; M1 rollups from ADVERT do not unblock message Heard without Tier 1. +- **Do not** invest in geo hop lines or auto-matcher heuristics before Tier 1 data lands. +- **Tier 2 shortcut:** staff manual segment annotation (M1 `PATCH …/segments/`) + `heard[]` lookup — demo map lines before safe auto-matcher ADR. + +--- + ## Passive path -- [ ] **Channel text `heard[]` without hops** — `channel_message` ingest often has `path_len` but no `path`; ADVERT-only `rx_log_data` has paths but no `TextMessage` link. Server-led ingest design: [packet-path-tracing-outstanding.md § Message path data chain](./packet-path-tracing/packet-path-tracing-outstanding.md#message-path-data-chain-confirmed--pre-prod-jun-2026). -- [ ] **Passive packet path subsystem** — rollups, resolution table, Neo4j export, realtime/history UI ([ADR-0001](./packet-path-tracing/adr/0001-meshcore-packet-path-tracing-subsystem.md)). -- [ ] **Proven hash → node matcher** — per [traceroute ADR §A](../traceroute/adr/0001-mc-path-hash-resolution.md); no unsafe heuristics in v1. +- [ ] **Tier 1 — message path data chain** — [#385](https://github.com/pskillen/meshflow-api/issues/385): `path_hashes` on observation tied to `TextMessage.original_mc_packet` for channel traffic. Detail: [packet-path-tracing-outstanding.md § Message path data chain](./packet-path-tracing/packet-path-tracing-outstanding.md#message-path-data-chain-confirmed--pre-prod-jun-2026). +- [ ] **Passive packet path subsystem (M1+)** — rollups, resolution table, Neo4j export, realtime/history UI ([ADR-0001](./packet-path-tracing/adr/0001-meshcore-packet-path-tracing-subsystem.md)); merge/deploy PRs [#378](https://github.com/pskillen/meshflow-api/pull/378), [bot#122](https://github.com/pskillen/meshflow-bot/pull/122), [ui#310](https://github.com/pskillen/meshflow-ui/pull/310). +- [ ] **Tier 2 — `heard[]` → segment resolution table** — augment `bulk_format_path_hops` in `text_messages/views.py` with `MeshCorePathSegmentResolution` (manual + resolved rows). +- [ ] **Proven hash → node matcher** — per [traceroute ADR §A](../traceroute/adr/0001-mc-path-hash-resolution.md); no unsafe heuristics in v1 ([#373](https://github.com/pskillen/meshflow-api/issues/373)). - [ ] **`GET /meshcore/packets/`** — optional `resolved_path` on list/detail (deferred). -- [ ] **[meshflow-ui#311](https://github.com/pskillen/meshflow-ui/issues/311)** — schematic hop chain per feeder in heard dialog. ### Bot ([meshflow-bot#119](https://github.com/pskillen/meshflow-bot/issues/119)) -Prefer **thin bot / fat server** — see packet-path-tracing outstanding. +Prefer **thin bot / fat server** — tracked under [#385](https://github.com/pskillen/meshflow-api/issues/385). -- [ ] Unit tests for `_path_hashes()` (1/2/3-byte `path_hash_size`). -- [ ] Optional: upload `rx_log_data` PATH (and related) frames for server-side `path` split. +- [ ] Upload `rx_log_data` TEXT_MSG / PATH (or raw pass-through) — **required for Tier 1**; no bot-side correlation. +- [ ] Unit tests for `_path_hashes()` (1/2/3-byte `path_hash_size`) when wire includes `path`. --- @@ -49,3 +70,4 @@ From [#267](https://github.com/pskillen/meshflow-api/issues/267) — not schedul *(Move items here when closing.)* - [x] **#304** — UI heard path map ([meshflow-ui#304](https://github.com/pskillen/meshflow-ui/issues/304)). +- [x] **#311** — schematic hop chain per feeder in heard dialog ([meshflow-ui#311](https://github.com/pskillen/meshflow-ui/issues/311)); blocked on Tier 1 data for production channel traffic. diff --git a/docs/features/meshcore/phase-3-progress.md b/docs/features/meshcore/phase-3-progress.md index 4a19418..c7019d5 100644 --- a/docs/features/meshcore/phase-3-progress.md +++ b/docs/features/meshcore/phase-3-progress.md @@ -37,10 +37,16 @@ MC **path/trace** or passive hop accumulation as a Meshtastic traceroute analog: --- +### Passive path — heard UI per-feeder schematic + +- [meshflow-ui#311](https://github.com/pskillen/meshflow-ui/issues/311) — `MeshCoreHeardPathsPanel` / `PathHopChain` per feeder in heard dialog. + +--- + ## In flight -- **Packet path subsystem** — ADR + rollups + resolution: [packet-path-tracing/](./packet-path-tracing/). -- **UI #311** — logical path per feeder in heard dialog ([meshflow-ui#311](https://github.com/pskillen/meshflow-ui/issues/311)). +- **Tier 1 MVP (message heard paths)** — [#385](https://github.com/pskillen/meshflow-api/issues/385): PRs [api#386](https://github.com/pskillen/meshflow-api/pull/386), [bot#124](https://github.com/pskillen/meshflow-bot/pull/124). Design: [tier-1-message-path-twin.md](./packet-path-tracing/tier-1-message-path-twin.md). Pre-prod verification pending. +- **Packet path subsystem (M1)** — ADR + rollups + staff segments API: [packet-path-tracing/](./packet-path-tracing/); PRs [#378](https://github.com/pskillen/meshflow-api/pull/378), [bot#122](https://github.com/pskillen/meshflow-bot/pull/122), [ui#310](https://github.com/pskillen/meshflow-ui/pull/310). --- @@ -48,15 +54,15 @@ MC **path/trace** or passive hop accumulation as a Meshtastic traceroute analog: | Repo | Issue | | --- | --- | -| meshflow-api | [#360](https://github.com/pskillen/meshflow-api/issues/360), [#369](https://github.com/pskillen/meshflow-api/issues/369) | -| meshflow-bot | [#119](https://github.com/pskillen/meshflow-bot/issues/119) | +| meshflow-api | [#360](https://github.com/pskillen/meshflow-api/issues/360), [#369](https://github.com/pskillen/meshflow-api/issues/369), **[#385](https://github.com/pskillen/meshflow-api/issues/385)** (Tier 1), [#372](https://github.com/pskillen/meshflow-api/issues/372) (M1) | +| meshflow-bot | [#119](https://github.com/pskillen/meshflow-bot/issues/119) (+ [#385](https://github.com/pskillen/meshflow-api/issues/385) upload surface) | | meshflow-ui | [#304](https://github.com/pskillen/meshflow-ui/issues/304), [#311](https://github.com/pskillen/meshflow-ui/issues/311) | --- ## References -- [phase-3-outstanding.md](./phase-3-outstanding.md) +- [phase-3-outstanding.md](./phase-3-outstanding.md) — MVP tiers (Tier 1 [#385](https://github.com/pskillen/meshflow-api/issues/385)) - [traceroute/README.md](../traceroute/README.md) § MeshCore path parity - [ADR-0001 — MC path hash resolution](../traceroute/adr/0001-mc-path-hash-resolution.md) - [packet-ingestion/meshcore.md](../packet-ingestion/meshcore.md) — what the bot uploads today diff --git a/docs/features/packet-ingestion/meshcore.md b/docs/features/packet-ingestion/meshcore.md index 213f6e3..b997a10 100644 --- a/docs/features/packet-ingestion/meshcore.md +++ b/docs/features/packet-ingestion/meshcore.md @@ -43,7 +43,7 @@ Code: `meshflow-bot/src/meshcore/serializers.py`, `src/api/StorageAPI.py` (`stor | `advert` | `ADVERT` (1) | `MeshCoreRawPacket` | `ObservedNode` upsert (pubkey / prefix); optional `Position` + `NodeLatestStatus` if `adv_lat`/`adv_lon` non-zero; `meshcore_adv_type` on node | | `channel_text` | `CHANNEL_TEXT` (2) | `MeshCoreTextPacket` | `TextMessage` (`original_mc_packet`, `protocol=MESHCORE`); channel FK via feeder `channel_idx` | | `contact_text` | `CONTACT_TEXT` (3) | `MeshCoreTextPacket` | `TextMessage` + sender `ObservedNode` from `from_pubkey_prefix`; node-claim path | -| `raw` | `RAW` (99) | `MeshCoreRawPacket` | **Serializer supports it; bot does not upload `raw` today** | +| `raw` | `RAW` (99) | `MeshCoreRawPacket` | `rx_log_data` **TEXT_MSG** / **PATH** (path + `pkt_hash`); twin-merge onto `channel_text` for Heard — see [tier-1-message-path-twin.md](../meshcore/packet-path-tracing/tier-1-message-path-twin.md) | Also creates or updates **MeshCorePacketObservation** per `(packet, observer)` (deduped). Repeater **path_hashes** are stored on the observation row only (not on deduped `MeshCoreRawPacket`), so two feeders reporting the same `pkt_hash` can keep different paths. See [ADR-0001 (path hash resolution)](../traceroute/adr/0001-mc-path-hash-resolution.md).