diff --git a/backend/scene-detector/src/core/settings.py b/backend/scene-detector/src/core/settings.py
index 8e196aa..a95a930 100644
--- a/backend/scene-detector/src/core/settings.py
+++ b/backend/scene-detector/src/core/settings.py
@@ -9,6 +9,7 @@ class Settings(BaseSettings):
# general config
LOG_LEVEL: str = "DEBUG"
LOG_FORMAT: str = "json"
+ HTTP_PORT: int = 9098
# Nats config
NATS_URL: str = "nats://localhost:4222"
diff --git a/backend/scene-detector/src/nats/__init__.py b/backend/scene-detector/src/handler/__init__.py
similarity index 100%
rename from backend/scene-detector/src/nats/__init__.py
rename to backend/scene-detector/src/handler/__init__.py
diff --git a/backend/scene-detector/src/nats/connection.py b/backend/scene-detector/src/handler/connection.py
similarity index 100%
rename from backend/scene-detector/src/nats/connection.py
rename to backend/scene-detector/src/handler/connection.py
diff --git a/backend/scene-detector/src/handler/http_server.py b/backend/scene-detector/src/handler/http_server.py
new file mode 100644
index 0000000..f19c14e
--- /dev/null
+++ b/backend/scene-detector/src/handler/http_server.py
@@ -0,0 +1,25 @@
+from http.server import HTTPServer
+from http.server import BaseHTTPRequestHandler
+import threading
+import json
+
+
+class HealthEnpointHandler(BaseHTTPRequestHandler):
+ def do_GET(self) -> None:
+ if self.path == "/health":
+ body = json.dumps({"status": "Healthy"}).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.end_headers()
+ self.wfile.write(body)
+ else:
+ self.send_response(404)
+ self.end_headers()
+
+
+def start_health_server(port: int) -> HTTPServer:
+ server = HTTPServer(("", port), HealthEnpointHandler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+
+ return server
diff --git a/backend/scene-detector/src/nats/messages.py b/backend/scene-detector/src/handler/messages.py
similarity index 100%
rename from backend/scene-detector/src/nats/messages.py
rename to backend/scene-detector/src/handler/messages.py
diff --git a/backend/scene-detector/src/nats/publisher.py b/backend/scene-detector/src/handler/publisher.py
similarity index 100%
rename from backend/scene-detector/src/nats/publisher.py
rename to backend/scene-detector/src/handler/publisher.py
diff --git a/backend/scene-detector/src/handler/subscriber.py b/backend/scene-detector/src/handler/subscriber.py
new file mode 100644
index 0000000..235ffc5
--- /dev/null
+++ b/backend/scene-detector/src/handler/subscriber.py
@@ -0,0 +1,70 @@
+from nats.js.errors import KeyNotFoundError
+from nats.js.kv import KeyValue
+from nats.js.api import ConsumerConfig
+from nats.aio.msg import Msg
+from ..core.logging import logger
+from ..core.settings import settings
+from ..processing.job import process_job
+from .messages import SceneSplitMessage
+from .publisher import scene_video_chunks
+from nats.js.client import JetStreamContext
+import json
+
+
+async def raw_videos(
+ js: JetStreamContext, msg_processed_kv: KeyValue, job_status_kv: KeyValue
+) -> None:
+ """Nats jetstream consumer that subscribes to subject to process videos"""
+ sub = await js.subscribe(
+ subject=settings.SCENE_SPLIT_SUBJECT,
+ durable=settings.NATS_SUB_QUEUE_NAME,
+ queue=settings.NATS_SUB_QUEUE_NAME,
+ config=ConsumerConfig(
+ max_deliver=settings.MAX_DELIVER_ATTEMPTS, ack_wait=settings.ACK_WAIT_S
+ ),
+ )
+
+ async for msg in sub.messages:
+ await _process_msg(js, msg_processed_kv, job_status_kv, msg)
+
+
+async def _process_msg(
+ js: JetStreamContext, msg_processed_kv: KeyValue, job_status_kv: KeyValue, msg: Msg
+) -> None:
+ """Processes a single scene-split message"""
+ try:
+ metadata = SceneSplitMessage.model_validate_json(msg.data.decode())
+
+ if await _is_already_processed(msg_processed_kv, metadata.job_id):
+ logger.debug("job already processed, skipping", job_id=metadata.job_id)
+ await msg.ack()
+ return
+
+ await _update_job_status(job_status_kv, metadata.job_id)
+
+ chunk_messages = await process_job(metadata)
+
+ await scene_video_chunks(js, chunk_messages)
+ await msg_processed_kv.put(metadata.job_id, b"done")
+ await msg.ack()
+ except Exception as e:
+ logger.error("unexpected error processing job", err=str(e))
+ await msg.nak()
+
+
+async def _is_already_processed(kv: KeyValue, job_id: str) -> bool:
+ """Checks if the job_id exists in the scene-split-processed so it doesnt reprocess"""
+ try:
+ await kv.get(job_id)
+ return True
+ except KeyNotFoundError:
+ return False
+
+
+async def _update_job_status(job_status_kv: KeyValue, job_id: str) -> None:
+ """Writes PROCESSING:scene-detector stage to the job-status KV bucket"""
+ try:
+ status = json.dumps({"state": "PROCESSING", "stage": "scene-detector"}).encode()
+ await job_status_kv.put(job_id, status)
+ except Exception as e:
+ logger.error("failed to update job status stage", job_id=job_id, err=str(e))
diff --git a/backend/scene-detector/src/nats/subscriber.py b/backend/scene-detector/src/nats/subscriber.py
deleted file mode 100644
index ecbe765..0000000
--- a/backend/scene-detector/src/nats/subscriber.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from nats.js.errors import KeyNotFoundError
-from nats.js.kv import KeyValue
-from nats.js.api import ConsumerConfig
-from ..core.logging import logger
-from ..core.settings import settings
-from ..processing.job import process_job
-from .messages import SceneSplitMessage
-from .publisher import scene_video_chunks
-from nats.js.client import JetStreamContext
-
-
-async def raw_videos(js: JetStreamContext, kv: KeyValue) -> None:
- """Nats jetstream consumer that subscribes to subject to process videos"""
- sub = await js.subscribe(
- subject=settings.SCENE_SPLIT_SUBJECT,
- durable=settings.NATS_SUB_QUEUE_NAME,
- queue=settings.NATS_SUB_QUEUE_NAME,
- config=ConsumerConfig(
- max_deliver=settings.MAX_DELIVER_ATTEMPTS, ack_wait=settings.ACK_WAIT_S
- ),
- )
-
- async for msg in sub.messages:
- try:
- metadata = SceneSplitMessage.model_validate_json(msg.data.decode())
- if await _is_already_processed(kv, metadata.job_id):
- logger.debug("job already processed, skipping", job_id=metadata.job_id)
- await msg.ack()
- continue
- chunk_messages = await process_job(metadata)
- await scene_video_chunks(js, chunk_messages)
- await kv.put(metadata.job_id, b"done")
- await msg.ack()
- except Exception as e:
- logger.error("unexpected error processing job", err=str(e))
- await msg.nak()
-
-
-async def _is_already_processed(kv: KeyValue, job_id: str) -> bool:
- """Checks if the job_id exists in the scene-split-processed so it doesnt reprocess"""
- try:
- await kv.get(job_id)
- return True
- except KeyNotFoundError:
- return False
diff --git a/backend/scene-detector/src/processing/job.py b/backend/scene-detector/src/processing/job.py
index 9246eb3..437009d 100644
--- a/backend/scene-detector/src/processing/job.py
+++ b/backend/scene-detector/src/processing/job.py
@@ -2,8 +2,8 @@
from ..storage.queries import fetch_video
from ..storage.queries import upload_video_chunks
from .video import split_into_chunks
-from ..nats.messages import SceneSplitMessage
-from ..nats.messages import VideoChunkMessage
+from ..handler.messages import SceneSplitMessage
+from ..handler.messages import VideoChunkMessage
from scenedetect import VideoOpenFailure
import asyncio
import shutil
diff --git a/backend/scene-detector/src/service.py b/backend/scene-detector/src/service.py
index a8f16f8..7b12cbc 100644
--- a/backend/scene-detector/src/service.py
+++ b/backend/scene-detector/src/service.py
@@ -1,6 +1,7 @@
+from src.handler.http_server import start_health_server
from nats.js.api import KeyValueConfig
-from .nats.subscriber import raw_videos
-from .nats.connection import nats_connect
+from .handler.subscriber import raw_videos
+from .handler.connection import nats_connect
from .storage.check_health import check_storage_health
from .core.logging import logger
from .core.settings import settings
@@ -11,6 +12,7 @@
async def start_service() -> None:
"""Start the python scene-detection service"""
check_storage_health()
+ health_server = start_health_server(settings.HTTP_PORT)
nc, js = await nats_connect()
@@ -29,7 +31,7 @@ async def start_service() -> None:
)
try:
- kv = await js.create_key_value(
+ msg_processed_kv = await js.create_key_value(
config=KeyValueConfig(
bucket="scene-split-processed",
description="key value bucket for scene detector to check if the job_id already processed for idempotency",
@@ -40,8 +42,16 @@ async def start_service() -> None:
raise RuntimeError(f"failed to create scene-split-processed KV bucket: {e}")
try:
- await raw_videos(js, kv)
+ job_status_kv = await js.key_value("job-status")
+ except js_errors.NotFoundError:
+ raise RuntimeError(
+ "job-status KV bucket not found, check video-status is running"
+ )
+
+ try:
+ await raw_videos(js, msg_processed_kv, job_status_kv)
finally:
+ health_server.shutdown()
await nc.drain()
diff --git a/backend/scene-detector/tests/conftest.py b/backend/scene-detector/tests/conftest.py
index 26937c9..293862f 100644
--- a/backend/scene-detector/tests/conftest.py
+++ b/backend/scene-detector/tests/conftest.py
@@ -3,4 +3,5 @@
"tests.fixtures.helpers",
"tests.fixtures.nats",
"tests.fixtures.storage",
+ "tests.fixtures.kv",
]
diff --git a/backend/scene-detector/tests/fixtures/helpers.py b/backend/scene-detector/tests/fixtures/helpers.py
index 17d4aaf..ef63e7e 100644
--- a/backend/scene-detector/tests/fixtures/helpers.py
+++ b/backend/scene-detector/tests/fixtures/helpers.py
@@ -1,6 +1,13 @@
-from src.storage import queries
+from typing import Any
+from typing import AsyncGenerator
from pathlib import Path
+from nats.js import JetStreamContext
+from unittest.mock import patch
+from src.handler.http_server import start_health_server
+from src.storage import queries
+import socket
import pytest
+import pytest_asyncio
@pytest.fixture(autouse=True)
@@ -9,6 +16,20 @@ def patch_temp_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(queries, "TEMP_DIR", str(tmp_path))
+@pytest_asyncio.fixture
+async def patched_start_service(
+ js_context: tuple[Any, JetStreamContext],
+) -> AsyncGenerator[tuple[Any, JetStreamContext], None]:
+ """Yields (nc, js) with check_storage_health, start_health_server, and nats_connect patched"""
+ nc, js = js_context
+ with (
+ patch("src.service.check_storage_health"),
+ patch("src.service.start_health_server"),
+ patch("src.service.nats_connect", return_value=(nc, js)),
+ ):
+ yield nc, js
+
+
@pytest.fixture
def chunk_files(tmp_path: Path) -> list[str]:
"""Creates a set of fake .mp4 chunk files in tmp_path"""
@@ -18,3 +39,37 @@ def chunk_files(tmp_path: Path) -> list[str]:
chunk.write_bytes(b"fake chunk content")
chunks.append(str(chunk))
return chunks
+
+
+@pytest.fixture
+def single_video_chunk(tmp_path: Path) -> str:
+ chunk = tmp_path / "chunk.mp4"
+ chunk.write_bytes(b"data")
+ return str(chunk)
+
+
+def _free_port() -> int:
+ with socket.socket() as s:
+ s.bind(("", 0))
+ return s.getsockname()[1]
+
+
+@pytest.fixture
+def live_http_server() -> Any:
+ port = _free_port()
+ server = start_health_server(port)
+ yield f"http://localhost:{port}"
+ server.shutdown()
+
+
+@pytest.fixture
+def spy_drain(js_context: tuple[Any, JetStreamContext]) -> tuple[Any, list[bool]]:
+ """Replaces nc.drain with a no-op spy (whatever that means)"""
+ nc, _ = js_context
+ called: list[bool] = []
+
+ async def _spy() -> None:
+ called.append(True)
+
+ nc.drain = _spy
+ return nc, called
diff --git a/backend/scene-detector/tests/fixtures/kv.py b/backend/scene-detector/tests/fixtures/kv.py
new file mode 100644
index 0000000..6fae093
--- /dev/null
+++ b/backend/scene-detector/tests/fixtures/kv.py
@@ -0,0 +1,11 @@
+from nats.js.errors import KeyNotFoundError
+from nats.js.kv import KeyValue
+from unittest.mock import AsyncMock
+import pytest
+
+
+@pytest.fixture
+def mock_kv() -> AsyncMock:
+ kv = AsyncMock(spec=KeyValue)
+ kv.get.side_effect = KeyNotFoundError()
+ return kv
diff --git a/backend/scene-detector/tests/fixtures/nats.py b/backend/scene-detector/tests/fixtures/nats.py
index 6743fab..98512a4 100644
--- a/backend/scene-detector/tests/fixtures/nats.py
+++ b/backend/scene-detector/tests/fixtures/nats.py
@@ -1,7 +1,10 @@
+from unittest.mock import AsyncMock
+from unittest.mock import MagicMock
from typing import Any
from typing import Generator
from typing import AsyncGenerator
from nats.js import JetStreamContext
+from nats.js.api import KeyValueConfig
from nats.aio.msg import Msg
from testcontainers.nats import NatsContainer
from src.core.settings import settings
@@ -32,6 +35,7 @@ async def js_context(
name="videos",
subjects=[settings.SCENE_SPLIT_SUBJECT, settings.VIDEO_CHUNKS_SUBJECT],
)
+ await js.create_key_value(config=KeyValueConfig(bucket="job-status"))
yield nc, js
await nc.close()
@@ -41,7 +45,7 @@ async def nats_video_chunks_subscriber(
js_context: tuple[Any, JetStreamContext], monkeypatch: Any
) -> AsyncGenerator[list[Any], None]:
monkeypatch.setattr(
- "src.nats.publisher.settings.VIDEO_CHUNKS_SUBJECT",
+ "src.handler.publisher.settings.VIDEO_CHUNKS_SUBJECT",
settings.VIDEO_CHUNKS_SUBJECT,
)
nc, js = js_context
@@ -53,3 +57,14 @@ async def handler(msg: Msg) -> None:
sub = await nc.subscribe(settings.VIDEO_CHUNKS_SUBJECT, cb=handler)
yield received
await sub.unsubscribe()
+
+
+@pytest.fixture
+def mock_nats() -> tuple[MagicMock, MagicMock]:
+ mock_js = MagicMock()
+ mock_js.find_stream_name_by_subject = AsyncMock()
+ mock_js.create_key_value = AsyncMock()
+ mock_js.key_value = AsyncMock()
+ mock_nc = MagicMock()
+ mock_nc.drain = AsyncMock()
+ return mock_nc, mock_js
diff --git a/backend/scene-detector/tests/integration/test_http_server.py b/backend/scene-detector/tests/integration/test_http_server.py
new file mode 100644
index 0000000..4a2fb89
--- /dev/null
+++ b/backend/scene-detector/tests/integration/test_http_server.py
@@ -0,0 +1,16 @@
+import json
+import urllib.request
+import urllib.error
+import pytest
+
+
+def test_health_endpoint_returns_200(live_http_server: str) -> None:
+ with urllib.request.urlopen(f"{live_http_server}/health") as resp:
+ assert resp.status == 200
+ assert json.loads(resp.read()) == {"status": "Healthy"}
+
+
+def test_unknown_path_returns_404(live_http_server: str) -> None:
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ urllib.request.urlopen(f"{live_http_server}/not-found")
+ assert exc_info.value.code == 404
diff --git a/backend/scene-detector/tests/integration/test_nats_connect.py b/backend/scene-detector/tests/integration/test_nats_connect.py
index 9562754..83ddb95 100644
--- a/backend/scene-detector/tests/integration/test_nats_connect.py
+++ b/backend/scene-detector/tests/integration/test_nats_connect.py
@@ -1,7 +1,7 @@
from typing import Any
from nats.aio.client import Client as NATSClient
from nats.js.client import JetStreamContext
-from src.nats.connection import nats_connect
+from src.handler.connection import nats_connect
import pytest
@@ -9,7 +9,7 @@
async def test_connect_returns_connected_clients(
nats_url: str, monkeypatch: Any
) -> None:
- monkeypatch.setattr("src.nats.connection.settings.NATS_URL", nats_url)
+ monkeypatch.setattr("src.handler.connection.settings.NATS_URL", nats_url)
nc, js = await nats_connect()
diff --git a/backend/scene-detector/tests/integration/test_publisher.py b/backend/scene-detector/tests/integration/test_publisher.py
index 7c4d17a..092c274 100644
--- a/backend/scene-detector/tests/integration/test_publisher.py
+++ b/backend/scene-detector/tests/integration/test_publisher.py
@@ -1,7 +1,7 @@
from typing import Any
from nats.js.client import JetStreamContext
-from src.nats.messages import VideoChunkMessage
-from src.nats.publisher import scene_video_chunks
+from src.handler.messages import VideoChunkMessage
+from src.handler.publisher import scene_video_chunks
import pytest
diff --git a/backend/scene-detector/tests/integration/test_start_service.py b/backend/scene-detector/tests/integration/test_start_service.py
index 832e878..e50e194 100644
--- a/backend/scene-detector/tests/integration/test_start_service.py
+++ b/backend/scene-detector/tests/integration/test_start_service.py
@@ -1,8 +1,9 @@
-from unittest.mock import patch, AsyncMock
-from nats.js import JetStreamContext
from typing import Any
+from unittest.mock import patch
+from unittest.mock import AsyncMock
+from nats.js import JetStreamContext
from src.service import start_service
-from src.nats.messages import VideoChunkMessage
+from src.handler.messages import VideoChunkMessage
from src.core.settings import settings
import asyncio
import json
@@ -12,17 +13,18 @@
@pytest.mark.asyncio
async def test_full_flow_publishes_chunks_downstream(
- js_context: tuple[Any, JetStreamContext],
+ patched_start_service: tuple[Any, JetStreamContext],
nats_video_chunks_subscriber: list[Any],
monkeypatch: Any,
) -> None:
"""Publishes to upstream topic -> process_job runs -> chunks appear on downstream topic"""
- nc, js = js_context
+ nc, js = patched_start_service
monkeypatch.setattr(
- "src.nats.subscriber.settings.SCENE_SPLIT_SUBJECT", settings.SCENE_SPLIT_SUBJECT
+ "src.handler.subscriber.settings.SCENE_SPLIT_SUBJECT",
+ settings.SCENE_SPLIT_SUBJECT,
)
monkeypatch.setattr(
- "src.nats.subscriber.settings.NATS_SUB_QUEUE_NAME", "test-full-flow-worker"
+ "src.handler.subscriber.settings.NATS_SUB_QUEUE_NAME", "test-full-flow-worker"
)
nc.drain = AsyncMock()
@@ -47,20 +49,18 @@ async def test_full_flow_publishes_chunks_downstream(
async def fake_process_job(_metadata: Any) -> list[VideoChunkMessage]:
return fake_chunks
- with (
- patch("src.service.check_storage_health"),
- patch("src.service.nats_connect", return_value=(nc, js)),
- patch("src.nats.subscriber.process_job", side_effect=fake_process_job),
- ):
+ with patch("src.handler.subscriber.process_job", side_effect=fake_process_job):
task = asyncio.create_task(start_service())
- payload = json.dumps(
- {
- "job_id": job_id,
- "storage_url": "/fake/video.mp4",
- "target_resolution": "480p",
- }
- ).encode()
- await nc.publish(settings.SCENE_SPLIT_SUBJECT, payload)
+ await nc.publish(
+ settings.SCENE_SPLIT_SUBJECT,
+ json.dumps(
+ {
+ "job_id": job_id,
+ "storage_url": "/fake/video.mp4",
+ "target_resolution": "480p",
+ }
+ ).encode(),
+ )
await asyncio.sleep(0.5)
task.cancel()
try:
@@ -87,76 +87,58 @@ async def fake_process_job(_metadata: Any) -> list[VideoChunkMessage]:
@pytest.mark.asyncio
async def test_raises_runtime_error_when_video_chunks_stream_not_found(
- js_context: tuple[Any, JetStreamContext],
+ patched_start_service: tuple[Any, JetStreamContext],
monkeypatch: Any,
) -> None:
"""Raises RuntimeError when no NATS stream covers the downstream chunks subject"""
- nc, js = js_context
+ nc, js = patched_start_service
monkeypatch.setattr(
"src.service.settings.VIDEO_CHUNKS_SUBJECT", "nonexistent.subject.xyz"
)
nc.drain = AsyncMock()
- with (
- patch("src.service.check_storage_health"),
- patch("src.service.nats_connect", return_value=(nc, js)),
- pytest.raises(RuntimeError, match="No stream found for video chunks"),
- ):
+ with pytest.raises(RuntimeError, match="No stream found for video chunks"):
await start_service()
@pytest.mark.asyncio
async def test_drain_called_in_finally_when_raw_videos_raises(
- js_context: tuple[Any, JetStreamContext],
+ patched_start_service: tuple[Any, JetStreamContext],
+ spy_drain: tuple[Any, list[bool]],
) -> None:
"""nc.drain() is called in the finally block even when raw_videos raises"""
- nc, js = js_context
- drain_called = False
-
- async def spy_drain() -> None:
- nonlocal drain_called
- drain_called = True
- # Don't call the real drain — the connection is shared with the fixture
+ nc, js = patched_start_service # this isnt used? maybe we dont need it
+ _, called = spy_drain
- nc.drain = spy_drain
-
- async def failing_raw_videos(_js: JetStreamContext, _kv: Any) -> None:
+ async def failing_raw_videos(
+ _js: JetStreamContext, _kv: Any, _job_status_kv: Any
+ ) -> None:
raise RuntimeError("subscriber failed unexpectedly")
with (
- patch("src.service.check_storage_health"),
- patch("src.service.nats_connect", return_value=(nc, js)),
patch("src.service.raw_videos", side_effect=failing_raw_videos),
pytest.raises(RuntimeError, match="subscriber failed unexpectedly"),
):
await start_service()
- assert drain_called
+ assert called
@pytest.mark.asyncio
async def test_drain_called_in_finally_on_cancellation(
- js_context: tuple[Any, JetStreamContext],
+ patched_start_service: tuple[Any, JetStreamContext],
+ spy_drain: tuple[Any, list[bool]],
) -> None:
"""nc.drain() is called in the finally block when the service task is cancelled"""
- nc, js = js_context
- drain_called = False
-
- async def spy_drain() -> None:
- nonlocal drain_called
- drain_called = True
- # Don't call the real drain — the connection is shared with the fixture
-
- nc.drain = spy_drain
+ nc, js = patched_start_service # this isnt used? maybe we dont need it
+ _, called = spy_drain
- async def hanging_raw_videos(_js: JetStreamContext, _kv: Any) -> None:
+ async def hanging_raw_videos(
+ _js: JetStreamContext, _kv: Any, _job_status_kv: Any
+ ) -> None:
await asyncio.sleep(30)
- with (
- patch("src.service.check_storage_health"),
- patch("src.service.nats_connect", return_value=(nc, js)),
- patch("src.service.raw_videos", side_effect=hanging_raw_videos),
- ):
+ with patch("src.service.raw_videos", side_effect=hanging_raw_videos):
task = asyncio.create_task(start_service())
await asyncio.sleep(0.05)
task.cancel()
@@ -165,24 +147,25 @@ async def hanging_raw_videos(_js: JetStreamContext, _kv: Any) -> None:
except asyncio.CancelledError:
pass
- assert drain_called
+ assert called
@pytest.mark.asyncio
async def test_service_can_be_cancelled_while_process_job_is_running(
- js_context: tuple[Any, JetStreamContext],
+ patched_start_service: tuple[Any, JetStreamContext],
monkeypatch: Any,
) -> None:
"""Service cancels promptly mid-processing, proving process_job does not block the event loop"""
- nc, js = js_context
+ nc, _ = patched_start_service
monkeypatch.setattr(
- "src.nats.subscriber.settings.SCENE_SPLIT_SUBJECT", settings.SCENE_SPLIT_SUBJECT
+ "src.handler.subscriber.settings.SCENE_SPLIT_SUBJECT",
+ settings.SCENE_SPLIT_SUBJECT,
)
# Unique consumer name: prevents unacked messages from leaking into other tests' durable consumers
monkeypatch.setattr(
- "src.nats.subscriber.settings.NATS_SUB_QUEUE_NAME", "test-cancellation-worker"
+ "src.handler.subscriber.settings.NATS_SUB_QUEUE_NAME",
+ "test-cancellation-worker",
)
- # No-op drain: prevents start_service's finally block from closing the shared fixture connection
nc.drain = AsyncMock()
processing_started = asyncio.Event()
@@ -192,11 +175,7 @@ async def slow_process_job(_metadata: Any) -> list[Any]:
await asyncio.sleep(30)
return []
- with (
- patch("src.service.check_storage_health"),
- patch("src.service.nats_connect", return_value=(nc, js)),
- patch("src.nats.subscriber.process_job", side_effect=slow_process_job),
- ):
+ with patch("src.handler.subscriber.process_job", side_effect=slow_process_job):
task = asyncio.create_task(start_service())
payload = json.dumps(
{
@@ -212,7 +191,7 @@ async def slow_process_job(_metadata: Any) -> list[Any]:
try:
await asyncio.wait_for(task, timeout=2.0)
except asyncio.CancelledError:
- pass # expected — task was properly cancelled
+ pass
except asyncio.TimeoutError:
pytest.fail(
"Service did not cancel within 2 seconds — process_job may be blocking the event loop"
@@ -221,11 +200,9 @@ async def slow_process_job(_metadata: Any) -> list[Any]:
@pytest.mark.asyncio
async def test_raises_before_nats_when_storage_unreachable(
- js_context: tuple[Any, JetStreamContext],
monkeypatch: Any,
) -> None:
"""Service raises and never connects to NATS when SeaweedFS is unreachable"""
- nc, js = js_context
monkeypatch.setattr(
"src.storage.check_health.settings.BASE_STORAGE_URL", "http://localhost:1"
)
diff --git a/backend/scene-detector/tests/integration/test_subscriber.py b/backend/scene-detector/tests/integration/test_subscriber.py
index b22664e..d7a127e 100644
--- a/backend/scene-detector/tests/integration/test_subscriber.py
+++ b/backend/scene-detector/tests/integration/test_subscriber.py
@@ -1,30 +1,66 @@
from typing import Any
from unittest.mock import patch
-from nats.js.client import JetStreamContext
+from nats.js.kv import KeyValue
from nats.js.api import KeyValueConfig
-from src.nats.subscriber import raw_videos
-from src.nats.messages import SceneSplitMessage
+from nats.js.client import JetStreamContext
+from src.handler.subscriber import raw_videos
+from src.handler.messages import SceneSplitMessage
import json
import pytest
import asyncio
import uuid
+async def _run_subscriber(
+ nc: Any,
+ js: JetStreamContext,
+ kv: KeyValue,
+ job_status_kv: KeyValue,
+ payload: bytes,
+) -> list[Any]:
+ """
+ Launch raw_videos as a task, pub one msg, wait, then cancel
+ returrns all processed jobs as a side effect of the process_job
+ """
+ processed_job: list[Any] = []
+
+ async def fake_process_job(metadata: Any) -> list[Any]:
+ processed_job.append(metadata)
+ return []
+
+ with patch("src.handler.subscriber.process_job", side_effect=fake_process_job):
+ task = asyncio.create_task(raw_videos(js, kv, job_status_kv))
+ await nc.publish("jobs.video.scene-split", payload)
+ await asyncio.sleep(0.5) # let the subscriber process the message
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+ return processed_job
+
+
@pytest.mark.asyncio
async def test_processes_published_message(
js_context: tuple[Any, JetStreamContext], monkeypatch: Any
) -> None:
"""Verifies subscriber receives a message and calls process_job with correct data"""
nc, js = js_context
+
monkeypatch.setattr(
- "src.nats.subscriber.settings.SCENE_SPLIT_SUBJECT", "jobs.video.scene-split"
+ "src.handler.subscriber.settings.SCENE_SPLIT_SUBJECT", "jobs.video.scene-split"
)
monkeypatch.setattr(
- "src.nats.subscriber.settings.NATS_SUB_QUEUE_NAME", "scene-detector-workers"
+ "src.handler.subscriber.settings.NATS_SUB_QUEUE_NAME", "scene-detector-workers"
)
+
kv = await js.create_key_value(
config=KeyValueConfig(bucket="test-scene-split-status-1")
)
+ job_status_kv = await js.create_key_value(
+ config=KeyValueConfig(bucket="test-job-status-sub-1")
+ )
job_id = str(uuid.uuid4())
payload = json.dumps(
@@ -34,24 +70,11 @@ async def test_processes_published_message(
"target_resolution": "480p",
}
).encode()
- received: list[Any] = []
- async def fake_process_job(metadata: Any) -> list[Any]:
- received.append(metadata)
- return []
+ recieved = await _run_subscriber(nc, js, kv, job_status_kv, payload)
- with patch("src.nats.subscriber.process_job", side_effect=fake_process_job):
- task = asyncio.create_task(raw_videos(js, kv))
- await nc.publish("jobs.video.scene-split", payload)
- await asyncio.sleep(0.5) # let the subscriber process the message
- task.cancel()
- try:
- await task
- except asyncio.CancelledError:
- pass
-
- assert len(received) == 1
- assert received[0] == SceneSplitMessage(
+ assert len(recieved) == 1
+ assert recieved[0] == SceneSplitMessage(
job_id=job_id, storage_url="/fake/video.mp4", target_resolution="480p"
)
@@ -62,16 +85,21 @@ async def test_skips_redelivered_message_for_already_processed_job(
) -> None:
"""Verifies subscriber acks and skips processing when job_id already exists in KV"""
nc, js = js_context
+
monkeypatch.setattr(
- "src.nats.subscriber.settings.SCENE_SPLIT_SUBJECT", "jobs.video.scene-split"
+ "src.handler.subscriber.settings.SCENE_SPLIT_SUBJECT", "jobs.video.scene-split"
)
monkeypatch.setattr(
- "src.nats.subscriber.settings.NATS_SUB_QUEUE_NAME",
+ "src.handler.subscriber.settings.NATS_SUB_QUEUE_NAME",
"scene-detector-workers-idempotency",
)
+
kv = await js.create_key_value(
config=KeyValueConfig(bucket="test-scene-split-status-2")
)
+ job_status_kv = await js.create_key_value(
+ config=KeyValueConfig(bucket="test-job-status-sub-2")
+ )
await kv.put("job-already-done", b"done")
payload = json.dumps(
@@ -81,20 +109,7 @@ async def test_skips_redelivered_message_for_already_processed_job(
"target_resolution": "480p",
}
).encode()
- process_calls: list[Any] = []
- async def fake_process_job(metadata: Any) -> list[Any]:
- process_calls.append(metadata)
- return []
-
- with patch("src.nats.subscriber.process_job", side_effect=fake_process_job):
- task = asyncio.create_task(raw_videos(js, kv))
- await nc.publish("jobs.video.scene-split", payload)
- await asyncio.sleep(0.5)
- task.cancel()
- try:
- await task
- except asyncio.CancelledError:
- pass
+ process_calls = await _run_subscriber(nc, js, kv, job_status_kv, payload)
assert len(process_calls) == 0
diff --git a/backend/scene-detector/tests/unit/test_http_server.py b/backend/scene-detector/tests/unit/test_http_server.py
new file mode 100644
index 0000000..98a7a4d
--- /dev/null
+++ b/backend/scene-detector/tests/unit/test_http_server.py
@@ -0,0 +1,63 @@
+from typing import Any
+from http.server import HTTPServer
+from unittest.mock import MagicMock, create_autospec, patch
+from src.handler.http_server import HealthEnpointHandler, start_health_server
+import json
+import pytest
+import threading
+
+
+def make_handler(path: str) -> MagicMock:
+ handler = create_autospec(HealthEnpointHandler, instance=True)
+ handler.path = path
+ handler.wfile = MagicMock()
+ return handler
+
+
+@pytest.mark.parametrize(
+ "path,expected_status,expected_body",
+ [
+ ("/health", 200, {"status": "Healthy"}),
+ ("/unknown", 404, None),
+ ],
+ ids=["health", "not_found"],
+)
+def test_endpoint(
+ path: str, expected_status: int, expected_body: dict[str, Any] | None
+) -> None:
+ handler = make_handler(path)
+ HealthEnpointHandler.do_GET(handler)
+
+ handler.send_response.assert_called_once_with(expected_status)
+ handler.end_headers.assert_called_once()
+ if expected_body is not None:
+ handler.send_header.assert_called_once_with("Content-Type", "application/json")
+ assert json.loads(handler.wfile.write.call_args[0][0]) == expected_body
+ else:
+ handler.wfile.write.assert_not_called()
+
+
+# ── server startup ────────────────────────────────────────────────────────────
+
+
+def test_start_health_server() -> None:
+ mock_server = MagicMock(spec=HTTPServer)
+ real_thread_cls = threading.Thread
+ created_threads: list[MagicMock] = []
+ captured_kwargs: list[dict[str, Any]] = []
+
+ def capture_thread(**kwargs: object) -> MagicMock:
+ captured_kwargs.append(kwargs) # type: ignore[arg-type]
+ t = MagicMock(spec=real_thread_cls)
+ created_threads.append(t)
+ return t
+
+ with (
+ patch("src.handler.http_server.HTTPServer", return_value=mock_server),
+ patch("src.handler.http_server.threading.Thread", side_effect=capture_thread),
+ ):
+ result = start_health_server(9099)
+
+ assert result is mock_server
+ assert captured_kwargs[0].get("daemon") is True
+ created_threads[0].start.assert_called_once()
diff --git a/backend/scene-detector/tests/unit/test_nats_connect.py b/backend/scene-detector/tests/unit/test_nats_connect.py
index c0ca6e0..eb7f376 100644
--- a/backend/scene-detector/tests/unit/test_nats_connect.py
+++ b/backend/scene-detector/tests/unit/test_nats_connect.py
@@ -7,7 +7,7 @@
from nats.errors import NoServersError
from nats.errors import AuthorizationError
from nats.js.client import JetStreamContext
-from src.nats.connection import nats_connect
+from src.handler.connection import nats_connect
import pytest
@@ -17,7 +17,7 @@
)
async def test_connect_raises_on_nats_failure(exc: Any) -> None:
"""It should raise the error when caught"""
- with patch("src.nats.connection.NATSClient") as mock_client_class:
+ with patch("src.handler.connection.NATSClient") as mock_client_class:
mock_instance = MagicMock(spec=NATSClient)
mock_instance.connect = AsyncMock(side_effect=exc)
mock_client_class.return_value = mock_instance
@@ -32,7 +32,7 @@ async def test_connect_returns_nats_and_jetstream() -> None:
mock_ns.connect = AsyncMock()
mock_ns.jetstream.return_value = mock_js
- with patch("src.nats.connection.NATSClient", return_value=mock_ns):
+ with patch("src.handler.connection.NATSClient", return_value=mock_ns):
nc, js = await nats_connect()
assert nc is mock_ns
diff --git a/backend/scene-detector/tests/unit/test_process_job.py b/backend/scene-detector/tests/unit/test_process_job.py
index ff706ff..89f5c70 100644
--- a/backend/scene-detector/tests/unit/test_process_job.py
+++ b/backend/scene-detector/tests/unit/test_process_job.py
@@ -1,7 +1,7 @@
from scenedetect import VideoOpenFailure
from unittest.mock import patch
from src.processing.job import process_job
-from src.nats.messages import SceneSplitMessage, VideoChunkMessage
+from src.handler.messages import SceneSplitMessage, VideoChunkMessage
import pytest
METADATA = SceneSplitMessage(
diff --git a/backend/scene-detector/tests/unit/test_publisher.py b/backend/scene-detector/tests/unit/test_publisher.py
index 919434e..bd0b153 100644
--- a/backend/scene-detector/tests/unit/test_publisher.py
+++ b/backend/scene-detector/tests/unit/test_publisher.py
@@ -3,8 +3,8 @@
from nats.errors import TimeoutError
from nats.js.errors import APIError
from nats.js.client import JetStreamContext
-from src.nats.publisher import scene_video_chunks
-from src.nats.messages import VideoChunkMessage
+from src.handler.publisher import scene_video_chunks
+from src.handler.messages import VideoChunkMessage
import pytest
diff --git a/backend/scene-detector/tests/unit/test_start_service.py b/backend/scene-detector/tests/unit/test_start_service.py
index 6beee67..fca1f21 100644
--- a/backend/scene-detector/tests/unit/test_start_service.py
+++ b/backend/scene-detector/tests/unit/test_start_service.py
@@ -1,43 +1,54 @@
-from unittest.mock import patch
-from unittest.mock import MagicMock
-from unittest.mock import AsyncMock
+from typing import Any
+from unittest.mock import patch, MagicMock, AsyncMock
from src.service import start_service
import pytest
import nats.js.errors as js_errors
-@pytest.mark.asyncio
-async def test_raises_on_runtime_error() -> None:
- """It should raise the RuntimeError when stream is not found"""
- mock_js = MagicMock()
- mock_js.find_stream_name_by_subject = AsyncMock(side_effect=js_errors.NotFoundError)
-
- mock_nc = MagicMock()
- mock_nc.drain = AsyncMock()
-
+@pytest.fixture
+def service_patches(mock_nats: tuple[MagicMock, MagicMock]) -> Any:
+ mock_nc, mock_js = mock_nats
with (
patch("src.service.check_storage_health"),
+ patch("src.service.start_health_server"),
patch("src.service.nats_connect", return_value=(mock_nc, mock_js)),
- pytest.raises(RuntimeError),
):
- await start_service()
+ yield mock_nc, mock_js
@pytest.mark.asyncio
-async def test_raises_runtime_error_when_kv_creation_fails() -> None:
- """It should raise RuntimeError when the KV bucket cannot be created"""
- mock_js = MagicMock()
- mock_js.find_stream_name_by_subject = AsyncMock()
- mock_js.create_key_value = AsyncMock(side_effect=js_errors.APIError())
-
- mock_nc = MagicMock()
- mock_nc.drain = AsyncMock()
-
- with (
- patch("src.service.check_storage_health"),
- patch("src.service.nats_connect", return_value=(mock_nc, mock_js)),
- pytest.raises(
- RuntimeError, match="failed to create scene-split-processed KV bucket"
+@pytest.mark.parametrize(
+ "setup_js,match",
+ [
+ (
+ lambda js: setattr(
+ js,
+ "find_stream_name_by_subject",
+ AsyncMock(side_effect=js_errors.NotFoundError),
+ ),
+ None,
),
- ):
+ (
+ lambda js: setattr(
+ js, "create_key_value", AsyncMock(side_effect=js_errors.APIError())
+ ),
+ "failed to create scene-split-processed KV bucket",
+ ),
+ (
+ lambda js: setattr(
+ js, "key_value", AsyncMock(side_effect=js_errors.NotFoundError)
+ ),
+ "job-status KV bucket not found",
+ ),
+ ],
+ ids=["stream_not_found", "kv_creation_fails", "job_status_kv_not_found"],
+)
+async def test_raises_runtime_error(
+ service_patches: Any,
+ setup_js: Any,
+ match: str | None,
+) -> None:
+ _, mock_js = service_patches
+ setup_js(mock_js)
+ with pytest.raises(RuntimeError, match=match):
await start_service()
diff --git a/backend/scene-detector/tests/unit/test_subscriber.py b/backend/scene-detector/tests/unit/test_subscriber.py
index 5e9864d..a1eb9b5 100644
--- a/backend/scene-detector/tests/unit/test_subscriber.py
+++ b/backend/scene-detector/tests/unit/test_subscriber.py
@@ -3,119 +3,128 @@
from nats.js.errors import APIError, KeyNotFoundError
from nats.js.client import JetStreamContext
from nats.js.kv import KeyValue
-from src.nats.subscriber import raw_videos
-from src.nats.messages import SceneSplitMessage, VideoChunkMessage
+from src.handler.subscriber import raw_videos
+from src.handler.messages import SceneSplitMessage, VideoChunkMessage
import json
import pytest
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+
def make_mock_msg(data: dict[str, Any]) -> AsyncMock:
msg = AsyncMock()
msg.data = json.dumps(data).encode()
return msg
-def make_mock_kv(already_processed: bool = False) -> MagicMock:
- mock_kv = AsyncMock(spec=KeyValue)
- if already_processed:
- mock_kv.get.return_value = MagicMock()
- else:
- mock_kv.get.side_effect = KeyNotFoundError()
- return mock_kv
-
-
async def async_iter(items: Any) -> AsyncGenerator[Any, None]:
for item in items:
yield item
-@pytest.mark.asyncio
-async def test_acks_on_success() -> None:
- msg = make_mock_msg(
- {"job_id": "1", "storage_url": "/fake/idk.mp4", "target_resolution": "480p"}
- )
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
+def make_mock_js(*msgs: AsyncMock) -> AsyncMock:
+ js = AsyncMock(spec=JetStreamContext)
+ sub = MagicMock()
+ sub.messages = async_iter(list(msgs))
+ js.subscribe.return_value = sub
+ return js
- with (
- patch(
- "src.nats.subscriber.process_job", new_callable=AsyncMock, return_value=[]
- ),
- patch("src.nats.subscriber.scene_video_chunks", new_callable=AsyncMock),
- ):
- await raw_videos(mock_js, mock_kv)
- msg.ack.assert_called_once()
- msg.nak.assert_not_called()
+# ── fixtures ─────────────────────────────────────────────────────────────────
-@pytest.mark.asyncio
-async def test_naks_when_process_job_fails() -> None:
- msg = make_mock_msg(
+@pytest.fixture
+def msg() -> AsyncMock:
+ return make_mock_msg(
{"job_id": "1", "storage_url": "/fake/idk.mp4", "target_resolution": "480p"}
)
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
+
+@pytest.mark.asyncio
+async def test_acks_on_success(mock_kv: AsyncMock, msg: AsyncMock) -> None:
with (
patch(
- "src.nats.subscriber.process_job",
+ "src.handler.subscriber.process_job",
new_callable=AsyncMock,
- side_effect=Exception("failed"),
+ return_value=[],
),
- patch("src.nats.subscriber.scene_video_chunks", new_callable=AsyncMock),
+ patch("src.handler.subscriber.scene_video_chunks", new_callable=AsyncMock),
):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(make_mock_js(msg), mock_kv, AsyncMock(spec=KeyValue))
- msg.nak.assert_called_once()
- msg.ack.assert_not_called()
+ msg.ack.assert_called_once()
+ msg.nak.assert_not_called()
@pytest.mark.asyncio
-async def test_naks_when_publish_fails() -> None:
- msg = make_mock_msg(
- {"job_id": "1", "storage_url": "/fake/idk.mp4", "target_resolution": "480p"}
- )
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
-
+@pytest.mark.parametrize(
+ "process_job_kwargs,publish_kwargs",
+ [
+ ({"side_effect": Exception("process failed")}, {}),
+ ({"return_value": []}, {"side_effect": Exception("publish failed")}),
+ ],
+ ids=["process_job_fails", "publish_fails"],
+)
+async def test_naks_on_failure(
+ mock_kv: AsyncMock,
+ msg: AsyncMock,
+ process_job_kwargs: dict[str, Any],
+ publish_kwargs: dict[str, Any],
+) -> None:
with (
patch(
- "src.nats.subscriber.process_job", new_callable=AsyncMock, return_value=[]
+ "src.handler.subscriber.process_job",
+ new_callable=AsyncMock,
+ **process_job_kwargs,
),
patch(
- "src.nats.subscriber.scene_video_chunks",
+ "src.handler.subscriber.scene_video_chunks",
new_callable=AsyncMock,
- side_effect=Exception("publish failed"),
+ **publish_kwargs,
),
):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(make_mock_js(msg), mock_kv, AsyncMock(spec=KeyValue))
msg.nak.assert_called_once()
msg.ack.assert_not_called()
@pytest.mark.asyncio
-async def test_raises_when_subscribe_fails() -> None:
+async def test_raises_when_subscribe_fails(mock_kv: AsyncMock) -> None:
mock_js = AsyncMock(spec=JetStreamContext)
mock_js.subscribe.side_effect = APIError()
- mock_kv = make_mock_kv()
with pytest.raises(APIError):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(mock_js, mock_kv, AsyncMock(spec=KeyValue))
@pytest.mark.asyncio
-async def test_calls_process_job_per_message() -> None:
+async def test_acks_and_skips_when_job_already_processed(msg: AsyncMock) -> None:
+ already_processed_kv = AsyncMock(spec=KeyValue)
+ already_processed_kv.get.return_value = MagicMock()
+ mock_js = make_mock_js(msg)
+
+ with (
+ patch(
+ "src.handler.subscriber.process_job",
+ new_callable=AsyncMock,
+ return_value=[],
+ ) as mock_process,
+ patch("src.handler.subscriber.scene_video_chunks", new_callable=AsyncMock),
+ ):
+ await raw_videos(mock_js, already_processed_kv, AsyncMock(spec=KeyValue))
+
+ msg.ack.assert_called_once()
+ msg.nak.assert_not_called()
+ mock_process.assert_not_called()
+
+
+# ── message routing ───────────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_calls_process_job_per_message(mock_kv: AsyncMock) -> None:
msgs = [
make_mock_msg(
{"job_id": "1", "storage_url": "/fake/a.mp4", "target_resolution": "480p"}
@@ -124,19 +133,17 @@ async def test_calls_process_job_per_message() -> None:
{"job_id": "2", "storage_url": "/fake/b.mp4", "target_resolution": "480p"}
),
]
- mock_sub = MagicMock()
- mock_sub.messages = async_iter(msgs)
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
+ mock_js = make_mock_js(*msgs)
with (
patch(
- "src.nats.subscriber.process_job", new_callable=AsyncMock, return_value=[]
+ "src.handler.subscriber.process_job",
+ new_callable=AsyncMock,
+ return_value=[],
) as mock_process,
- patch("src.nats.subscriber.scene_video_chunks", new_callable=AsyncMock),
+ patch("src.handler.subscriber.scene_video_chunks", new_callable=AsyncMock),
):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(mock_js, mock_kv, AsyncMock(spec=KeyValue))
assert mock_process.call_count == 2
assert mock_process.call_args_list[0][0][0] == SceneSplitMessage(
@@ -148,7 +155,7 @@ async def test_calls_process_job_per_message() -> None:
@pytest.mark.asyncio
-async def test_passes_chunk_messages_to_publisher() -> None:
+async def test_passes_chunk_messages_to_publisher(mock_kv: AsyncMock) -> None:
chunk_messages = [
VideoChunkMessage(
job_id="1",
@@ -161,51 +168,23 @@ async def test_passes_chunk_messages_to_publisher() -> None:
msg = make_mock_msg(
{"job_id": "1", "storage_url": "/fake/idk.mp4", "target_resolution": "480p"}
)
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
+ mock_js = make_mock_js(msg)
with (
patch(
- "src.nats.subscriber.process_job",
+ "src.handler.subscriber.process_job",
new_callable=AsyncMock,
return_value=chunk_messages,
),
patch(
- "src.nats.subscriber.scene_video_chunks", new_callable=AsyncMock
+ "src.handler.subscriber.scene_video_chunks", new_callable=AsyncMock
) as mock_publish,
):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(mock_js, mock_kv, AsyncMock(spec=KeyValue))
mock_publish.assert_called_once_with(mock_js, chunk_messages)
-@pytest.mark.asyncio
-async def test_acks_and_skips_when_job_already_processed() -> None:
- msg = make_mock_msg(
- {"job_id": "1", "storage_url": "/fake/idk.mp4", "target_resolution": "480p"}
- )
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv(already_processed=True)
-
- with (
- patch(
- "src.nats.subscriber.process_job", new_callable=AsyncMock, return_value=[]
- ) as mock_process,
- patch("src.nats.subscriber.scene_video_chunks", new_callable=AsyncMock),
- ):
- await raw_videos(mock_js, mock_kv)
-
- msg.ack.assert_called_once()
- msg.nak.assert_not_called()
- mock_process.assert_not_called()
-
-
@pytest.mark.asyncio
async def test_writes_to_kv_on_success() -> None:
msg = make_mock_msg(
@@ -215,68 +194,110 @@ async def test_writes_to_kv_on_success() -> None:
"target_resolution": "480p",
}
)
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
+ mock_kv = AsyncMock(spec=KeyValue)
+ mock_kv.get.side_effect = KeyNotFoundError()
+ mock_js = make_mock_js(msg)
with (
patch(
- "src.nats.subscriber.process_job", new_callable=AsyncMock, return_value=[]
+ "src.handler.subscriber.process_job",
+ new_callable=AsyncMock,
+ return_value=[],
),
- patch("src.nats.subscriber.scene_video_chunks", new_callable=AsyncMock),
+ patch("src.handler.subscriber.scene_video_chunks", new_callable=AsyncMock),
):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(mock_js, mock_kv, AsyncMock(spec=KeyValue))
mock_kv.put.assert_called_once_with("abc-123", b"done")
@pytest.mark.asyncio
-async def test_does_not_write_to_kv_when_process_job_fails() -> None:
- msg = make_mock_msg(
- {"job_id": "1", "storage_url": "/fake/idk.mp4", "target_resolution": "480p"}
- )
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
-
+@pytest.mark.parametrize(
+ "process_job_kwargs,publish_kwargs",
+ [
+ ({"side_effect": Exception("process failed")}, {}),
+ ({"return_value": []}, {"side_effect": Exception("publish failed")}),
+ ],
+ ids=["process_job_fails", "publish_fails"],
+)
+async def test_does_not_write_to_kv_on_failure(
+ mock_kv: AsyncMock,
+ msg: AsyncMock,
+ process_job_kwargs: dict[str, Any],
+ publish_kwargs: dict[str, Any],
+) -> None:
with (
patch(
- "src.nats.subscriber.process_job",
+ "src.handler.subscriber.process_job",
+ new_callable=AsyncMock,
+ **process_job_kwargs,
+ ),
+ patch(
+ "src.handler.subscriber.scene_video_chunks",
new_callable=AsyncMock,
- side_effect=Exception("failed"),
+ **publish_kwargs,
),
- patch("src.nats.subscriber.scene_video_chunks", new_callable=AsyncMock),
):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(make_mock_js(msg), mock_kv, AsyncMock(spec=KeyValue))
mock_kv.put.assert_not_called()
@pytest.mark.asyncio
-async def test_does_not_write_to_kv_when_publish_fails() -> None:
- msg = make_mock_msg(
- {"job_id": "1", "storage_url": "/fake/idk.mp4", "target_resolution": "480p"}
- )
- mock_sub = MagicMock()
- mock_sub.messages = async_iter([msg])
- mock_js = AsyncMock(spec=JetStreamContext)
- mock_js.subscribe.return_value = mock_sub
- mock_kv = make_mock_kv()
+async def test_update_job_status_error_logs_and_continues(
+ mock_kv: AsyncMock, msg: AsyncMock
+) -> None:
+ """When job_status_kv.put raises, the error is logged and message is still acked"""
+ mock_job_status_kv = AsyncMock(spec=KeyValue)
+ mock_job_status_kv.put.side_effect = Exception("kv write failed")
with (
patch(
- "src.nats.subscriber.process_job", new_callable=AsyncMock, return_value=[]
- ),
- patch(
- "src.nats.subscriber.scene_video_chunks",
+ "src.handler.subscriber.process_job",
new_callable=AsyncMock,
- side_effect=Exception("publish failed"),
+ return_value=[],
),
+ patch("src.handler.subscriber.scene_video_chunks", new_callable=AsyncMock),
):
- await raw_videos(mock_js, mock_kv)
+ await raw_videos(make_mock_js(msg), mock_kv, mock_job_status_kv)
- mock_kv.put.assert_not_called()
+ msg.ack.assert_called_once()
+ msg.nak.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_stage_written_to_job_status_kv_before_processing(
+ mock_kv: AsyncMock,
+) -> None:
+ """job_status_kv.put is called with PROCESSING:scene-detector before process_job runs"""
+ msg = make_mock_msg(
+ {
+ "job_id": "abc-123",
+ "storage_url": "/fake/idk.mp4",
+ "target_resolution": "480p",
+ }
+ )
+ mock_js = make_mock_js(msg)
+ mock_job_status_kv = AsyncMock(spec=KeyValue)
+ call_order: list[str] = []
+
+ async def fake_process_job(_metadata: Any) -> list[Any]:
+ call_order.append("process_job")
+ return []
+
+ async def fake_job_status_put(key: str, value: bytes) -> None:
+ call_order.append("job_status_put")
+
+ mock_job_status_kv.put.side_effect = fake_job_status_put
+
+ with (
+ patch("src.handler.subscriber.process_job", side_effect=fake_process_job),
+ patch("src.handler.subscriber.scene_video_chunks", new_callable=AsyncMock),
+ ):
+ await raw_videos(mock_js, mock_kv, mock_job_status_kv)
+
+ expected_payload = json.dumps(
+ {"state": "PROCESSING", "stage": "scene-detector"}
+ ).encode()
+ mock_job_status_kv.put.assert_called_once_with("abc-123", expected_payload)
+ assert call_order == ["job_status_put", "process_job"]
diff --git a/backend/scene-detector/tests/unit/test_upload_video_chunks.py b/backend/scene-detector/tests/unit/test_upload_video_chunks.py
index 0ae04df..2409cb6 100644
--- a/backend/scene-detector/tests/unit/test_upload_video_chunks.py
+++ b/backend/scene-detector/tests/unit/test_upload_video_chunks.py
@@ -1,31 +1,28 @@
-from unittest.mock import patch
-from unittest.mock import MagicMock
from pathlib import Path
+from unittest.mock import patch, MagicMock
from src.storage.queries import upload_video_chunks
import requests
import pytest
-def test_raises_file_not_found(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+@pytest.fixture
+def fake_base_url(monkeypatch: pytest.MonkeyPatch) -> str:
+ url = "http://fake:8888"
+ monkeypatch.setattr("src.storage.queries.settings.BASE_STORAGE_URL", url)
+ return url
+
+
+def test_raises_file_not_found(fake_base_url: str, tmp_path: Path) -> None:
"""Raises FileNotFoundError when a chunk path does not exist on disk"""
- monkeypatch.setattr(
- "src.storage.queries.settings.BASE_STORAGE_URL", "http://fake:8888"
- )
with pytest.raises(FileNotFoundError):
upload_video_chunks("job-1", [str(tmp_path / "missing.mp4")])
@pytest.mark.parametrize("status_code", [400, 404, 500, 503])
def test_raises_on_http_error(
- tmp_path: Path, status_code: int, monkeypatch: pytest.MonkeyPatch
+ fake_base_url: str, single_video_chunk: str, status_code: int
) -> None:
"""Raises HTTPError when SeaweedFS returns 4xx/5xx on upload"""
- monkeypatch.setattr(
- "src.storage.queries.settings.BASE_STORAGE_URL", "http://fake:8888"
- )
- chunk = tmp_path / "chunk.mp4"
- chunk.write_bytes(b"data")
-
mock_response = MagicMock()
mock_response.status_code = status_code
mock_response.raise_for_status.side_effect = requests.HTTPError(
@@ -36,34 +33,23 @@ def test_raises_on_http_error(
patch("src.storage.queries.requests.put", return_value=mock_response),
pytest.raises(requests.HTTPError),
):
- upload_video_chunks("job-1", [str(chunk)])
+ upload_video_chunks("job-1", [single_video_chunk])
def test_raises_on_connection_error(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ fake_base_url: str, single_video_chunk: str
) -> None:
"""Raises ConnectionError when SeaweedFS is unreachable during upload"""
- monkeypatch.setattr(
- "src.storage.queries.settings.BASE_STORAGE_URL", "http://fake:8888"
- )
- chunk = tmp_path / "chunk.mp4"
- chunk.write_bytes(b"data")
-
with (
patch("src.storage.queries.requests.put", side_effect=requests.ConnectionError),
pytest.raises(requests.ConnectionError),
):
- upload_video_chunks("job-1", [str(chunk)])
+ upload_video_chunks("job-1", [single_video_chunk])
-def test_returns_correct_storage_urls(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
+def test_returns_correct_storage_urls(fake_base_url: str, tmp_path: Path) -> None:
"""Returns list of SeaweedFS URLs matching {base}/{job_id}/{filename}"""
- base_url = "http://fake:8888"
job_id = "job-abc"
- monkeypatch.setattr("src.storage.queries.settings.BASE_STORAGE_URL", base_url)
-
chunks = []
for name in ["chunk-001.mp4", "chunk-002.mp4"]:
f = tmp_path / name
@@ -77,6 +63,6 @@ def test_returns_correct_storage_urls(
urls = upload_video_chunks(job_id, chunks)
assert urls == [
- f"{base_url}/{job_id}/chunk-001.mp4",
- f"{base_url}/{job_id}/chunk-002.mp4",
+ f"{fake_base_url}/{job_id}/chunk-001.mp4",
+ f"{fake_base_url}/{job_id}/chunk-002.mp4",
]
diff --git a/backend/transcoder-worker/cmd/helpers_test.go b/backend/transcoder-worker/cmd/helpers_test.go
index dcb57f7..c2736a7 100644
--- a/backend/transcoder-worker/cmd/helpers_test.go
+++ b/backend/transcoder-worker/cmd/helpers_test.go
@@ -10,18 +10,21 @@ import (
"github.com/stretchr/testify/require"
)
-func patchExit(t *testing.T) *int {
+func patchOsExit(t *testing.T) *int {
t.Helper()
- code := -1
- osExit = func(c int) { code = c }
+ code := new(int)
+ *code = -1
+ osExit = func(c int) {
+ *code = c
+ }
t.Cleanup(func() { osExit = os.Exit })
- return &code
+ return code
}
// writeEnvFile creates ../.env with the given content and removes it on cleanup.
func writeEnvFile(t *testing.T, content string) {
t.Helper()
- for _, key := range []string{"NATS_URL", "PROD_MODE", "BASE_STORAGE_URL"} {
+ for _, key := range []string{"NATS_URL", "PROD_MODE", "BASE_STORAGE_URL", "HTTP_PORT"} {
if old, set := os.LookupEnv(key); set {
t.Cleanup(func() { os.Setenv(key, old) })
} else {
diff --git a/backend/transcoder-worker/cmd/main.go b/backend/transcoder-worker/cmd/main.go
index cba128e..73badb1 100644
--- a/backend/transcoder-worker/cmd/main.go
+++ b/backend/transcoder-worker/cmd/main.go
@@ -1,16 +1,15 @@
package main
import (
- "context"
"fmt"
"log"
"log/slog"
"os"
"os/signal"
"syscall"
- "time"
"transcoder-worker/internal/handler"
+ "transcoder-worker/internal/observability"
"transcoder-worker/internal/storage"
"github.com/joho/godotenv"
@@ -26,6 +25,7 @@ type Config struct {
NatsURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"`
ProdMode bool `envconfig:"PROD_MODE" default:"false"`
BaseStorageURL string `envconfig:"BASE_STORAGE_URL" default:"http://localhost:8888"`
+ HTTPPort string `envconfig:"HTTP_PORT" default:"9095"`
}
func main() {
@@ -34,7 +34,7 @@ func main() {
log.Fatalf("failed to load config values: %v", err)
}
- logger := newLogger(cfg)
+ logger := observability.StructuredLogger(cfg.ProdMode)
err = storage.CheckHealth(cfg.BaseStorageURL, logger)
if err != nil {
@@ -57,20 +57,13 @@ func main() {
return
}
- kv, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
- Bucket: "transcode-chunk-job-processed",
- Description: "tracks already completed video chunk for the jobID is already processed for idempotency",
- TTL: 3 * time.Hour,
- })
- if err != nil {
- logger.Error("failed to ccreate transcode-chunk-job-processed kv bucket", "err", err)
- osExit(1)
- }
+ processedKV := handler.CreateMsgProcessedKV(js, logger)
+ jobStatusKV := handler.ConnectJobStatusKV(js, logger)
quit := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
- err = runProcessing(cfg.BaseStorageURL, js, nc, kv, logger, quit)
+ err = runProcessing(cfg.BaseStorageURL, cfg.HTTPPort, processedKV, jobStatusKV, js, nc, logger, quit)
if err != nil {
logger.Error("error flushing remaining msgs", "err", err)
}
@@ -82,36 +75,31 @@ type ncDrainer interface {
// run the subscriber and publisher and blocks so main doesnt exit after consumevideochunk retunrs
func runProcessing(
- baseStorageURL string,
+ baseStorageURL, httpPort string,
+ processedKV, jobStatusKV jetstream.KeyValue,
js jetstream.JetStream,
nc ncDrainer,
- kv jetstream.KeyValue,
logger *slog.Logger,
quit <-chan os.Signal,
) error {
logger.Debug("starting service")
- consCtx, err := handler.ConsumeVideoChunk(baseStorageURL, js, kv, logger)
+ server := handler.StartHttpServer(logger, httpPort)
+
+ consCtx, err := handler.ConsumeVideoChunk(baseStorageURL, js, processedKV, jobStatusKV, logger)
if err != nil {
+ handler.ShutdownHttpServer(server, logger)
return fmt.Errorf("failed to start consumer: %w", err)
}
<-quit
+ handler.ShutdownHttpServer(server, logger)
+
consCtx.Stop() // stop recieving new msgs from jetstream
return nc.Drain()
}
-func newLogger(cfg *Config) *slog.Logger {
- level := slog.LevelDebug
- if cfg.ProdMode {
- level = slog.LevelInfo
- }
- h := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})
-
- return slog.New(h).With("service", "transcoder-worker")
-}
-
func loadConfig() (*Config, error) {
err := godotenv.Load("../.env")
if err != nil {
diff --git a/backend/transcoder-worker/cmd/main_integration_test.go b/backend/transcoder-worker/cmd/main_integration_test.go
index f4d568f..a68b275 100644
--- a/backend/transcoder-worker/cmd/main_integration_test.go
+++ b/backend/transcoder-worker/cmd/main_integration_test.go
@@ -37,11 +37,12 @@ func TestRunProcessingI(t *testing.T) {
t.Run("quit signal exits cleanly", func(t *testing.T) {
js, nc := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
quit := make(chan os.Signal, 1)
done := make(chan error, 1)
go func() {
- done <- runProcessing(sharedFilerURL, js, nc, kv, test.SilentLogger(), quit)
+ done <- runProcessing(sharedFilerURL, "0", kv, jobStatusKV, js, nc, test.SilentLogger(), quit)
}()
time.Sleep(200 * time.Millisecond)
@@ -82,9 +83,10 @@ func TestRunProcessingI(t *testing.T) {
quit := make(chan os.Signal, 1)
done := make(chan error, 1)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
go func() {
- done <- runProcessing(sharedFilerURL, js, nc, kv, test.SilentLogger(), quit)
+ done <- runProcessing(sharedFilerURL, "0", kv, jobStatusKV, js, nc, test.SilentLogger(), quit)
}()
time.Sleep(500 * time.Millisecond)
@@ -140,7 +142,9 @@ func TestRunProcessingI(t *testing.T) {
require.NoError(t, err)
quit := make(chan os.Signal, 1)
- err = runProcessing(sharedFilerURL, js, nc, &test.MockKV{}, test.SilentLogger(), quit)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
+
+ err = runProcessing(sharedFilerURL, "0", &test.MockKV{}, jobStatusKV, js, nc, test.SilentLogger(), quit)
assert.Error(t, err)
})
@@ -161,8 +165,8 @@ func TestKVSetup(t *testing.T) {
func TestMainI(t *testing.T) {
t.Run("exits on NATS connect error", func(t *testing.T) {
- code := patchExit(t)
- writeEnvFile(t, fmt.Sprintf("BASE_STORAGE_URL=%s\nNATS_URL=nats://localhost:1\n", sharedFilerURL))
+ code := patchOsExit(t)
+ writeEnvFile(t, fmt.Sprintf("BASE_STORAGE_URL=%s\nNATS_URL=nats://localhost:1\nHTTP_PORT=0\n", sharedFilerURL))
main()
@@ -178,9 +182,17 @@ func TestMainI(t *testing.T) {
natsURL, err := container.ConnectionString(ctx)
require.NoError(t, err)
- // No stream configured — ConsumeVideoChunk fails, main() logs the error and returns (no os.Exit)
- code := patchExit(t)
- writeEnvFile(t, fmt.Sprintf("BASE_STORAGE_URL=%s\nNATS_URL=%s\n", sharedFilerURL, natsURL))
+ // Pre-create job-status bucket (video-status would have done this in prod).
+ // No stream configured — ConsumeVideoChunk fails, main() logs error and returns without osExit.
+ setupNC, err := nats.Connect(natsURL)
+ require.NoError(t, err)
+ defer setupNC.Close()
+ setupJS, err := jetstream.New(setupNC)
+ require.NoError(t, err)
+ test.SetupJobStatusKV(t, setupJS)
+
+ code := patchOsExit(t)
+ writeEnvFile(t, fmt.Sprintf("BASE_STORAGE_URL=%s\nNATS_URL=%s\nHTTP_PORT=0\n", sharedFilerURL, natsURL))
main()
diff --git a/backend/transcoder-worker/cmd/main_unit_test.go b/backend/transcoder-worker/cmd/main_unit_test.go
index 23cad2d..abbb457 100644
--- a/backend/transcoder-worker/cmd/main_unit_test.go
+++ b/backend/transcoder-worker/cmd/main_unit_test.go
@@ -3,10 +3,10 @@
package main
import (
- "context"
- "log/slog"
+ "net"
"os"
"path/filepath"
+ "strconv"
"testing"
"time"
"transcoder-worker/internal/test"
@@ -20,32 +20,13 @@ func okJS() *test.MockJS {
return &test.MockJS{JStream: &test.MockStream{Cons: &test.MockConsumer{}}}
}
-func okKV() *test.MockKV {
- return &test.MockKV{}
-}
-
-func TestNewLogger(t *testing.T) {
- t.Run("dev mode enables debug level", func(t *testing.T) {
- logger := newLogger(&Config{ProdMode: false})
-
- assert.True(t, logger.Enabled(context.Background(), slog.LevelDebug))
- })
-
- t.Run("prod mode disables debug level", func(t *testing.T) {
- logger := newLogger(&Config{ProdMode: true})
-
- assert.False(t, logger.Enabled(context.Background(), slog.LevelDebug))
- assert.True(t, logger.Enabled(context.Background(), slog.LevelInfo))
- })
-}
-
func TestRunProcessing(t *testing.T) {
t.Run("consumer setup error returns error", func(t *testing.T) {
js := &test.MockJS{JStreamNameErr: assert.AnError}
nc := &test.MockDrainer{}
quit := make(chan os.Signal, 1)
- err := runProcessing("http://storage", js, nc, okKV(), test.SilentLogger(), quit)
+ err := runProcessing("http://storage", "0", &test.MockKV{}, &test.MockKV{}, js, nc, test.SilentLogger(), quit)
require.ErrorIs(t, err, assert.AnError)
assert.False(t, nc.DrainCalled, "Drain should not be called if consumer setup fails")
@@ -56,7 +37,7 @@ func TestRunProcessing(t *testing.T) {
done := make(chan error, 1)
go func() {
- done <- runProcessing("http://storage", okJS(), &test.MockDrainer{}, okKV(), test.SilentLogger(), quit)
+ done <- runProcessing("http://storage", "0", &test.MockKV{}, &test.MockKV{}, okJS(), &test.MockDrainer{}, test.SilentLogger(), quit)
}()
select {
@@ -81,7 +62,7 @@ func TestRunProcessing(t *testing.T) {
quit := make(chan os.Signal, 1)
quit <- os.Interrupt
- require.NoError(t, runProcessing("http://storage", js, &test.MockDrainer{}, okKV(), test.SilentLogger(), quit))
+ require.NoError(t, runProcessing("http://storage", "0", &test.MockKV{}, &test.MockKV{}, js, &test.MockDrainer{}, test.SilentLogger(), quit))
require.NotNil(t, consumer.Ctx)
assert.True(t, consumer.Ctx.Stopped)
@@ -92,17 +73,34 @@ func TestRunProcessing(t *testing.T) {
quit := make(chan os.Signal, 1)
quit <- os.Interrupt
- require.NoError(t, runProcessing("http://storage", okJS(), nc, okKV(), test.SilentLogger(), quit))
+ require.NoError(t, runProcessing("http://storage", "0", &test.MockKV{}, &test.MockKV{}, okJS(), nc, test.SilentLogger(), quit))
assert.True(t, nc.DrainCalled)
})
+ t.Run("server shuts down when consumer setup fails", func(t *testing.T) {
+ ln, err := net.Listen("tcp", ":0")
+ require.NoError(t, err)
+ port := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)
+ ln.Close()
+
+ js := &test.MockJS{JStreamNameErr: assert.AnError}
+ quit := make(chan os.Signal, 1)
+
+ runProcessing("http://storage", port, &test.MockKV{}, &test.MockKV{}, js, &test.MockDrainer{}, test.SilentLogger(), quit) //nolint:errcheck
+
+ // If server was properly shut down, the port should be free to bind again.
+ ln2, err := net.Listen("tcp", ":"+port)
+ require.NoError(t, err, "port should be free after server shutdown")
+ ln2.Close()
+ })
+
t.Run("drain error is returned", func(t *testing.T) {
nc := &test.MockDrainer{DrainErr: assert.AnError}
quit := make(chan os.Signal, 1)
quit <- os.Interrupt
- err := runProcessing("http://storage", okJS(), nc, okKV(), test.SilentLogger(), quit)
+ err := runProcessing("http://storage", "0", &test.MockKV{}, &test.MockKV{}, okJS(), nc, test.SilentLogger(), quit)
assert.ErrorIs(t, err, assert.AnError)
})
@@ -144,7 +142,7 @@ func TestLoadConfig(t *testing.T) {
func TestMainFunc(t *testing.T) {
t.Run("exits on storage health check failure", func(t *testing.T) {
- code := patchExit(t)
+ code := patchOsExit(t)
writeEnvFile(t, "BASE_STORAGE_URL=http://localhost:1\n")
main()
diff --git a/backend/transcoder-worker/internal/handler/http.go b/backend/transcoder-worker/internal/handler/http.go
new file mode 100644
index 0000000..95ac727
--- /dev/null
+++ b/backend/transcoder-worker/internal/handler/http.go
@@ -0,0 +1,51 @@
+package handler
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "time"
+)
+
+// starts the http server with /health endpoint
+func StartHttpServer(logger *slog.Logger, httpPort string) *http.Server {
+ router := http.NewServeMux()
+
+ router.HandleFunc("GET /health", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+
+ err := json.NewEncoder(w).Encode(map[string]string{"status": "Healthy"})
+ if err != nil {
+ logger.Error("failed to encode health status msg", "err", err)
+ }
+ })
+
+ server := &http.Server{
+ Addr: ":" + httpPort,
+ Handler: router,
+ }
+
+ go func() {
+ fmt.Printf("server running on http://localhost:%s\n", httpPort)
+
+ err := server.ListenAndServe()
+ if err != nil && err != http.ErrServerClosed {
+ logger.Error("http server error", "err", err)
+ osExit(1)
+ }
+ }()
+
+ return server
+}
+
+func ShutdownHttpServer(server *http.Server, logger *slog.Logger) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ err := server.Shutdown(ctx)
+ if err != nil {
+ logger.Error("error shutting down http server", "err", err)
+ }
+}
diff --git a/backend/transcoder-worker/internal/handler/http_unit_test.go b/backend/transcoder-worker/internal/handler/http_unit_test.go
new file mode 100644
index 0000000..a3d49a1
--- /dev/null
+++ b/backend/transcoder-worker/internal/handler/http_unit_test.go
@@ -0,0 +1,46 @@
+//go:build unit
+
+package handler_test
+
+import (
+ "encoding/json"
+ "net/http"
+ "testing"
+ "time"
+ "transcoder-worker/internal/handler"
+ "transcoder-worker/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// health endpoint returns healthy status
+func TestStartHttpServer(t *testing.T) {
+ port := test.FreePort(t)
+ server := handler.StartHttpServer(test.SilentLogger(), port)
+ t.Cleanup(func() { handler.ShutdownHttpServer(server, test.SilentLogger()) })
+
+ time.Sleep(50 * time.Millisecond)
+
+ resp, err := http.Get("http://localhost:" + port + "/health")
+ require.NoError(t, err)
+ defer resp.Body.Close()
+
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+ var body map[string]string
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
+ assert.Equal(t, "Healthy", body["status"])
+}
+
+// server stops accepting connections after shutdown
+func TestShutdownHttpServer(t *testing.T) {
+ port := test.FreePort(t)
+ server := handler.StartHttpServer(test.SilentLogger(), port)
+ time.Sleep(50 * time.Millisecond)
+
+ handler.ShutdownHttpServer(server, test.SilentLogger())
+
+ _, err := http.Get("http://localhost:" + port + "/health")
+ assert.Error(t, err, "server should no longer accept connections after shutdown")
+}
diff --git a/backend/transcoder-worker/internal/handler/job_status_kv.go b/backend/transcoder-worker/internal/handler/job_status_kv.go
new file mode 100644
index 0000000..63f00a2
--- /dev/null
+++ b/backend/transcoder-worker/internal/handler/job_status_kv.go
@@ -0,0 +1,47 @@
+package handler
+
+import (
+ "context"
+ "encoding/json"
+ "log/slog"
+ "time"
+
+ "github.com/nats-io/nats.go/jetstream"
+)
+
+// connect to existing job status kv to publishing the processing stage update msgs
+func ConnectJobStatusKV(js jetstream.JetStream, logger *slog.Logger) jetstream.KeyValue {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ kv, err := js.KeyValue(ctx, "job-status")
+ if err != nil {
+ logger.Error("failed to create recombine-chunk-recieved kv bucket", "err", err)
+ osExit(1)
+ return nil
+ }
+
+ return kv
+}
+
+func UpdateJobStatusKV(jobStatusKV jetstream.KeyValue, JobID string, logger *slog.Logger) error {
+ status, err := json.Marshal(struct {
+ State string `json:"state"`
+ Stage string `json:"stage"`
+ }{State: "PROCESSING", Stage: "transcoder"})
+ if err != nil {
+ logger.Error("error marshalling status text", "err", err)
+ return err
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ _, err = jobStatusKV.Put(ctx, JobID, status)
+ if err != nil {
+ logger.Error("failed to write job status to jobStatus kv", "job_id", JobID, "err", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/backend/transcoder-worker/internal/handler/job_status_kv_integration_test.go b/backend/transcoder-worker/internal/handler/job_status_kv_integration_test.go
new file mode 100644
index 0000000..eda7f9a
--- /dev/null
+++ b/backend/transcoder-worker/internal/handler/job_status_kv_integration_test.go
@@ -0,0 +1,42 @@
+//go:build integration
+
+package handler
+
+import (
+ "context"
+ "os"
+ "testing"
+ "transcoder-worker/internal/test"
+
+ "github.com/nats-io/nats.go/jetstream"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// connects to existing job-status bucket
+func TestConnectJobStatusKV(t *testing.T) {
+ t.Run("connects to existing job-status bucket", func(t *testing.T) {
+ js, _ := test.SetupNats(t)
+
+ _, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
+ Bucket: "job-status",
+ })
+ require.NoError(t, err)
+
+ kv := ConnectJobStatusKV(js, test.SilentLogger())
+
+ assert.NotNil(t, kv)
+ })
+
+ t.Run("exits when job-status bucket does not exist", func(t *testing.T) {
+ js, _ := test.SetupNats(t)
+
+ code := -1
+ osExit = func(c int) { code = c }
+ t.Cleanup(func() { osExit = os.Exit })
+
+ ConnectJobStatusKV(js, test.SilentLogger())
+
+ assert.Equal(t, 1, code)
+ })
+}
diff --git a/backend/transcoder-worker/internal/handler/job_status_kv_unit_test.go b/backend/transcoder-worker/internal/handler/job_status_kv_unit_test.go
new file mode 100644
index 0000000..3f57120
--- /dev/null
+++ b/backend/transcoder-worker/internal/handler/job_status_kv_unit_test.go
@@ -0,0 +1,47 @@
+//go:build unit
+
+package handler_test
+
+import (
+ "errors"
+ "testing"
+ "transcoder-worker/internal/handler"
+ "transcoder-worker/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestUpdateJobStatusKV(t *testing.T) {
+ tests := []struct {
+ name string
+ kv *test.MockKV
+ wantErr bool
+ wantKey string
+ }{
+ {
+ name: "success returns nil and writes job_id as key",
+ kv: &test.MockKV{},
+ wantErr: false,
+ wantKey: "job-1",
+ },
+ {
+ name: "KV Put error returns error",
+ kv: &test.MockKV{PutErr: errors.New("kv unavailable")},
+ wantErr: true,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ err := handler.UpdateJobStatusKV(tc.kv, "job-1", test.SilentLogger())
+
+ if tc.wantErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ assert.Equal(t, tc.wantKey, tc.kv.PutKey)
+ }
+ })
+ }
+}
diff --git a/backend/transcoder-worker/internal/service/chunk_kv.go b/backend/transcoder-worker/internal/handler/msg_processed_kv.go
similarity index 58%
rename from backend/transcoder-worker/internal/service/chunk_kv.go
rename to backend/transcoder-worker/internal/handler/msg_processed_kv.go
index dc5e4ed..600c9ce 100644
--- a/backend/transcoder-worker/internal/service/chunk_kv.go
+++ b/backend/transcoder-worker/internal/handler/msg_processed_kv.go
@@ -1,14 +1,37 @@
-package service
+package handler
import (
"context"
"errors"
"fmt"
+ "log/slog"
+ "os"
"time"
"github.com/nats-io/nats.go/jetstream"
)
+var osExit = os.Exit
+
+// Create the Msg Processed KV store for idempotency
+func CreateMsgProcessedKV(js jetstream.JetStream, logger *slog.Logger) jetstream.KeyValue {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+ Bucket: "transcode-chunk-job-processed",
+ Description: "tracks already completed video chunk for the jobID is already processed for idempotency",
+ TTL: 3 * time.Hour,
+ })
+ if err != nil {
+ logger.Error("failed to create transcode-chunk-job-processed kv bucket", "err", err)
+ osExit(1)
+ return nil
+ }
+
+ return kv
+}
+
// check if a jobID chunk already is processed, returns a bool based on if it exists in the KV
func CheckChunkProcessed(kv jetstream.KeyValue, jobID string, chunkIndex int) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
diff --git a/backend/transcoder-worker/internal/handler/msg_processed_kv_integration_test.go b/backend/transcoder-worker/internal/handler/msg_processed_kv_integration_test.go
new file mode 100644
index 0000000..69f464c
--- /dev/null
+++ b/backend/transcoder-worker/internal/handler/msg_processed_kv_integration_test.go
@@ -0,0 +1,21 @@
+//go:build integration
+
+package handler
+
+import (
+ "testing"
+ "transcoder-worker/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// creates bucket with correct name and TTL
+func TestCreateMsgProcessedKV(t *testing.T) {
+ js, _ := test.SetupNats(t)
+
+ kv := CreateMsgProcessedKV(js, test.SilentLogger())
+
+ require.NotNil(t, kv)
+ assert.Equal(t, "transcode-chunk-job-processed", kv.Bucket())
+}
diff --git a/backend/transcoder-worker/internal/service/chunk_kv_unit_test.go b/backend/transcoder-worker/internal/handler/msg_processed_kv_unit_test.go
similarity index 77%
rename from backend/transcoder-worker/internal/service/chunk_kv_unit_test.go
rename to backend/transcoder-worker/internal/handler/msg_processed_kv_unit_test.go
index 90d4c76..e7df2d4 100644
--- a/backend/transcoder-worker/internal/service/chunk_kv_unit_test.go
+++ b/backend/transcoder-worker/internal/handler/msg_processed_kv_unit_test.go
@@ -1,11 +1,11 @@
//go:build unit
-package service_test
+package handler_test
import (
"errors"
"testing"
- "transcoder-worker/internal/service"
+ "transcoder-worker/internal/handler"
"transcoder-worker/internal/test"
"github.com/nats-io/nats.go/jetstream"
@@ -17,7 +17,7 @@ func TestCheckChunkProcessed(t *testing.T) {
t.Run("returns false when key not found", func(t *testing.T) {
kv := &test.MockKV{GetFound: false}
- processed, err := service.CheckChunkProcessed(kv, "job-1", 0)
+ processed, err := handler.CheckChunkProcessed(kv, "job-1", 0)
require.NoError(t, err)
assert.False(t, processed)
@@ -26,7 +26,7 @@ func TestCheckChunkProcessed(t *testing.T) {
t.Run("returns true when key exists", func(t *testing.T) {
kv := &test.MockKV{GetFound: true}
- processed, err := service.CheckChunkProcessed(kv, "job-1", 0)
+ processed, err := handler.CheckChunkProcessed(kv, "job-1", 0)
require.NoError(t, err)
assert.True(t, processed)
@@ -35,7 +35,7 @@ func TestCheckChunkProcessed(t *testing.T) {
t.Run("returns error on unexpected kv failure", func(t *testing.T) {
kv := &test.MockKV{GetErr: errors.New("kv unavailable")}
- _, err := service.CheckChunkProcessed(kv, "job-1", 0)
+ _, err := handler.CheckChunkProcessed(kv, "job-1", 0)
require.Error(t, err)
assert.ErrorContains(t, err, "failed")
@@ -44,7 +44,7 @@ func TestCheckChunkProcessed(t *testing.T) {
t.Run("does not return error for ErrKeyNotFound", func(t *testing.T) {
kv := &test.MockKV{GetErr: jetstream.ErrKeyNotFound}
- processed, err := service.CheckChunkProcessed(kv, "job-1", 0)
+ processed, err := handler.CheckChunkProcessed(kv, "job-1", 0)
require.NoError(t, err)
assert.False(t, processed)
@@ -55,7 +55,7 @@ func TestCheckChunkProcessed(t *testing.T) {
// We verify by having GetFound=true and confirming no error path is hit.
kv := &test.MockKV{GetFound: true}
- processed, err := service.CheckChunkProcessed(kv, "abc", 3)
+ processed, err := handler.CheckChunkProcessed(kv, "abc", 3)
require.NoError(t, err)
assert.True(t, processed)
@@ -66,7 +66,7 @@ func TestAddChunkProcessed(t *testing.T) {
t.Run("returns nil on success", func(t *testing.T) {
kv := &test.MockKV{}
- err := service.AddChunkProcessed(kv, "job-1", 0)
+ err := handler.AddChunkProcessed(kv, "job-1", 0)
require.NoError(t, err)
})
@@ -74,7 +74,7 @@ func TestAddChunkProcessed(t *testing.T) {
t.Run("writes correct key job_id.chunk_index", func(t *testing.T) {
kv := &test.MockKV{}
- err := service.AddChunkProcessed(kv, "job-abc", 2)
+ err := handler.AddChunkProcessed(kv, "job-abc", 2)
require.NoError(t, err)
assert.Equal(t, "job-abc.2", kv.PutKey)
@@ -83,7 +83,7 @@ func TestAddChunkProcessed(t *testing.T) {
t.Run("returns error on kv failure", func(t *testing.T) {
kv := &test.MockKV{PutErr: errors.New("put failed")}
- err := service.AddChunkProcessed(kv, "job-1", 0)
+ err := handler.AddChunkProcessed(kv, "job-1", 0)
require.Error(t, err)
assert.ErrorContains(t, err, "failed")
diff --git a/backend/transcoder-worker/internal/handler/subscriber.go b/backend/transcoder-worker/internal/handler/subscriber.go
index 33dd28a..06eefb4 100644
--- a/backend/transcoder-worker/internal/handler/subscriber.go
+++ b/backend/transcoder-worker/internal/handler/subscriber.go
@@ -20,7 +20,7 @@ var removeAll = os.RemoveAll
// consume video chunk from nats jetstream and process it
func ConsumeVideoChunk(
- baseStorageURL string, js jetstream.JetStream, kv jetstream.KeyValue, logger *slog.Logger,
+ baseStorageURL string, js jetstream.JetStream, processedKV, jobStatusKV jetstream.KeyValue, logger *slog.Logger,
) (jetstream.ConsumeContext, error) {
ctx := context.Background()
@@ -62,7 +62,7 @@ func ConsumeVideoChunk(
return
}
- exists, err := service.CheckChunkProcessed(kv, payload.JobID, payload.ChunkIndex)
+ exists, err := CheckChunkProcessed(processedKV, payload.JobID, payload.ChunkIndex)
if err != nil {
logger.Error("failed to check chunk processed", "err", err)
return
@@ -78,6 +78,11 @@ func ConsumeVideoChunk(
return
}
+ err = UpdateJobStatusKV(jobStatusKV, payload.JobID, logger)
+ if err != nil {
+ logger.Error("failed to update job_status stage", "job_id", payload.JobID, "err", err)
+ }
+
filePath, err := storage.GetUnprocessedVideoChunk(payload.StorageURL, payload.JobID)
if err != nil {
logger.Error("error fetching unprocessed video chunk", "job_id", payload.JobID, "err", err)
@@ -138,7 +143,7 @@ func ConsumeVideoChunk(
return
}
- err = service.AddChunkProcessed(kv, payload.JobID, payload.ChunkIndex)
+ err = AddChunkProcessed(processedKV, payload.JobID, payload.ChunkIndex)
if err != nil {
logger.Error("failed to mark job chunk as processed", "err", err)
return
diff --git a/backend/transcoder-worker/internal/handler/subscriber_integration_test.go b/backend/transcoder-worker/internal/handler/subscriber_integration_test.go
index 950a470..a463461 100644
--- a/backend/transcoder-worker/internal/handler/subscriber_integration_test.go
+++ b/backend/transcoder-worker/internal/handler/subscriber_integration_test.go
@@ -49,8 +49,9 @@ func TestConsumeVideoChunk(t *testing.T) {
js, err := jetstream.New(nc)
require.NoError(t, err)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
assert.Error(t, err)
})
@@ -58,8 +59,9 @@ func TestConsumeVideoChunk(t *testing.T) {
t.Run("returns non-nil consume context", func(t *testing.T) {
js, _ := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- consCtx, err := ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ consCtx, err := ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
assert.NotNil(t, consCtx)
@@ -69,8 +71,9 @@ func TestConsumeVideoChunk(t *testing.T) {
ctx := context.Background()
js, _ := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err := ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
stream, err := js.Stream(ctx, "jobs")
@@ -91,8 +94,9 @@ func TestConsumeVideoChunk(t *testing.T) {
t.Run("invalid JSON does not publish downstream", func(t *testing.T) {
js, nc := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err := ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
received := make(chan struct{}, 1)
@@ -130,8 +134,9 @@ func TestConsumeVideoChunk(t *testing.T) {
sub, err := nc.Subscribe("jobs.chunks.complete", func(m *nats.Msg) { received <- m.Data })
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
test.PublishVideoChunk(t, js, service.VideoChunkMessage{
@@ -189,8 +194,9 @@ func TestConsumeVideoChunkNaksOnError(t *testing.T) {
})
storageURL := test.SeedUnprocessedVideo(t, sharedFilerURL, jobID, tc.fileName, tc.videoContent(t))
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := ConsumeVideoChunk(tc.baseStorageURL, js, kv, test.SilentLogger())
+ _, err := ConsumeVideoChunk(tc.baseStorageURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
test.PublishVideoChunk(t, js, service.VideoChunkMessage{
@@ -238,8 +244,9 @@ func TestConsumeVideoChunkPublishFails(t *testing.T) {
videoContent, err := os.ReadFile("../test/test_video.mp4")
require.NoError(t, err)
storageURL := test.SeedUnprocessedVideo(t, sharedFilerURL, jobID, "test_video.mp4", videoContent)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
test.PublishVideoChunk(t, js, service.VideoChunkMessage{
@@ -260,8 +267,9 @@ func TestConsumeVideoChunkCleanup(t *testing.T) {
videoContent, err := os.ReadFile("../test/test_video.mp4")
require.NoError(t, err)
storageURL := test.SeedUnprocessedVideo(t, sharedFilerURL, jobID, "test_video.mp4", videoContent)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
received := make(chan struct{}, 1)
@@ -331,8 +339,9 @@ func TestConsumeVideoChunkIdempotency(t *testing.T) {
sub, err := nc.Subscribe("jobs.chunks.complete", func(_ *nats.Msg) { received <- struct{}{} })
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
test.PublishVideoChunk(t, js, service.VideoChunkMessage{
@@ -360,8 +369,9 @@ func TestConsumeVideoChunkIdempotency(t *testing.T) {
videoContent, err := os.ReadFile("../test/test_video.mp4")
require.NoError(t, err)
storageURL := test.SeedUnprocessedVideo(t, sharedFilerURL, jobID, "test_video.mp4", videoContent)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err = ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
test.PublishVideoChunk(t, js, service.VideoChunkMessage{
@@ -388,8 +398,9 @@ func TestConsumeVideoChunkIdempotency(t *testing.T) {
// Seed invalid video so transcoding fails.
storageURL := test.SeedUnprocessedVideo(t, sharedFilerURL, jobID, "bad.mp4", []byte("not a video"))
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := ConsumeVideoChunk(sharedFilerURL, js, kv, test.SilentLogger())
+ _, err := ConsumeVideoChunk(sharedFilerURL, js, kv, jobStatusKV, test.SilentLogger())
require.NoError(t, err)
test.PublishVideoChunk(t, js, service.VideoChunkMessage{
diff --git a/backend/transcoder-worker/internal/handler/subscriber_unit_test.go b/backend/transcoder-worker/internal/handler/subscriber_unit_test.go
index 9c41266..2cb3e83 100644
--- a/backend/transcoder-worker/internal/handler/subscriber_unit_test.go
+++ b/backend/transcoder-worker/internal/handler/subscriber_unit_test.go
@@ -11,26 +11,10 @@ import (
"transcoder-worker/internal/service"
"transcoder-worker/internal/test"
- "github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
-// mockMsg stubs jetstream.Msg for message-handling tests.
-// It is kept here rather than in internal/test because it is only
-// needed for subscriber behaviour and carries no value elsewhere.
-type mockMsg struct {
- jetstream.Msg
- data []byte
- nakCalled bool
- ackCalled bool
- nakErr error
-}
-
-func (m *mockMsg) Data() []byte { return m.data }
-func (m *mockMsg) Nak() error { m.nakCalled = true; return m.nakErr }
-func (m *mockMsg) Ack() error { m.ackCalled = true; return nil }
-
func validPayload(t *testing.T, jobID string) []byte {
t.Helper()
data, err := json.Marshal(service.VideoChunkMessage{
@@ -78,7 +62,7 @@ func TestReturnError(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
- _, err := handler.ConsumeVideoChunk("http://storage", tc.js, &test.MockKV{}, test.SilentLogger())
+ _, err := handler.ConsumeVideoChunk("http://storage", tc.js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger())
require.Error(t, err)
assert.ErrorIs(t, err, tc.wantErr)
@@ -88,80 +72,80 @@ func TestReturnError(t *testing.T) {
func TestAckAndNacking(t *testing.T) {
t.Run("invalid JSON naks and does not ack", func(t *testing.T) {
- msg := &mockMsg{data: []byte("not valid json")}
+ msg := &test.MockMsg{Payload: []byte("not valid json")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
- consCtx, err := handler.ConsumeVideoChunk("http://storage", js, &test.MockKV{}, test.SilentLogger())
+ consCtx, err := handler.ConsumeVideoChunk("http://storage", js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger())
require.NoError(t, err)
assert.NotNil(t, consCtx)
- assert.True(t, msg.nakCalled)
- assert.False(t, msg.ackCalled)
+ assert.True(t, msg.NakCalled)
+ assert.False(t, msg.AckCalled)
})
t.Run("invalid JSON with nak error logs and returns", func(t *testing.T) {
nakErr := errors.New("nak failed")
- msg := &mockMsg{data: []byte("not valid json"), nakErr: nakErr}
+ msg := &test.MockMsg{Payload: []byte("not valid json"), NakErr: nakErr}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
- consCtx, err := handler.ConsumeVideoChunk("http://storage", js, &test.MockKV{}, test.SilentLogger())
+ consCtx, err := handler.ConsumeVideoChunk("http://storage", js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger())
require.NoError(t, err)
assert.NotNil(t, consCtx)
- assert.True(t, msg.nakCalled)
+ assert.True(t, msg.NakCalled)
})
t.Run("fetch failure naks", func(t *testing.T) {
- msg := &mockMsg{data: validPayload(t, "job-1")}
+ msg := &test.MockMsg{Payload: validPayload(t, "job-1")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
- _, err := handler.ConsumeVideoChunk("http://storage", js, &test.MockKV{}, test.SilentLogger())
+ _, err := handler.ConsumeVideoChunk("http://storage", js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger())
require.NoError(t, err)
- assert.True(t, msg.nakCalled)
+ assert.True(t, msg.NakCalled)
})
}
func TestIdempotency(t *testing.T) {
t.Run("already processed chunk acks and skips processing", func(t *testing.T) {
- msg := &mockMsg{data: validPayload(t, "job-1")}
+ msg := &test.MockMsg{Payload: validPayload(t, "job-1")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{GetFound: true}
- _, err := handler.ConsumeVideoChunk("http://storage", js, kv, test.SilentLogger())
+ _, err := handler.ConsumeVideoChunk("http://storage", js, kv, &test.MockKV{}, test.SilentLogger())
require.NoError(t, err)
- assert.True(t, msg.ackCalled)
- assert.False(t, msg.nakCalled)
+ assert.True(t, msg.AckCalled)
+ assert.False(t, msg.NakCalled)
})
t.Run("already processed chunk does not write to kv again", func(t *testing.T) {
- msg := &mockMsg{data: validPayload(t, "job-1")}
+ msg := &test.MockMsg{Payload: validPayload(t, "job-1")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{GetFound: true}
- _, err := handler.ConsumeVideoChunk("http://storage", js, kv, test.SilentLogger())
+ _, err := handler.ConsumeVideoChunk("http://storage", js, kv, &test.MockKV{}, test.SilentLogger())
require.NoError(t, err)
assert.Empty(t, kv.PutKey)
})
t.Run("kv check error does not ack or nak", func(t *testing.T) {
- msg := &mockMsg{data: validPayload(t, "job-1")}
+ msg := &test.MockMsg{Payload: validPayload(t, "job-1")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{GetErr: errors.New("kv unavailable")}
- _, err := handler.ConsumeVideoChunk("http://storage", js, kv, test.SilentLogger())
+ _, err := handler.ConsumeVideoChunk("http://storage", js, kv, &test.MockKV{}, test.SilentLogger())
require.NoError(t, err)
- assert.False(t, msg.ackCalled)
- assert.False(t, msg.nakCalled)
+ assert.False(t, msg.AckCalled)
+ assert.False(t, msg.NakCalled)
})
t.Run("writes kv with correct key on success", func(t *testing.T) {
@@ -173,12 +157,12 @@ func TestIdempotency(t *testing.T) {
})
require.NoError(t, err)
- msg := &mockMsg{data: payload}
+ msg := &test.MockMsg{Payload: payload}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{}
- _, _ = handler.ConsumeVideoChunk("http://localhost:1", js, kv, test.SilentLogger())
+ _, _ = handler.ConsumeVideoChunk("http://localhost:1", js, kv, &test.MockKV{}, test.SilentLogger())
assert.Empty(t, kv.PutKey, "kv.Put should not be called when processing fails")
})
diff --git a/backend/transcoder-worker/internal/observability/logging.go b/backend/transcoder-worker/internal/observability/logging.go
new file mode 100644
index 0000000..ef5af93
--- /dev/null
+++ b/backend/transcoder-worker/internal/observability/logging.go
@@ -0,0 +1,17 @@
+package observability
+
+import (
+ "log/slog"
+ "os"
+)
+
+// General Structured logger for code
+func StructuredLogger(prodMode bool) *slog.Logger {
+ level := slog.LevelDebug
+ if prodMode {
+ level = slog.LevelInfo
+ }
+ h := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})
+
+ return slog.New(h).With("service", "transcoder-worker")
+}
diff --git a/backend/transcoder-worker/internal/observability/logging_unit_test.go b/backend/transcoder-worker/internal/observability/logging_unit_test.go
new file mode 100644
index 0000000..8987b52
--- /dev/null
+++ b/backend/transcoder-worker/internal/observability/logging_unit_test.go
@@ -0,0 +1,28 @@
+//go:build unit
+
+package observability_test
+
+import (
+ "context"
+ "log/slog"
+ "testing"
+ "transcoder-worker/internal/observability"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestStructuredLogger(t *testing.T) {
+
+ t.Run("prod mode set to false should enable debug level", func(t *testing.T) {
+ logger := observability.StructuredLogger(false)
+
+ assert.True(t, logger.Enabled(context.Background(), slog.LevelDebug))
+ })
+
+ t.Run("prod mode set to true should disable debug level", func(t *testing.T) {
+ logger := observability.StructuredLogger(true)
+
+ assert.False(t, logger.Enabled(context.Background(), slog.LevelDebug))
+ assert.True(t, logger.Enabled(context.Background(), slog.LevelInfo))
+ })
+}
diff --git a/backend/transcoder-worker/internal/test/handler_helpers.go b/backend/transcoder-worker/internal/test/handler_helpers.go
index c9856f8..7e73119 100644
--- a/backend/transcoder-worker/internal/test/handler_helpers.go
+++ b/backend/transcoder-worker/internal/test/handler_helpers.go
@@ -5,6 +5,8 @@ import (
"encoding/json"
"io"
"log/slog"
+ "net"
+ "strconv"
"testing"
"time"
"transcoder-worker/internal/service"
@@ -45,3 +47,13 @@ func AssertNacked(t *testing.T, js jetstream.JetStream, msg string) {
return info.NumAckPending > 0
}, 30*time.Second, 200*time.Millisecond, msg)
}
+
+func FreePort(t *testing.T) string {
+ t.Helper()
+ ln, err := net.Listen("tcp", ":0")
+ require.NoError(t, err)
+ port := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)
+ err = ln.Close()
+ require.NoError(t, err)
+ return port
+}
diff --git a/backend/transcoder-worker/internal/test/jetstream_mocks.go b/backend/transcoder-worker/internal/test/jetstream_mocks.go
index 8ec56df..ef4e137 100644
--- a/backend/transcoder-worker/internal/test/jetstream_mocks.go
+++ b/backend/transcoder-worker/internal/test/jetstream_mocks.go
@@ -123,3 +123,18 @@ func (m *MockDrainer) Drain() error {
m.DrainCalled = true
return m.DrainErr
}
+
+// MockMsg stubs jetstream.Msg for message-handling tests.
+// It is kept here rather than in internal/test because it is only
+// needed for subscriber behaviour and carries no value elsewhere.
+type MockMsg struct {
+ jetstream.Msg
+ Payload []byte
+ NakCalled bool
+ AckCalled bool
+ NakErr error
+}
+
+func (m *MockMsg) Data() []byte { return m.Payload }
+func (m *MockMsg) Nak() error { m.NakCalled = true; return m.NakErr }
+func (m *MockMsg) Ack() error { m.AckCalled = true; return nil }
diff --git a/backend/transcoder-worker/internal/test/nats_fixtures.go b/backend/transcoder-worker/internal/test/nats_fixtures.go
index 1c55380..6f3791a 100644
--- a/backend/transcoder-worker/internal/test/nats_fixtures.go
+++ b/backend/transcoder-worker/internal/test/nats_fixtures.go
@@ -83,3 +83,12 @@ func SetupKV(t *testing.T, js jetstream.JetStream) jetstream.KeyValue {
require.NoError(t, err)
return kv
}
+
+func SetupJobStatusKV(t *testing.T, js jetstream.JetStream) jetstream.KeyValue {
+ t.Helper()
+ kv, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
+ Bucket: "job-status",
+ })
+ require.NoError(t, err)
+ return kv
+}
diff --git a/backend/video-recombiner/cmd/helpers_test.go b/backend/video-recombiner/cmd/helpers_test.go
index 8287f6b..ac14892 100644
--- a/backend/video-recombiner/cmd/helpers_test.go
+++ b/backend/video-recombiner/cmd/helpers_test.go
@@ -22,7 +22,7 @@ func patchExit(t *testing.T) *int {
// writeEnvFile creates ../.env with the given content and removes it on cleanup.
func writeEnvFile(t *testing.T, content string) {
t.Helper()
- for _, key := range []string{"NATS_URL", "PROD_MODE", "BASE_STORAGE_URL"} {
+ for _, key := range []string{"NATS_URL", "PROD_MODE", "BASE_STORAGE_URL", "HTTP_PORT"} {
if old, set := os.LookupEnv(key); set {
t.Cleanup(func() { os.Setenv(key, old) })
} else {
@@ -39,7 +39,3 @@ func writeEnvFile(t *testing.T, content string) {
func okJS() *test.MockJS {
return &test.MockJS{JStream: &test.MockStream{Cons: &test.MockConsumer{}}}
}
-
-func okKV() *test.MockKV {
- return &test.MockKV{}
-}
diff --git a/backend/video-recombiner/cmd/main.go b/backend/video-recombiner/cmd/main.go
index 0600d12..f22102a 100644
--- a/backend/video-recombiner/cmd/main.go
+++ b/backend/video-recombiner/cmd/main.go
@@ -1,14 +1,12 @@
package main
import (
- "context"
"fmt"
"log"
"log/slog"
"os"
"os/signal"
"syscall"
- "time"
"video-recombiner/internal/handler"
"video-recombiner/internal/observability"
"video-recombiner/internal/storage"
@@ -22,6 +20,7 @@ import (
var osExit = os.Exit
type Config struct {
+ HTTPPort string `envconfig:"HTTP_PORT" default:"9090"`
NatsURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"`
ProdMode bool `envconfig:"PROD_MODE" default:"false"`
BaseStorageURL string `envconfig:"BASE_STORAGE_URL" default:"http://localhost:8888"`
@@ -56,20 +55,13 @@ func main() {
return
}
- kv, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
- Bucket: "recombine-chunk-recieved",
- Description: "tracks video chunk for the jobID is already recieved for idempotency",
- TTL: 3 * time.Hour,
- })
- if err != nil {
- logger.Error("failed to create recombine-chunk-recieved kv bucket", "err", err)
- osExit(1)
- }
+ msgRecievedKV := handler.CreateMsgRecievedKV(js, logger)
+ jobStatusKV := handler.ConnectJobStatusKV(js, logger)
quit := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
- err = runCombiner(js, nc, kv, logger, cfg.BaseStorageURL, quit)
+ err = runCombiner(js, nc, msgRecievedKV, jobStatusKV, logger, cfg.BaseStorageURL, cfg.HTTPPort, quit)
if err != nil {
logger.Error("error flushing remaining msgs", "err", err)
}
@@ -82,20 +74,25 @@ type ncDrainer interface {
func runCombiner(
js jetstream.JetStream,
nc ncDrainer,
- kv jetstream.KeyValue,
+ msgRecievedKV, jobStatusKV jetstream.KeyValue,
logger *slog.Logger,
- baseStorageURL string,
+ baseStorageURL, httpPort string,
quit <-chan os.Signal,
) error {
logger.Debug("starting service...")
- consCtx, err := handler.RecombineVideo(js, kv, logger, baseStorageURL)
+ server := handler.StartHttpServer(logger, httpPort)
+
+ consCtx, err := handler.RecombineVideo(js, msgRecievedKV, jobStatusKV, logger, baseStorageURL)
if err != nil {
+ handler.ShutdownHttpServer(server, logger)
return fmt.Errorf("failed to start subscriber/publisher: %w", err)
}
<-quit
+ handler.ShutdownHttpServer(server, logger)
+
consCtx.Stop()
return nc.Drain()
}
diff --git a/backend/video-recombiner/cmd/main_integration_test.go b/backend/video-recombiner/cmd/main_integration_test.go
index 36d1431..13e60d3 100644
--- a/backend/video-recombiner/cmd/main_integration_test.go
+++ b/backend/video-recombiner/cmd/main_integration_test.go
@@ -37,11 +37,12 @@ func TestRunCombinerI(t *testing.T) {
t.Run("quit signal exits cleanly", func(t *testing.T) {
js, nc := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
quit := make(chan os.Signal, 1)
done := make(chan error, 1)
go func() {
- done <- runCombiner(js, nc, kv, test.SilentLogger(), sharedFilerURL, quit)
+ done <- runCombiner(js, nc, kv, jobStatusKV, test.SilentLogger(), sharedFilerURL, "0", quit)
}()
time.Sleep(200 * time.Millisecond)
@@ -73,7 +74,7 @@ func TestRunCombinerI(t *testing.T) {
require.NoError(t, err)
quit := make(chan os.Signal, 1)
- err = runCombiner(js, nc, nil, test.SilentLogger(), sharedFilerURL, quit)
+ err = runCombiner(js, nc, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), sharedFilerURL, "0", quit)
assert.Error(t, err)
})
@@ -86,6 +87,7 @@ func TestRunCombinerI(t *testing.T) {
js, nc := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
jobID := "job-full-flow"
t.Cleanup(func() {
@@ -110,7 +112,7 @@ func TestRunCombinerI(t *testing.T) {
done := make(chan error, 1)
go func() {
- done <- runCombiner(js, nc, kv, test.SilentLogger(), sharedFilerURL, quit)
+ done <- runCombiner(js, nc, kv, jobStatusKV, test.SilentLogger(), sharedFilerURL, "0", quit)
}()
time.Sleep(500 * time.Millisecond)
@@ -181,7 +183,7 @@ func TestMainI(t *testing.T) {
assert.Equal(t, 1, *code)
})
- t.Run("no stream logs error and returns", func(t *testing.T) {
+ t.Run("reaches runCombiner and logs error on no stream", func(t *testing.T) {
ctx := context.Background()
container, err := natstc.Run(ctx, "nats:2.10-alpine")
require.NoError(t, err)
@@ -190,8 +192,16 @@ func TestMainI(t *testing.T) {
natsURL, err := container.ConnectionString(ctx)
require.NoError(t, err)
+ // Pre-create job-status bucket
+ setupNC, err := nats.Connect(natsURL)
+ require.NoError(t, err)
+ defer setupNC.Close()
+ setupJS, err := jetstream.New(setupNC)
+ require.NoError(t, err)
+ test.SetupJobStatusKV(t, setupJS)
+
code := patchExit(t)
- writeEnvFile(t, fmt.Sprintf("BASE_STORAGE_URL=%s\nNATS_URL=%s\n", sharedFilerURL, natsURL))
+ writeEnvFile(t, fmt.Sprintf("BASE_STORAGE_URL=%s\nNATS_URL=%s\nHTTP_PORT=0\n", sharedFilerURL, natsURL))
main()
diff --git a/backend/video-recombiner/cmd/main_unit_test.go b/backend/video-recombiner/cmd/main_unit_test.go
index cb76d98..b96f172 100644
--- a/backend/video-recombiner/cmd/main_unit_test.go
+++ b/backend/video-recombiner/cmd/main_unit_test.go
@@ -3,6 +3,7 @@
package main
import (
+ "net"
"os"
"path/filepath"
"testing"
@@ -14,28 +15,28 @@ import (
)
func TestRunCombiner(t *testing.T) {
- t.Run("consume video chunk error should return error", func(t *testing.T) {
+ t.Run("consumer setup error returns error", func(t *testing.T) {
js := &test.MockJS{JStreamNameErr: assert.AnError}
nc := &test.MockDrainer{}
quit := make(chan os.Signal, 1)
- err := runCombiner(js, nc, okKV(), test.SilentLogger(), "http://storage", quit)
+ err := runCombiner(js, nc, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), "http://storage", "0", quit)
require.ErrorIs(t, err, assert.AnError)
assert.False(t, nc.DrainCalled, "Drain should not be called if consumer setup fails")
})
- t.Run("it should block from returning until quit signal is recieved", func(t *testing.T) {
+ t.Run("blocks until quit signal", func(t *testing.T) {
quit := make(chan os.Signal, 1)
done := make(chan error, 1)
go func() {
- done <- runCombiner(okJS(), &test.MockDrainer{}, okKV(), test.SilentLogger(), "http://storage", quit)
+ done <- runCombiner(okJS(), &test.MockDrainer{}, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), "http://storage", "0", quit)
}()
select {
case <-done:
- t.Fatal("runProcessing returned before quit signal was sent")
+ t.Fatal("runCombiner returned before quit signal was sent")
case <-time.After(100 * time.Millisecond):
}
@@ -45,41 +46,54 @@ func TestRunCombiner(t *testing.T) {
case err := <-done:
require.NoError(t, err)
case <-time.After(time.Second):
- t.Fatal("runProcessing did not return after quit signal")
+ t.Fatal("runCombiner did not return after quit signal")
}
})
- t.Run("it should stop consumer on quit signal", func(t *testing.T) {
+ t.Run("stops consumer on quit", func(t *testing.T) {
consumer := &test.MockConsumer{}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
quit := make(chan os.Signal, 1)
quit <- os.Interrupt
- require.NoError(t, runCombiner(js, &test.MockDrainer{}, okKV(), test.SilentLogger(), "http://storage", quit))
+ require.NoError(t, runCombiner(js, &test.MockDrainer{}, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), "http://storage", "0", quit))
require.NotNil(t, consumer.Ctx)
assert.True(t, consumer.Ctx.Stopped)
})
- t.Run("it should drain nats messages on quit", func(t *testing.T) {
+ t.Run("drains NATS on quit", func(t *testing.T) {
nc := &test.MockDrainer{}
quit := make(chan os.Signal, 1)
quit <- os.Interrupt
- require.NoError(t, runCombiner(okJS(), nc, okKV(), test.SilentLogger(), "http://storage", quit))
+ require.NoError(t, runCombiner(okJS(), nc, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), "http://storage", "0", quit))
assert.True(t, nc.DrainCalled)
})
- t.Run("it should handle drain errors", func(t *testing.T) {
+ t.Run("drain error is returned", func(t *testing.T) {
nc := &test.MockDrainer{DrainErr: assert.AnError}
quit := make(chan os.Signal, 1)
quit <- os.Interrupt
- err := runCombiner(okJS(), nc, okKV(), test.SilentLogger(), "http://storage", quit)
+ err := runCombiner(okJS(), nc, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), "http://storage", "0", quit)
assert.ErrorIs(t, err, assert.AnError)
})
+
+ t.Run("server shuts down when consumer setup fails", func(t *testing.T) {
+ port := test.FreePort(t)
+ js := &test.MockJS{JStreamNameErr: assert.AnError}
+ quit := make(chan os.Signal, 1)
+
+ runCombiner(js, &test.MockDrainer{}, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), "http://storage", port, quit) //nolint:errcheck
+
+ // If server was properly shut down, the port should be free to bind again.
+ ln, err := net.Listen("tcp", ":"+port)
+ require.NoError(t, err, "port should be free after server shutdown")
+ ln.Close()
+ })
}
func TestLoadConfig(t *testing.T) {
@@ -94,7 +108,7 @@ func TestLoadConfig(t *testing.T) {
})
t.Run("reads all values from env file", func(t *testing.T) {
- test.WriteEnvFile(t, "NATS_URL=nats://test:9999\nPROD_MODE=true\nBASE_STORAGE_URL=http://localhost:9333\nHTTP_PORT=9090\n")
+ writeEnvFile(t, "NATS_URL=nats://test:9999\nPROD_MODE=true\nBASE_STORAGE_URL=http://localhost:9333\nHTTP_PORT=9090\n")
cfg, err := loadConfig()
@@ -105,7 +119,7 @@ func TestLoadConfig(t *testing.T) {
})
t.Run("empty env file uses struct defaults", func(t *testing.T) {
- test.WriteEnvFile(t, "")
+ writeEnvFile(t, "")
cfg, err := loadConfig()
@@ -115,3 +129,14 @@ func TestLoadConfig(t *testing.T) {
assert.Equal(t, "http://localhost:8888", cfg.BaseStorageURL)
})
}
+
+func TestMainFunc(t *testing.T) {
+ t.Run("exits on storage health check failure", func(t *testing.T) {
+ code := patchExit(t)
+ writeEnvFile(t, "BASE_STORAGE_URL=http://localhost:1\n")
+
+ main()
+
+ assert.Equal(t, 1, *code)
+ })
+}
diff --git a/backend/video-recombiner/internal/handler/http.go b/backend/video-recombiner/internal/handler/http.go
new file mode 100644
index 0000000..95ac727
--- /dev/null
+++ b/backend/video-recombiner/internal/handler/http.go
@@ -0,0 +1,51 @@
+package handler
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "time"
+)
+
+// starts the http server with /health endpoint
+func StartHttpServer(logger *slog.Logger, httpPort string) *http.Server {
+ router := http.NewServeMux()
+
+ router.HandleFunc("GET /health", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+
+ err := json.NewEncoder(w).Encode(map[string]string{"status": "Healthy"})
+ if err != nil {
+ logger.Error("failed to encode health status msg", "err", err)
+ }
+ })
+
+ server := &http.Server{
+ Addr: ":" + httpPort,
+ Handler: router,
+ }
+
+ go func() {
+ fmt.Printf("server running on http://localhost:%s\n", httpPort)
+
+ err := server.ListenAndServe()
+ if err != nil && err != http.ErrServerClosed {
+ logger.Error("http server error", "err", err)
+ osExit(1)
+ }
+ }()
+
+ return server
+}
+
+func ShutdownHttpServer(server *http.Server, logger *slog.Logger) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ err := server.Shutdown(ctx)
+ if err != nil {
+ logger.Error("error shutting down http server", "err", err)
+ }
+}
diff --git a/backend/video-recombiner/internal/handler/http_unit_test.go b/backend/video-recombiner/internal/handler/http_unit_test.go
new file mode 100644
index 0000000..d44b098
--- /dev/null
+++ b/backend/video-recombiner/internal/handler/http_unit_test.go
@@ -0,0 +1,46 @@
+//go:build unit
+
+package handler_test
+
+import (
+ "encoding/json"
+ "net/http"
+ "testing"
+ "time"
+ "video-recombiner/internal/handler"
+ "video-recombiner/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// health endpoint returns healthy status
+func TestStartHttpServer(t *testing.T) {
+ port := test.FreePort(t)
+ server := handler.StartHttpServer(test.SilentLogger(), port)
+ t.Cleanup(func() { handler.ShutdownHttpServer(server, test.SilentLogger()) })
+
+ time.Sleep(50 * time.Millisecond)
+
+ resp, err := http.Get("http://localhost:" + port + "/health")
+ require.NoError(t, err)
+ defer resp.Body.Close()
+
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+ var body map[string]string
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
+ assert.Equal(t, "Healthy", body["status"])
+}
+
+// server stops accepting connections after shutdown
+func TestShutdownHttpServer(t *testing.T) {
+ port := test.FreePort(t)
+ server := handler.StartHttpServer(test.SilentLogger(), port)
+ time.Sleep(50 * time.Millisecond)
+
+ handler.ShutdownHttpServer(server, test.SilentLogger())
+
+ _, err := http.Get("http://localhost:" + port + "/health")
+ assert.Error(t, err, "server should no longer accept connections after shutdown")
+}
diff --git a/backend/video-recombiner/internal/handler/job_status_kv.go b/backend/video-recombiner/internal/handler/job_status_kv.go
new file mode 100644
index 0000000..cc4904e
--- /dev/null
+++ b/backend/video-recombiner/internal/handler/job_status_kv.go
@@ -0,0 +1,49 @@
+package handler
+
+import (
+ "context"
+ "encoding/json"
+ "log/slog"
+ "os"
+ "time"
+
+ "github.com/nats-io/nats.go/jetstream"
+)
+
+var osExit = os.Exit
+
+// connect to existing job status kv to publishing the processing stage update msgs
+func ConnectJobStatusKV(js jetstream.JetStream, logger *slog.Logger) jetstream.KeyValue {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ kv, err := js.KeyValue(ctx, "job-status")
+ if err != nil {
+ logger.Error("failed to connect to job-status kv bucket", "err", err)
+ osExit(1)
+ }
+
+ return kv
+}
+
+func UpdateJobStatusKV(jobStatusKV jetstream.KeyValue, JobID string, logger *slog.Logger) error {
+ status, err := json.Marshal(struct {
+ State string `json:"state"`
+ Stage string `json:"stage"`
+ }{State: "PROCESSING", Stage: "video-recombiner"})
+ if err != nil {
+ logger.Error("error marshalling status text", "err", err)
+ return err
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ _, err = jobStatusKV.Put(ctx, JobID, status)
+ if err != nil {
+ logger.Error("failed to write job status to jobStatus kv", "job_id", JobID, "err", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/backend/video-recombiner/internal/handler/job_status_kv_integration_test.go b/backend/video-recombiner/internal/handler/job_status_kv_integration_test.go
new file mode 100644
index 0000000..c1ebf0e
--- /dev/null
+++ b/backend/video-recombiner/internal/handler/job_status_kv_integration_test.go
@@ -0,0 +1,42 @@
+//go:build integration
+
+package handler
+
+import (
+ "context"
+ "os"
+ "testing"
+ "video-recombiner/internal/test"
+
+ "github.com/nats-io/nats.go/jetstream"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// connects to existing job-status bucket
+func TestConnectJobStatusKV(t *testing.T) {
+ t.Run("connects to existing job-status bucket", func(t *testing.T) {
+ js, _ := test.SetupNats(t)
+
+ _, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
+ Bucket: "job-status",
+ })
+ require.NoError(t, err)
+
+ kv := ConnectJobStatusKV(js, test.SilentLogger())
+
+ assert.NotNil(t, kv)
+ })
+
+ t.Run("exits when job-status bucket does not exist", func(t *testing.T) {
+ js, _ := test.SetupNats(t)
+
+ code := -1
+ osExit = func(c int) { code = c }
+ t.Cleanup(func() { osExit = os.Exit })
+
+ ConnectJobStatusKV(js, test.SilentLogger())
+
+ assert.Equal(t, 1, code)
+ })
+}
diff --git a/backend/video-recombiner/internal/handler/job_status_kv_unit_test.go b/backend/video-recombiner/internal/handler/job_status_kv_unit_test.go
new file mode 100644
index 0000000..dbd6268
--- /dev/null
+++ b/backend/video-recombiner/internal/handler/job_status_kv_unit_test.go
@@ -0,0 +1,47 @@
+//go:build unit
+
+package handler_test
+
+import (
+ "errors"
+ "testing"
+ "video-recombiner/internal/handler"
+ "video-recombiner/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestUpdateJobStatusKV(t *testing.T) {
+ tests := []struct {
+ name string
+ kv *test.MockKV
+ wantErr bool
+ wantKey string
+ }{
+ {
+ name: "success returns nil and writes job_id as key",
+ kv: &test.MockKV{},
+ wantErr: false,
+ wantKey: "job-1",
+ },
+ {
+ name: "KV Put error returns error",
+ kv: &test.MockKV{PutErr: errors.New("kv unavailable")},
+ wantErr: true,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ err := handler.UpdateJobStatusKV(tc.kv, "job-1", test.SilentLogger())
+
+ if tc.wantErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ assert.Equal(t, tc.wantKey, tc.kv.PutKey)
+ }
+ })
+ }
+}
diff --git a/backend/video-recombiner/internal/handler/msg_processed_kv_integration_test.go b/backend/video-recombiner/internal/handler/msg_processed_kv_integration_test.go
new file mode 100644
index 0000000..82d2fde
--- /dev/null
+++ b/backend/video-recombiner/internal/handler/msg_processed_kv_integration_test.go
@@ -0,0 +1,21 @@
+//go:build integration
+
+package handler
+
+import (
+ "testing"
+ "video-recombiner/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// creates bucket with correct name and TTL
+func TestCreateMsgRecievedKV(t *testing.T) {
+ js, _ := test.SetupNats(t)
+
+ kv := CreateMsgRecievedKV(js, test.SilentLogger())
+
+ require.NotNil(t, kv)
+ assert.Equal(t, "recombine-chunk-recieved", kv.Bucket())
+}
diff --git a/backend/video-recombiner/internal/service/chunk_kv.go b/backend/video-recombiner/internal/handler/msg_recieved_kv.go
similarity index 61%
rename from backend/video-recombiner/internal/service/chunk_kv.go
rename to backend/video-recombiner/internal/handler/msg_recieved_kv.go
index f3bbd5a..d0d89ed 100644
--- a/backend/video-recombiner/internal/service/chunk_kv.go
+++ b/backend/video-recombiner/internal/handler/msg_recieved_kv.go
@@ -1,14 +1,33 @@
-package service
+package handler
import (
"context"
"errors"
"fmt"
+ "log/slog"
"time"
"github.com/nats-io/nats.go/jetstream"
)
+// Create the Msg Recieved KV store for idempotency
+func CreateMsgRecievedKV(js jetstream.JetStream, logger *slog.Logger) jetstream.KeyValue {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+ Bucket: "recombine-chunk-recieved",
+ Description: "tracks video chunk for the jobID is already recieved for idempotency",
+ TTL: 3 * time.Hour,
+ })
+ if err != nil {
+ logger.Error("failed to create recombine-chunk-recieved kv bucket", "err", err)
+ osExit(1)
+ }
+
+ return kv
+}
+
// check if a jobID chunk already is recieved, returns a bool based on if it exists in the KV
func CheckChunkRecieved(kv jetstream.KeyValue, jobID string, chunkIndex int) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
diff --git a/backend/video-recombiner/internal/service/chunk_kv_unit_test.go b/backend/video-recombiner/internal/handler/msg_recieved_kv_unit_test.go
similarity index 77%
rename from backend/video-recombiner/internal/service/chunk_kv_unit_test.go
rename to backend/video-recombiner/internal/handler/msg_recieved_kv_unit_test.go
index 298ad4a..f1bfba1 100644
--- a/backend/video-recombiner/internal/service/chunk_kv_unit_test.go
+++ b/backend/video-recombiner/internal/handler/msg_recieved_kv_unit_test.go
@@ -1,11 +1,11 @@
//go:build unit
-package service_test
+package handler_test
import (
"errors"
"testing"
- "video-recombiner/internal/service"
+ "video-recombiner/internal/handler"
"video-recombiner/internal/test"
"github.com/nats-io/nats.go/jetstream"
@@ -17,7 +17,7 @@ func TestCheckChunkRecieved(t *testing.T) {
t.Run("returns false when key not found", func(t *testing.T) {
kv := &test.MockKV{GetFound: false}
- processed, err := service.CheckChunkRecieved(kv, "job-1", 0)
+ processed, err := handler.CheckChunkRecieved(kv, "job-1", 0)
require.NoError(t, err)
assert.False(t, processed)
@@ -26,7 +26,7 @@ func TestCheckChunkRecieved(t *testing.T) {
t.Run("returns true when key exists", func(t *testing.T) {
kv := &test.MockKV{GetFound: true}
- processed, err := service.CheckChunkRecieved(kv, "job-1", 0)
+ processed, err := handler.CheckChunkRecieved(kv, "job-1", 0)
require.NoError(t, err)
assert.True(t, processed)
@@ -35,7 +35,7 @@ func TestCheckChunkRecieved(t *testing.T) {
t.Run("returns error on unexpected kv failure", func(t *testing.T) {
kv := &test.MockKV{GetErr: errors.New("kv unavailable")}
- _, err := service.CheckChunkRecieved(kv, "job-1", 0)
+ _, err := handler.CheckChunkRecieved(kv, "job-1", 0)
require.Error(t, err)
assert.ErrorContains(t, err, "failed")
@@ -44,7 +44,7 @@ func TestCheckChunkRecieved(t *testing.T) {
t.Run("does not return error for ErrKeyNotFound", func(t *testing.T) {
kv := &test.MockKV{GetErr: jetstream.ErrKeyNotFound}
- processed, err := service.CheckChunkRecieved(kv, "job-1", 0)
+ processed, err := handler.CheckChunkRecieved(kv, "job-1", 0)
require.NoError(t, err)
assert.False(t, processed)
@@ -55,7 +55,7 @@ func TestCheckChunkRecieved(t *testing.T) {
// We verify by having GetFound=true and confirming no error path is hit.
kv := &test.MockKV{GetFound: true}
- processed, err := service.CheckChunkRecieved(kv, "abc", 3)
+ processed, err := handler.CheckChunkRecieved(kv, "abc", 3)
require.NoError(t, err)
assert.True(t, processed)
@@ -66,7 +66,7 @@ func TestAddChunkRecieved(t *testing.T) {
t.Run("returns nil on success", func(t *testing.T) {
kv := &test.MockKV{}
- err := service.AddChunkRecieved(kv, "job-1", 0)
+ err := handler.AddChunkRecieved(kv, "job-1", 0)
require.NoError(t, err)
})
@@ -74,7 +74,7 @@ func TestAddChunkRecieved(t *testing.T) {
t.Run("writes correct key job_id.chunk_index", func(t *testing.T) {
kv := &test.MockKV{}
- err := service.AddChunkRecieved(kv, "job-abc", 2)
+ err := handler.AddChunkRecieved(kv, "job-abc", 2)
require.NoError(t, err)
assert.Equal(t, "job-abc.2", kv.PutKey)
@@ -83,7 +83,7 @@ func TestAddChunkRecieved(t *testing.T) {
t.Run("returns error on kv failure", func(t *testing.T) {
kv := &test.MockKV{PutErr: errors.New("put failed")}
- err := service.AddChunkRecieved(kv, "job-1", 0)
+ err := handler.AddChunkRecieved(kv, "job-1", 0)
require.Error(t, err)
assert.ErrorContains(t, err, "failed")
diff --git a/backend/video-recombiner/internal/handler/subscriber.go b/backend/video-recombiner/internal/handler/subscriber.go
index c348d4d..a94c4b2 100644
--- a/backend/video-recombiner/internal/handler/subscriber.go
+++ b/backend/video-recombiner/internal/handler/subscriber.go
@@ -16,7 +16,7 @@ const subSubject = "jobs.chunks.complete"
// recombines video chunks back into one video
func RecombineVideo(
- js jetstream.JetStream, kv jetstream.KeyValue, logger *slog.Logger, baseStorageURL string,
+ js jetstream.JetStream, msgRecievedKV, jobStatusKV jetstream.KeyValue, logger *slog.Logger, baseStorageURL string,
) (jetstream.ConsumeContext, error) {
ctx := context.Background()
@@ -58,7 +58,7 @@ func RecombineVideo(
return
}
- recieved, err := service.CheckChunkRecieved(kv, payload.JobID, payload.ChunkIndex)
+ recieved, err := CheckChunkRecieved(msgRecievedKV, payload.JobID, payload.ChunkIndex)
if err != nil {
logger.Error("failed to check chunk recieved", "err", err)
return
@@ -74,6 +74,11 @@ func RecombineVideo(
return
}
+ err = UpdateJobStatusKV(jobStatusKV, payload.JobID, logger)
+ if err != nil {
+ logger.Error("failed to update job_status stage", "job_id", payload.JobID, "err", err)
+ }
+
ready, chunks := tracker.Add(payload.JobID, payload.ChunkIndex, payload.StorageURL, payload.TotalChunks)
err = msg.Ack()
@@ -82,7 +87,7 @@ func RecombineVideo(
return
}
- err = service.AddChunkRecieved(kv, payload.JobID, payload.ChunkIndex)
+ err = AddChunkRecieved(msgRecievedKV, payload.JobID, payload.ChunkIndex)
if err != nil {
logger.Error("failed to mark job chunk as recieved", "err", err)
return
diff --git a/backend/video-recombiner/internal/handler/subscriber_integration_test.go b/backend/video-recombiner/internal/handler/subscriber_integration_test.go
index 4c70228..4f50834 100644
--- a/backend/video-recombiner/internal/handler/subscriber_integration_test.go
+++ b/backend/video-recombiner/internal/handler/subscriber_integration_test.go
@@ -50,7 +50,7 @@ func TestRecombineVideo(t *testing.T) {
js, err := jetstream.New(nc)
require.NoError(t, err)
- _, err = handler.RecombineVideo(js, nil, test.SilentLogger(), t.TempDir())
+ _, err = handler.RecombineVideo(js, nil, nil, test.SilentLogger(), t.TempDir())
assert.Error(t, err)
})
@@ -58,8 +58,9 @@ func TestRecombineVideo(t *testing.T) {
t.Run("returns consume context", func(t *testing.T) {
js, _ := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- consCtx, err := handler.RecombineVideo(js, kv, test.SilentLogger(), t.TempDir())
+ consCtx, err := handler.RecombineVideo(js, kv, jobStatusKV, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
assert.NotNil(t, consCtx)
@@ -69,8 +70,9 @@ func TestRecombineVideo(t *testing.T) {
ctx := context.Background()
js, _ := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := handler.RecombineVideo(js, kv, test.SilentLogger(), t.TempDir())
+ _, err := handler.RecombineVideo(js, kv, jobStatusKV, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
stream, err := js.Stream(ctx, "jobs")
@@ -96,8 +98,9 @@ func TestMessageHandlingI(t *testing.T) {
t.Run("invalid JSON does not publish downstream", func(t *testing.T) {
js, nc := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := handler.RecombineVideo(js, kv, test.SilentLogger(), t.TempDir())
+ _, err := handler.RecombineVideo(js, kv, jobStatusKV, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
received := make(chan struct{}, 1)
@@ -120,8 +123,9 @@ func TestMessageHandlingI(t *testing.T) {
t.Run("partial chunk does not publish downstream", func(t *testing.T) {
js, nc := test.SetupNats(t)
kv := test.SetupKV(t, js)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := handler.RecombineVideo(js, kv, test.SilentLogger(), t.TempDir())
+ _, err := handler.RecombineVideo(js, kv, jobStatusKV, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
received := make(chan struct{}, 1)
@@ -160,7 +164,9 @@ func TestMessageHandlingI(t *testing.T) {
test.SeedProcessedVideo(t, sharedFilerURL, "job-combine", "chunk-0.mp4", videoData)
test.SeedProcessedVideo(t, sharedFilerURL, "job-combine", "chunk-1.mp4", videoData)
- _, err = handler.RecombineVideo(js, kv, test.SilentLogger(), sharedFilerURL)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
+
+ _, err = handler.RecombineVideo(js, kv, jobStatusKV, test.SilentLogger(), sharedFilerURL)
require.NoError(t, err)
received := make(chan struct{}, 1)
@@ -203,7 +209,9 @@ func TestRecombineVideoIdempotency(t *testing.T) {
_, err := kv.Put(context.Background(), fmt.Sprintf("%s.%d", jobID, 0), []byte("received"))
require.NoError(t, err)
- _, err = handler.RecombineVideo(js, kv, test.SilentLogger(), sharedFilerURL)
+ jobStatusKV := test.SetupJobStatusKV(t, js)
+
+ _, err = handler.RecombineVideo(js, kv, jobStatusKV, test.SilentLogger(), sharedFilerURL)
require.NoError(t, err)
secondComplete := make(chan struct{}, 1)
@@ -235,8 +243,9 @@ func TestRecombineVideoIdempotency(t *testing.T) {
kv := test.SetupKV(t, js)
jobID := "job-idempotency-write"
+ jobStatusKV := test.SetupJobStatusKV(t, js)
- _, err := handler.RecombineVideo(js, kv, test.SilentLogger(), sharedFilerURL)
+ _, err := handler.RecombineVideo(js, kv, jobStatusKV, test.SilentLogger(), sharedFilerURL)
require.NoError(t, err)
// Partial chunk (TotalChunks:2) so combine never fires — KV write still happens after ack.
diff --git a/backend/video-recombiner/internal/handler/subscriber_unit_test.go b/backend/video-recombiner/internal/handler/subscriber_unit_test.go
index 4278d26..a4bd66f 100644
--- a/backend/video-recombiner/internal/handler/subscriber_unit_test.go
+++ b/backend/video-recombiner/internal/handler/subscriber_unit_test.go
@@ -11,23 +11,10 @@ import (
"video-recombiner/internal/service"
"video-recombiner/internal/test"
- "github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
-type mockMsg struct {
- jetstream.Msg
- data []byte
- ackErr error
- nakCalled bool
- ackCalled bool
-}
-
-func (m *mockMsg) Data() []byte { return m.data }
-func (m *mockMsg) Nak() error { m.nakCalled = true; return nil }
-func (m *mockMsg) Ack() error { m.ackCalled = true; return m.ackErr }
-
func validPayload(t *testing.T, jobID string) []byte {
t.Helper()
data, err := json.Marshal(service.ChunkCompleteMessage{
@@ -75,7 +62,7 @@ func TestReturnError(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
- _, err := handler.RecombineVideo(tc.js, &test.MockKV{}, test.SilentLogger(), "http://storage")
+ _, err := handler.RecombineVideo(tc.js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), "http://storage")
require.Error(t, err)
assert.ErrorIs(t, err, tc.wantErr)
@@ -85,16 +72,16 @@ func TestReturnError(t *testing.T) {
func TestMessageHandling(t *testing.T) {
t.Run("invalid JSON naks and does not ack", func(t *testing.T) {
- msg := &mockMsg{data: []byte("not valid json")}
+ msg := &test.MockMsg{Payload: []byte("not valid json")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
- consCtx, err := handler.RecombineVideo(js, &test.MockKV{}, test.SilentLogger(), t.TempDir())
+ consCtx, err := handler.RecombineVideo(js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
assert.NotNil(t, consCtx)
- assert.True(t, msg.nakCalled)
- assert.False(t, msg.ackCalled)
+ assert.True(t, msg.NakCalled)
+ assert.False(t, msg.AckCalled)
})
t.Run("partial chunk acks without combining", func(t *testing.T) {
@@ -107,16 +94,16 @@ func TestMessageHandling(t *testing.T) {
})
require.NoError(t, err)
- msg := &mockMsg{data: payload}
+ msg := &test.MockMsg{Payload: payload}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
- consCtx, err := handler.RecombineVideo(js, &test.MockKV{}, test.SilentLogger(), t.TempDir())
+ consCtx, err := handler.RecombineVideo(js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
assert.NotNil(t, consCtx)
- assert.True(t, msg.ackCalled)
- assert.False(t, msg.nakCalled)
+ assert.True(t, msg.AckCalled)
+ assert.False(t, msg.NakCalled)
})
t.Run("all chunks ready acks and triggers combine even if download fails", func(t *testing.T) {
@@ -130,16 +117,16 @@ func TestMessageHandling(t *testing.T) {
})
require.NoError(t, err)
- msg := &mockMsg{data: payload}
+ msg := &test.MockMsg{Payload: payload}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
- consCtx, err := handler.RecombineVideo(js, &test.MockKV{}, test.SilentLogger(), t.TempDir())
+ consCtx, err := handler.RecombineVideo(js, &test.MockKV{}, &test.MockKV{}, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
assert.NotNil(t, consCtx)
- assert.True(t, msg.ackCalled)
- assert.False(t, msg.nakCalled)
+ assert.True(t, msg.AckCalled)
+ assert.False(t, msg.NakCalled)
})
t.Run("ack failure does not trigger combine or write kv", func(t *testing.T) {
@@ -152,58 +139,58 @@ func TestMessageHandling(t *testing.T) {
})
require.NoError(t, err)
- msg := &mockMsg{data: payload, ackErr: errors.New("ack failed")}
+ msg := &test.MockMsg{Payload: payload, AckErr: errors.New("ack failed")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{}
- consCtx, err := handler.RecombineVideo(js, kv, test.SilentLogger(), t.TempDir())
+ consCtx, err := handler.RecombineVideo(js, kv, &test.MockKV{}, test.SilentLogger(), t.TempDir())
require.NoError(t, err)
assert.NotNil(t, consCtx)
- assert.True(t, msg.ackCalled)
- assert.False(t, msg.nakCalled)
+ assert.True(t, msg.AckCalled)
+ assert.False(t, msg.NakCalled)
assert.Empty(t, kv.PutKey)
})
}
func TestIdempotency(t *testing.T) {
t.Run("already processed chunk acks and skips processing", func(t *testing.T) {
- msg := &mockMsg{data: validPayload(t, "job-1")}
+ msg := &test.MockMsg{Payload: validPayload(t, "job-1")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{GetFound: true}
- _, err := handler.RecombineVideo(js, kv, test.SilentLogger(), "http://storage")
+ _, err := handler.RecombineVideo(js, kv, &test.MockKV{}, test.SilentLogger(), "http://storage")
require.NoError(t, err)
- assert.True(t, msg.ackCalled)
- assert.False(t, msg.nakCalled)
+ assert.True(t, msg.AckCalled)
+ assert.False(t, msg.NakCalled)
})
t.Run("already processed chunk does not write to kv again", func(t *testing.T) {
- msg := &mockMsg{data: validPayload(t, "job-1")}
+ msg := &test.MockMsg{Payload: validPayload(t, "job-1")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{GetFound: true}
- _, err := handler.RecombineVideo(js, kv, test.SilentLogger(), "http://storage")
+ _, err := handler.RecombineVideo(js, kv, &test.MockKV{}, test.SilentLogger(), "http://storage")
require.NoError(t, err)
assert.Empty(t, kv.PutKey)
})
t.Run("kv check error does not ack or nak", func(t *testing.T) {
- msg := &mockMsg{data: validPayload(t, "job-1")}
+ msg := &test.MockMsg{Payload: validPayload(t, "job-1")}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{GetErr: errors.New("kv unavailable")}
- _, err := handler.RecombineVideo(js, kv, test.SilentLogger(), "http://storage")
+ _, err := handler.RecombineVideo(js, kv, &test.MockKV{}, test.SilentLogger(), "http://storage")
require.NoError(t, err)
- assert.False(t, msg.ackCalled)
- assert.False(t, msg.nakCalled)
+ assert.False(t, msg.AckCalled)
+ assert.False(t, msg.NakCalled)
})
t.Run("writes kv with correct key after ack", func(t *testing.T) {
@@ -215,12 +202,12 @@ func TestIdempotency(t *testing.T) {
})
require.NoError(t, err)
- msg := &mockMsg{data: payload}
+ msg := &test.MockMsg{Payload: payload}
consumer := &test.MockConsumerWithMsg{Msg: msg}
js := &test.MockJS{JStream: &test.MockStream{Cons: consumer}}
kv := &test.MockKV{}
- _, err = handler.RecombineVideo(js, kv, test.SilentLogger(), "http://storage")
+ _, err = handler.RecombineVideo(js, kv, &test.MockKV{}, test.SilentLogger(), "http://storage")
require.NoError(t, err)
assert.Equal(t, "job-abc.2", kv.PutKey)
diff --git a/backend/video-recombiner/internal/test/handler_helpers.go b/backend/video-recombiner/internal/test/handler_helpers.go
new file mode 100644
index 0000000..a241ea7
--- /dev/null
+++ b/backend/video-recombiner/internal/test/handler_helpers.go
@@ -0,0 +1,19 @@
+package test
+
+import (
+ "net"
+ "strconv"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func FreePort(t *testing.T) string {
+ t.Helper()
+ ln, err := net.Listen("tcp", ":0")
+ require.NoError(t, err)
+ port := strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)
+ err = ln.Close()
+ require.NoError(t, err)
+ return port
+}
diff --git a/backend/video-recombiner/internal/test/jetstream_mocks.go b/backend/video-recombiner/internal/test/jetstream_mocks.go
index 8ec56df..efe178c 100644
--- a/backend/video-recombiner/internal/test/jetstream_mocks.go
+++ b/backend/video-recombiner/internal/test/jetstream_mocks.go
@@ -123,3 +123,15 @@ func (m *MockDrainer) Drain() error {
m.DrainCalled = true
return m.DrainErr
}
+
+type MockMsg struct {
+ jetstream.Msg
+ Payload []byte
+ AckErr error
+ NakCalled bool
+ AckCalled bool
+}
+
+func (m *MockMsg) Data() []byte { return m.Payload }
+func (m *MockMsg) Nak() error { m.NakCalled = true; return nil }
+func (m *MockMsg) Ack() error { m.AckCalled = true; return m.AckErr }
diff --git a/backend/video-recombiner/internal/test/nats_fixtures.go b/backend/video-recombiner/internal/test/nats_fixtures.go
index 74370d3..eb4fa36 100644
--- a/backend/video-recombiner/internal/test/nats_fixtures.go
+++ b/backend/video-recombiner/internal/test/nats_fixtures.go
@@ -78,3 +78,12 @@ func SetupKV(t *testing.T, js jetstream.JetStream) jetstream.KeyValue {
require.NoError(t, err)
return kv
}
+
+func SetupJobStatusKV(t *testing.T, js jetstream.JetStream) jetstream.KeyValue {
+ t.Helper()
+ kv, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
+ Bucket: "job-status",
+ })
+ require.NoError(t, err)
+ return kv
+}
diff --git a/backend/video-status/cmd/main.go b/backend/video-status/cmd/main.go
index b21a0eb..dd8a7e4 100644
--- a/backend/video-status/cmd/main.go
+++ b/backend/video-status/cmd/main.go
@@ -19,9 +19,12 @@ import (
)
type Config struct {
- NatsURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"`
- ProdMode bool `envconfig:"PROD_MODE" default:"false"`
- HTTPPort string `envconfig:"HTTP_PORT" default:"8081"`
+ NatsURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"`
+ ProdMode bool `envconfig:"PROD_MODE" default:"false"`
+ HTTPPort string `envconfig:"HTTP_PORT" default:"8085"`
+ SceneDetectorURL string `envconfig:"SCENE_DETECTOR_URL" default:"http://localhost:9098"`
+ TranscoderURL string `envconfig:"TRANSCODER_URL" default:"http://localhost:9095"`
+ RecombinerURL string `envconfig:"RECOMBINER_URL" default:"http://localhost:9090"`
}
var osExit = os.Exit
@@ -46,22 +49,15 @@ func main() {
osExit(1)
}
- kv, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
- Bucket: "job-status",
- Description: "tracks job state across the pipeline",
- })
- if err != nil {
- logger.Error("failed to create job-status kv bucket", "err", err)
- osExit(1)
- }
+ jobStatusKV := handler.CreateJobStatusKV(js, logger)
- advisorySub, err := handler.ListenAdvisoriesFailure(nc, js, kv, logger)
+ advisorySub, err := handler.ListenAdvisoriesFailure(nc, js, jobStatusKV, logger)
if err != nil {
logger.Error("failed to subscribe to advisories", "err", err)
osExit(1)
}
- jobCompleteSub, err := handler.ListenJobComplete(js, kv, logger)
+ jobCompleteSub, err := handler.ListenJobComplete(js, jobStatusKV, logger)
if err != nil {
logger.Error("failed to subscribe to job complete stream", "err", err)
osExit(1)
@@ -72,7 +68,7 @@ func main() {
logger.Debug("starting service...")
- server := startHttpApi(logger, kv, cfg)
+ server := startHttpApi(logger, jobStatusKV, cfg)
<-quit
@@ -96,10 +92,18 @@ func main() {
}
}
-func startHttpApi(logger *slog.Logger, kv jetstream.KeyValue, cfg *Config) *http.Server {
+func startHttpApi(logger *slog.Logger, jobStatusKV jetstream.KeyValue, cfg *Config) *http.Server {
router := http.NewServeMux()
- jh := &handler.JobStatusHandler{Logger: logger, KV: kv}
+ jh := &handler.JobStatusHandler{
+ Logger: logger,
+ KV: jobStatusKV,
+ URLs: handler.ServiceURLs{
+ SceneDetector: cfg.SceneDetectorURL,
+ Transcoder: cfg.TranscoderURL,
+ Recombiner: cfg.RecombinerURL,
+ },
+ }
router.HandleFunc("GET /jobs/{id}/status", jh.PollJobStatus)
diff --git a/backend/video-status/cmd/main_unit_test.go b/backend/video-status/cmd/main_unit_test.go
index f57a838..fb557d2 100644
--- a/backend/video-status/cmd/main_unit_test.go
+++ b/backend/video-status/cmd/main_unit_test.go
@@ -31,7 +31,7 @@ func TestLoadConfig(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, "nats://localhost:4222", cfg.NatsURL)
assert.Equal(t, false, cfg.ProdMode)
- assert.Equal(t, "8081", cfg.HTTPPort)
+ assert.Equal(t, "8085", cfg.HTTPPort)
})
t.Run("env var overrides", func(t *testing.T) {
diff --git a/backend/video-status/internal/handler/http.go b/backend/video-status/internal/handler/http.go
index 2cc25de..8945516 100644
--- a/backend/video-status/internal/handler/http.go
+++ b/backend/video-status/internal/handler/http.go
@@ -2,7 +2,7 @@ package handler
import (
"encoding/json"
- "errors"
+ "fmt"
"log/slog"
"net/http"
@@ -15,22 +15,26 @@ const (
StateProcessing JobState = "PROCESSING"
StateComplete JobState = "COMPLETE"
StateFailed JobState = "FAILED"
+ StateDegraded JobState = "DEGRADED"
)
type JobStatus struct {
State JobState `json:"state"`
+ Stage string `json:"stage"`
Error string `json:"error,omitempty"`
}
type jobStatusResponse struct {
JobID string `json:"job_id"`
State JobState `json:"state"`
+ Stage string `json:"stage"`
Error string `json:"error,omitempty"`
}
type JobStatusHandler struct {
Logger *slog.Logger
KV jetstream.KeyValue
+ URLs ServiceURLs
}
func (j *JobStatusHandler) PollJobStatus(w http.ResponseWriter, r *http.Request) {
@@ -41,14 +45,11 @@ func (j *JobStatusHandler) PollJobStatus(w http.ResponseWriter, r *http.Request)
return
}
- entry, err := j.KV.Get(r.Context(), jobID)
+ kh := KVHandler{logger: j.Logger, kv: j.KV}
+
+ entry, httpStatusCode, err := kh.getJobStatusKV(r.Context(), jobID)
if err != nil {
- if errors.Is(err, jetstream.ErrKeyNotFound) {
- http.Error(w, "job not found", http.StatusNotFound)
- return
- }
- j.Logger.Error("failed to get job status from kv", "job_id", jobID, "err", err)
- http.Error(w, "failed to get job status", http.StatusInternalServerError)
+ http.Error(w, err.Error(), httpStatusCode)
return
}
@@ -60,9 +61,35 @@ func (j *JobStatusHandler) PollJobStatus(w http.ResponseWriter, r *http.Request)
return
}
+ if status.State == StateProcessing || status.State == StateDegraded {
+ status = checkServiceHealth(status, j.URLs, kh.logger)
+ err := kh.updateJobStatusKV(r.Context(), jobID, status)
+ if err != nil {
+ j.Logger.Error("failed to update job status KV", "job_id", jobID, "err", err)
+ return
+ }
+ }
+
w.Header().Set("Content-Type", "application/json")
- err = json.NewEncoder(w).Encode(jobStatusResponse{JobID: jobID, State: status.State, Error: status.Error})
+ err = json.NewEncoder(w).Encode(jobStatusResponse{JobID: jobID, State: status.State, Stage: status.Stage, Error: status.Error})
if err != nil {
j.Logger.Error("error encoding job status response", "err", err)
}
}
+
+func checkServiceHealth(status JobStatus, urls ServiceURLs, logger *slog.Logger) JobStatus {
+ serviceURL, ok := urls.forStage(status.Stage)
+ if !ok {
+ return status
+ }
+
+ if isServiceHealthy(serviceURL, logger) {
+ status.State = StateProcessing
+ status.Error = ""
+ } else {
+ status.State = StateDegraded
+ status.Error = fmt.Sprintf("service unavailable at stage: %s", nextService[status.Stage])
+ }
+
+ return status
+}
diff --git a/backend/video-status/internal/handler/http_integration_test.go b/backend/video-status/internal/handler/http_integration_test.go
index b97b7cd..1c0f78b 100644
--- a/backend/video-status/internal/handler/http_integration_test.go
+++ b/backend/video-status/internal/handler/http_integration_test.go
@@ -22,10 +22,14 @@ type statusResponse struct {
Error string `json:"error,omitempty"`
}
-func newTestServer(t *testing.T) *httptest.Server {
+func newTestServer(t *testing.T, urls ...ServiceURLs) *httptest.Server {
t.Helper()
+ var u ServiceURLs
+ if len(urls) > 0 {
+ u = urls[0]
+ }
mux := http.NewServeMux()
- h := &JobStatusHandler{Logger: test.SilentLogger(), KV: sharedKV}
+ h := &JobStatusHandler{Logger: test.SilentLogger(), KV: sharedKV, URLs: u}
mux.HandleFunc("GET /jobs/{id}/status", h.PollJobStatus)
ts := httptest.NewServer(mux)
t.Cleanup(ts.Close)
@@ -52,25 +56,33 @@ func TestResponse(t *testing.T) {
{
name: "PROCESSING job returns 200 with correct state",
jobID: "job-processing",
- status: JobStatus{State: StateProcessing},
+ status: JobStatus{State: StateProcessing, Stage: "scene-detector"},
wantCode: http.StatusOK,
wantState: "PROCESSING",
},
{
name: "COMPLETE job returns 200 with correct state",
jobID: "job-complete",
- status: JobStatus{State: StateComplete},
+ status: JobStatus{State: StateComplete, Stage: "transcoder"},
wantCode: http.StatusOK,
wantState: "COMPLETE",
},
{
name: "FAILED job returns 200 with error field populated",
jobID: "job-failed",
- status: JobStatus{State: StateFailed, Error: "pipeline failed at stage: transcoder-worker"},
+ status: JobStatus{State: StateFailed, Stage: "transcoder", Error: "pipeline failed at stage: transcoder-worker"},
wantCode: http.StatusOK,
wantState: "FAILED",
wantErr: "pipeline failed at stage: transcoder-worker",
},
+ {
+ name: "DEGRADED job returns 200 with error field and stage",
+ jobID: "job-degraded",
+ status: JobStatus{State: StateDegraded, Stage: "scene-detector", Error: "service unavailable at stage: transcoder"},
+ wantCode: http.StatusOK,
+ wantState: "DEGRADED",
+ wantErr: "service unavailable at stage: transcoder",
+ },
}
for _, tc := range tests {
@@ -121,9 +133,9 @@ func TestConnectionDrop(t *testing.T) {
jobID string
status JobStatus
}{
- {"does not panic on dropped connection (PROCESSING)", "drop-processing", JobStatus{State: StateProcessing}},
- {"does not panic on dropped connection (COMPLETE)", "drop-complete", JobStatus{State: StateComplete}},
- {"does not panic on dropped connection (FAILED)", "drop-failed", JobStatus{State: StateFailed, Error: "something broke"}},
+ {"does not panic on dropped connection (PROCESSING)", "drop-processing", JobStatus{State: StateProcessing, Stage: "scene-detector"}},
+ {"does not panic on dropped connection (COMPLETE)", "drop-complete", JobStatus{State: StateComplete, Stage: "transcoder"}},
+ {"does not panic on dropped connection (FAILED)", "drop-failed", JobStatus{State: StateFailed, Stage: "transcoder", Error: "something broke"}},
{"does not panic on dropped connection (not found)", "drop-notfound", JobStatus{}},
}
@@ -146,7 +158,7 @@ func TestConnectionDrop(t *testing.T) {
func TestConcurrentRequests(t *testing.T) {
t.Run("concurrent requests for a completed job return consistent state", func(t *testing.T) {
- seedStatus(t, "concurrent-job", JobStatus{State: StateComplete})
+ seedStatus(t, "concurrent-job", JobStatus{State: StateComplete, Stage: "transcoder"})
ts := newTestServer(t)
const goroutines = 20
@@ -204,19 +216,38 @@ func TestConcurrentRequests(t *testing.T) {
})
}
+// continues serving requests after a client disconnects
func TestServerContinuesAfterDisconnect(t *testing.T) {
- t.Run("server continues serving requests after a client disconnects", func(t *testing.T) {
- seedStatus(t, "reconnect-job", JobStatus{State: StateProcessing})
- ts := newTestServer(t)
+ seedStatus(t, "reconnect-job", JobStatus{State: StateProcessing, Stage: "scene-detector"})
+ ts := newTestServer(t)
- firstResp, err := http.Get(fmt.Sprintf("%s/jobs/reconnect-job/status", ts.URL))
- require.NoError(t, err)
- firstResp.Body.Close()
+ firstResp, err := http.Get(fmt.Sprintf("%s/jobs/reconnect-job/status", ts.URL))
+ require.NoError(t, err)
+ firstResp.Body.Close()
- secondResp, err := http.Get(fmt.Sprintf("%s/jobs/reconnect-job/status", ts.URL))
- require.NoError(t, err)
- defer secondResp.Body.Close()
+ secondResp, err := http.Get(fmt.Sprintf("%s/jobs/reconnect-job/status", ts.URL))
+ require.NoError(t, err)
+ defer secondResp.Body.Close()
- assert.Equal(t, http.StatusOK, secondResp.StatusCode)
- })
+ assert.Equal(t, http.StatusOK, secondResp.StatusCode)
+}
+
+// degraded job recovers to PROCESSING when service comes back up
+func TestPollJobStatus_DegradedRecovery(t *testing.T) {
+ healthySrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer healthySrv.Close()
+
+ seedStatus(t, "job-recovery", JobStatus{State: StateDegraded, Stage: "scene-detector", Error: "service unavailable at stage: transcoder"})
+ ts := newTestServer(t, ServiceURLs{Transcoder: healthySrv.URL})
+
+ resp, err := http.Get(fmt.Sprintf("%s/jobs/job-recovery/status", ts.URL))
+ require.NoError(t, err)
+ defer resp.Body.Close()
+
+ var body statusResponse
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
+ assert.Equal(t, "PROCESSING", body.State)
+ assert.Empty(t, body.Error)
}
diff --git a/backend/video-status/internal/handler/http_unit_test.go b/backend/video-status/internal/handler/http_unit_test.go
index 19bf7f8..fc02cba 100644
--- a/backend/video-status/internal/handler/http_unit_test.go
+++ b/backend/video-status/internal/handler/http_unit_test.go
@@ -14,8 +14,12 @@ import (
"github.com/stretchr/testify/require"
)
-func newHandler(kv *test.MockKV) *JobStatusHandler {
- return &JobStatusHandler{Logger: test.SilentLogger(), KV: kv}
+func newHandler(kv *test.MockKV, urls ...ServiceURLs) *JobStatusHandler {
+ var u ServiceURLs
+ if len(urls) > 0 {
+ u = urls[0]
+ }
+ return &JobStatusHandler{Logger: test.SilentLogger(), KV: kv, URLs: u}
}
func mustMarshalStatus(t *testing.T, status JobStatus) []byte {
@@ -105,25 +109,31 @@ func TestPollJobStatus_States(t *testing.T) {
}{
{
name: "PROCESSING state",
- status: JobStatus{State: StateProcessing},
+ status: JobStatus{State: StateProcessing, Stage: "scene-detector"},
wantState: StateProcessing,
},
{
name: "COMPLETE state",
- status: JobStatus{State: StateComplete},
+ status: JobStatus{State: StateComplete, Stage: "scene-detector"},
wantState: StateComplete,
},
{
name: "FAILED state includes error message",
- status: JobStatus{State: StateFailed, Error: "pipeline failed at stage: transcoder-worker"},
+ status: JobStatus{State: StateFailed, Stage: "scene-detector", Error: "pipeline failed at stage: transcoder-worker"},
wantState: StateFailed,
wantErrMsg: "pipeline failed at stage: transcoder-worker",
},
{
name: "FAILED with empty error field",
- status: JobStatus{State: StateFailed},
+ status: JobStatus{State: StateFailed, Stage: "transcoder"},
wantState: StateFailed,
},
+ {
+ name: "DEGRADED state includes error message",
+ status: JobStatus{State: StateDegraded, Stage: "scene-detector", Error: "service unavailable at stage: transcoder"},
+ wantState: StateDegraded,
+ wantErrMsg: "service unavailable at stage: transcoder",
+ },
}
for _, tc := range tests {
@@ -149,11 +159,12 @@ func TestPollJobStatus_States(t *testing.T) {
func TestPollJobStatus_ResponseShape(t *testing.T) {
tests := []struct {
- name string
- jobID string
+ name string
+ jobID string
+ wantStage string
}{
- {"echoes job_id in response", "my-specific-job"},
- {"echoes different job_id", "another-job-456"},
+ {"echoes job_id in response", "my-specific-job", ""},
+ {"echoes different job_id", "another-job-456", ""},
}
for _, tc := range tests {
@@ -174,6 +185,7 @@ func TestPollJobStatus_ResponseShape(t *testing.T) {
require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
assert.Equal(t, tc.jobID, resp.JobID)
assert.NotEmpty(t, resp.State)
+ assert.Equal(t, tc.wantStage, resp.Stage)
})
}
}
@@ -183,9 +195,9 @@ func TestPollJobStatus_DroppedConnection(t *testing.T) {
name string
status JobStatus
}{
- {"does not panic on dropped connection (PROCESSING)", JobStatus{State: StateProcessing}},
- {"does not panic on dropped connection (COMPLETE)", JobStatus{State: StateComplete}},
- {"does not panic on dropped connection (FAILED)", JobStatus{State: StateFailed, Error: "something broke"}},
+ {"does not panic on dropped connection (PROCESSING)", JobStatus{State: StateProcessing, Stage: "scene-detector"}},
+ {"does not panic on dropped connection (COMPLETE)", JobStatus{State: StateComplete, Stage: "scene-detector"}},
+ {"does not panic on dropped connection (FAILED)", JobStatus{State: StateFailed, Stage: "transcoder", Error: "something broke"}},
}
for _, tc := range tests {
@@ -203,3 +215,83 @@ func TestPollJobStatus_DroppedConnection(t *testing.T) {
})
}
}
+
+func TestPollJobStatus_HealthCheck(t *testing.T) {
+ healthySrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer healthySrv.Close()
+
+ tests := []struct {
+ name string
+ status JobStatus
+ urls ServiceURLs
+ wantState JobState
+ }{
+ {
+ name: "PROCESSING with service down becomes DEGRADED",
+ status: JobStatus{State: StateProcessing, Stage: "scene-detector"},
+ urls: ServiceURLs{Transcoder: "http://localhost:19999"},
+ wantState: StateDegraded,
+ },
+ {
+ name: "PROCESSING with service up stays PROCESSING",
+ status: JobStatus{State: StateProcessing, Stage: "scene-detector"},
+ urls: ServiceURLs{Transcoder: healthySrv.URL},
+ wantState: StateProcessing,
+ },
+ {
+ name: "DEGRADED with service recovered returns PROCESSING",
+ status: JobStatus{State: StateDegraded, Stage: "scene-detector", Error: "service unavailable at stage: transcoder"},
+ urls: ServiceURLs{Transcoder: healthySrv.URL},
+ wantState: StateProcessing,
+ },
+ {
+ name: "COMPLETE skips health check",
+ status: JobStatus{State: StateComplete},
+ urls: ServiceURLs{},
+ wantState: StateComplete,
+ },
+ {
+ name: "FAILED skips health check",
+ status: JobStatus{State: StateFailed, Error: "pipeline failed"},
+ urls: ServiceURLs{},
+ wantState: StateFailed,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ kv := test.NewMockKV()
+ kv.Seed("job-1", mustMarshalStatus(t, tc.status))
+ h := &JobStatusHandler{Logger: test.SilentLogger(), KV: kv, URLs: tc.urls}
+
+ req := httptest.NewRequest(http.MethodGet, "/jobs/job-1/status", nil)
+ req.SetPathValue("id", "job-1")
+ rec := httptest.NewRecorder()
+
+ h.PollJobStatus(rec, req)
+
+ require.Equal(t, http.StatusOK, rec.Code)
+ var resp jobStatusResponse
+ require.NoError(t, json.NewDecoder(rec.Body).Decode(&resp))
+ assert.Equal(t, tc.wantState, resp.State)
+ })
+ }
+
+ t.Run("updateJobStatusKV failure during health check returns early", func(t *testing.T) {
+ kv := test.NewMockKV()
+ kv.Seed("job-1", mustMarshalStatus(t, JobStatus{State: StateProcessing, Stage: "scene-detector"}))
+ kv.PutErr = errors.New("kv unavailable")
+ h := &JobStatusHandler{Logger: test.SilentLogger(), KV: kv, URLs: ServiceURLs{Transcoder: "http://localhost:19999"}}
+
+ req := httptest.NewRequest(http.MethodGet, "/jobs/job-1/status", nil)
+ req.SetPathValue("id", "job-1")
+ rec := httptest.NewRecorder()
+
+ h.PollJobStatus(rec, req)
+
+ require.Equal(t, http.StatusOK, rec.Code)
+ assert.Empty(t, rec.Body.String())
+ })
+}
diff --git a/backend/video-status/internal/handler/job_status_kv.go b/backend/video-status/internal/handler/job_status_kv.go
new file mode 100644
index 0000000..683896e
--- /dev/null
+++ b/backend/video-status/internal/handler/job_status_kv.go
@@ -0,0 +1,66 @@
+package handler
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "log/slog"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/nats-io/nats.go/jetstream"
+)
+
+var osExit = os.Exit
+
+// create a job status kv to publishing the processing stage update msgs
+func CreateJobStatusKV(js jetstream.JetStream, logger *slog.Logger) jetstream.KeyValue {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+ Bucket: "job-status",
+ Description: "tracks job state across the pipeline",
+ })
+ if err != nil {
+ logger.Error("failed to create job-status kv bucket", "err", err)
+ osExit(1)
+ }
+
+ return kv
+}
+
+type KVHandler struct {
+ logger *slog.Logger
+ kv jetstream.KeyValue
+}
+
+func (h *KVHandler) getJobStatusKV(ctx context.Context, jobID string) (jetstream.KeyValueEntry, int, error) {
+ entry, err := h.kv.Get(ctx, jobID)
+ if err != nil {
+ if errors.Is(err, jetstream.ErrKeyNotFound) {
+ return nil, http.StatusNotFound, errors.New("job not found")
+ }
+ h.logger.Error("failed to get job status from kv", "job_id", jobID, "err", err)
+ return nil, http.StatusInternalServerError, errors.New("failed to get job status")
+ }
+
+ return entry, http.StatusOK, nil
+}
+
+func (h *KVHandler) updateJobStatusKV(ctx context.Context, JobID string, status JobStatus) error {
+ data, err := json.Marshal(status)
+ if err != nil {
+ h.logger.Error("error marshalling status", "err", err)
+ return err
+ }
+
+ _, err = h.kv.Put(ctx, JobID, data)
+ if err != nil {
+ h.logger.Error("failed to write job status to jobStatus kv", "job_id", JobID, "err", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/backend/video-status/internal/handler/job_status_kv_integration_test.go b/backend/video-status/internal/handler/job_status_kv_integration_test.go
new file mode 100644
index 0000000..5958dc1
--- /dev/null
+++ b/backend/video-status/internal/handler/job_status_kv_integration_test.go
@@ -0,0 +1,17 @@
+//go:build integration
+
+package handler
+
+import (
+ "testing"
+ "video-status/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCreateJobStatusKV(t *testing.T) {
+ kv := CreateJobStatusKV(sharedJS, test.SilentLogger())
+ require.NotNil(t, kv)
+ assert.Equal(t, "job-status", kv.Bucket())
+}
diff --git a/backend/video-status/internal/handler/job_status_kv_unit_test.go b/backend/video-status/internal/handler/job_status_kv_unit_test.go
new file mode 100644
index 0000000..8024a3d
--- /dev/null
+++ b/backend/video-status/internal/handler/job_status_kv_unit_test.go
@@ -0,0 +1,98 @@
+//go:build unit
+
+package handler
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "testing"
+ "video-status/internal/test"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetJobStatusKV(t *testing.T) {
+ tests := []struct {
+ name string
+ kv *test.MockKV
+ wantStatus int
+ wantErr string
+ }{
+ {
+ name: "key not found returns 404",
+ kv: test.NewMockKV(),
+ wantStatus: http.StatusNotFound,
+ wantErr: "job not found",
+ },
+ {
+ name: "generic KV error returns 500",
+ kv: func() *test.MockKV {
+ m := test.NewMockKV()
+ m.GetErr = errors.New("kv unavailable")
+ return m
+ }(),
+ wantStatus: http.StatusInternalServerError,
+ wantErr: "failed to get job status",
+ },
+ {
+ name: "success returns entry and 200",
+ kv: func() *test.MockKV {
+ m := test.NewMockKV()
+ m.Seed("job-1", []byte(`{"state":"PROCESSING"}`))
+ return m
+ }(),
+ wantStatus: http.StatusOK,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ h := &KVHandler{logger: test.SilentLogger(), kv: tc.kv}
+ entry, code, err := h.getJobStatusKV(context.Background(), "job-1")
+
+ assert.Equal(t, tc.wantStatus, code)
+ if tc.wantErr != "" {
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), tc.wantErr)
+ assert.Nil(t, entry)
+ } else {
+ require.NoError(t, err)
+ assert.NotNil(t, entry)
+ }
+ })
+ }
+}
+
+func TestUpdateJobStatusKV(t *testing.T) {
+ tests := []struct {
+ name string
+ kv *test.MockKV
+ wantErr bool
+ }{
+ {name: "success returns nil", kv: test.NewMockKV(), wantErr: false},
+ {
+ name: "KV Put error returns error",
+ kv: func() *test.MockKV {
+ m := test.NewMockKV()
+ m.PutErr = errors.New("kv unavailable")
+ return m
+ }(),
+ wantErr: true,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ h := &KVHandler{logger: test.SilentLogger(), kv: tc.kv}
+ err := h.updateJobStatusKV(context.Background(), "job-1", JobStatus{State: StateProcessing, Stage: "scene-detector"})
+
+ if tc.wantErr {
+ assert.Error(t, err)
+ } else {
+ assert.NoError(t, err)
+ }
+ })
+ }
+}
diff --git a/backend/video-status/internal/handler/subscriber_integration_test.go b/backend/video-status/internal/handler/subscriber_integration_test.go
index d9976dd..428d7d2 100644
--- a/backend/video-status/internal/handler/subscriber_integration_test.go
+++ b/backend/video-status/internal/handler/subscriber_integration_test.go
@@ -175,7 +175,7 @@ func TestListenAdvisoriesFailure_KVPutFails(t *testing.T) {
})
}
-func TestListenJobComplete_NoStream(t *testing.T) {
+func TestListenJobCompleteI(t *testing.T) {
t.Run("returns error when no stream covers jobs.complete", func(t *testing.T) {
ctx := context.Background()
@@ -197,39 +197,37 @@ func TestListenJobComplete_NoStream(t *testing.T) {
assert.Error(t, err)
})
-}
-
-func TestListenJobComplete_ReturnsSub(t *testing.T) {
- consCtx, err := ListenJobComplete(sharedJS, sharedKV, test.SilentLogger())
-
- require.NoError(t, err)
- assert.NotNil(t, consCtx)
- t.Cleanup(consCtx.Stop)
-}
+ t.Run("returns sub", func(t *testing.T) {
+ consCtx, err := ListenJobComplete(sharedJS, sharedKV, test.SilentLogger())
-func TestListenJobComplete_ConsumerConfig(t *testing.T) {
- ctx := context.Background()
+ require.NoError(t, err)
+ assert.NotNil(t, consCtx)
+ t.Cleanup(consCtx.Stop)
+ })
+ t.Run("Consumer config", func(t *testing.T) {
+ ctx := context.Background()
- consCtx, err := ListenJobComplete(sharedJS, sharedKV, test.SilentLogger())
- require.NoError(t, err)
- t.Cleanup(consCtx.Stop)
+ consCtx, err := ListenJobComplete(sharedJS, sharedKV, test.SilentLogger())
+ require.NoError(t, err)
+ t.Cleanup(consCtx.Stop)
- stream, err := sharedJS.Stream(ctx, "jobs")
- require.NoError(t, err)
- cons, err := stream.Consumer(ctx, "video-status-complete")
- require.NoError(t, err)
- info, err := cons.Info(ctx)
- require.NoError(t, err)
+ stream, err := sharedJS.Stream(ctx, "jobs")
+ require.NoError(t, err)
+ cons, err := stream.Consumer(ctx, "video-status-complete")
+ require.NoError(t, err)
+ info, err := cons.Info(ctx)
+ require.NoError(t, err)
- assert.Equal(t, "video-status-complete", info.Config.Name)
- assert.Equal(t, "video-status-complete", info.Config.Durable)
- assert.Equal(t, "jobs.complete", info.Config.FilterSubject)
- assert.Equal(t, jetstream.AckExplicitPolicy, info.Config.AckPolicy)
- assert.Equal(t, 3, info.Config.MaxDeliver)
- assert.Equal(t, 30*time.Second, info.Config.AckWait)
+ assert.Equal(t, "video-status-complete", info.Config.Name)
+ assert.Equal(t, "video-status-complete", info.Config.Durable)
+ assert.Equal(t, "jobs.complete", info.Config.FilterSubject)
+ assert.Equal(t, jetstream.AckExplicitPolicy, info.Config.AckPolicy)
+ assert.Equal(t, 3, info.Config.MaxDeliver)
+ assert.Equal(t, 30*time.Second, info.Config.AckWait)
+ })
}
-func TestListenJobComplete_WritesKV(t *testing.T) {
+func TestListenJobComplete(t *testing.T) {
t.Run("valid jobs.complete message writes COMPLETE to KV and acks", func(t *testing.T) {
consCtx, err := ListenJobComplete(sharedJS, sharedKV, test.SilentLogger())
require.NoError(t, err)
@@ -241,9 +239,7 @@ func TestListenJobComplete_WritesKV(t *testing.T) {
test.AssertKVComplete(t, sharedKV, jobID)
})
-}
-func TestListenJobComplete_InvalidJSON(t *testing.T) {
t.Run("invalid JSON does not write KV", func(t *testing.T) {
consCtx, err := ListenJobComplete(sharedJS, sharedKV, test.SilentLogger())
require.NoError(t, err)
@@ -254,9 +250,7 @@ func TestListenJobComplete_InvalidJSON(t *testing.T) {
test.AssertKVEmpty(t, sharedKV, "jc-bad-json")
})
-}
-func TestListenJobComplete_KVPutFails(t *testing.T) {
t.Run("KV Put failure is handled without panic", func(t *testing.T) {
mockKV := test.NewMockKV()
mockKV.PutErr = errors.New("kv unavailable")
diff --git a/backend/video-status/internal/handler/watcher.go b/backend/video-status/internal/handler/watcher.go
new file mode 100644
index 0000000..191d03e
--- /dev/null
+++ b/backend/video-status/internal/handler/watcher.go
@@ -0,0 +1,58 @@
+package handler
+
+import (
+ "log/slog"
+ "net/http"
+ "time"
+)
+
+// need this because we are checking the next service
+// from the current processing stage and used for the
+// error msg
+var nextService = map[string]string{
+ "upload": "scene-detector",
+ "scene-detector": "transcoder",
+ "transcoder": "video-recombiner",
+}
+
+type ServiceURLs struct {
+ SceneDetector string
+ Transcoder string
+ Recombiner string
+}
+
+func (s ServiceURLs) forStage(stage string) (string, bool) {
+ next, ok := nextService[stage]
+ if !ok {
+ return "", false
+ }
+
+ urls := map[string]string{
+ "scene-detector": s.SceneDetector,
+ "transcoder": s.Transcoder,
+ "video-recombiner": s.Recombiner,
+ }
+
+ url, ok := urls[next]
+ if !ok || url == "" {
+ return "", false
+ }
+ return url, true
+}
+
+func isServiceHealthy(baseURL string, logger *slog.Logger) bool {
+ c := http.Client{Timeout: 3 * time.Second}
+
+ resp, err := c.Get(baseURL + "/health")
+ if err != nil {
+ return false
+ }
+ defer func() {
+ err := resp.Body.Close()
+ if err != nil {
+ logger.Error("error closing resp body", "err", err)
+ }
+ }()
+
+ return resp.StatusCode == http.StatusOK
+}
diff --git a/backend/video-status/internal/handler/watcher_unit_test.go b/backend/video-status/internal/handler/watcher_unit_test.go
new file mode 100644
index 0000000..72bfe45
--- /dev/null
+++ b/backend/video-status/internal/handler/watcher_unit_test.go
@@ -0,0 +1,159 @@
+//go:build unit
+
+package handler
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "video-status/internal/test"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestNextServiceMap(t *testing.T) {
+ tests := []struct {
+ stage string
+ want string
+ }{
+ {"upload", "scene-detector"},
+ {"scene-detector", "transcoder"},
+ {"transcoder", "video-recombiner"},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.stage, func(t *testing.T) {
+ got, ok := nextService[tc.stage]
+ assert.True(t, ok)
+ assert.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestForStage(t *testing.T) {
+ urls := ServiceURLs{
+ SceneDetector: "http://scene:9098",
+ Transcoder: "http://transcoder:9095",
+ Recombiner: "http://recombiner:9090",
+ }
+
+ tests := []struct {
+ stage string
+ wantURL string
+ wantOK bool
+ }{
+ {"upload", "http://scene:9098", true},
+ {"scene-detector", "http://transcoder:9095", true},
+ {"transcoder", "http://recombiner:9090", true},
+ {"video-recombine", "", false},
+ {"unknown", "", false},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.stage, func(t *testing.T) {
+ url, ok := urls.forStage(tc.stage)
+ assert.Equal(t, tc.wantOK, ok)
+ assert.Equal(t, tc.wantURL, url)
+ })
+ }
+
+ t.Run("empty URL returns false", func(t *testing.T) {
+ url, ok := ServiceURLs{}.forStage("scene-detector")
+ assert.False(t, ok)
+ assert.Empty(t, url)
+ })
+}
+
+func TestIsServiceHealthy(t *testing.T) {
+ tests := []struct {
+ name string
+ handler http.HandlerFunc
+ want bool
+ }{
+ {
+ name: "200 response returns true",
+ handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) },
+ want: true,
+ },
+ {
+ name: "503 response returns false",
+ handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) },
+ want: false,
+ },
+ {
+ name: "500 response returns false",
+ handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) },
+ want: false,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ srv := httptest.NewServer(tc.handler)
+ defer srv.Close()
+ assert.Equal(t, tc.want, isServiceHealthy(srv.URL, test.SilentLogger()))
+ })
+ }
+
+ t.Run("connection refused returns false", func(t *testing.T) {
+ assert.False(t, isServiceHealthy("http://localhost:19999", test.SilentLogger()))
+ })
+}
+
+func TestCheckServiceHealth(t *testing.T) {
+ healthySrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer healthySrv.Close()
+
+ downURL := "http://localhost:19999"
+
+ tests := []struct {
+ name string
+ status JobStatus
+ urls ServiceURLs
+ wantState JobState
+ wantErrMsg string
+ }{
+ {
+ name: "PROCESSING with healthy service stays PROCESSING",
+ status: JobStatus{State: StateProcessing, Stage: "scene-detector"},
+ urls: ServiceURLs{Transcoder: healthySrv.URL},
+ wantState: StateProcessing,
+ },
+ {
+ name: "PROCESSING with service down becomes DEGRADED",
+ status: JobStatus{State: StateProcessing, Stage: "scene-detector"},
+ urls: ServiceURLs{Transcoder: downURL},
+ wantState: StateDegraded,
+ wantErrMsg: "service unavailable at stage: transcoder",
+ },
+ {
+ name: "DEGRADED with service recovered becomes PROCESSING and clears error",
+ status: JobStatus{State: StateDegraded, Stage: "scene-detector", Error: "service unavailable at stage: transcoder"},
+ urls: ServiceURLs{Transcoder: healthySrv.URL},
+ wantState: StateProcessing,
+ },
+ {
+ name: "DEGRADED with service still down stays DEGRADED",
+ status: JobStatus{State: StateDegraded, Stage: "scene-detector"},
+ urls: ServiceURLs{Transcoder: downURL},
+ wantState: StateDegraded,
+ wantErrMsg: "service unavailable at stage: transcoder",
+ },
+ {
+ name: "unknown stage returns status unchanged",
+ status: JobStatus{State: StateProcessing, Stage: "unknown-stage"},
+ urls: ServiceURLs{},
+ wantState: StateProcessing,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ result := checkServiceHealth(tc.status, tc.urls, test.SilentLogger())
+ assert.Equal(t, tc.wantState, result.State)
+ assert.Equal(t, tc.wantErrMsg, result.Error)
+ })
+ }
+}
diff --git a/backend/video-status/internal/test/handler_helpers.go b/backend/video-status/internal/test/handler_helpers.go
index 0cbfaaf..f60b09b 100644
--- a/backend/video-status/internal/test/handler_helpers.go
+++ b/backend/video-status/internal/test/handler_helpers.go
@@ -18,6 +18,7 @@ import (
// Kept minimal — only the fields needed for assertions.
type AssertJobStatus struct {
State string `json:"state"`
+ Stage string `json:"stage"`
Error string `json:"error,omitempty"`
}
@@ -60,3 +61,23 @@ func AssertKVComplete(t *testing.T, kv jetstream.KeyValue, jobID string) {
return json.Unmarshal(entry.Value(), &s) == nil && s.State == "COMPLETE"
}, 5*time.Second, 100*time.Millisecond, "KV entry for %q never reached COMPLETE state", jobID)
}
+
+func AssertKVDegraded(t *testing.T, kv jetstream.KeyValue, jobID, wantErrContains string) {
+ t.Helper()
+ require.Eventually(t, func() bool {
+ entry, err := kv.Get(context.Background(), jobID)
+ if err != nil {
+ return false
+ }
+
+ var s AssertJobStatus
+ return json.Unmarshal(entry.Value(), &s) == nil && s.State == "DEGRADED"
+ }, 5*time.Second, 100*time.Millisecond, "KV entry for %q never reached DEGRADED state", jobID)
+
+ entry, err := kv.Get(context.Background(), jobID)
+ require.NoError(t, err)
+
+ var s AssertJobStatus
+ require.NoError(t, json.Unmarshal(entry.Value(), &s))
+ assert.Contains(t, s.Error, wantErrContains)
+}
diff --git a/backend/video-upload/cmd/helpers_test.go b/backend/video-upload/cmd/helpers_test.go
new file mode 100644
index 0000000..aa93ff6
--- /dev/null
+++ b/backend/video-upload/cmd/helpers_test.go
@@ -0,0 +1,47 @@
+//go:build unit
+
+package main
+
+import (
+ "context"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "testing"
+ "video-upload/internal/test"
+
+ "github.com/nats-io/nats.go/jetstream"
+ "github.com/stretchr/testify/require"
+)
+
+// freePort returns a port number that is not currently in use.
+func freePort(t *testing.T) string {
+ t.Helper()
+
+ l, err := net.Listen("tcp", ":0")
+ require.NoError(t, err)
+ port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
+
+ err = l.Close()
+ require.NoError(t, err)
+
+ return port
+}
+
+// startTestServer calls startHttpApi with a free port and a temp output dir,
+// registers a Cleanup to shut the server down, and returns the server and cfg.
+func startTestServer(t *testing.T, kv jetstream.KeyValue) (*http.Server, *Config) {
+ t.Helper()
+
+ fakeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+
+ t.Cleanup(fakeSrv.Close)
+
+ cfg := &Config{HTTPPort: freePort(t), StorageURL: fakeSrv.URL}
+ server := startHttpApi(test.SilentLogger(), &test.MockJS{}, kv, cfg)
+
+ t.Cleanup(func() { server.Shutdown(context.Background()) }) //nolint:errcheck
+
+ return server, cfg
+}
\ No newline at end of file
diff --git a/backend/video-upload/cmd/main.go b/backend/video-upload/cmd/main.go
index 64b9bab..a672bdd 100644
--- a/backend/video-upload/cmd/main.go
+++ b/backend/video-upload/cmd/main.go
@@ -53,15 +53,7 @@ func main() {
os.Exit(1)
}
- kv, err := js.CreateOrUpdateKeyValue(context.Background(), jetstream.KeyValueConfig{
- Bucket: "job-status",
- Description: "tracks job state across the pipeline",
- TTL: 3 * time.Hour,
- })
- if err != nil {
- logger.Error("failed to ccreate job-status kv bucket", "err", err)
- os.Exit(1)
- }
+ kv := handler.ConnectJobStatusKV(js, logger)
quit := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
@@ -98,7 +90,8 @@ func startHttpApi(logger *slog.Logger, js jetstream.JetStream, kv jetstream.KeyV
go func() {
fmt.Printf("server running on http://localhost:%s\n", cfg.HTTPPort)
- if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ err := server.ListenAndServe()
+ if err != nil && err != http.ErrServerClosed {
log.Fatalf("http server error: %v", err)
}
}()
diff --git a/backend/video-upload/cmd/main_unit_test.go b/backend/video-upload/cmd/main_unit_test.go
index 677a094..3f407ce 100644
--- a/backend/video-upload/cmd/main_unit_test.go
+++ b/backend/video-upload/cmd/main_unit_test.go
@@ -3,15 +3,12 @@
package main
import (
- "context"
"errors"
- "net"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
- "strconv"
"strings"
"testing"
"video-upload/internal/test"
@@ -20,28 +17,6 @@ import (
"github.com/stretchr/testify/require"
)
-// freePort returns a port number that is not currently in use.
-func freePort(t *testing.T) string {
- t.Helper()
- l, err := net.Listen("tcp", ":0")
- require.NoError(t, err)
- port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
- l.Close()
- return port
-}
-
-// startTestServer calls startHttpApi with a free port and a temp output dir,
-// registers a Cleanup to shut the server down, and returns the server and cfg.
-func startTestServer(t *testing.T) (*http.Server, *Config) {
- t.Helper()
- fakeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
- t.Cleanup(fakeSrv.Close)
- cfg := &Config{HTTPPort: freePort(t), StorageURL: fakeSrv.URL}
- server := startHttpApi(test.SilentLogger(), &test.MockJS{}, &test.MockKV{}, cfg)
- t.Cleanup(func() { server.Shutdown(context.Background()) }) //nolint:errcheck
- return server, cfg
-}
-
func TestLoadConfig(t *testing.T) {
t.Run("missing env file shouldnt return error", func(t *testing.T) {
if _, err := os.Stat(filepath.Join("..", ".env")); err == nil {
@@ -80,20 +55,20 @@ func TestLoadConfig(t *testing.T) {
func TestStartHttp(t *testing.T) {
t.Run("returns non-nil server with address derived from config", func(t *testing.T) {
- server, cfg := startTestServer(t)
+ server, cfg := startTestServer(t, &test.MockKV{})
require.NotNil(t, server)
assert.Equal(t, ":"+cfg.HTTPPort, server.Addr)
})
t.Run("server handler is non-nil", func(t *testing.T) {
- server, _ := startTestServer(t)
+ server, _ := startTestServer(t, &test.MockKV{})
assert.NotNil(t, server.Handler)
})
t.Run("unregistered path returns 404", func(t *testing.T) {
- server, _ := startTestServer(t)
+ server, _ := startTestServer(t, &test.MockKV{})
req := httptest.NewRequest(http.MethodGet, "/nonexistent", nil)
w := httptest.NewRecorder()
@@ -104,26 +79,14 @@ func TestStartHttp(t *testing.T) {
}
func TestMainFunc(t *testing.T) {
- t.Run("exits on config load error", func(t *testing.T) {
- if os.Getenv("RUN_MAIN") == "config_error" {
- os.Chdir("/") //nolint:errcheck
- main()
- return
- }
- cmd := exec.Command(os.Args[0], "-test.run=TestMain/exits_on_config_load_error", "-test.count=1")
- cmd.Env = append(os.Environ(), "RUN_MAIN=config_error")
- err := cmd.Run()
- var exitErr *exec.ExitError
- require.ErrorAs(t, err, &exitErr)
- assert.Equal(t, 1, exitErr.ExitCode())
- })
-
t.Run("exits on NATS connect error", func(t *testing.T) {
if os.Getenv("RUN_MAIN") == "nats_error" {
main()
return
}
+
test.WriteEnvFile(t, "NATS_URL=nats://localhost:1\n")
+
var env []string
for _, e := range os.Environ() {
if !strings.HasPrefix(e, "NATS_URL=") && !strings.HasPrefix(e, "PROD_MODE=") &&
@@ -131,21 +94,20 @@ func TestMainFunc(t *testing.T) {
env = append(env, e)
}
}
+
cmd := exec.Command(os.Args[0], "-test.run=TestMain/exits_on_NATS_connect_error", "-test.count=1")
cmd.Env = append(env, "RUN_MAIN=nats_error")
err := cmd.Run()
+
var exitErr *exec.ExitError
+
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, 1, exitErr.ExitCode())
})
t.Run("returns 500 when KV.Put fails during upload", func(t *testing.T) {
- fakeSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
- t.Cleanup(fakeSrv.Close)
- cfg := &Config{HTTPPort: freePort(t), StorageURL: fakeSrv.URL}
kv := &test.MockKV{PutErr: errors.New("kv unavailable")}
- server := startHttpApi(test.SilentLogger(), &test.MockJS{}, kv, cfg)
- t.Cleanup(func() { server.Shutdown(context.Background()) }) //nolint:errcheck
+ server, _ := startTestServer(t, kv)
req := test.NewUploadRequest(t, "/jobs/upload", "video.mp4", []byte("data"), "1080p")
w := httptest.NewRecorder()
diff --git a/backend/video-upload/internal/handler/job_status_kv.go b/backend/video-upload/internal/handler/job_status_kv.go
new file mode 100644
index 0000000..c9751a3
--- /dev/null
+++ b/backend/video-upload/internal/handler/job_status_kv.go
@@ -0,0 +1,46 @@
+package handler
+
+import (
+ "context"
+ "encoding/json"
+ "log/slog"
+ "os"
+ "time"
+
+ "github.com/nats-io/nats.go/jetstream"
+)
+
+var osExit = os.Exit
+
+// connect to existing job status kv to publishing the processing stage update msgs
+func ConnectJobStatusKV(js jetstream.JetStream, logger *slog.Logger) jetstream.KeyValue {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ kv, err := js.KeyValue(ctx, "job-status")
+ if err != nil {
+ logger.Error("failed to connect to job-status kv bucket", "err", err)
+ osExit(1)
+ }
+
+ return kv
+}
+
+func updateJobStatusKV(ctx context.Context, jobID string, kv jetstream.KeyValue, logger *slog.Logger) error {
+ status, err := json.Marshal(struct {
+ State string `json:"state"`
+ Stage string `json:"stage"`
+ }{State: "PROCESSING", Stage: "upload"})
+ if err != nil {
+ logger.Error("error marshalling PROCESSING:upload text", "err", err)
+ return err
+ }
+
+ _, err = kv.Put(ctx, jobID, status)
+ if err != nil {
+ logger.Error("failed to write job status to jetstream kv", "job_id", jobID, "err", err)
+ return err
+ }
+
+ return nil
+}
\ No newline at end of file
diff --git a/backend/video-upload/internal/handler/video.go b/backend/video-upload/internal/handler/video.go
index 4e42cc8..3eb6e66 100644
--- a/backend/video-upload/internal/handler/video.go
+++ b/backend/video-upload/internal/handler/video.go
@@ -81,19 +81,9 @@ func (v *VideoHandler) UploadVideo(w http.ResponseWriter, r *http.Request) {
return
}
- status, err := json.Marshal(struct {
- State string `json:"state"`
- }{State: "PROCESSING"})
- if err != nil {
- http.Error(w, "failed to build job status", http.StatusInternalServerError)
- v.Logger.Error("error marshalling PROCESSING text", "err", err)
- return
- }
-
- _, err = v.KV.Put(r.Context(), result.JobID, status)
+ err = updateJobStatusKV(r.Context(), result.JobID, v.KV, v.Logger)
if err != nil {
http.Error(w, "failed to record job status", http.StatusInternalServerError)
- v.Logger.Error("failed to write job status to jetstream kv", "job_id", result.JobID, "err", err)
return
}
diff --git a/frontend/src/api/lib/basePath.ts b/frontend/src/api/lib/basePath.ts
index 5f245a9..847a340 100644
--- a/frontend/src/api/lib/basePath.ts
+++ b/frontend/src/api/lib/basePath.ts
@@ -2,4 +2,4 @@ import axios from "axios";
export const VideoApi = axios.create({baseURL: 'http://localhost:8080'});
-export const StatusApi = axios.create({baseURL: 'http://localhost:8081'});
\ No newline at end of file
+export const StatusApi = axios.create({baseURL: 'http://localhost:8085'});
\ No newline at end of file
diff --git a/frontend/src/components/uploadVideosList.tsx b/frontend/src/components/uploadVideosList.tsx
index 37d7ac5..4b9c757 100644
--- a/frontend/src/components/uploadVideosList.tsx
+++ b/frontend/src/components/uploadVideosList.tsx
@@ -1,7 +1,8 @@
-import { CheckCheck, Loader, Video, X } from "lucide-react"
+import { CheckCheck, Video, X } from "lucide-react"
import type { JobStatus, UploadedVideo } from "../types/video"
import { formatSize, truncateName } from "../utils/fileDisplay"
import { useVideoQueueStore } from "../state/videoQueue"
+import VideoProgressBar from "./videoProgressBar"
interface UploadVideoListProps {
videos: UploadedVideo[]
@@ -14,9 +15,8 @@ const UploadVideoList = ({ videos, onRemove }: UploadVideoListProps) => {
const { uploadedVideos, resetVideo, setResolution } = useVideoQueueStore()
function StatusIcon({ status }: { status: JobStatus }) {
- if (status === 'processing') return
if (status === 'complete') return
- if (status === 'error') return
+ if (status === 'error') return
return
}
@@ -26,7 +26,7 @@ const UploadVideoList = ({ videos, onRemove }: UploadVideoListProps) => {
if (video?.status === "error") {
resetVideo(id)
}
-
+
setResolution(id, resolution)
}
@@ -74,6 +74,10 @@ const UploadVideoList = ({ videos, onRemove }: UploadVideoListProps) => {
/>
)}
+
+ {(video.status === 'processing' || video.status === 'degraded' || video.status === 'error') && (
+
+ )}
))}
diff --git a/frontend/src/components/videoProgressBar.tsx b/frontend/src/components/videoProgressBar.tsx
new file mode 100644
index 0000000..7154c6b
--- /dev/null
+++ b/frontend/src/components/videoProgressBar.tsx
@@ -0,0 +1,47 @@
+import type { UploadedVideo } from "../types/video"
+
+const STAGE_PROGRESS: Record = {
+ '': 5,
+ 'scene-detector': 33,
+ 'transcoder': 77,
+ 'video-recombiner': 90,
+}
+
+const STAGE_LABELS: Record = {
+ '': 'Queued',
+ 'scene-detector': 'Detecting scenes',
+ 'transcoder': 'Transcoding',
+ 'video-recombiner': 'Recombining',
+}
+
+const DEGRADED_LABELS: Record = {
+ 'upload': 'Degraded: scene-detector is down',
+ 'scene-detector': 'Degraded: transcoder is down',
+ 'transcoder': 'Degraded: video-recombiner is down',
+}
+
+const VideoProgressBar = ({ video }: { video: UploadedVideo }) => {
+ const stage = video.stage ?? ''
+ const pct = STAGE_PROGRESS[stage] ?? 5
+ const label = video.status === 'degraded' ? (DEGRADED_LABELS[stage] ?? 'Degraded, backend service is down...')
+ : video.status === 'error' ? 'Failed'
+ : (STAGE_LABELS[stage] ?? stage)
+
+ let barColor = 'bg-green-500'
+ if (video.status === 'degraded') barColor = 'bg-orange-400'
+ else if (video.status === 'error') barColor = 'bg-red-500'
+
+ return (
+
+ )
+}
+
+export default VideoProgressBar
\ No newline at end of file
diff --git a/frontend/src/hooks/useJobPolling.ts b/frontend/src/hooks/useJobPolling.ts
index ac33bc4..588ddd8 100644
--- a/frontend/src/hooks/useJobPolling.ts
+++ b/frontend/src/hooks/useJobPolling.ts
@@ -8,28 +8,32 @@ async function pollVideo(video: UploadedVideo) {
try {
const data = await VideoService.status(video.jobId!)
if (data.state === 'COMPLETE') markComplete(video)
- else if (data.state === 'ERROR') updateVideo(video.id, { status: 'error', error: data.error })
+ else if (data.state === 'FAILED') updateVideo(video.id, { status: 'error', error: data.error })
+ else if (data.state === 'DEGRADED') updateVideo(video.id, { status: 'degraded', stage: data.stage, error: data.error })
+ else if (data.state === 'PROCESSING') updateVideo(video.id, { status: 'processing', stage: data.stage })
} catch {
updateVideo(video.id, { status: 'error' })
}
}
+const isActivePoll = (v: UploadedVideo) => (v.status === 'processing' || v.status === 'degraded') && !!v.jobId
+
export function useJobPolling() {
- const processingCount = useVideoQueueStore(
- s => s.uploadedVideos.filter(v => v.status === 'processing' && v.jobId).length
+ const activeCount = useVideoQueueStore(
+ s => s.uploadedVideos.filter(isActivePoll).length
)
useEffect(() => {
- if (processingCount === 0) return
+ if (activeCount === 0) return
const interval = setInterval(() => {
const { uploadedVideos } = useVideoQueueStore.getState()
uploadedVideos
- .filter(v => v.status === 'processing' && v.jobId)
+ .filter(isActivePoll)
.forEach(pollVideo)
}, 1000)
return () => clearInterval(interval)
- }, [processingCount])
+ }, [activeCount])
}
\ No newline at end of file
diff --git a/frontend/src/types/video.ts b/frontend/src/types/video.ts
index 6a5c571..ac384f7 100644
--- a/frontend/src/types/video.ts
+++ b/frontend/src/types/video.ts
@@ -1,4 +1,4 @@
-export type JobStatus = 'pending' | 'uploading' | 'processing' | 'complete' | 'error'
+export type JobStatus = 'pending' | 'uploading' | 'processing' | 'complete' | 'error' | 'degraded'
export interface UploadedVideo {
id: number
@@ -8,5 +8,6 @@ export interface UploadedVideo {
status: JobStatus
uploadProgress: number
jobId: string | null
+ stage?: string
error?: string
}