Feat: Added video upscaling with Real-ESRGAN nn upscaling

.github/workflows/ci.yaml

@@ -21,7 +21,8 @@ jobs:
         video_recombiner: ${{ steps.changes.outputs.video_recombiner }}
         video_upload: ${{ steps.changes.outputs.video_upload }}
         video_status: ${{ steps.changes.outputs.video_status }}
-        shared: ${{ steps.changes.outputs.shared }}
+        shared_go: ${{ steps.changes.outputs.shared_go }}
+        shared_python: ${{ steps.changes.outputs.shared_python }}
     steps:
       - uses: actions/checkout@v4
       - uses: dorny/paths-filter@v3
@@ -40,8 +41,10 @@ jobs:
               - 'backend/video-upload/**'
             video_status:
               - 'backend/video-status/**'
-            shared:
-              - 'backend/shared/**'
+            shared_go:
+              - 'backend/shared/go/**'
+            shared_python:
+              - 'backend/shared/python/**'
   frontend-ci:
     needs: path-filter
     if: ${{ github.event_name == 'pull_request' && needs.path-filter.outputs.frontend == 'true' }}
@@ -226,9 +229,9 @@ jobs:
         working-directory: ./backend/video-status/cmd
         run: make test_all
 
-  shared-ci:
+  shared-go-ci:
     needs: path-filter
-    if: ${{ github.event_name == 'pull_request' && needs.path-filter.outputs.shared == 'true' }}
+    if: ${{ github.event_name == 'pull_request' && needs.path-filter.outputs.shared_go == 'true' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -241,19 +244,55 @@ jobs:
           cache: 'true'
 
       - name: Download Go modules
-        working-directory: ./backend/shared
+        working-directory: ./backend/shared/go
         run: go mod download
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v9
         with:
           version: v2.11
-          working-directory: ./backend/shared
+          working-directory: ./backend/shared/go
 
       - name: run tests
-        working-directory: ./backend/shared
+        working-directory: ./backend/shared/go
         run: make test_all
 
+  shared-python-ci:
+    needs: path-filter
+    if: ${{ github.event_name == 'pull_request' && needs.path-filter.outputs.shared_python == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.8.19"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install the project
+        run: uv sync --locked --all-extras --dev
+        working-directory: ./backend/shared/python
+
+      - name: Run Linter and Formatting
+        run: |
+          uv run ruff check .
+          uv run ruff format --check
+        working-directory: ./backend/shared/python
+
+      - name: Run Type Checking
+        run: uv run pyrefly check
+        working-directory: ./backend/shared/python
+
+      - name: Run Unit/Integration Tests
+        run: make test_all
+        working-directory: ./backend/shared/python
+
   coverage:
     needs: path-filter
     runs-on: ubuntu-latest
@@ -264,7 +303,8 @@ jobs:
         needs.path-filter.outputs.video_recombiner == 'true' ||
         needs.path-filter.outputs.video_upload == 'true' ||
         needs.path-filter.outputs.video_status == 'true' ||
-        needs.path-filter.outputs.shared == 'true'
+        needs.path-filter.outputs.shared_go == 'true' ||
+        needs.path-filter.outputs.shared_python == 'true'
       )
     steps:
       - uses: actions/checkout@v4
@@ -301,8 +341,12 @@ jobs:
         working-directory: ./backend/video-status/cmd
         run: make coverage
 
-      - name: Run shared coverage
-        working-directory: ./backend/shared
+      - name: Run shared go coverage
+        working-directory: ./backend/shared/go
+        run: make coverage
+
+      - name: Run shared python coverage
+        working-directory: ./backend/shared/python
         run: make coverage
 
       - name: Upload coverage to Codecov                                                                       
@@ -313,6 +357,7 @@ jobs:
             ./backend/video-recombiner/cmd/coverage.out,
             ./backend/video-upload/cmd/coverage.out,
             ./backend/video-status/cmd/coverage.out,
-            ./backend/shared/coverage.out,
-            ./backend/scene-detector/coverage.xml                                                                       
+            ./backend/scene-detector/coverage.xml,
+            ./backend/shared/go/coverage.out,
+            ./backend/shared/python/coverage.out                                                                      
           token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -33,6 +33,7 @@ docs/
 __pycache__/
 .pytest_cache/
 .ruff_cache/
+shared_python.egg-info
 
 # Logs
 logs

backend/go.work

@@ -1,7 +1,7 @@
 go 1.26.2
 
 use (
-    ./shared
+    ./shared/go
     ./pipeline-tests
     ./transcoder-worker
     ./video-recombiner

backend/go.work.sum

@@ -40,6 +40,7 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
 go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
 go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
 go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=

backend/pipeline-tests/e2e_integration_test.go

@@ -9,6 +9,8 @@ import (
 	"os/exec"
 	"path/filepath"
 	"pipeline-tests/helpers"
+
+	shelpers "shared/test"
 	"testing"
 	"time"
 	"github.com/stretchr/testify/assert"
@@ -18,7 +20,7 @@ import (
 var sharedFilerURL string
 
 func TestMain(m *testing.M) {
-	filerURL, cleanup := helpers.StartSeaweedFSFiler()
+	filerURL, cleanup := shelpers.StartSeaweedFSFiler()
 	sharedFilerURL = filerURL
 
 	code := m.Run()

backend/scene-detector/pyproject.toml

@@ -10,8 +10,12 @@ dependencies = [
     "scenedetect[opencv-headless]>=0.6.7.1",
     "structlog>=25.5.0",
     "types-requests>=2.33.0.20260402",
+    "shared_python"
 ]
 
+[tool.uv.sources]
+shared_python = { path = "../shared/python", editable = true }
+
 [tool.pyright]
 venvPath = "."
 venv = ".venv"
@@ -22,7 +26,8 @@ extraPaths = ["."]
 [tool.pyrefly]
 python-version = "3.13"
 untyped-def-behavior = "check-and-infer-return-any"
-search-path = ["."]
+search-path = [".", "../shared/python/src"]
+venv = ".venv"
 
 [tool.pyrefly.errors]
 unannotated-parameter = "error"

backend/scene-detector/src/core/settings.py

@@ -7,24 +7,13 @@
 
 class Settings(BaseSettings):
     # general config
-    LOG_LEVEL: str = "DEBUG"
-    LOG_FORMAT: str = "json"
     HTTP_PORT: int = 9098
+    SERVICE_NAME: str = "scene-detector"
 
     # Nats config
-    NATS_URL: str = "nats://localhost:4222"
-    NATS_SUB_QUEUE_NAME: str = "scene-detector-workers"
-    SCENE_SPLIT_SUBJECT: str = (
-        "jobs.video.scene-split"  # topic containing Job ID + storage path in MinIO
-    )
-    VIDEO_CHUNKS_SUBJECT: str = "jobs.video.chunks"
-
-    MAX_RECONNECT_ATTEMPT: int = 5
-    RECONNECT_TIME_WAIT_S: int = 2
-
-    MAX_DELIVER_ATTEMPTS: int = 3
-    ACK_WAIT_S: int = 30
-    KV_BUCKET_TTL_S: int = 3 * 60 * 60  # 3 hour TTL
+    SUB_QUEUE_NAME: str = "scene-detector-workers"
+    SUB_SUBJECT: str = "jobs.video.scene-split"
+    PUB_SUBJECT: str = "jobs.video.chunks"
 
     BASE_STORAGE_URL: str = "http://localhost:8888"