Refactor package structure

.github/workflows/mkdocs.yml

@@ -1,4 +1,4 @@
-name: Deploy MkDocs and Doorstop to GitHub Pages
+name: Deploy MkDocs to GitHub Pages
 
 on:
   push:
@@ -7,7 +7,6 @@ on:
     paths:
       - 'src/**'
       - 'docs/mkdocs/**'
-      - 'docs/doorstop/**'
   workflow_dispatch:
 
 jobs:
@@ -25,11 +24,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install mkdocs mkdocs-material mkdocstrings[python] doorstop
-
-      - name: Generate Doorstop HTML
-        run: |
-          doorstop publish all docs/mkdocs/doorstop
+          pip install mkdocs mkdocs-material mkdocstrings[python]
 
       - name: Build and deploy MkDocs
         run: |

.gitignore

@@ -12,4 +12,5 @@ __pycache__
 .env
 wandb
 .wandb
-results
+results
+wireguard

Justfile

@@ -6,6 +6,25 @@ default:
 install:
     uv sync
 
+# Connect to VPN
+vpn_connect:
+    sudo wg-quick up wg0
+
+# Disconnect from VPN
+vpn_disconnect:
+    sudo wg-quick down wg0
+
+# Download shape predictor model weights
+download_shape_predictor:
+    mkdir -p models/dlib
+    wget -O models/dlib/shape_predictor_68_face_landmarks.dat.bz2 https://raw.githubusercontent.com/davisking/dlib-models/master/shape_predictor_68_face_landmarks.dat.bz2
+    bzip2 -d models/dlib/shape_predictor_68_face_landmarks.dat.bz2
+
+# Download opencv face detector model weights
+download_opencv_face_detector:
+    mkdir -p models/opencv
+    wget -O models/opencv/haarcascade_frontalface_default.xml https://raw.githubusercontent.com/opencv/opencv/4.x/data/haarcascades/haarcascade_frontalface_default.xml
+
 # === Scripts from src.scripts package ===
 
 # Run benchmark
@@ -36,7 +55,7 @@ test:
 
 # Run tests with coverage
 cov:
-    uv run pytest --cov=src
+    uv run pytest --cov=src --cov-report=term-missing
 
 # === Static Analysis ===
 
@@ -93,10 +112,6 @@ metrics_all: metrics_raw_src metrics_raw_tests metrics_raw_notebooks metrics_cyc
 
 # === Documentation ===
 
-# Build doorstop documentation
-doorstop_build:
-    uv run doorstop publish all site/doorstop
-
 # Serve documentation locally
 docs:
     uv run mkdocs serve

docs/mkdocs/api/common.md

@@ -0,0 +1,3 @@
+# Common
+
+::: src.common

docs/mkdocs/api/evaluation.md

@@ -0,0 +1,3 @@
+# Evaluation
+
+::: src.evaluation

docs/mkdocs/api/plots.md

@@ -0,0 +1,3 @@
+# Plots
+
+::: src.plots

docs/mkdocs/api/scripts.md

@@ -0,0 +1,3 @@
+# Scripts
+
+::: src.scripts

docs/mkdocs/api/training.md

@@ -0,0 +1,5 @@
+# Training
+
+::: src.training
+    options:
+      show_submodules: true

docs/mkdocs/api/video_processing.md

@@ -0,0 +1,3 @@
+# Video Processing
+
+::: src.video_processing

docs/mkdocs/index.md

@@ -1,3 +1,3 @@
-# Dokumentacja projektu
+# Project documentation
 
-Wygenerowana przy pomocy MkDocs
+Generated with MkDocs.

mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: "Dokumentacja projektu"
+site_name: "Project documentation"
 docs_dir: "docs/mkdocs"
 theme:
   name: "material"
@@ -21,9 +21,11 @@ plugins:
             show_function_docstring: true
 nav:
   - Home: index.md
-  - API:
-      - Augmentation: api/augmentation.md
+  - API Reference:
+      - Common: api/common.md
       - Dataset: api/dataset.md
-      - Loss: api/loss.md
-      - Validation: api/validation.md
-  - Wymagania: doorstop/index.html
+      - Evaluation: api/evaluation.md
+      - Plots: api/plots.md
+      - Scripts: api/scripts.md
+      - Training: api/training.md
+      - Video Processing: api/video_processing.md

notebooks/04-fine_tuning.ipynb

@@ -44,15 +44,15 @@
     "from torchvision import transforms\n",
     "from torchvision.utils import make_grid\n",
     "\n",
-    "from src.augmentation.mask import GenerateMask\n",
-    "from src.augmentation.random_rectangle import (\n",
+    "from src.training.augmentation import GenerateMask\n",
+    "from src.training.augmentation.random_rectangle import (\n",
     "    AddRandomRectangleAverageColor,\n",
     "    AddRandomRectangleRandomColor,\n",
     "    AddRandomBlackRectangle,\n",
     ")\n",
     "from src.dataset.lfw import LFWDataset\n",
     "from src.dataset.rof import ROFDataset\n",
-    "from src.loss.contrastive_loss import ContrastiveLoss"
+    "from src.training.loss import ContrastiveLoss"
    ],
    "id": "8d8ee930d7320f79",
    "outputs": [],

notebooks/07-face_landmark_based_augmentation.ipynb

@@ -43,7 +43,7 @@
     "\n",
     "os.chdir(\"..\")\n",
     "\n",
-    "from src.augmentation import AddMaskAugmentation\n",
+    "from src.training.augmentation import AddMaskAugmentation\n",
     "\n",
     "os.getcwd()"
    ]

notebooks/08-glasses-augmentation.ipynb

@@ -26,8 +26,8 @@
     "\n",
     "os.chdir(\"..\")\n",
     "\n",
-    "from src.training.data_loaders import get_lfw_loaders\n",
-    "from src.augmentation import AddGlassesAugmentation\n",
+    "from src.dataset.data_loaders import get_lfw_loaders\n",
+    "from src.training.augmentation import AddGlassesAugmentation\n",
     "\n",
     "os.getcwd()"
    ]

notebooks/09-all-augmentations-demo.ipynb

@@ -31,7 +31,7 @@
     "\n",
     "os.chdir(\"..\")\n",
     "\n",
-    "from src.augmentation import get_augmentation_transform\n",
+    "from src.training.augmentation import get_augmentation_transform\n",
     "\n",
     "os.getcwd()"
    ]

pyproject.toml

@@ -33,6 +33,12 @@ profile = "black"
 
 [tool.pytest.ini_options]
 pythonpath = ["src"]
+markers = [
+    "integration: integration tests that may require external resources (e.g. model weights)",
+]
+filterwarnings = [
+    "ignore::pydantic._internal._generate_schema.UnsupportedFieldAttributeWarning",  # Ignore warning from libraries
+]
 
 [dependency-groups]
 dev = [
@@ -43,7 +49,6 @@ dev = [
     "mkdocs>=1.6.1",
     "mkdocstrings[python]>=0.29.1",
     "mkdocs-material>=9.6.13",
-    "doorstop>=3.0",
     "pytest-cov>=6.1.1",
 ]
 

src/common/__init__.py

@@ -0,0 +1,15 @@
+from .config import default_device_str
+from .models import (
+    get_dlib_frontal_face_detector,
+    get_dlib_shape_predictor,
+    get_facenet_pytorch_inception_resnet_v1,
+    get_facenet_pytorch_mtcnn_detector,
+)
+
+__all__ = [
+    "default_device_str",
+    "get_facenet_pytorch_inception_resnet_v1",
+    "get_facenet_pytorch_mtcnn_detector",
+    "get_dlib_frontal_face_detector",
+    "get_dlib_shape_predictor",
+]

src/dataset/__init__.py

@@ -0,0 +1,21 @@
+from .data_loaders import (
+    get_dummy_loaders,
+    get_lfw_loaders,
+    get_rococo_loaders,
+)
+from .dummy import DummyFacePairsDataset
+from .face_pairs import FacePairsDataset
+from .lfw import LFWDataset
+from .rococo_training import RococoTrainingDataset
+from .rof import ROFDataset
+
+__all__ = [
+    "get_lfw_loaders",
+    "get_rococo_loaders",
+    "get_dummy_loaders",
+    "DummyFacePairsDataset",
+    "FacePairsDataset",
+    "LFWDataset",
+    "RococoTrainingDataset",
+    "ROFDataset",
+]

src/training/data_loaders.py → src/dataset/data_loaders.py

@@ -1,7 +1,8 @@
 from torch.utils.data import DataLoader
 
-from src.dataset.lfw import LFWDataset
-from src.dataset.rococo_training import RococoTrainingDataset
+from .dummy import DummyFacePairsDataset
+from .lfw import LFWDataset
+from .rococo_training import RococoTrainingDataset
 
 
 def get_lfw_loaders(
@@ -58,7 +59,6 @@ def get_rococo_loaders(
     num_workers=4,
     pin_memory=True,
 ) -> dict[str, DataLoader]:
-
     train_set = RococoTrainingDataset.from_match_and_mismatch_pairs(
         root_dir=rococo_root,
         match_pairs_file="data/rococo2v3-dev/train_match_pairs.csv",
@@ -99,3 +99,34 @@ def get_rococo_loaders(
         "train": train_loader,
         "val": val_loader,
     }
+
+
+def get_dummy_loaders(
+    transform,
+    batch_size=32,
+    num_workers=0,
+    pin_memory=False,
+) -> dict[str, DataLoader]:
+    train_set = DummyFacePairsDataset(transform)
+    val_set = DummyFacePairsDataset(transform)
+
+    train_loader = DataLoader(
+        train_set,
+        batch_size=batch_size,
+        shuffle=False,
+        num_workers=num_workers,
+        pin_memory=pin_memory,
+    )
+
+    val_loader = DataLoader(
+        val_set,
+        batch_size=batch_size,
+        shuffle=False,
+        num_workers=num_workers,
+        pin_memory=pin_memory,
+    )
+
+    return {
+        "train": train_loader,
+        "val": val_loader,
+    }