From 6402eb71540574023d043016dca5b5d27e26f3b0 Mon Sep 17 00:00:00 2001 From: Andrew Yager Date: Sun, 8 Mar 2026 16:36:52 +1100 Subject: [PATCH 1/7] feat: replace icontains search with PostgreSQL full-text search Consolidate all asset search logic into build_asset_search() in services/search.py with FTS ranking on PostgreSQL and icontains fallback on SQLite. Barcode exact matches are boosted above text matches. DRY up duplicate Q() filter blocks across views, bulk service, and resolve service. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/assets/services/bulk.py | 21 +- src/assets/services/resolve.py | 20 +- src/assets/services/search.py | 139 +++++++++++-- src/assets/tests/test_services.py | 314 ++++++++++++++++++++++++++---- src/assets/tests/test_views.py | 2 +- src/assets/views.py | 35 +--- 6 files changed, 419 insertions(+), 112 deletions(-) diff --git a/src/assets/services/bulk.py b/src/assets/services/bulk.py index 3718f21..22a2b6c 100644 --- a/src/assets/services/bulk.py +++ b/src/assets/services/bulk.py @@ -2,9 +2,10 @@ from django.contrib.auth import get_user_model from django.db import transaction as db_transaction -from django.db.models import F, Q +from django.db.models import F from ..models import Asset, AssetSerial, Category, Location, Transaction +from .search import build_asset_search User = get_user_model() @@ -52,16 +53,7 @@ def build_asset_filter_queryset(filters: dict): q = filters.get("q", "") if q: - queryset = queryset.filter( - Q(name__icontains=q) - | Q(description__icontains=q) - | Q(barcode__icontains=q) - | Q(tags__name__icontains=q) - | Q( - nfc_tags__tag_id__icontains=q, - nfc_tags__removed_at__isnull=True, - ) - ).distinct() + queryset = build_asset_search(queryset, q, include_nfc=True) department = filters.get("department", "") if department: @@ -121,12 +113,7 @@ def build_bulk_queryset( queryset = queryset.filter(status=filters["status"]) q = filters.get("q", "") if q: - queryset = queryset.filter( - Q(name__icontains=q) - | Q(description__icontains=q) - | Q(barcode__icontains=q) - | Q(tags__name__icontains=q) - ).distinct() + queryset = build_asset_search(queryset, q, include_nfc=True) if filters.get("department"): queryset = queryset.filter( category__department_id=filters["department"] diff --git a/src/assets/services/resolve.py b/src/assets/services/resolve.py index ce4c90a..8b0f851 100644 --- a/src/assets/services/resolve.py +++ b/src/assets/services/resolve.py @@ -5,9 +5,8 @@ flexible asset identifiers. """ -from django.db.models import Q - from assets.models import Asset, AssetSerial, NFCTag +from assets.services.search import build_asset_search def _truncate(value, max_len=100): @@ -128,16 +127,13 @@ def resolve_asset_from_input(asset_id=None, search=None, barcode=None): ) # 3e. Broad text match (name, description, tags, category) - matches = ( - Asset.objects.filter(status="active") - .filter( - Q(name__icontains=search) - | Q(description__icontains=search) - | Q(tags__name__icontains=search) - | Q(category__name__icontains=search) - ) - .distinct()[:2] - ) + base_qs = Asset.objects.filter(status="active") + matches = build_asset_search( + base_qs, + search, + include_nfc=False, + include_category=True, + )[:2] results = list(matches) if len(results) == 1: return results[0], None diff --git a/src/assets/services/search.py b/src/assets/services/search.py index f6585d0..b7a50e1 100644 --- a/src/assets/services/search.py +++ b/src/assets/services/search.py @@ -1,30 +1,133 @@ -"""Asset text search helpers.""" +"""Asset text search helpers. -from django.db.models import Q +Provides PostgreSQL full-text search (FTS) for Asset querysets, with +icontains fallback for identifier fields (barcode, NFC tag IDs) that +don't benefit from stemming/tokenisation. + +Falls back to icontains-only search on non-PostgreSQL backends (e.g. +SQLite in tests). +""" + +from django.db import connection +from django.db.models import Case, Q, Value, When +from django.db.models.fields import FloatField MAX_SEARCH_WORDS = 20 -def build_asset_text_query(q): - """Build Q object matching all words in q across asset text fields. +def _is_postgres(): + return connection.vendor == "postgresql" - Each word must appear in at least one of: name, description, barcode, - or tag name. Words are ANDed together so "blue bonnet" requires both - "blue" and "bonnet" to appear (possibly in different fields). - At most ``MAX_SEARCH_WORDS`` words are considered; additional words - are silently ignored to bound query complexity. - """ - words = q.split()[:MAX_SEARCH_WORDS] - if not words: - return Q(pk__in=[]) - combined = Q() +def _build_fts_search(queryset, words, search_text, icontains_q): + """FTS search path for PostgreSQL.""" + from django.contrib.postgres.search import ( + SearchQuery, + SearchRank, + SearchVector, + ) + + vector = ( + SearchVector("name", weight="A") + + SearchVector("description", weight="B") + + SearchVector("tags__name", weight="B") + ) + search_query = SearchQuery(search_text, search_type="websearch") + + fts_filter = Q(fts_rank__gt=0) + combined_filter = fts_filter | icontains_q + + barcode_exact = Case( + When(barcode__iexact=search_text, then=Value(10.0)), + default=Value(0.0), + output_field=FloatField(), + ) + + return ( + queryset.annotate( + fts_rank=SearchRank(vector, search_query), + barcode_boost=barcode_exact, + ) + .filter(combined_filter) + .distinct() + .order_by("-barcode_boost", "-fts_rank") + ) + + +def _build_icontains_search(queryset, words, search_text, icontains_q): + """icontains fallback for non-PostgreSQL backends.""" + # Build word-AND query across text fields + text_q = Q() for word in words: - word_q = ( + text_q &= ( Q(name__icontains=word) | Q(description__icontains=word) - | Q(barcode__icontains=word) | Q(tags__name__icontains=word) ) - combined &= word_q - return combined + + combined_filter = text_q | icontains_q + + barcode_exact = Case( + When(barcode__iexact=search_text, then=Value(10.0)), + default=Value(0.0), + output_field=FloatField(), + ) + + return ( + queryset.annotate(barcode_boost=barcode_exact) + .filter(combined_filter) + .distinct() + .order_by("-barcode_boost") + ) + + +def build_asset_search( + queryset, + q, + include_nfc=True, + include_category=False, +): + """Apply search to an Asset queryset. + + Uses PostgreSQL FTS when available, falls back to icontains on other + backends. + + - FTS/icontains on: name, description, tag names + - icontains on: barcode (identifier, not prose) + - icontains on: NFC tag IDs (if include_nfc=True) + - icontains on: category name (if include_category=True) + + Args: + queryset: Base Asset queryset to filter. + q: Search string (space-separated words, ANDed). + include_nfc: Include NFC tag ID substring matching. + include_category: Include category name substring matching. + + Returns: + Filtered, distinct queryset ordered by relevance. + """ + words = q.split()[:MAX_SEARCH_WORDS] + if not words: + return queryset.none() + + search_text = " ".join(words) + + # Identifier fields: always icontains (not prose, no stemming benefit) + icontains_q = Q() + for word in words: + word_q = Q(barcode__icontains=word) + if include_nfc: + word_q |= Q( + nfc_tags__tag_id__icontains=word, + nfc_tags__removed_at__isnull=True, + ) + if include_category: + word_q |= Q(category__name__icontains=word) + icontains_q &= word_q + + if _is_postgres(): + return _build_fts_search(queryset, words, search_text, icontains_q) + else: + return _build_icontains_search( + queryset, words, search_text, icontains_q + ) diff --git a/src/assets/tests/test_services.py b/src/assets/tests/test_services.py index b005bcc..e87baf5 100644 --- a/src/assets/tests/test_services.py +++ b/src/assets/tests/test_services.py @@ -242,12 +242,218 @@ def test_merge_moves_transactions(self, asset, user, category, location): assert asset.transactions.filter(action="audit").exists() -class TestBuildAssetTextQuery: - """Tests for build_asset_text_query search helper.""" +class TestBuildAssetSearch: + """Tests for build_asset_search FTS search service.""" - def test_multi_word_requires_all_words(self, category, location, user): + def test_search_by_name(self, category, location, user): + """Search matches asset by name (case-insensitive).""" + from assets.services.search import build_asset_search + + hit = AssetFactory( + name="Victorian Chandelier", + category=category, + current_location=location, + created_by=user, + ) + miss = AssetFactory( + name="Wooden Table", + category=category, + current_location=location, + created_by=user, + ) + results = build_asset_search(Asset.objects.all(), "chandelier") + assert hit in results + assert miss not in results + + def test_search_by_name_case_insensitive(self, category, location, user): + """FTS search is case-insensitive.""" + from assets.services.search import build_asset_search + + hit = AssetFactory( + name="Victorian Chandelier", + category=category, + current_location=location, + created_by=user, + ) + results = build_asset_search(Asset.objects.all(), "CHANDELIER") + assert hit in results + + def test_search_by_description(self, category, location, user): + """Search matches asset by description text.""" + from assets.services.search import build_asset_search + + hit = AssetFactory( + name="Prop Item", + description="A beautiful ornate golden frame", + category=category, + current_location=location, + created_by=user, + ) + miss = AssetFactory( + name="Other Item", + description="A simple wooden box", + category=category, + current_location=location, + created_by=user, + ) + results = build_asset_search(Asset.objects.all(), "ornate golden") + assert hit in results + assert miss not in results + + def test_search_by_barcode_substring(self, category, location, user): + """Barcode search uses substring match (not FTS).""" + from assets.services.search import build_asset_search + + hit = AssetFactory( + name="Some Asset", + barcode="PROP-00142", + category=category, + current_location=location, + created_by=user, + ) + miss = AssetFactory( + name="Other Asset", + barcode="PROP-00999", + category=category, + current_location=location, + created_by=user, + ) + results = build_asset_search(Asset.objects.all(), "PROP-001") + assert hit in results + assert miss not in results + + def test_search_by_tag_name(self, category, location, user): + """Search matches asset by associated tag name.""" + from assets.services.search import build_asset_search + + hit = AssetFactory( + name="Tagged Asset", + category=category, + current_location=location, + created_by=user, + ) + tag = TagFactory(name="fragile") + hit.tags.add(tag) + + miss = AssetFactory( + name="Untagged Asset", + category=category, + current_location=location, + created_by=user, + ) + results = build_asset_search(Asset.objects.all(), "fragile") + assert hit in results + assert miss not in results + + def test_search_by_nfc_tag_id(self, category, location, user): + """Search matches asset by NFC tag ID when include_nfc=True.""" + from assets.services.search import build_asset_search + + hit = AssetFactory( + name="NFC Asset", + category=category, + current_location=location, + created_by=user, + ) + NFCTagFactory( + tag_id="04:A3:B2:C1:D4:E5:F6", asset=hit, assigned_by=user + ) + + miss = AssetFactory( + name="Other Asset", + category=category, + current_location=location, + created_by=user, + ) + results = build_asset_search( + Asset.objects.all(), "04:A3:B2", include_nfc=True + ) + assert hit in results + assert miss not in results + + def test_search_nfc_excluded_by_default_false( + self, category, location, user + ): + """NFC tag search excluded when include_nfc=False.""" + from assets.services.search import build_asset_search + + asset = AssetFactory( + name="NFC Only Asset", + category=category, + current_location=location, + created_by=user, + ) + NFCTagFactory( + tag_id="04:A3:B2:C1:D4:E5:F6", asset=asset, assigned_by=user + ) + results = build_asset_search( + Asset.objects.all(), "04:A3:B2", include_nfc=False + ) + assert asset not in results + + def test_search_excludes_removed_nfc_tags(self, category, location, user): + """Removed NFC tags are excluded from search results.""" + from django.utils import timezone + + from assets.services.search import build_asset_search + + asset = AssetFactory( + name="Removed NFC Asset", + category=category, + current_location=location, + created_by=user, + ) + NFCTagFactory( + tag_id="04:AA:BB:CC:DD:EE:FF", + asset=asset, + assigned_by=user, + removed_at=timezone.now(), + ) + results = build_asset_search( + Asset.objects.all(), "04:AA:BB", include_nfc=True + ) + assert asset not in results + + def test_search_by_category_name(self, category, location, user): + """Search matches asset by category name when include_category=True.""" + from assets.services.search import build_asset_search + + hit = AssetFactory( + name="Some Prop", + category=category, + current_location=location, + created_by=user, + ) + # category fixture has a name — search for it + results = build_asset_search( + Asset.objects.all(), + category.name, + include_category=True, + ) + assert hit in results + + def test_search_category_excluded_by_default( + self, category, location, user + ): + """Category search excluded when include_category=False (default).""" + from assets.services.search import build_asset_search + + AssetFactory( + name="Unique Zephyr", + category=category, + current_location=location, + created_by=user, + ) + # Search by category name only — should not match + results = build_asset_search( + Asset.objects.all(), category.name, include_category=False + ) + # The asset name doesn't contain the category name, so no match + assert results.count() == 0 + + def test_multi_word_and_semantics(self, category, location, user): """Multi-word query ANDs all words: both must match.""" - from assets.services.search import build_asset_text_query + from assets.services.search import build_asset_search hit = AssetFactory( name="Blue Bonnet Hat", @@ -261,55 +467,87 @@ def test_multi_word_requires_all_words(self, category, location, user): current_location=location, created_by=user, ) - q = build_asset_text_query("blue bonnet") - results = Asset.objects.filter(q).distinct() + results = build_asset_search(Asset.objects.all(), "blue bonnet") assert hit in results assert miss not in results + def test_empty_query_returns_empty(self, asset): + """Empty string query returns empty queryset.""" + from assets.services.search import build_asset_search + + results = build_asset_search(Asset.objects.all(), "") + assert results.count() == 0 + + def test_whitespace_only_returns_empty(self, asset): + """Whitespace-only query returns empty queryset.""" + from assets.services.search import build_asset_search + + results = build_asset_search(Asset.objects.all(), " ") + assert results.count() == 0 + + def test_no_duplicates_with_multiple_tags(self, category, location, user): + """Asset with multiple matching tags appears only once.""" + from assets.services.search import build_asset_search + + asset = AssetFactory( + name="Multi Tag Asset", + category=category, + current_location=location, + created_by=user, + ) + tag1 = TagFactory(name="vintage") + tag2 = TagFactory(name="vintage-style") + asset.tags.add(tag1, tag2) + + results = build_asset_search(Asset.objects.all(), "vintage") + assert list(results.filter(pk=asset.pk)).count(asset) == 1 + def test_max_search_words_truncation(self, category, location, user): """The 21st word in a query is silently ignored.""" - from assets.services.search import ( - MAX_SEARCH_WORDS, - build_asset_text_query, - ) + from assets.services.search import MAX_SEARCH_WORDS, build_asset_search - # Create asset matching 20 words but not the 21st - words_20 = [f"w{i}" for i in range(MAX_SEARCH_WORDS)] - name = " ".join(words_20[:5]) - desc = " ".join(words_20[5:]) + words_20 = [f"wordtest{i}" for i in range(MAX_SEARCH_WORDS)] asset = AssetFactory( - name=name, - description=desc, + name=" ".join(words_20[:5]), + description=" ".join(words_20[5:]), category=category, current_location=location, created_by=user, ) - # Query with 20 words: should match - q_20 = build_asset_text_query(" ".join(words_20)) - assert ( - Asset.objects.filter(q_20).distinct().filter(pk=asset.pk).exists() - ) + # 20 words: should match + results = build_asset_search(Asset.objects.all(), " ".join(words_20)) + assert asset in results - # Query with 21 words where the extra word doesn't match - q_21 = build_asset_text_query(" ".join(words_20) + " nonexistentword") - # 21st word is ignored, so result should still match - assert ( - Asset.objects.filter(q_21).distinct().filter(pk=asset.pk).exists() + # 21 words where extra doesn't match — still matches (21st ignored) + results = build_asset_search( + Asset.objects.all(), " ".join(words_20) + " nonexistentxyz" ) + assert asset in results - def test_empty_query_returns_no_results(self, asset): - """Empty string query returns nothing-matching Q.""" - from assets.services.search import build_asset_text_query - - q = build_asset_text_query("") - assert Asset.objects.filter(q).count() == 0 - - def test_whitespace_only_returns_no_results(self, asset): - """Whitespace-only query returns nothing-matching Q.""" - from assets.services.search import build_asset_text_query + def test_barcode_exact_match_ranks_above_name( + self, category, location, user + ): + """Asset with exact barcode match ranks higher than name-only match.""" + from assets.services.search import build_asset_search - q = build_asset_text_query(" ") - assert Asset.objects.filter(q).count() == 0 + barcode_hit = AssetFactory( + name="Generic Item", + barcode="PROP-00142", + category=category, + current_location=location, + created_by=user, + ) + name_hit = AssetFactory( + name="PROP-00142 Label Backup", + barcode="PROP-99999", + category=category, + current_location=location, + created_by=user, + ) + results = list(build_asset_search(Asset.objects.all(), "PROP-00142")) + assert barcode_hit in results + # Barcode exact match should be first + assert results.index(barcode_hit) < results.index(name_hit) class TestExportService: diff --git a/src/assets/tests/test_views.py b/src/assets/tests/test_views.py index 2a520fa..0839e15 100644 --- a/src/assets/tests/test_views.py +++ b/src/assets/tests/test_views.py @@ -7680,7 +7680,7 @@ def test_long_input_truncated_in_error(self): @pytest.mark.django_db class TestExportWithWordSearch: - """Export view uses build_asset_text_query for multi-word search.""" + """Export view uses build_asset_search for multi-word search.""" def test_export_multi_word_search_finds_matching_asset( self, admin_client, category, location, user diff --git a/src/assets/views.py b/src/assets/views.py index 0635d76..d9ac0d9 100644 --- a/src/assets/views.py +++ b/src/assets/views.py @@ -69,7 +69,7 @@ can_handover_asset, get_user_role, ) -from .services.search import build_asset_text_query +from .services.search import build_asset_search BARCODE_PATTERN = re.compile(r"^[A-Z]+-[A-Z0-9]+$", re.IGNORECASE) @@ -313,13 +313,7 @@ def asset_list(request): q = request.GET.get("q", "").strip()[:200] if q: - text_q = build_asset_text_query(q) - # Also match NFC tags (full-phrase, not word-split) - nfc_q = Q( - nfc_tags__tag_id__icontains=q, - nfc_tags__removed_at__isnull=True, - ) - queryset = queryset.filter(text_q | nfc_q).distinct() + queryset = build_asset_search(queryset, q, include_nfc=True) # Filters department = request.GET.get("department") @@ -3562,9 +3556,10 @@ def asset_search(request): except (ValueError, TypeError): limit = 20 - text_q = build_asset_text_query(q) - # Also match category name (full phrase, not word-split) - category_q = Q(category__name__icontains=q) + base_qs = Asset.objects.filter(status="active") + filtered_qs = build_asset_search( + base_qs, q, include_nfc=False, include_category=True + ) primary_image_prefetch = Prefetch( "images", @@ -3572,10 +3567,7 @@ def asset_search(request): to_attr="primary_images", ) qs = ( - Asset.objects.filter(status="active") - .filter(text_q | category_q) - .distinct() - .annotate( + filtered_qs.annotate( relevance=Case( When(barcode__iexact=q, then=Value(1)), When(barcode__icontains=q, then=Value(2)), @@ -4190,7 +4182,7 @@ def export_assets(request): q = request.GET.get("q", "").strip()[:200] if q: - queryset = queryset.filter(build_asset_text_query(q)).distinct() + queryset = build_asset_search(queryset, q, include_nfc=False) buffer = export_assets_xlsx(queryset) @@ -4548,16 +4540,7 @@ def print_all_filtered_labels(request): q = request.GET.get("q", "") if q: - queryset = queryset.filter( - Q(name__icontains=q) - | Q(description__icontains=q) - | Q(barcode__icontains=q) - | Q(tags__name__icontains=q) - | Q( - nfc_tags__tag_id__icontains=q, - nfc_tags__removed_at__isnull=True, - ) - ).distinct() + queryset = build_asset_search(queryset, q, include_nfc=True) department = request.GET.get("department") if department: From 1f4aa8fb5f5a3d5187159dbfbeb0de507f2e39c3 Mon Sep 17 00:00:00 2001 From: Andrew Yager Date: Sun, 8 Mar 2026 16:37:31 +1100 Subject: [PATCH 2/7] feat: add Sentry error tracking for backend, frontend, and admin Enable sentry-sdk 2.54.0 with Django and Celery integrations, conditional on SENTRY_DSN env var. Add Sentry Browser SDK 9.5.0 via CDN to both the main frontend (base.html) and admin (base_site.html), conditional on SENTRY_DSN_JS. Supports self-hosted Sentry with configurable environment and trace sample rate. Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 7 +++ requirements.in | 3 ++ requirements.txt | 75 ++++++++++++++++-------------- src/props/celery.py | 5 ++ src/props/context_processors.py | 2 + src/props/settings.py | 22 +++++++++ src/templates/admin/base_site.html | 28 +++++++++++ src/templates/base.html | 15 ++++++ 8 files changed, 123 insertions(+), 34 deletions(-) create mode 100644 src/templates/admin/base_site.html diff --git a/.env.example b/.env.example index ed377ed..700f1bb 100644 --- a/.env.example +++ b/.env.example @@ -80,6 +80,13 @@ CELERY_RESULT_BACKEND=redis://redis:6379/0 # SERVER_EMAIL=errors@example.com # ADMIN_EMAIL=admin@example.com +# Sentry error tracking (feature disabled if SENTRY_DSN not set) +# For self-hosted Sentry, use your instance's DSN +# SENTRY_DSN=https://examplePublicKey@sentry.yourdomain.com/1 +# SENTRY_DSN_JS=https://examplePublicKey@sentry.yourdomain.com/2 +# SENTRY_ENVIRONMENT=production +# SENTRY_TRACES_SAMPLE_RATE=0.1 + # Production-only # DOMAIN=assets.yourdomain.com # ACME_EMAIL=admin@yourdomain.com diff --git a/requirements.in b/requirements.in index 6759d5f..737cd7e 100644 --- a/requirements.in +++ b/requirements.in @@ -42,6 +42,9 @@ weasyprint # AI (optional at runtime, required at build) anthropic +# Error tracking +sentry-sdk[django,celery] + # Colour coloraide diff --git a/requirements.txt b/requirements.txt index 452f71f..2313e73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,14 +2,14 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --output-file=/Users/andrewya/dev/props/.worktrees/issue-10/requirements.txt /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in +# pip-compile requirements.in # amqp==5.3.1 # via kombu annotated-types==0.7.0 # via pydantic anthropic==0.78.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in anyio==4.12.1 # via # anthropic @@ -32,7 +32,7 @@ automat==25.4.16 billiard==4.2.4 # via celery black==26.1.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in boto3==1.42.43 # via django-storages botocore==1.42.43 @@ -45,22 +45,24 @@ build==1.4.0 # via pip-tools celery[redis]==5.6.2 # via - # -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # -r requirements.in # django-celery-beat + # sentry-sdk certifi==2026.1.4 # via # httpcore # httpx + # sentry-sdk cffi==2.0.0 # via # cryptography # weasyprint channels==4.3.2 # via - # -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # -r requirements.in # channels-redis channels-redis==4.3.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in click==8.3.1 # via # black @@ -76,7 +78,7 @@ click-plugins==1.1.1.2 click-repl==0.3.0 # via celery coloraide==8.3 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in constantly==23.10.4 # via twisted coverage[toml]==7.13.3 @@ -91,32 +93,33 @@ cryptography==46.0.5 cssselect2==0.9.0 # via weasyprint daphne==4.2.1 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in distro==1.9.0 # via anthropic django==5.2.11 # via - # -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # -r requirements.in # channels # django-celery-beat # django-htmx # django-storages # django-timezone-field # django-unfold + # sentry-sdk django-celery-beat==2.8.1 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in django-gravatar2==1.4.4 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in django-htmx==1.27.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in django-ratelimit==4.1.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in django-storages[boto3]==1.14.6 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in django-timezone-field==7.2.1 # via django-celery-beat django-unfold==0.78.1 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in docstring-parser==0.17.0 # via anthropic et-xmlfile==2.0.0 @@ -124,15 +127,15 @@ et-xmlfile==2.0.0 execnet==2.1.2 # via pytest-xdist factory-boy==3.3.3 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in faker==40.1.2 # via factory-boy flake8==7.3.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in fonttools[woff]==4.61.1 # via weasyprint gunicorn==25.0.2 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in h11==0.16.0 # via httpcore httpcore==1.0.9 @@ -154,7 +157,7 @@ incremental==24.11.0 iniconfig==2.3.0 # via pytest isort==7.0.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in jiter==0.13.0 # via anthropic jmespath==1.1.0 @@ -170,7 +173,7 @@ msgpack==1.1.2 mypy-extensions==1.1.0 # via black openpyxl==3.1.5 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in packaging==26.0 # via # black @@ -183,16 +186,16 @@ packaging==26.0 pathspec==1.0.4 # via black pi-heif==1.2.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in pillow==12.1.1 # via - # -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # -r requirements.in # pi-heif # python-barcode # qrcode # weasyprint pip-tools==7.5.2 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in platformdirs==4.5.1 # via black pluggy==1.6.0 @@ -202,7 +205,7 @@ pluggy==1.6.0 prompt-toolkit==3.0.52 # via click-repl psycopg[binary]==3.3.2 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in psycopg-binary==3.3.2 # via psycopg pyasn1==0.6.2 @@ -240,15 +243,15 @@ pytest==9.0.2 # pytest-django # pytest-xdist pytest-asyncio==1.3.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in pytest-cov==7.0.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in pytest-django==4.11.1 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in pytest-xdist==3.8.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in python-barcode[images]==0.16.1 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in python-crontab==3.3.0 # via django-celery-beat python-dateutil==2.9.0.post0 @@ -258,14 +261,16 @@ python-dateutil==2.9.0.post0 pytokens==0.4.1 # via black qrcode[pil]==8.2 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in redis==6.4.0 # via - # -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # -r requirements.in # channels-redis # kombu s3transfer==0.16.0 # via boto3 +sentry-sdk[celery,django]==2.54.0 + # via -r requirements.in service-identity==24.2.0 # via twisted six==1.17.0 @@ -301,7 +306,9 @@ tzdata==2025.3 tzlocal==5.3.1 # via celery urllib3==2.6.3 - # via botocore + # via + # botocore + # sentry-sdk vine==5.1.0 # via # amqp @@ -310,7 +317,7 @@ vine==5.1.0 wcwidth==0.5.3 # via prompt-toolkit weasyprint==68.1 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in webencodings==0.5.1 # via # cssselect2 @@ -319,7 +326,7 @@ webencodings==0.5.1 wheel==0.46.3 # via pip-tools whitenoise==6.11.0 - # via -r /Users/andrewya/dev/props/.worktrees/issue-10/requirements.in + # via -r requirements.in zope-interface==8.2 # via twisted zopfli==0.4.1 diff --git a/src/props/celery.py b/src/props/celery.py index cbd1eaf..038578e 100644 --- a/src/props/celery.py +++ b/src/props/celery.py @@ -9,3 +9,8 @@ app = Celery("props") app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() + +# Sentry integration for Celery is auto-configured by sentry-sdk +# when the Django integration is active and SENTRY_DSN is set. +# The sentry_sdk.init() call in settings.py handles this via the +# CeleryIntegration that ships with sentry-sdk[celery]. diff --git a/src/props/context_processors.py b/src/props/context_processors.py index 5383240..208fe3f 100644 --- a/src/props/context_processors.py +++ b/src/props/context_processors.py @@ -62,6 +62,8 @@ def site_settings(request): "logo_url": logo_url, "color_mode": color_mode, "app_version": settings.APP_VERSION, + "SENTRY_DSN_JS": getattr(settings, "SENTRY_DSN_JS", ""), + "SENTRY_ENVIRONMENT": getattr(settings, "SENTRY_ENVIRONMENT", ""), } diff --git a/src/props/settings.py b/src/props/settings.py index cc486b2..dec061b 100644 --- a/src/props/settings.py +++ b/src/props/settings.py @@ -3,6 +3,8 @@ import os from pathlib import Path +import sentry_sdk + from django.urls import reverse_lazy BASE_DIR = Path(__file__).resolve().parent.parent @@ -37,6 +39,7 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", + "django.contrib.postgres", "django_htmx", "django_celery_beat", "django_gravatar", @@ -538,6 +541,25 @@ }, } +# Sentry error tracking (enabled when SENTRY_DSN is set) +SENTRY_DSN = os.environ.get("SENTRY_DSN", "") +SENTRY_DSN_JS = os.environ.get("SENTRY_DSN_JS", "") +SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") +SENTRY_TRACES_SAMPLE_RATE = float( + os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1") +) + +if SENTRY_DSN: + sentry_sdk.init( + dsn=SENTRY_DSN, + environment=SENTRY_ENVIRONMENT, + release=APP_VERSION, + traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, + send_default_pii=True, + # Profile 100% of sampled transactions + profiles_sample_rate=1.0, + ) + # Logging — ensure tracebacks appear in container logs even with DEBUG=False LOGGING = { "version": 1, diff --git a/src/templates/admin/base_site.html b/src/templates/admin/base_site.html new file mode 100644 index 0000000..99740cf --- /dev/null +++ b/src/templates/admin/base_site.html @@ -0,0 +1,28 @@ +{% extends "admin/base.html" %} + +{% block title %}{% if subtitle %}{{ subtitle }} | {% endif %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} + +{% block branding %} + {% include "unfold/helpers/site_branding.html" %} +{% endblock %} + +{% block extrahead %} + {{ block.super }} + {% if SENTRY_DSN_JS %} + + + {% endif %} +{% endblock %} + +{% block nav-global %}{% endblock %} diff --git a/src/templates/base.html b/src/templates/base.html index ac80727..555f10a 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -38,6 +38,21 @@ } {% endif %} + {% if SENTRY_DSN_JS %} + + + {% endif %} {% block extra_head %}{% endblock %} From 42ce803b2f39c2b33539e56bf3f2231c498d82d5 Mon Sep 17 00:00:00 2001 From: Andrew Yager Date: Sun, 8 Mar 2026 16:43:28 +1100 Subject: [PATCH 3/7] chore: upgrade all pinned dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Notable upgrades: Django 5.2.11→5.2.12, django-unfold 0.78.1→0.83.1, anthropic 0.78.0→0.84.0, django-celery-beat 2.8.1→2.9.0, black 26.1.0→26.3.0, isort 7.0.0→8.0.1. Co-Authored-By: Claude Opus 4.6 (1M context) --- requirements.txt | 64 +++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2313e73..40f57d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ amqp==5.3.1 # via kombu annotated-types==0.7.0 # via pydantic -anthropic==0.78.0 +anthropic==0.84.0 # via -r requirements.in anyio==4.12.1 # via @@ -25,17 +25,17 @@ attrs==25.4.0 # via # service-identity # twisted -autobahn==24.4.2 +autobahn==25.12.2 # via daphne automat==25.4.16 # via twisted billiard==4.2.4 # via celery -black==26.1.0 +black==26.3.0 # via -r requirements.in -boto3==1.42.43 +boto3==1.42.63 # via django-storages -botocore==1.42.43 +botocore==1.42.63 # via # boto3 # s3transfer @@ -43,18 +43,21 @@ brotli==1.2.0 # via fonttools build==1.4.0 # via pip-tools +cbor2==5.8.0 + # via autobahn celery[redis]==5.6.2 # via # -r requirements.in # django-celery-beat # sentry-sdk -certifi==2026.1.4 +certifi==2026.2.25 # via # httpcore # httpx # sentry-sdk cffi==2.0.0 # via + # autobahn # cryptography # weasyprint channels==4.3.2 @@ -77,13 +80,13 @@ click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -coloraide==8.3 +coloraide==8.6 # via -r requirements.in constantly==23.10.4 # via twisted -coverage[toml]==7.13.3 +coverage[toml]==7.13.4 # via pytest-cov -cron-descriptor==2.0.6 +cron-descriptor==1.4.5 # via django-celery-beat cryptography==46.0.5 # via @@ -96,7 +99,7 @@ daphne==4.2.1 # via -r requirements.in distro==1.9.0 # via anthropic -django==5.2.11 +django==5.2.12 # via # -r requirements.in # channels @@ -106,9 +109,9 @@ django==5.2.11 # django-timezone-field # django-unfold # sentry-sdk -django-celery-beat==2.8.1 +django-celery-beat==2.9.0 # via -r requirements.in -django-gravatar2==1.4.4 +django-gravatar2==1.4.5 # via -r requirements.in django-htmx==1.27.0 # via -r requirements.in @@ -118,7 +121,7 @@ django-storages[boto3]==1.14.6 # via -r requirements.in django-timezone-field==7.2.1 # via django-celery-beat -django-unfold==0.78.1 +django-unfold==0.83.1 # via -r requirements.in docstring-parser==0.17.0 # via anthropic @@ -128,13 +131,13 @@ execnet==2.1.2 # via pytest-xdist factory-boy==3.3.3 # via -r requirements.in -faker==40.1.2 +faker==40.8.0 # via factory-boy flake8==7.3.0 # via -r requirements.in fonttools[woff]==4.61.1 # via weasyprint -gunicorn==25.0.2 +gunicorn==25.1.0 # via -r requirements.in h11==0.16.0 # via httpcore @@ -156,7 +159,7 @@ incremental==24.11.0 # via twisted iniconfig==2.3.0 # via pytest -isort==7.0.0 +isort==8.0.1 # via -r requirements.in jiter==0.13.0 # via anthropic @@ -169,7 +172,9 @@ kombu[redis]==5.6.2 mccabe==0.7.0 # via flake8 msgpack==1.1.2 - # via channels-redis + # via + # autobahn + # channels-redis mypy-extensions==1.1.0 # via black openpyxl==3.1.5 @@ -185,7 +190,7 @@ packaging==26.0 # wheel pathspec==1.0.4 # via black -pi-heif==1.2.0 +pi-heif==1.3.0 # via -r requirements.in pillow==12.1.1 # via @@ -194,9 +199,9 @@ pillow==12.1.1 # python-barcode # qrcode # weasyprint -pip-tools==7.5.2 +pip-tools==7.5.3 # via -r requirements.in -platformdirs==4.5.1 +platformdirs==4.9.4 # via black pluggy==1.6.0 # via @@ -204,10 +209,12 @@ pluggy==1.6.0 # pytest-cov prompt-toolkit==3.0.52 # via click-repl -psycopg[binary]==3.3.2 +psycopg[binary]==3.3.3 # via -r requirements.in -psycopg-binary==3.3.2 +psycopg-binary==3.3.3 # via psycopg +py-ubjson==0.16.1 + # via autobahn pyasn1==0.6.2 # via # pyasn1-modules @@ -246,7 +253,7 @@ pytest-asyncio==1.3.0 # via -r requirements.in pytest-cov==7.0.0 # via -r requirements.in -pytest-django==4.11.1 +pytest-django==4.12.0 # via -r requirements.in pytest-xdist==3.8.0 # via -r requirements.in @@ -283,16 +290,15 @@ tinycss2==1.5.1 # via # cssselect2 # weasyprint -tinyhtml5==2.0.0 +tinyhtml5==2.1.0 # via weasyprint twisted[tls]==25.5.0 # via daphne -txaio==25.9.2 +txaio==25.12.2 # via autobahn typing-extensions==4.15.0 # via # anthropic - # cron-descriptor # pydantic # pydantic-core # twisted @@ -305,6 +311,8 @@ tzdata==2025.3 # kombu tzlocal==5.3.1 # via celery +ujson==5.11.0 + # via autobahn urllib3==2.6.3 # via # botocore @@ -314,7 +322,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.5.3 +wcwidth==0.6.0 # via prompt-toolkit weasyprint==68.1 # via -r requirements.in @@ -325,7 +333,7 @@ webencodings==0.5.1 # tinyhtml5 wheel==0.46.3 # via pip-tools -whitenoise==6.11.0 +whitenoise==6.12.0 # via -r requirements.in zope-interface==8.2 # via twisted From 199568d374e095500b138ed84b1ec0bade3cd5fd Mon Sep 17 00:00:00 2001 From: Andrew Yager Date: Sun, 8 Mar 2026 18:14:41 +1100 Subject: [PATCH 4/7] fix: FTS ts_rank returns non-zero for non-matches, use @@ operator PostgreSQL ts_rank() returns ~1e-20 for non-matching content rather than exactly 0, so filtering on fts_rank > 0 matched everything. Switch to the @@ full-text match operator via Django's filter(search=query) for correct boolean matching. Also move tags__name out of SearchVector to avoid M2M JOIN row duplication and update .dockerignore to exclude .worktrees/ (1.5GB) and bin/. Co-Authored-By: Claude Opus 4.6 (1M context) --- .dockerignore | 17 ++++++++++++++++- src/assets/services/search.py | 22 ++++++++++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index fecdb94..5f3cc6b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -43,6 +43,11 @@ docs/ # Implementation tracking .impl-tracker* +.impl-preferences* +.impl-verification/ + +# Binaries (Tailwind standalone CLI for local dev) +bin/ # Old codebase _old/ @@ -52,13 +57,23 @@ _old/ # Claude Code .claude/ +CLAUDE.local.md + +# Issue pipeline and worktrees +.issue-pipeline/ +.worktrees/ + +# Image cache +.image-cache/ # Docker (avoid recursive copies) docker-compose.yml Dockerfile -# Media (runtime data, not build artifact) +# Runtime/dev data (not build artifacts) src/media/ +src/staticfiles/ +src/db.sqlite3 # Traefik config traefik/ diff --git a/src/assets/services/search.py b/src/assets/services/search.py index b7a50e1..57ad4f4 100644 --- a/src/assets/services/search.py +++ b/src/assets/services/search.py @@ -27,15 +27,24 @@ def _build_fts_search(queryset, words, search_text, icontains_q): SearchVector, ) - vector = ( - SearchVector("name", weight="A") - + SearchVector("description", weight="B") - + SearchVector("tags__name", weight="B") + # Only direct fields in the FTS vector — M2M joins (tags) cause + # row duplication that breaks DISTINCT + ORDER BY on annotations. + # Tags are handled via icontains below instead. + vector = SearchVector("name", weight="A") + SearchVector( + "description", weight="B" ) search_query = SearchQuery(search_text, search_type="websearch") - fts_filter = Q(fts_rank__gt=0) - combined_filter = fts_filter | icontains_q + # Use the @@ match operator (via annotate + filter) rather than + # ts_rank > 0, because ts_rank returns ~1e-20 for non-matches. + fts_filter = Q(search=search_query) + + # Tags via icontains (short strings, stemming adds little value) + tag_q = Q() + for word in words: + tag_q &= Q(tags__name__icontains=word) + + combined_filter = fts_filter | tag_q | icontains_q barcode_exact = Case( When(barcode__iexact=search_text, then=Value(10.0)), @@ -45,6 +54,7 @@ def _build_fts_search(queryset, words, search_text, icontains_q): return ( queryset.annotate( + search=vector, fts_rank=SearchRank(vector, search_query), barcode_boost=barcode_exact, ) From fbfaddd1c0d5172fd0a70710d5e871abe5c32446 Mon Sep 17 00:00:00 2001 From: Andrew Yager Date: Sun, 8 Mar 2026 18:24:15 +1100 Subject: [PATCH 5/7] fix: resolve Docker-only test failures (AI eager tasks, email branding) - Clear ANTHROPIC_API_KEY in conftest.py so eager Celery tasks don't hit the real API when ANTHROPIC_API_KEY is set in Docker .env - Fix AI memory test: use real uploaded file instead of fake path, and use image dimensions (8000x6001) that actually exceed the 48M pixel limit - Fix email branding test: use settings.SITE_NAME instead of hardcoded "PROPS" Co-Authored-By: Claude Opus 4.6 (1M context) --- src/assets/tests/test_ai.py | 29 ++++++++++++++++++++------ src/conftest.py | 6 ++++++ src/props/tests/test_infrastructure.py | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/assets/tests/test_ai.py b/src/assets/tests/test_ai.py index 1aecf0a..1b1cf45 100644 --- a/src/assets/tests/test_ai.py +++ b/src/assets/tests/test_ai.py @@ -1066,24 +1066,41 @@ def test_dashboard_shows_ai_daily_usage_and_remaining( class TestAIEdgeCases: """S7.11 — AI image analysis edge cases.""" - def test_vv755_large_image_memory_check(self, admin_user): + @override_settings(ANTHROPIC_API_KEY="test-key") + def test_vv755_large_image_memory_check(self, admin_user, tmp_path): """VV755: Very large image should fail AI analysis gracefully, not crash the worker.""" + import io + + from PIL import Image as PILImage + + from django.core.files.uploadedfile import SimpleUploadedFile + asset = AssetFactory(name="Big Image Asset") + + # Create a real image file so FieldFile.read() works + buf = io.BytesIO() + PILImage.new("RGB", (10, 10), "red").save(buf, format="JPEG") + buf.seek(0) + upload = SimpleUploadedFile( + "big.jpg", buf.read(), content_type="image/jpeg" + ) img = AssetImage.objects.create( asset=asset, - image="assets/test.jpg", + image=upload, is_primary=True, ai_processing_status="pending", ) from assets.services.ai import analyse_image - mock_img = MagicMock() - mock_img.size = (8000, 6000) - mock_img.mode = "RGB" + mock_pil_img = MagicMock() + mock_pil_img.size = (8000, 6001) # 48_008_000 > 48_000_000 limit + mock_pil_img.mode = "RGB" - with patch("PIL.Image.open", return_value=mock_img): + # PIL.Image.open returns oversized dimensions to trigger + # the "too large" guard in the task + with patch("PIL.Image.open", return_value=mock_pil_img): with patch("anthropic.Anthropic"): try: analyse_image(img.pk) diff --git a/src/conftest.py b/src/conftest.py index e7b1be4..aafbb36 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -26,6 +26,12 @@ settings.CELERY_TASK_ALWAYS_EAGER = True settings.CELERY_TASK_EAGER_PROPAGATES = True +# Disable AI analysis in tests — prevents eager Celery tasks from +# hitting the real Anthropic API when ANTHROPIC_API_KEY is set in +# the Docker environment. Tests that specifically exercise AI +# behaviour mock the API client directly. +settings.ANTHROPIC_API_KEY = "" + # Use in-memory cache for tests (avoids Redis connection errors) settings.CACHES = { "default": { diff --git a/src/props/tests/test_infrastructure.py b/src/props/tests/test_infrastructure.py index c9837de..5d43ad2 100644 --- a/src/props/tests/test_infrastructure.py +++ b/src/props/tests/test_infrastructure.py @@ -397,7 +397,7 @@ def capture_init(self, *args, **kwargs): assert len(sent_messages) == 1 msg = sent_messages[0] - assert "PROPS" in msg.body + assert settings.SITE_NAME in msg.body @patch("django.core.mail.EmailMultiAlternatives.send") def test_handles_list_recipient(self, mock_send, db): From cdd3d95e8fe7d5805c1091c5daf178874f68b7ce Mon Sep 17 00:00:00 2001 From: Andrew Yager Date: Sun, 8 Mar 2026 18:30:13 +1100 Subject: [PATCH 6/7] fix: address Sentry review feedback (escapejs, dynamic sample rate) - Use |escapejs filter on all JS string interpolations to prevent XSS from misconfigured DSN/environment values - Make frontend tracesSampleRate dynamic via SENTRY_TRACES_SAMPLE_RATE context variable instead of hardcoded 0.1 - Parse SENTRY_TRACES_SAMPLE_RATE defensively with try/except - Clarify settings comments re: SENTRY_DSN vs SENTRY_DSN_JS scope Co-Authored-By: Claude Opus 4.6 (1M context) --- src/props/context_processors.py | 3 +++ src/props/settings.py | 13 +++++++++---- src/templates/admin/base_site.html | 8 ++++---- src/templates/base.html | 8 ++++---- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/props/context_processors.py b/src/props/context_processors.py index 208fe3f..e833028 100644 --- a/src/props/context_processors.py +++ b/src/props/context_processors.py @@ -64,6 +64,9 @@ def site_settings(request): "app_version": settings.APP_VERSION, "SENTRY_DSN_JS": getattr(settings, "SENTRY_DSN_JS", ""), "SENTRY_ENVIRONMENT": getattr(settings, "SENTRY_ENVIRONMENT", ""), + "SENTRY_TRACES_SAMPLE_RATE": getattr( + settings, "SENTRY_TRACES_SAMPLE_RATE", 0.1 + ), } diff --git a/src/props/settings.py b/src/props/settings.py index dec061b..f8cba4a 100644 --- a/src/props/settings.py +++ b/src/props/settings.py @@ -541,13 +541,18 @@ }, } -# Sentry error tracking (enabled when SENTRY_DSN is set) +# Sentry error tracking +# SENTRY_DSN enables backend (Python/Django/Celery) reporting. +# SENTRY_DSN_JS enables browser JS reporting (frontend + admin). SENTRY_DSN = os.environ.get("SENTRY_DSN", "") SENTRY_DSN_JS = os.environ.get("SENTRY_DSN_JS", "") SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") -SENTRY_TRACES_SAMPLE_RATE = float( - os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1") -) +try: + SENTRY_TRACES_SAMPLE_RATE = float( + os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1") + ) +except (TypeError, ValueError): + SENTRY_TRACES_SAMPLE_RATE = 0.1 if SENTRY_DSN: sentry_sdk.init( diff --git a/src/templates/admin/base_site.html b/src/templates/admin/base_site.html index 99740cf..28896e4 100644 --- a/src/templates/admin/base_site.html +++ b/src/templates/admin/base_site.html @@ -15,11 +15,11 @@ > {% endif %} diff --git a/src/templates/base.html b/src/templates/base.html index 555f10a..eaf3394 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -45,11 +45,11 @@ > {% endif %} From ce3f318e4a9343be344d931d5ebfb69c5194ee29 Mon Sep 17 00:00:00 2001 From: Andrew Yager Date: Sun, 8 Mar 2026 18:47:17 +1100 Subject: [PATCH 7/7] feat: add SearchVectorField with GIN index for FTS performance Pre-compute tsvector on Asset model instead of building it per query. PostgreSQL trigger auto-updates the vector on name/description changes. Migration backfills existing rows and creates a GIN index for fast lookups. Co-Authored-By: Claude Opus 4.6 --- .../migrations/0039_asset_search_vector.py | 90 +++++++++++++++++++ src/assets/models.py | 8 ++ src/assets/services/search.py | 29 +++--- 3 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 src/assets/migrations/0039_asset_search_vector.py diff --git a/src/assets/migrations/0039_asset_search_vector.py b/src/assets/migrations/0039_asset_search_vector.py new file mode 100644 index 0000000..9df7ef1 --- /dev/null +++ b/src/assets/migrations/0039_asset_search_vector.py @@ -0,0 +1,90 @@ +"""Add SearchVectorField with GIN index and PostgreSQL trigger to Asset.""" + +import django.contrib.postgres.indexes +from django.contrib.postgres.search import SearchVectorField +from django.db import migrations + + +def backfill_search_vector(apps, schema_editor): + """Populate search_vector for existing rows (PostgreSQL only).""" + if schema_editor.connection.vendor != "postgresql": + return + schema_editor.execute(""" + UPDATE assets_asset + SET search_vector = + setweight(to_tsvector('english', coalesce(name, '')), 'A') || + setweight(to_tsvector('english', coalesce(description, '')), 'B') + """) + + +def create_trigger(apps, schema_editor): + """Create PostgreSQL trigger to auto-update search_vector.""" + if schema_editor.connection.vendor != "postgresql": + return + schema_editor.execute(""" + CREATE OR REPLACE FUNCTION assets_asset_search_vector_update() + RETURNS trigger AS $$ + BEGIN + NEW.search_vector := + setweight(to_tsvector('english', coalesce(NEW.name, '')), 'A') || + setweight( + to_tsvector('english', coalesce(NEW.description, '')), 'B' + ); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + DROP TRIGGER IF EXISTS assets_asset_search_vector_trigger + ON assets_asset; + + CREATE TRIGGER assets_asset_search_vector_trigger + BEFORE INSERT OR UPDATE OF name, description + ON assets_asset + FOR EACH ROW + EXECUTE FUNCTION assets_asset_search_vector_update(); + """) + + +def drop_trigger(apps, schema_editor): + """Drop the search_vector trigger and function (reverse).""" + if schema_editor.connection.vendor != "postgresql": + return + schema_editor.execute(""" + DROP TRIGGER IF EXISTS assets_asset_search_vector_trigger + ON assets_asset; + DROP FUNCTION IF EXISTS assets_asset_search_vector_update(); + """) + + +class Migration(migrations.Migration): + + dependencies = [ + ("assets", "0038_location_created_at_not_null"), + ] + + operations = [ + # 1. Add the SearchVectorField column + migrations.AddField( + model_name="asset", + name="search_vector", + field=SearchVectorField(editable=False, null=True), + ), + # 2. Add GIN index on the new column + migrations.AddIndex( + model_name="asset", + index=django.contrib.postgres.indexes.GinIndex( + fields=["search_vector"], + name="idx_asset_search_vector", + ), + ), + # 3. Backfill existing rows + migrations.RunPython( + backfill_search_vector, + reverse_code=migrations.RunPython.noop, + ), + # 4. Create trigger for automatic updates + migrations.RunPython( + create_trigger, + reverse_code=drop_trigger, + ), + ] diff --git a/src/assets/models.py b/src/assets/models.py index 95bcd5c..9b255d9 100644 --- a/src/assets/models.py +++ b/src/assets/models.py @@ -7,6 +7,8 @@ from barcode.writer import ImageWriter from django.conf import settings +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVectorField from django.core.cache import cache from django.core.exceptions import ValidationError from django.core.files.base import ContentFile @@ -357,6 +359,8 @@ class Asset(models.Model): related_name="created_assets", ) + search_vector = SearchVectorField(null=True, editable=False) + objects = AssetManager() class Meta: @@ -375,6 +379,10 @@ class Meta: fields=["is_serialised"], name="idx_asset_is_serialised", ), + GinIndex( + fields=["search_vector"], + name="idx_asset_search_vector", + ), ] permissions = [ ("can_checkout_asset", "Can check out assets"), diff --git a/src/assets/services/search.py b/src/assets/services/search.py index 57ad4f4..c4def0b 100644 --- a/src/assets/services/search.py +++ b/src/assets/services/search.py @@ -20,26 +20,20 @@ def _is_postgres(): def _build_fts_search(queryset, words, search_text, icontains_q): - """FTS search path for PostgreSQL.""" - from django.contrib.postgres.search import ( - SearchQuery, - SearchRank, - SearchVector, - ) + """FTS search path for PostgreSQL. + + Uses the pre-computed search_vector field (updated by a DB trigger) + with a GIN index for fast full-text search. + """ + from django.contrib.postgres.search import SearchQuery, SearchRank - # Only direct fields in the FTS vector — M2M joins (tags) cause - # row duplication that breaks DISTINCT + ORDER BY on annotations. - # Tags are handled via icontains below instead. - vector = SearchVector("name", weight="A") + SearchVector( - "description", weight="B" - ) search_query = SearchQuery(search_text, search_type="websearch") - # Use the @@ match operator (via annotate + filter) rather than - # ts_rank > 0, because ts_rank returns ~1e-20 for non-matches. - fts_filter = Q(search=search_query) + # Filter on the stored search_vector field (GIN-indexed) + fts_filter = Q(search_vector=search_query) - # Tags via icontains (short strings, stemming adds little value) + # Tags via icontains (short strings, stemming adds little value; + # M2M joins in SearchVector cause row duplication issues) tag_q = Q() for word in words: tag_q &= Q(tags__name__icontains=word) @@ -54,8 +48,7 @@ def _build_fts_search(queryset, words, search_text, icontains_q): return ( queryset.annotate( - search=vector, - fts_rank=SearchRank(vector, search_query), + fts_rank=SearchRank("search_vector", search_query), barcode_boost=barcode_exact, ) .filter(combined_filter)