From 77ac141c937efcc48bbdb47ae14d600099e9694d Mon Sep 17 00:00:00 2001 From: FlintWave Date: Thu, 2 Jul 2026 11:35:24 -0700 Subject: [PATCH 1/6] feat(engines): port Google-style query operator parsing from Android Parses "exact phrase", -term, site:/-site:, intitle:, inurl:, filetype: (alias ext:), before:/after: dates, and OR/| into an engine query (operators the engines understand, forwarded verbatim), an operator-free clean text (for relevance, correction, and the summary lookup), and the structural filters that are enforced locally over the merged results. Mirrors engine/query/QueryOperators.kt, including tokenizer robustness against unterminated quotes and blank phrases. Co-Authored-By: Claude Fable 5 --- .../engines/query_operators.py | 388 +++++++++++++++++ tests/engines/test_query_operators.py | 394 ++++++++++++++++++ 2 files changed, 782 insertions(+) create mode 100644 src/searchmob_desktop/engines/query_operators.py create mode 100644 tests/engines/test_query_operators.py diff --git a/src/searchmob_desktop/engines/query_operators.py b/src/searchmob_desktop/engines/query_operators.py new file mode 100644 index 0000000..a0cf891 --- /dev/null +++ b/src/searchmob_desktop/engines/query_operators.py @@ -0,0 +1,388 @@ +"""Google-style search operator parsing, ported from the Android `QueryOperators.kt`. + +A raw query is parsed once into a `ParsedQuery`: `engine_query` is what actually goes upstream +(operators the engines themselves understand, `site:`, quoted phrases, `-exclusions`, `OR`/`|`, +are forwarded verbatim so the engine's own index does the heavy lifting; operators no public +engine implements consistently, `intitle:`, `inurl:`, `before:`, `after:`, are either turned into +a plain recall hint or dropped, and the actual constraint is enforced locally by `matches` over +the aggregated results). `clean_text` is the query with every operator stripped to plain words, +for anything that reasons about "what is the user asking about" rather than "how do I fetch it" +(on-device relevance, spelling correction, the contextual-summary lookup). + +Pure and deterministic: no I/O, no locale/clock dependence beyond UTC date math, so the same +input always parses the same way. The server truncates queries at 512 characters before they ever +reach here, so this has to tolerate garbage (an unterminated quote, a bare `-`, an empty `site:`) +without raising, never silently dropping a token it does not understand; anything it cannot make +sense of falls back to being treated as ordinary query text. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime +from urllib.parse import urlsplit + +from searchmob_desktop.engines.rank.ranker import host_of_url + +__all__ = ["ParsedQuery", "parse_query_operators"] + +# Operator names this parser recognizes left of a `:`; anything else stays ordinary text. +_RECOGNIZED_OPS = frozenset({"site", "intitle", "inurl", "filetype", "ext", "before", "after"}) + +_YEAR = re.compile(r"^(\d{4})$") +_YEAR_MONTH = re.compile(r"^(\d{4})-(\d{2})$") +_FULL_DASH = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$") +_FULL_SLASH = re.compile(r"^(\d{4})/(\d{2})/(\d{2})$") + +# Maximal letter/digit runs (unicode-aware, underscore excluded), for whole-word exclusion +# matching. Same token shape as the relevance module's tokenizer. +_WORD = re.compile(r"[^\W_]+", re.UNICODE) + + +@dataclass(frozen=True, slots=True) +class ParsedQuery: + """A query parsed into its Google-style operators plus the leftover free text. + + See the module docstring for what `engine_query` and `clean_text` are for. The remaining + fields are the structural filters `matches` enforces locally over the merged results. + `after_ms`/`before_ms` are UTC epoch milliseconds at the start of the named day. + """ + + raw: str + clean_text: str + engine_query: str + phrases: tuple[str, ...] = () + excluded_terms: tuple[str, ...] = () + excluded_phrases: tuple[str, ...] = () + include_sites: tuple[str, ...] = () + exclude_sites: tuple[str, ...] = () + in_title: tuple[str, ...] = () + not_in_title: tuple[str, ...] = () + in_url: tuple[str, ...] = () + not_in_url: tuple[str, ...] = () + file_types: tuple[str, ...] = () + after_ms: int | None = None + before_ms: int | None = None + + @property + def has_filters(self) -> bool: + """True when any operator is enforced locally by `matches` (everything but plain + terms/phrases/OR).""" + return bool( + self.excluded_terms + or self.excluded_phrases + or self.include_sites + or self.exclude_sites + or self.in_title + or self.not_in_title + or self.in_url + or self.not_in_url + or self.file_types + or self.after_ms is not None + or self.before_ms is not None + ) + + @property + def has_operators(self) -> bool: + """True when the query carried any operator syntax at all. + + Used to skip the on-device spell corrector, which reasons about word spelling, not query + syntax: an operator-laden query would just get its operators mangled. + """ + return self.engine_query != self.clean_text or self.has_filters + + def matches(self, title: str, url: str, snippet: str, published: int | None) -> bool: + """True when an aggregated result survives every locally-enforced operator in this query. + + Positive `phrases` and plain terms are deliberately NOT checked here: they were already + sent upstream in `engine_query` and drive lexical relevance. A snippet is a partial + excerpt of the page, so demanding the phrase appear in the title/snippet would reject + results whose full body matches but whose short excerpt happens not to quote it; the + engines and the relevance ranker are trusted to have already done that job. + + A date-window bound (`after_ms`/`before_ms`) excludes a result with no known `published`, + deliberately: the user explicitly asked for a window, so an undated result cannot be + confirmed to be in it and is treated as a miss rather than let through. + """ + host = host_of_url(url) + if self.include_sites and ( + host is None or not any(_site_matches(entry, host) for entry in self.include_sites) + ): + return False + if ( + self.exclude_sites + and host is not None + and any(_site_matches(entry, host) for entry in self.exclude_sites) + ): + return False + title_lower = title.lower() + if any(needle.lower() not in title_lower for needle in self.in_title): + return False + if any(needle.lower() in title_lower for needle in self.not_in_title): + return False + url_lower = url.lower() + if any(needle.lower() not in url_lower for needle in self.in_url): + return False + if any(needle.lower() in url_lower for needle in self.not_in_url): + return False + if self.file_types: + extension = _extension_of(url) + if extension is None or extension not in self.file_types: + return False + if self.after_ms is not None or self.before_ms is not None: + if published is None: + return False + if self.after_ms is not None and published < self.after_ms: + return False + if self.before_ms is not None and published >= self.before_ms: + return False + if self.excluded_terms: + words = _words_of(f"{title} {snippet} {host or ''}") + if any(term.lower() in words for term in self.excluded_terms): + return False + haystack = f"{title} {snippet}".lower() + if any(phrase.lower() in haystack for phrase in self.excluded_phrases): + return False + return True + + +def _site_matches(entry: str, host: str) -> bool: + """Whether `entry` (a normalized `site:`/`-site:` value) covers `host`. + + An entry starting with `.` (a bare TLD like `.edu`) matches any host ending in it; otherwise + the entry must equal the host or be one of its parent domains (`example.com` covers + `docs.example.com` but not `notexample.com`). + """ + if entry.startswith("."): + return host.endswith(entry) + return host == entry or host.endswith(f".{entry}") + + +def _extension_of(url: str) -> str | None: + """The lowercased extension of `url`'s last path segment (query string and fragment ignored). + + None when the path has no segment or that segment has no dot. Parsed via `urlsplit` rather + than naive string-splitting so a bare `https://example.com` never misreads its TLD (the + `.com` in the host) as a file extension. + """ + try: + path = urlsplit(url.strip()).path + except ValueError: + return None + last_segment = path.rsplit("/", 1)[-1] + if "." not in last_segment: + return None + extension = last_segment.rsplit(".", 1)[-1].lower() + return extension or None + + +def _words_of(text: str) -> set[str]: + """Lowercased maximal letter/digit runs in `text`, for whole-word exclusion matching.""" + return {w.lower() for w in _WORD.findall(text)} + + +@dataclass(slots=True) +class _Accumulator: + """The mutable lists `parse_query_operators` fills as it walks the tokens, in token order.""" + + clean_parts: list[str] = field(default_factory=list) + engine_parts: list[str] = field(default_factory=list) + phrases: list[str] = field(default_factory=list) + excluded_terms: list[str] = field(default_factory=list) + excluded_phrases: list[str] = field(default_factory=list) + include_sites: list[str] = field(default_factory=list) + exclude_sites: list[str] = field(default_factory=list) + in_title: list[str] = field(default_factory=list) + not_in_title: list[str] = field(default_factory=list) + in_url: list[str] = field(default_factory=list) + not_in_url: list[str] = field(default_factory=list) + file_types: list[str] = field(default_factory=list) + after_ms: int | None = None + before_ms: int | None = None + + +def parse_query_operators(raw: str) -> ParsedQuery: + """Parse Google-style search operators out of `raw` into a `ParsedQuery`. + + Recognizes `"exact phrase"`, `-term`, `site:`/`-site:`, `intitle:`, `inurl:`, + `filetype:`/`ext:`, `before:`/`after:` dates (year, year-month, or full date), and `OR`/`|`. + Total and fail-soft: malformed input is kept as ordinary query text, never dropped or raised + on. `+word` is not special here (the server's scope-token pass runs before this parser). + """ + acc = _Accumulator() + for token in _tokenize(raw): + if token in ("OR", "|"): + acc.engine_parts.append(token) + continue + + negated = len(token) > 1 and token.startswith("-") + body = token[1:] if negated else token + + if body.startswith('"'): + # A blank phrase (a stray `"` or `-"`) is dropped entirely: an empty excluded phrase + # would match every result (`"" in text` is always True) and filter everything out. + phrase = _unquote(body) + if not phrase.strip(): + continue + if negated: + acc.excluded_phrases.append(phrase) + acc.engine_parts.append(f'-"{phrase}"') + else: + acc.phrases.append(phrase) + acc.clean_parts.append(phrase) + acc.engine_parts.append(f'"{phrase}"') + continue + + colon_index = body.find(":") + op_name = body[:colon_index].lower() if colon_index > 0 else None + if op_name is not None and op_name in _RECOGNIZED_OPS: + value_raw = body[colon_index + 1 :] + if _apply_operator(op_name, negated, token, value_raw, acc): + continue + + if negated: + acc.excluded_terms.append(body) + acc.engine_parts.append(token) + else: + acc.clean_parts.append(token) + acc.engine_parts.append(token) + + return ParsedQuery( + raw=raw, + clean_text=" ".join(acc.clean_parts), + engine_query=" ".join(acc.engine_parts), + phrases=tuple(acc.phrases), + excluded_terms=tuple(acc.excluded_terms), + excluded_phrases=tuple(acc.excluded_phrases), + include_sites=tuple(acc.include_sites), + exclude_sites=tuple(acc.exclude_sites), + in_title=tuple(acc.in_title), + not_in_title=tuple(acc.not_in_title), + in_url=tuple(acc.in_url), + not_in_url=tuple(acc.not_in_url), + file_types=tuple(acc.file_types), + after_ms=acc.after_ms, + before_ms=acc.before_ms, + ) + + +def _apply_operator( + op_name: str, negated: bool, token: str, value_raw: str, acc: _Accumulator +) -> bool: + """Handle one `op:value` token, mutating `acc` for the recognized operators. + + Returns False to signal that the token was NOT actually consumed as an operator (a `-` + negated `filetype:`/`ext:`/`before:`/`after:` has no defined negated meaning), so the caller + falls back to treating it as a plain `-word` exclusion. + """ + if op_name == "site": + value = _normalize_site(_unquote_or_raw(value_raw)) + if not value: + return True # an empty operator is dropped entirely + if negated: + acc.exclude_sites.append(value) + acc.engine_parts.append(f"-site:{value}") + else: + acc.include_sites.append(value) + acc.engine_parts.append(f"site:{value}") + elif op_name in ("filetype", "ext"): + if negated: + return False # no defined negated meaning; caller treats it as -word + value = _normalize_file_type(_unquote_or_raw(value_raw)) + if not value: + return True + acc.file_types.append(value) + acc.engine_parts.append(f"filetype:{value}") + elif op_name in ("intitle", "inurl"): + value = _unquote_or_raw(value_raw) + if not value: + return True + if negated: + # -intitle:/-inurl: are locally-enforced only; dropped from the upstream query. + (acc.not_in_title if op_name == "intitle" else acc.not_in_url).append(value) + else: + (acc.in_title if op_name == "intitle" else acc.in_url).append(value) + acc.clean_parts.append(value) + acc.engine_parts.append(value) + elif op_name in ("before", "after"): + if negated: + return False # no defined negated meaning; caller treats it as -word + value = _unquote_or_raw(value_raw) + if not value: + return True + millis = _parse_date(value) + if millis is None: + # Never silently drop something we could not parse: keep the whole token as text. + acc.clean_parts.append(token) + acc.engine_parts.append(token) + elif op_name == "after": + acc.after_ms = millis + else: + # before:/after: are locally-enforced only; dropped from the upstream query. + acc.before_ms = millis + return True + + +def _tokenize(raw: str) -> list[str]: + """Split `raw` on whitespace, except inside `"..."`; an unterminated quote runs to the end.""" + tokens: list[str] = [] + current: list[str] = [] + in_quotes = False + for ch in raw: + if ch == '"': + in_quotes = not in_quotes + current.append(ch) + elif ch.isspace() and not in_quotes: + if current: + tokens.append("".join(current)) + current.clear() + else: + current.append(ch) + if current: + tokens.append("".join(current)) + return tokens + + +def _unquote(text: str) -> str: + """Strip the surrounding quotes off `text` (which starts with `"`); an unterminated one runs + to the end.""" + closing = text.find('"', 1) + return text[1:closing] if closing >= 0 else text[1:] + + +def _unquote_or_raw(text: str) -> str: + return _unquote(text) if text.startswith('"') else text + + +def _normalize_site(value: str) -> str: + """`*.example.com` / `example.com.` -> `example.com`.""" + v = value.strip() + if v.startswith("*."): + v = v[2:] + return v.rstrip(".").lower() + + +def _normalize_file_type(value: str) -> str: + """`.PDF` / `PDF` -> `pdf`.""" + return value.strip().lower().removeprefix(".") + + +def _parse_date(value: str) -> int | None: + """`YYYY`, `YYYY-MM`, `YYYY-MM-DD`, or `YYYY/MM/DD` -> UTC epoch millis at the start of that + day (year/month default to day 1 / January 1). None for anything that does not match one of + those shapes or names an impossible calendar date (e.g. month 13). + """ + if _YEAR.match(value): + parts = (int(value), 1, 1) + elif match := _YEAR_MONTH.match(value): + parts = (int(match.group(1)), int(match.group(2)), 1) + elif match := _FULL_DASH.match(value) or _FULL_SLASH.match(value): + parts = (int(match.group(1)), int(match.group(2)), int(match.group(3))) + else: + return None + try: + moment = datetime(parts[0], parts[1], parts[2], tzinfo=UTC) + except ValueError: + return None + return int(moment.timestamp() * 1000) diff --git a/tests/engines/test_query_operators.py b/tests/engines/test_query_operators.py new file mode 100644 index 0000000..c1156a6 --- /dev/null +++ b/tests/engines/test_query_operators.py @@ -0,0 +1,394 @@ +"""Google-style operator parsing and local enforcement, ported from `QueryOperatorsTest.kt`. + +Covers the parse split (clean_text vs engine_query vs structural filters), tokenizer robustness +against garbage input, and `matches` enforcing every locally-checked operator over a merged +result. Everything here is pure; no I/O. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from searchmob_desktop.engines.query_operators import parse_query_operators + + +def _epoch_ms(year: int, month: int = 1, day: int = 1) -> int: + return int(datetime(year, month, day, tzinfo=UTC).timestamp() * 1000) + + +# --- plain terms / phrases --------------------------------------------------------------------- + + +def test_plain_terms_pass_through_unchanged() -> None: + p = parse_query_operators("wireless mouse") + assert p.clean_text == "wireless mouse" + assert p.engine_query == "wireless mouse" + assert not p.has_filters + assert not p.has_operators + + +def test_leading_plus_is_not_special() -> None: + # The server's scope-token pass runs before this parser, so `+word` and `c++` are plain text. + p = parse_query_operators("+word c++") + assert p.clean_text == "+word c++" + assert p.engine_query == "+word c++" + + +def test_quoted_phrase_feeds_phrases_clean_text_and_engine_query() -> None: + p = parse_query_operators('mouse "gaming grade" wireless') + assert p.phrases == ("gaming grade",) + assert p.clean_text == "mouse gaming grade wireless" + assert p.engine_query == 'mouse "gaming grade" wireless' + + +def test_negated_phrase_goes_to_excluded_phrases_only_and_is_hidden_from_clean_text() -> None: + p = parse_query_operators('mouse -"black friday"') + assert p.excluded_phrases == ("black friday",) + assert p.phrases == () + assert p.clean_text == "mouse" + assert p.engine_query == 'mouse -"black friday"' + + +def test_negated_term_goes_to_excluded_terms_and_is_hidden_from_clean_text() -> None: + p = parse_query_operators("mouse -bluetooth") + assert p.excluded_terms == ("bluetooth",) + assert p.clean_text == "mouse" + assert p.engine_query == "mouse -bluetooth" + + +def test_bare_dash_is_a_plain_term() -> None: + p = parse_query_operators("mouse -") + assert p.excluded_terms == () + assert p.clean_text == "mouse -" + assert p.engine_query == "mouse -" + + +def test_unrecognized_colon_token_is_a_plain_term() -> None: + # "10:30" and unknown pseudo-operators must not be mistaken for a recognized op. + p = parse_query_operators("meet at 10:30 badop:xyz") + assert p.clean_text == "meet at 10:30 badop:xyz" + assert p.engine_query == "meet at 10:30 badop:xyz" + assert not p.has_filters + + +# --- site: --------------------------------------------------------------------------------------- + + +def test_site_operator_parses_and_normalizes() -> None: + p = parse_query_operators("foo site:*.Example.COM.") + assert p.include_sites == ("example.com",) + assert p.engine_query == "foo site:example.com" + # site: is a pure operator; it never appears in clean_text. + assert p.clean_text == "foo" + assert p.has_filters + + +def test_negated_site_operator_parses() -> None: + p = parse_query_operators("foo -site:aliexpress.com") + assert p.exclude_sites == ("aliexpress.com",) + assert p.engine_query == "foo -site:aliexpress.com" + + +def test_empty_site_operator_is_dropped_entirely() -> None: + p = parse_query_operators("foo site: bar") + assert p.include_sites == () + assert p.clean_text == "foo bar" + assert p.engine_query == "foo bar" + + +# --- intitle: / inurl: ---------------------------------------------------------------------------- + + +def test_intitle_and_inurl_become_bare_recall_hints_and_local_filters() -> None: + p = parse_query_operators("intitle:review inurl:shop") + assert p.in_title == ("review",) + assert p.in_url == ("shop",) + assert p.clean_text == "review shop" + assert p.engine_query == "review shop" + + +def test_negated_intitle_and_inurl_are_local_only_and_dropped_from_engine_query() -> None: + p = parse_query_operators("mouse -intitle:sponsored -inurl:ad") + assert p.not_in_title == ("sponsored",) + assert p.not_in_url == ("ad",) + assert p.clean_text == "mouse" + assert p.engine_query == "mouse" + + +def test_quoted_intitle_value_is_one_entry_with_spaces() -> None: + p = parse_query_operators('intitle:"foo bar"') + assert p.in_title == ("foo bar",) + assert p.clean_text == "foo bar" + assert p.engine_query == "foo bar" + + +# --- filetype: / ext: ----------------------------------------------------------------------------- + + +def test_filetype_operator_parses_and_strips_leading_dot() -> None: + p = parse_query_operators("manual filetype:.PDF") + assert p.file_types == ("pdf",) + assert p.engine_query == "manual filetype:pdf" + assert p.clean_text == "manual" + + +def test_ext_is_an_alias_for_filetype() -> None: + p = parse_query_operators("manual ext:DOC") + assert p.file_types == ("doc",) + assert p.engine_query == "manual filetype:doc" + + +def test_negated_filetype_has_no_defined_meaning_and_falls_back_to_excluded_term() -> None: + p = parse_query_operators("-filetype:exe") + assert p.file_types == () + assert p.excluded_terms == ("filetype:exe",) + assert p.engine_query == "-filetype:exe" + + +# --- before: / after: ----------------------------------------------------------------------------- + + +def test_date_forms_parse_to_utc_start_of_period() -> None: + assert parse_query_operators("after:2023").after_ms == _epoch_ms(2023) + assert parse_query_operators("after:2023-06").after_ms == _epoch_ms(2023, 6) + assert parse_query_operators("after:2023-06-15").after_ms == _epoch_ms(2023, 6, 15) + assert parse_query_operators("after:2023/06/15").after_ms == _epoch_ms(2023, 6, 15) + assert parse_query_operators("before:2024").before_ms == _epoch_ms(2024) + + +def test_date_operators_are_dropped_from_clean_text_and_engine_query() -> None: + p = parse_query_operators("news after:2023 before:2024") + assert p.clean_text == "news" + assert p.engine_query == "news" + + +def test_repeated_date_operator_last_one_wins() -> None: + p = parse_query_operators("after:2020 after:2023") + assert p.after_ms == _epoch_ms(2023) + + +def test_invalid_date_shape_is_kept_as_plain_term_never_dropped_silently() -> None: + p = parse_query_operators("before:notadate") + assert p.before_ms is None + assert p.clean_text == "before:notadate" + assert p.engine_query == "before:notadate" + + +def test_impossible_calendar_date_is_kept_as_plain_term() -> None: + p = parse_query_operators("before:2023-13") + assert p.before_ms is None + assert p.clean_text == "before:2023-13" + assert p.engine_query == "before:2023-13" + + +def test_negated_date_operator_has_no_defined_meaning_and_falls_back_to_excluded_term() -> None: + p = parse_query_operators("-after:2023") + assert p.after_ms is None + assert p.excluded_terms == ("after:2023",) + assert p.engine_query == "-after:2023" + + +# --- OR / | --------------------------------------------------------------------------------------- + + +def test_or_and_pipe_are_kept_in_engine_query_but_dropped_from_clean_text() -> None: + p = parse_query_operators("cats OR dogs | birds") + assert p.clean_text == "cats dogs birds" + assert p.engine_query == "cats OR dogs | birds" + + +def test_lowercase_or_is_a_plain_term_not_the_operator() -> None: + # Only the exact uppercase token "OR" is the operator. + p = parse_query_operators("cats or dogs") + assert p.clean_text == "cats or dogs" + assert p.engine_query == "cats or dogs" + + +# --- tokenizer robustness ------------------------------------------------------------------------- + + +def test_unterminated_quote_runs_to_end_of_string_without_crashing() -> None: + p = parse_query_operators('foo "bar baz qux') + assert p.phrases == ("bar baz qux",) + assert p.clean_text == "foo bar baz qux" + assert p.engine_query == 'foo "bar baz qux"' + + +def test_unterminated_quote_on_a_negated_operator_value_does_not_crash() -> None: + p = parse_query_operators('intitle:"unterminated value') + assert p.in_title == ("unterminated value",) + + +# --- engine_query order preservation & composite -------------------------------------------------- + + +def test_engine_query_preserves_original_token_order_across_mixed_operators() -> None: + p = parse_query_operators( + 'wireless mouse "gaming grade" -bluetooth site:amazon.com -site:aliexpress.com ' + "intitle:review -intitle:sponsored filetype:pdf before:2024 badtoken:xyz OR keyboards" + ) + assert p.engine_query == ( + 'wireless mouse "gaming grade" -bluetooth site:amazon.com -site:aliexpress.com ' + "review filetype:pdf badtoken:xyz OR keyboards" + ) + assert p.clean_text == "wireless mouse gaming grade review badtoken:xyz keyboards" + assert p.excluded_terms == ("bluetooth",) + assert p.include_sites == ("amazon.com",) + assert p.exclude_sites == ("aliexpress.com",) + assert p.in_title == ("review",) + assert p.not_in_title == ("sponsored",) + assert p.file_types == ("pdf",) + assert p.before_ms == _epoch_ms(2024) + + +# --- matches(): site: / -site: -------------------------------------------------------------------- + + +def test_matches_enforces_site_as_a_suffix_on_the_host() -> None: + p = parse_query_operators("foo site:example.com") + assert p.matches("T", "https://docs.example.com/page", "", None) + assert p.matches("T", "https://example.com/page", "", None) + assert not p.matches("T", "https://notexample.com/page", "", None) + + +def test_matches_site_supports_bare_tld_entries() -> None: + p = parse_query_operators("research site:.edu") + assert p.matches("T", "https://mit.edu/x", "", None) + assert not p.matches("T", "https://mit.com/x", "", None) + + +def test_matches_exclude_site_rejects_matching_hosts() -> None: + p = parse_query_operators("foo -site:pinterest.com") + assert not p.matches("T", "https://pinterest.com/x", "", None) + assert not p.matches("T", "https://sub.pinterest.com/x", "", None) + assert p.matches("T", "https://other.com/x", "", None) + + +def test_matches_include_site_rejects_an_unparsable_host() -> None: + p = parse_query_operators("foo site:example.com") + assert not p.matches("T", "not a url", "", None) + + +# --- matches(): intitle: / inurl: ----------------------------------------------------------------- + + +def test_matches_enforces_intitle_and_not_in_title() -> None: + p = parse_query_operators("intitle:review -intitle:sponsored") + assert p.matches("Full Review of X", "https://x.com", "", None) + assert not p.matches("Just a post", "https://x.com", "", None) + assert not p.matches("Sponsored Review", "https://x.com", "", None) + + +def test_matches_enforces_inurl_and_not_in_url() -> None: + p = parse_query_operators("inurl:shop -inurl:ad") + assert p.matches("T", "https://example.com/shop/item", "", None) + assert not p.matches("T", "https://example.com/other", "", None) + assert not p.matches("T", "https://example.com/shop/ad/item", "", None) + + +# --- matches(): filetype: ------------------------------------------------------------------------- + + +def test_matches_extracts_extension_ignoring_query_string() -> None: + p = parse_query_operators("filetype:pdf") + assert p.matches("T", "https://x.y/file.pdf?dl=1", "", None) + assert not p.matches("T", "https://x.y/file.docx?dl=1", "", None) + + +def test_matches_filetype_rejects_a_url_with_no_extension() -> None: + p = parse_query_operators("filetype:pdf") + assert not p.matches("T", "https://example.com/", "", None) + # A dot in the host (its TLD) must not be misread as a file extension. + assert not p.matches("T", "https://example.com", "", None) + + +# --- matches(): before: / after: ------------------------------------------------------------------ + + +def test_matches_enforces_date_window_inclusive_lower_exclusive_upper() -> None: + p = parse_query_operators("after:2023 before:2024") + lower = _epoch_ms(2023) + upper = _epoch_ms(2024) + assert p.matches("T", "https://x.com", "", lower) # inclusive lower bound + assert p.matches("T", "https://x.com", "", upper - 1) + assert not p.matches("T", "https://x.com", "", upper) # exclusive upper bound + assert not p.matches("T", "https://x.com", "", lower - 1) + + +def test_matches_excludes_an_undated_result_when_a_date_bound_is_set() -> None: + p = parse_query_operators("after:2023") + assert not p.matches("T", "https://x.com", "", None) + + +# --- matches(): -term / -"phrase" ----------------------------------------------------------------- + + +def test_matches_enforces_excluded_term_as_a_whole_word() -> None: + p = parse_query_operators("-cat") + assert p.matches("Category page", "https://x.com", "listing categories", None) + assert not p.matches("I love cat food", "https://x.com", "", None) + # The host is also checked (excluding a term that only appears as the domain name). + assert not p.matches("T", "https://cat.example.com", "", None) + + +def test_matches_enforces_excluded_phrase_as_a_substring() -> None: + p = parse_query_operators('-"black friday"') + assert not p.matches("Black Friday Deals", "https://x.com", "", None) + assert p.matches("Regular Deals", "https://x.com", "", None) + + +# --- matches(): positive terms/phrases are NOT locally enforced ----------------------------------- + + +def test_matches_does_not_enforce_positive_terms_or_phrases() -> None: + p = parse_query_operators('mouse "gaming grade"') + assert not p.has_filters + assert p.matches("Totally unrelated title", "https://x.com", "no overlap here either", None) + + +# --- has_filters ---------------------------------------------------------------------------------- + + +def test_has_filters_is_false_for_plain_terms_phrases_and_or() -> None: + assert not parse_query_operators("just plain words").has_filters + assert not parse_query_operators('"a phrase" OR term').has_filters + + +def test_has_filters_is_true_for_every_locally_enforced_operator() -> None: + for query in ( + "-excluded", + '-"excluded phrase"', + "site:example.com", + "-site:example.com", + "intitle:x", + "-intitle:x", + "inurl:x", + "-inurl:x", + "filetype:pdf", + "after:2023", + "before:2023", + ): + assert parse_query_operators(query).has_filters, query + + +def test_has_operators_is_true_for_forwarded_only_syntax() -> None: + # A quoted phrase or OR changes engine_query without setting a structural filter; the + # corrector-skip check must still see it as operator syntax. + assert parse_query_operators('mouse "gaming grade"').has_operators + assert parse_query_operators("cats OR dogs").has_operators + assert not parse_query_operators("plain words").has_operators + + +# --- degenerate quotes ---------------------------------------------------------------------------- + + +def test_blank_phrase_is_dropped_instead_of_excluding_everything() -> None: + # A stray `-"` used to add an empty excluded phrase, and `"" in text` is always True, so a + # single stray character filtered out every result. A stray `"` likewise polluted clean_text. + negated = parse_query_operators('mouse -"') + assert negated.excluded_phrases == () + assert negated.clean_text == "mouse" + assert negated.matches("Any title", "https://example.com/a", "any snippet", None) + positive = parse_query_operators('mouse "') + assert positive.phrases == () + assert positive.clean_text == "mouse" From c17605de8a9eacf90caddffbb8e3744bc1ef3a07 Mon Sep 17 00:00:00 2001 From: FlintWave Date: Thu, 2 Jul 2026 11:36:44 -0700 Subject: [PATCH 2/6] feat(engines): score relevance on operator-free ranking terms EngineContext gains a ranking_terms field (mirroring the Android SearchQuery.rankingTerms): when the fetched query carries scoping clauses, the aggregator's lexical blend and language affinity reason about the operator-free text instead, so a vertical's internal site: OR group or a user's operators never count as subject matter in the match score. None falls back to the query, keeping existing callers unchanged. Co-Authored-By: Claude Fable 5 --- src/searchmob_desktop/engines/aggregator.py | 9 ++-- src/searchmob_desktop/engines/types.py | 7 +++ tests/engines/test_aggregator.py | 47 +++++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/searchmob_desktop/engines/aggregator.py b/src/searchmob_desktop/engines/aggregator.py index 3d63ad8..8440001 100644 --- a/src/searchmob_desktop/engines/aggregator.py +++ b/src/searchmob_desktop/engines/aggregator.py @@ -166,8 +166,11 @@ def _published_of(item: SearchResult) -> int | None: # Fold a lexical query-match score into the RRF score so the final order leads with relevance # (does the result actually contain the query's content words, especially the title) and keeps # engine consensus as a strong secondary signal. Without this, near-tied RRF scores let an - # irrelevant result one engine ranked highly sit among the top hits. - terms = content_terms(ctx.query) + # irrelevant result one engine ranked highly sit among the top hits. Scoring reasons about the + # operator-free `ranking_terms` when set, so a scoping clause (a vertical's `site:` OR group, + # a user operator) never pollutes the lexical match; `ctx.query` is the fallback. + ranking_text = ctx.ranking_terms if ctx.ranking_terms is not None else ctx.query + terms = content_terms(ranking_text) def _final_score(b: _Bucket) -> float: # Navigational promotion: when the squished query names this result's domain (query @@ -178,7 +181,7 @@ def _final_score(b: _Bucket) -> float: blended_score( b.score, lexical_score(b.title, b.snippet, terms), - language_affinity(ctx.query, b.title, b.snippet), + language_affinity(ranking_text, b.title, b.snippet), ) * nav ) diff --git a/src/searchmob_desktop/engines/types.py b/src/searchmob_desktop/engines/types.py index 403090a..6f0623f 100644 --- a/src/searchmob_desktop/engines/types.py +++ b/src/searchmob_desktop/engines/types.py @@ -53,9 +53,16 @@ class EngineContext: distinct rows after dedup. `timeout_seconds` is the httpx client timeout, applied uniformly. `language_region`, when set, carries per-engine language/region parameters that tailor results to the UI language (DuckDuckGo `kl`, Brave `country`/`search_lang`/`ui_lang`); None is neutral. + + `ranking_terms` is the operator-free text the aggregator's lexical scorer should reason about + when `query` carries scoping clauses (`site:` groups from the verticals, user operators): a + scoping clause is fetch plumbing, not subject matter, and must never pollute the match score. + None (the default) scores against `query` itself, which keeps operator-free callers unchanged. + Mirrors `SearchQuery.rankingTerms` in the Android engine. """ query: str max_results: int = 10 timeout_seconds: float = 5.0 language_region: LanguageRegion | None = None + ranking_terms: str | None = None diff --git a/tests/engines/test_aggregator.py b/tests/engines/test_aggregator.py index bc5867a..6c9fa6f 100644 --- a/tests/engines/test_aggregator.py +++ b/tests/engines/test_aggregator.py @@ -119,6 +119,53 @@ async def test_respects_max_results() -> None: assert len(results) == 2 +async def _fake_scoring_bait(_client: httpx.AsyncClient, _ctx: EngineContext) -> list[SearchResult]: + # The first result matches the scoping clause's own words ("site", "reddit"), not the subject; + # the second matches the actual subject. Rank order gives the bait the higher RRF score. + return [ + SearchResult( + title="Reddit site index", + url="https://reddit.example/r/all", + snippet="site list", + engine="x", + ), + SearchResult( + title="Rust tutorial for beginners", + url="https://docs.example/rust", + snippet="a rust tutorial", + engine="x", + ), + ] + + +@pytest.mark.asyncio +async def test_ranking_terms_keep_scoping_clauses_out_of_the_lexical_score() -> None: + # A vertical (or a user operator) appends `site:` clauses to the fetched query. The lexical + # scorer must reason about the operator-free text, so the scored order and relevance values are + # identical to a plain search for the same subject; the clause words never count as a match. + scoped = "rust tutorial (site:reddit.example OR site:forum.example)" + plain_results = await aggregate( + EngineContext(query="rust tutorial", max_results=10), [_fake_scoring_bait] + ) + scoped_results = await aggregate( + EngineContext(query=scoped, max_results=10, ranking_terms="rust tutorial"), + [_fake_scoring_bait], + ) + assert [r.url for r in scoped_results] == [r.url for r in plain_results] + assert [r.relevance for r in scoped_results] == [r.relevance for r in plain_results] + # The genuinely matching result leads despite the bait's better engine rank. + assert scoped_results[0].url == "https://docs.example/rust" + + +@pytest.mark.asyncio +async def test_without_ranking_terms_the_query_itself_is_scored() -> None: + # Back-compat: callers that set no ranking_terms keep the old behavior (the full query drives + # the lexical score), so raw `aggregate` uses stay byte-identical. + ctx = EngineContext(query="reddit site", max_results=10) + results = await aggregate(ctx, [_fake_scoring_bait]) + assert results[0].url == "https://reddit.example/r/all" + + @pytest.mark.asyncio async def test_surfaced_url_is_tracker_stripped() -> None: """The clicked link must not carry trackers, even when the result is unique (no dedup).""" From c868b996b2fba7a4a50bdf2872292e60c84c8a6c Mon Sep 17 00:00:00 2001 From: FlintWave Date: Thu, 2 Jul 2026 11:41:17 -0700 Subject: [PATCH 3/6] feat(search): wire Google-style operators through the server and GUI Both /search routes and the in-app search parse the query once at the boundary (right after the scope-token pass): the engine query is forwarded upstream (scoped for the active vertical), every structural filter is enforced locally over the merged results before sort/ personalization/rules (drops only, scores untouched), and the operator-free clean text drives the summary lookup, the freshness heuristic, click training, and lexical scoring. The on-device spell corrector is skipped for operator-laden queries, whose syntax it would just mangle. Mirrors MetaSearchResultProvider.kt. Co-Authored-By: Claude Fable 5 --- src/searchmob_desktop/gui/main_window.py | 49 ++++++++-- src/searchmob_desktop/server/app.py | 56 +++++++---- tests/server/test_operator_route.py | 119 +++++++++++++++++++++++ 3 files changed, 197 insertions(+), 27 deletions(-) create mode 100644 tests/server/test_operator_route.py diff --git a/src/searchmob_desktop/gui/main_window.py b/src/searchmob_desktop/gui/main_window.py index ce1d82b..3c86109 100644 --- a/src/searchmob_desktop/gui/main_window.py +++ b/src/searchmob_desktop/gui/main_window.py @@ -62,6 +62,7 @@ detect_category, promote_media, ) +from searchmob_desktop.engines.query_operators import parse_query_operators from searchmob_desktop.engines.rank import ( PersonalizationModel, RankRule, @@ -186,6 +187,11 @@ def __init__( history_terms=lambda: [e.query for e in self._history_store.recent(500)] ) self._last_query = "" + # The operator-free text of the last query (see `query_operators`) and whether it carried + # any operator syntax. The clean text drives sort/personalization/click training; the flag + # skips the corrector, whose word-spelling logic would just mangle operators. + self._last_clean_query = "" + self._last_has_operators = False # Bind per the saved network-mode preference so a profile with network mode on listens on # the LAN from launch (the Settings toggle still rebinds on the next server restart). self._server = LocalServerController( @@ -874,23 +880,39 @@ def _on_submit(self) -> None: prefs = self._prefs_store.load() engines = build_engines_from_prefs(prefs) - # Scope the engine query for the active vertical; the summary, correction, and sort all keep - # the original query so freshness keywords and entity lookups are unaffected by operators. + # Google-style operators (site:/intitle:/before:/etc., see query_operators) are parsed once + # here: the engine query (scoped for the active vertical) goes upstream, while the + # operator-free clean text drives scoring, sort, the summary lookup, and the corrector so + # a scoping clause never throws those off. + parsed = parse_query_operators(query) + self._last_clean_query = parsed.clean_text + self._last_has_operators = parsed.has_operators ctx = EngineContext( - query=transform_query(query, self._vertical), + query=transform_query(parsed.engine_query, self._vertical), max_results=DEFAULT_POOL_SIZE, timeout_seconds=5.0, + ranking_terms=parsed.clean_text, ) summary_enabled = prefs.summary_enabled async def _run() -> tuple[list[SearchResult], SummaryBox | None, AggregateOutcome]: # Fetch the contextual summary concurrently with the metasearch so it adds no latency. + # The lookup uses the operator-free text: an operator-laden query never names an entity. summary_task = ( - asyncio.ensure_future(summary_for_query(query)) if summary_enabled else None + asyncio.ensure_future(summary_for_query(parsed.clean_text)) + if summary_enabled + else None ) outcome = await aggregate_with_status(ctx, engines) + # Operators the engines cannot be trusted to honor are enforced locally over the merged + # results (drops only; scores untouched), before sort/personalization/rules see them. + results = outcome.results + if parsed.has_filters: + results = [ + r for r in results if parsed.matches(r.title, r.url, r.snippet, r.published) + ] summary = await summary_task if summary_task is not None else None - return outcome.results, summary, outcome + return results, summary, outcome worker: AsyncWorker[tuple[list[SearchResult], SummaryBox | None, AggregateOutcome]] = ( AsyncWorker(_run) @@ -951,14 +973,16 @@ def _apply_ranking_and_show(self) -> None: """Sort, then re-rank the last raw results with the current rules; update view + status.""" # Sort first (relevance/date/freshness blend), then bucket by the user's rules so PIN/RAISE # are honored while preserving the chosen order within each bucket. + # The freshness heuristic and the click model reason about subject terms, so both get the + # operator-free clean text (a site:/filetype: clause is not a freshness keyword). now_ms = int(time.time() * 1000) - ordered = sort_results(self._raw_results, self._sort_mode, self._last_query, now_ms) + ordered = sort_results(self._raw_results, self._sort_mode, self._last_clean_query, now_ms) # Then nudge by the learned click model (between sort and rules, so PIN/RAISE/BLOCK win). if self._personalization_enabled: ordered = personalize_reorder( ordered, lambda r: host_of_url(r.url), - self._last_query, + self._last_clean_query, self._personalization, now_ms, ) @@ -1080,13 +1104,20 @@ def _on_result_activated(self, url: str, row: int) -> None: self._personalization, hosts, row, - query_terms(self._last_query), + query_terms(self._last_clean_query), int(time.time() * 1000), ) save_personalization(self._personalization) def _maybe_show_correction(self) -> None: - """Offer a 'Did you mean: X' link when the on-device corrector suggests one.""" + """Offer a 'Did you mean: X' link when the on-device corrector suggests one. + + Skipped for an operator-laden query: the corrector reasons about English word spelling, + not query syntax, and would just mangle the operators. + """ + if self._last_has_operators: + self._didyoumean.hide() + return suggestion = None try: suggestion = self._corrector.suggest(self._last_query) diff --git a/src/searchmob_desktop/server/app.py b/src/searchmob_desktop/server/app.py index c2687c4..b239d01 100644 --- a/src/searchmob_desktop/server/app.py +++ b/src/searchmob_desktop/server/app.py @@ -65,6 +65,7 @@ detect_category, promote_media, ) +from searchmob_desktop.engines.query_operators import ParsedQuery, parse_query_operators from searchmob_desktop.engines.rank import ( Lens, PersonalizationModel, @@ -491,7 +492,7 @@ def _resolve_locale(request: Request) -> str: return locale async def _run_metasearch( - query: str, + parsed: ParsedQuery, sort_mode: SortMode = SortMode.FRESH_RELEVANT, vertical: Vertical = Vertical.WEB, *, @@ -503,11 +504,13 @@ async def _run_metasearch( # (pin/raise/block still win). None disables promotion (e.g. the JSON endpoint). summary_task: Awaitable[SummaryBox | None] | None = None, ) -> tuple[list[SearchResult], tuple[EngineOutcome, ...]]: - if not query.strip() or not engines: + if not parsed.raw.strip() or not engines: return [], () - # Scope the query for the chosen vertical (a `site:` OR group the engines understand); the - # original query still drives sort/summary/correction so freshness keywords are detected. - scoped = transform_query(query, vertical) + # Scope the engine query for the chosen vertical (a `site:` OR group the engines + # understand), on top of the already-parsed operators (forwarded as-is; intitle:/inurl: + # turned into a bare recall hint). The operator-free text still drives scoring, sort, + # summary, and correction so a scoping clause never throws those off. + scoped = transform_query(parsed.engine_query, vertical) # Tailor results to the UI language: a non-English locale carries per-engine region/language # params (DuckDuckGo `kl`, Brave country/search_lang/ui_lang); None leaves results neutral. ctx = EngineContext( @@ -515,6 +518,7 @@ async def _run_metasearch( max_results=max_results, timeout_seconds=timeout_seconds, language_region=language_region_for(locale), + ranking_terms=parsed.clean_text, ) # The runner returns either a bare result list (a test fake) or an `AggregateOutcome` with # the per-engine status (the real `aggregate_with_status`); accept both. @@ -523,6 +527,11 @@ async def _run_metasearch( results, engine_outcomes = raw.results, raw.engines else: results, engine_outcomes = raw, () + # Operators the engines cannot be trusted to honor (intitle:/inurl:/filetype:/before:/ + # after:/-exclusions/site: scoping) are enforced locally here, before sort/personalization/ + # rules see the list. Like `apply_ranking`, this only drops rows; scores are untouched. + if parsed.has_filters: + results = [r for r in results if parsed.matches(r.title, r.url, r.snippet, r.published)] # The AI-slop filter mode is taken live from prefs when wired, so the Settings toggle takes # effect on the next search; otherwise the static build-time value is used. prefs = _load_prefs() @@ -531,10 +540,10 @@ async def _run_metasearch( # when a model is passed, which the caller does for the loopback owner), then apply the # user's rules so the served results match the in-app results and PIN/RAISE preserve order. now_ms = int(time.time() * 1000) - ordered = sort_results(results, sort_mode, query, now_ms) + ordered = sort_results(results, sort_mode, parsed.clean_text, now_ms) if model is not None: ordered = personalize_reorder( - ordered, lambda r: host_of_url(r.url), query, model, now_ms + ordered, lambda r: host_of_url(r.url), parsed.clean_text, model, now_ms ) # Media promotion: for a resolved media entity, nudge its canonical platforms up (bounded), # after relevance/personalization and before the user's rules, so pin/raise/block still win. @@ -583,13 +592,15 @@ def _register_render(query: str, results: list[SearchResult]) -> str: _render_cache.popitem(last=False) # evict the oldest render return rid - def _correction(query: str) -> str | None: + def _correction(parsed: ParsedQuery) -> str | None: # On-device "did you mean". `suggest` is fail-soft and already returns None when the - # corrected query equals the input, so any non-None result is a genuine suggestion. - if corrector is None or not query.strip(): + # corrected query equals the input, so any non-None result is a genuine suggestion. The + # corrector reasons about word spelling, not query syntax: an operator-laden query would + # just get its operators mangled, so it is skipped entirely (Android does the same). + if corrector is None or not parsed.raw.strip() or parsed.has_operators: return None try: - suggestion = corrector.suggest(query) + suggestion = corrector.suggest(parsed.raw) except Exception: return None return suggestion.corrected if suggestion is not None else None @@ -663,6 +674,10 @@ async def search_html(request: Request) -> Response: # persisting it. The engines, summary, and correction run on the cleaned query; the original # text is echoed in the box so the token round-trips and re-running re-applies the scope. query, scope = parse_scope_token(raw_query, rules_provider()) + # Google-style operators (site:/intitle:/before:/etc., see query_operators) are parsed once + # here so the operator-free text can drive anything that reasons about "what is the user + # asking about" rather than "how do I fetch it" (summary, correction, relevance, sort). + parsed = parse_query_operators(query) vertical = Vertical.from_value(request.query_params.get("vertical")) # An explicit `?sort=` wins. Absent it, a non-default vertical keeps its sensible default # (e.g. News favors Date); the plain Web view honors the user's configured default sort from @@ -676,11 +691,12 @@ async def search_html(request: Request) -> Response: else: sort_mode = default_sort(vertical) # Fetch the contextual summary concurrently with the metasearch so the box never adds - # latency to the results path. - summary_task = asyncio.ensure_future(_maybe_summary(query)) + # latency to the results path. Uses the operator-free text: an operator-laden query would + # never resolve a Wikipedia entity. + summary_task = asyncio.ensure_future(_maybe_summary(parsed.clean_text)) model = _owner_model(request) results, engine_outcomes = await _run_metasearch( - query, + parsed, sort_mode, vertical, model=model, @@ -700,13 +716,15 @@ async def search_html(request: Request) -> Response: # can train the model; everyone else (and a disabled owner) gets the plain destination link. link_builder: Callable[[int, str], str] | None = None if model is not None and results: - rid = _register_render(query, results) + # Click training reasons about subject terms, so the render remembers the + # operator-free text (a site: clause is not something the model should learn). + rid = _register_render(parsed.clean_text, results) link_builder = lambda pos, _url, _rid=rid: f"/click?rid={_rid}&pos={pos}" # noqa: E731 body = render_results_page( raw_query, results, is_safe_http_url, - correction=_correction(query), + correction=_correction(parsed), rules=rules_provider(), editable=_is_owner(request), summary=summary, @@ -934,11 +952,13 @@ async def search_json(request: Request) -> Response: # Same inline `+name` scope token as the HTML route: applied to this request only, with the # engines/correction run on the cleaned query and the original echoed back as `query`. query, scope = parse_scope_token(raw_query, rules_provider()) + # Same one-shot operator parse as the HTML route (see `search_html`). + parsed = parse_query_operators(query) vertical = Vertical.from_value(request.query_params.get("vertical")) sort_param = request.query_params.get("sort") sort_mode = SortMode.from_value(sort_param) if sort_param else default_sort(vertical) results, _ = await _run_metasearch( - query, + parsed, sort_mode, vertical, model=_owner_model(request), @@ -952,7 +972,7 @@ async def search_json(request: Request) -> Response: "results": [ {k: v for k, v in asdict(result).items() if k != "relevance"} for result in results ], - "correction": _correction(query), + "correction": _correction(parsed), } return JSONResponse(payload) diff --git a/tests/server/test_operator_route.py b/tests/server/test_operator_route.py new file mode 100644 index 0000000..e9d22bd --- /dev/null +++ b/tests/server/test_operator_route.py @@ -0,0 +1,119 @@ +"""Google-style operators on the served `/search` and `/api/search` endpoints. + +The routes parse the operators once: the engine query (operators the engines understand) goes +upstream, the structural filters are enforced locally over the merged results, the echoed `query` +keeps the original text, and the on-device corrector is skipped for operator-laden queries. The +fake metasearch records the `EngineContext` so the forwarded query and the operator-free +`ranking_terms` are both observable. +""" + +from __future__ import annotations + +from starlette.testclient import TestClient + +from searchmob_desktop.engines import EngineContext, SearchResult +from searchmob_desktop.engines.correct import Correction +from searchmob_desktop.server.app import build_app + +_RESULTS = [ + SearchResult(title="Guide", url="https://docs.example/guide.html", snippet="s", engine="e"), + SearchResult(title="Thread", url="https://forum.example/t/1", snippet="s", engine="e"), + SearchResult(title="Manual", url="https://docs.example/manual.pdf", snippet="s", engine="e"), +] + + +class _Recorder: + """Remembers the context the route handed to the metasearch runner.""" + + def __init__(self, results: list[SearchResult] | None = None) -> None: + self.ctx: EngineContext | None = None + self.results = results if results is not None else list(_RESULTS) + + +def _client(recorder: _Recorder, corrector: object = None) -> TestClient: + async def _metasearch(ctx: EngineContext, _engines: object) -> list[SearchResult]: + recorder.ctx = ctx + return list(recorder.results) + + app = build_app( + [lambda _c, _ctx: []], + bound_port_getter=lambda: 8787, + bound_host_getter=lambda: "127.0.0.1", + corrector=corrector, # type: ignore[arg-type] + metasearch=_metasearch, # type: ignore[arg-type] + host_allowlist_enabled=False, + ) + return TestClient(app, client=("127.0.0.1", 9)) + + +def test_engine_query_is_forwarded_and_local_filters_are_enforced() -> None: + recorder = _Recorder() + with _client(recorder) as client: + resp = client.get("/api/search", params={"q": "rust tutorial -site:forum.example"}) + body = resp.json() + # The engines see the operators they understand; the exclusion is also enforced locally. + assert recorder.ctx is not None + assert recorder.ctx.query == "rust tutorial -site:forum.example" + urls = [r["url"] for r in body["results"]] + assert "https://forum.example/t/1" not in urls + assert "https://docs.example/guide.html" in urls + # The echo keeps the original text. + assert body["query"] == "rust tutorial -site:forum.example" + + +def test_ranking_terms_carry_the_operator_free_text() -> None: + recorder = _Recorder() + with _client(recorder) as client: + client.get("/api/search", params={"q": "rust tutorial site:docs.example filetype:pdf"}) + assert recorder.ctx is not None + # The lexical scorer reasons about the clean text, never the scoping clauses. + assert recorder.ctx.ranking_terms == "rust tutorial" + assert recorder.ctx.query == "rust tutorial site:docs.example filetype:pdf" + + +def test_local_only_operators_are_stripped_from_the_engine_query() -> None: + recorder = _Recorder() + with _client(recorder) as client: + resp = client.get( + "/api/search", params={"q": "rust intitle:tutorial -inurl:forum after:2023"} + ) + assert recorder.ctx is not None + # intitle: becomes a bare recall hint; -inurl:/after: are dropped from the upstream query. + assert recorder.ctx.query == "rust tutorial" + # after:2023 excludes every undated fake result, so the filters demonstrably ran locally. + assert resp.json()["results"] == [] + + +def test_filetype_filter_applies_over_merged_results() -> None: + recorder = _Recorder() + with _client(recorder) as client: + body = client.get("/api/search", params={"q": "manual filetype:pdf"}).json() + urls = [r["url"] for r in body["results"]] + assert urls == ["https://docs.example/manual.pdf"] + + +def test_html_route_enforces_filters_and_echoes_the_original_query() -> None: + recorder = _Recorder() + with _client(recorder) as client: + resp = client.get("/search", params={"q": "rust tutorial -site:forum.example"}) + assert resp.status_code == 200 + assert "docs.example" in resp.text + assert "forum.example/t/1" not in resp.text + # The search box round-trips the original operator-laden text. + assert "rust tutorial -site:forum.example" in resp.text + + +class _AlwaysCorrector: + """Suggests a fixed rewrite for anything, to prove the operator skip.""" + + def suggest(self, _query: str) -> Correction: + return Correction(corrected="rust tutorial", confidence=0.9) + + +def test_corrector_is_skipped_for_operator_laden_queries() -> None: + with _client(_Recorder(), corrector=_AlwaysCorrector()) as client: + with_ops = client.get("/api/search", params={"q": "rust tutoriall site:docs.example"}) + plain = client.get("/api/search", params={"q": "rust tutoriall"}) + # The operator query gets no on-device suggestion; the plain one still does. + assert with_ops.json()["correction"] is None + assert plain.json()["correction"] == "rust tutorial" From 18492959fd9ccfb092ca3253a7c2548d2e2b6c1b Mon Sep 17 00:00:00 2001 From: FlintWave Date: Thu, 2 Jul 2026 11:44:36 -0700 Subject: [PATCH 4/6] fix(server): harden the served pages against CSRF and token probing Port the Android served-page security pass: - Reject a literal "Origin: null" on mutation POSTs. An attacker page can force that opaque value onto a genuinely cross-site POST (a page served with Referrer-Policy: no-referrer, or a sandboxed iframe), and it used to slip through because urlsplit("null") yields an empty host and an empty host is allowed for HTTP/1.0 clients. - Serve Referrer-Policy: same-origin instead of no-referrer, so our own forms carry a real origin the CSRF check can verify. Cross-origin navigation still sends nothing, and result links keep rel=noreferrer. - Send Content-Security-Policy, Cache-Control: no-store, and Permissions-Policy on every response, so queries never persist in a browser cache and no external script/style/frame can ever load. - Compare the network-access token in constant time (secrets.compare_digest) and accept it via Authorization: Bearer or X-SearchMob-Token headers as well as ?token= (which stays for the OpenSearch templates but leaks into history and bookmarks). Co-Authored-By: Claude Fable 5 --- src/searchmob_desktop/server/__init__.py | 4 + src/searchmob_desktop/server/app.py | 106 +++++++++++++++++++--- src/searchmob_desktop/server/templates.py | 5 +- tests/server/test_rules_endpoints.py | 22 ++++- tests/server/test_security_helpers.py | 33 ++++++- tests/server/test_server_routes.py | 49 +++++++++- 6 files changed, 197 insertions(+), 22 deletions(-) diff --git a/src/searchmob_desktop/server/__init__.py b/src/searchmob_desktop/server/__init__.py index 3eb8fce..d71c654 100644 --- a/src/searchmob_desktop/server/__init__.py +++ b/src/searchmob_desktop/server/__init__.py @@ -22,7 +22,9 @@ is_loopback_host, is_safe_http_url, local_hostnames, + presented_token, requires_token, + token_matches, ) from searchmob_desktop.server.runner import serve @@ -37,6 +39,8 @@ "is_loopback_host", "is_safe_http_url", "local_hostnames", + "presented_token", "requires_token", "serve", + "token_matches", ] diff --git a/src/searchmob_desktop/server/app.py b/src/searchmob_desktop/server/app.py index b239d01..12d499d 100644 --- a/src/searchmob_desktop/server/app.py +++ b/src/searchmob_desktop/server/app.py @@ -143,20 +143,45 @@ def _no_suggestions(_query: str, _limit: int) -> list[str]: return [] +# Content-Security-Policy sent on every response, mirroring the Android server's. Every piece of +# dynamic content the templates render (query text, titles, snippets, settings values, ...) is +# HTML-escaped, so the inline theme/reveal `" "" diff --git a/tests/server/test_server_routes.py b/tests/server/test_server_routes.py index 5ba0d96..c8bed0d 100644 --- a/tests/server/test_server_routes.py +++ b/tests/server/test_server_routes.py @@ -88,6 +88,19 @@ def test_home_advertises_opensearch_descriptor_link() -> None: assert 'title="SearchMob"' in body +def test_home_page_advertises_the_search_operators_help() -> None: + with _build_client() as client: + body = client.get("/").text + # The collapsible cheat sheet: a native
card listing the supported operators. + assert '
' in body + assert "Search operators" in body + assert "filetype:pdf" in body + assert "site:example.com" in body + assert "after:2022" in body + # Operator examples are escaped like everything else; the quoted-phrase row survives escaping. + assert ""exact phrase"" in body + + def test_opensearch_descriptor_advertises_both_search_and_suggestions() -> None: with _build_client(port=9999, host="127.0.0.1") as client: response = client.get("/opensearch.xml") From 3010a4c1d0ea995bae964b7df7be67dc9e27518e Mon Sep 17 00:00:00 2001 From: FlintWave Date: Thu, 2 Jul 2026 11:50:44 -0700 Subject: [PATCH 6/6] docs(changelog): record the operators, security, and Material 3 work Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd039d9..7cfe0f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to SearchMob Desktop are documented here. The version scheme ## [Unreleased] +### Added +- **Google-style search operators.** `"exact phrase"`, `-term`, `site:` / `-site:`, `intitle:`, + `inurl:`, `filetype:` (or `ext:`), `before:` / `after:` dates, and `OR`. Operators the upstream + engines understand are forwarded to them, and every structural filter is also enforced on your + device over the merged results, so they behave consistently across engines, in the app and on + the served pages. The served home page gains a collapsible operator cheat sheet, and relevance + ranking no longer lets scoping clauses (like the verticals' internal `site:` groups) pollute the + match score. + +### Security +- **Cross-site request forgery fix for the served pages.** A literal `Origin: null`, which an + attacker page can forge on a cross-site POST, was treated as same-origin; it is now rejected, + and the server sends `Referrer-Policy: same-origin` so its own forms carry a real origin + (cross-origin navigation still leaks nothing). +- Every served response now carries `Content-Security-Policy`, `Cache-Control: no-store`, and + `Permissions-Policy` headers, so queries never persist in browser caches and no external + script/style/frame can ever load. +- The network-mode access token is compared in constant time and can be sent via + `Authorization: Bearer` or `X-SearchMob-Token` headers instead of only the `?token=` query + parameter (which stays for OpenSearch templates but leaks into browser history). + +### Changed +- **The served pages are restyled to Material 3**: elevated search bar with focus states, rounded + cards, pill buttons, chip state layers, and theme-derived colors in every palette. + ## 26.06.07 — 2026-06-19 ### Fixed