From 5c44c41b20f751cf86417dccaea24bb2a3d2cb98 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Jul 2026 09:50:23 +0000 Subject: [PATCH 1/2] feat(engine): Google-style search operators parsed, forwarded, and enforced on-device Adds a pure QueryOperators parser for "exact phrase", -term, -"phrase", site:/-site:, intitle:/-intitle:, inurl:/-inurl:, filetype:/ext:, before:/after: (YYYY[-MM[-DD]] and YYYY/MM/DD), and OR/|. Operators the upstream engines understand (quotes, exclusions, site:, filetype:, OR) are forwarded in original token order; the rest become recall hints or are dropped, and every structural filter is enforced locally over the aggregated results so behavior is consistent across engines that ignore an operator. The operator-free text now drives lexical relevance (a vertical's site: clause no longer pollutes it), sorting, personalization, the contextual summary, and the on-device spell corrector (skipped for operator-laden queries). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016AYAgNYmBkxsEYcgzSAjgT --- .../org/searchmob/engine/EngineAdapter.kt | 2 + .../engine/MetaSearchResultProvider.kt | 64 ++- .../searchmob/engine/aggregate/Aggregator.kt | 5 +- .../searchmob/engine/query/QueryOperators.kt | 353 +++++++++++++++ .../engine/SearchModifiersProviderTest.kt | 98 +++++ .../engine/query/QueryOperatorsTest.kt | 412 ++++++++++++++++++ 6 files changed, 919 insertions(+), 15 deletions(-) create mode 100644 app/src/main/java/org/searchmob/engine/query/QueryOperators.kt create mode 100644 app/src/test/java/org/searchmob/engine/SearchModifiersProviderTest.kt create mode 100644 app/src/test/java/org/searchmob/engine/query/QueryOperatorsTest.kt diff --git a/app/src/main/java/org/searchmob/engine/EngineAdapter.kt b/app/src/main/java/org/searchmob/engine/EngineAdapter.kt index d3bf53a..61b27e2 100644 --- a/app/src/main/java/org/searchmob/engine/EngineAdapter.kt +++ b/app/src/main/java/org/searchmob/engine/EngineAdapter.kt @@ -10,6 +10,8 @@ enum class SearchCategory { GENERAL, } data class SearchQuery( val terms: String, val category: SearchCategory = SearchCategory.GENERAL, + /** Operator-free text used for on-device relevance ranking; defaults to [terms]. */ + val rankingTerms: String = terms, ) /** A normalized result from one engine, with the rank (0-based position) it held in that engine's list. */ diff --git a/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt b/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt index e23dc7e..b9ebb54 100644 --- a/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt +++ b/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt @@ -10,6 +10,7 @@ import org.searchmob.engine.aggregate.EngineOutcome import org.searchmob.engine.correct.NoopSpellCorrector import org.searchmob.engine.correct.SpellCorrector import org.searchmob.engine.http.HttpClientFactory +import org.searchmob.engine.query.QueryOperators import org.searchmob.engine.rank.DomainRanker import org.searchmob.engine.rank.PersonalizationModel import org.searchmob.engine.rank.Personalizer @@ -27,11 +28,18 @@ import org.searchmob.server.SearchResultProvider * Bridges the metasearch engine to the server's [SearchResultProvider] contract, so the engine drops * in behind the unchanged HTTP routes. * + * The raw query is run through [QueryOperators] before anything else: Google-fu style operators + * (`site:`, `-exclude`, `"phrase"`, `intitle:`/`inurl:`, `filetype:`/`ext:`, `before:`/`after:`, + * `OR`/`|`) are split off into an operator-free `cleanText` (for relevance/correction/the contextual + * summary) and an `engineQuery` sent upstream; whatever no engine can be trusted to honor is enforced + * locally as a post-filter over the aggregated results. + * * It also surfaces a "did you mean" correction: it prefers the consensus correction the upstream * engines reported (parsed from the responses already fetched, no extra request) and otherwise falls - * back to the offline [corrector]. When the original query returns results, the correction is offered - * as a non-binding suggestion; when it returns nothing and a confident correction exists, the - * correction is searched automatically and reported via [SearchOutcome.showingResultsFor]. + * back to the offline [corrector] (skipped entirely for an operator-laden query, whose syntax the + * corrector would just mangle). When the original query returns results, the correction is offered as + * a non-binding suggestion; when it returns nothing and a confident correction exists, the correction + * is searched automatically and reported via [SearchOutcome.showingResultsFor]. */ class MetaSearchResultProvider( private val registryProvider: suspend () -> EngineRegistry, @@ -95,8 +103,13 @@ class MetaSearchResultProvider( coroutineScope { if (query.isBlank()) return@coroutineScope SearchOutcome(emptyList()) - // Fetch the contextual summary concurrently with the metasearch so it adds no latency. - val summaryDeferred = async { runCatching { summaryFetcher(query) }.getOrNull() } + // Google-fu style operators (site:/intitle:/before:/etc., see QueryOperators) are parsed + // once here so the operator-free text can drive anything that reasons about "what is the + // user asking about" rather than "how do I fetch it". + val parsed = QueryOperators.parse(query) + // Fetch the contextual summary concurrently with the metasearch so it adds no latency. Uses + // the operator-free text: an operator-laden query would never resolve a Wikipedia entity. + val summaryDeferred = async { runCatching { summaryFetcher(parsed.cleanText) }.getOrNull() } // An inline `+name` scope token (parsed by the route) overrides the saved active scope // for this one search only; the persisted selection is never written. val rules = @@ -120,7 +133,12 @@ class MetaSearchResultProvider( ) val upstreamCorrection = upstreamRaw?.takeIf { !it.equals(query, ignoreCase = true) } - val onDevice = corrector.suggest(query) + // The on-device corrector reasons about English word spelling, not query syntax: an + // operator-laden query (parsed.engineQuery differing from parsed.cleanText, or any + // locally-enforced filter) would just get its operators mangled, so it is skipped and the + // suggestion relies on whatever the upstream engines themselves reported. + val hasOperators = parsed.engineQuery != parsed.cleanText || parsed.hasFilters + val onDevice = if (hasOperators) null else corrector.suggest(query) val suggestion = upstreamCorrection ?: onDevice?.corrected?.takeIf { !it.equals(query, ignoreCase = true) } val summary = summaryDeferred.await() @@ -178,7 +196,10 @@ class MetaSearchResultProvider( } } - /** Aggregate [query], sort, apply [rules] locally; return (results, correction, per-engine status). */ + /** + * Aggregate [query] (parsing any Google-fu operators it contains — see [QueryOperators]), sort, + * apply [rules] locally; return (results, correction, per-engine status). + */ private suspend fun aggregateRanked( query: String, vertical: Vertical, @@ -192,20 +213,35 @@ class MetaSearchResultProvider( // rules (pin/raise/block still win); null disables promotion. summaryDeferred: Deferred? = null, ): Triple, String?, List> { - // Scope the query for the chosen vertical (a `site:` OR group the engines understand). The - // original query still drives sort/summary/correction so freshness keywords are detected. - val scoped = Verticals.transformQuery(query, vertical) + val parsed = QueryOperators.parse(query) + // Scope the query for the chosen vertical (a `site:` OR group the engines understand), on top + // of the already-parsed engine query (operators forwarded as-is; intitle:/inurl: turned into a + // bare recall hint). The operator-free text still drives sort/summary/correction so freshness + // keywords are detected without a `site:` clause throwing that off. + val scoped = Verticals.transformQuery(parsed.engineQuery, vertical) // Tailor results to the active UI language (region-neutral when English/unset). val region = languageRegionFor(languageProvider()) val aggregated = - aggregator.aggregate(SearchQuery(scoped), registryProvider().activeEngines(httpClient, region)) + aggregator.aggregate( + SearchQuery(scoped, rankingTerms = parsed.cleanText), + registryProvider().activeEngines(httpClient, region), + ) + // Operators the engines cannot be trusted to honor (intitle:/inurl:/filetype:/before:/after:/ + // -exclusions/site: scoping) are enforced locally here, before sort/personalization/rules see + // the list. + val filtered = + if (parsed.hasFilters) { + aggregated.results.filter { parsed.matches(it.title, it.url, it.snippet, it.publishedMillis) } + } else { + aggregated.results + } // Sort first (relevance/date/freshness blend), then nudge by the owner's learned click model // (between sort and rules, so PIN/RAISE/BLOCK still win), then bucket by rules. val sorted = ResultSorter.sort( - aggregated.results, + filtered, sortMode, - query, + parsed.cleanText, System.currentTimeMillis(), relevanceOf = { it.score }, publishedOf = { it.publishedMillis }, @@ -215,7 +251,7 @@ class MetaSearchResultProvider( Personalizer.reorder( sorted, { DomainRanker.host(it.url) }, - query, + parsed.cleanText, personalization, System.currentTimeMillis(), ) diff --git a/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt b/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt index 82edc6f..13e02b9 100644 --- a/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt +++ b/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt @@ -97,7 +97,10 @@ class Aggregator( else -> EngineOutcome(adapter.id, EngineStatus.FAILED) } } - val results = rank(successes.flatMap { it.items }, query.terms) + // rankingTerms is the operator-free text (site:/filetype:/etc. clauses stripped, see + // QueryOperators) so a scoping clause never pollutes lexical relevance; the correction check + // still compares against the full terms actually sent upstream. + val results = rank(successes.flatMap { it.items }, query.rankingTerms) return AggregationResult(results, consensusCorrection(query.terms, successes), engineStatus) } diff --git a/app/src/main/java/org/searchmob/engine/query/QueryOperators.kt b/app/src/main/java/org/searchmob/engine/query/QueryOperators.kt new file mode 100644 index 0000000..a52bffe --- /dev/null +++ b/app/src/main/java/org/searchmob/engine/query/QueryOperators.kt @@ -0,0 +1,353 @@ +package org.searchmob.engine.query + +import org.searchmob.engine.rank.DomainRanker +import java.net.URI +import java.time.LocalDate +import java.time.ZoneOffset + +/** + * A query parsed into its Google-fu style operators plus the leftover free text. + * + * [engineQuery] is what actually goes upstream: operators the engines themselves understand + * (`site:`, quoted phrases, `-exclusions`, `OR`/`|`) are forwarded verbatim so the engine's own index + * does the heavy lifting; operators no public engine implements consistently (`intitle:`, `inurl:`, + * `before:`, `after:`) are either turned into a plain recall hint or dropped, and the actual constraint + * is enforced locally by [matches] over the aggregated results. [cleanText] is the query with every + * operator stripped to plain words, for anything that reasons about "what is the user asking about" + * rather than "how do I fetch it" (on-device relevance, spelling correction, the contextual-summary + * lookup). + */ +data class ParsedQuery( + val raw: String, + val cleanText: String, + val engineQuery: String, + val phrases: List, + val excludedTerms: List, + val excludedPhrases: List, + val includeSites: List, + val excludeSites: List, + val inTitle: List, + val notInTitle: List, + val inUrl: List, + val notInUrl: List, + val fileTypes: List, + val afterMillis: Long?, + val beforeMillis: Long?, +) { + /** True when any operator is enforced locally by [matches] (everything but plain terms/phrases/OR). */ + val hasFilters: Boolean + get() = + excludedTerms.isNotEmpty() || excludedPhrases.isNotEmpty() || + includeSites.isNotEmpty() || excludeSites.isNotEmpty() || + inTitle.isNotEmpty() || notInTitle.isNotEmpty() || + inUrl.isNotEmpty() || notInUrl.isNotEmpty() || + fileTypes.isNotEmpty() || afterMillis != null || beforeMillis != null + + /** + * True when an aggregated result survives every locally-enforced operator in this query. + * + * Positive [phrases] and plain terms are deliberately NOT checked here: they were already sent + * upstream in [engineQuery] and drive lexical relevance ([org.searchmob.engine.aggregate.Relevance]). + * A snippet is a partial excerpt of the page, so demanding the phrase appear in the title/snippet + * would reject results whose full body matches but whose short excerpt happens not to quote it; + * the engines and the relevance ranker are trusted to have already done that job. + * + * A date-window bound ([afterMillis]/[beforeMillis]) excludes a result with no known + * [publishedMillis], deliberately: the user explicitly asked for a window, so an undated result + * cannot be confirmed to be in it and is treated as a miss rather than let through. + */ + fun matches( + title: String, + url: String, + snippet: String, + publishedMillis: Long?, + ): Boolean { + val host = DomainRanker.host(url) + if (includeSites.isNotEmpty() && (host == null || includeSites.none { siteMatches(it, host) })) return false + if (excludeSites.isNotEmpty() && host != null && excludeSites.any { siteMatches(it, host) }) return false + if (inTitle.any { !title.contains(it, ignoreCase = true) }) return false + if (notInTitle.any { title.contains(it, ignoreCase = true) }) return false + val lowerUrl = url.lowercase() + if (inUrl.any { !lowerUrl.contains(it.lowercase()) }) return false + if (notInUrl.any { lowerUrl.contains(it.lowercase()) }) return false + if (fileTypes.isNotEmpty()) { + val extension = extensionOf(url) + if (extension == null || extension !in fileTypes) return false + } + if (afterMillis != null || beforeMillis != null) { + if (publishedMillis == null) return false + if (afterMillis != null && publishedMillis < afterMillis) return false + if (beforeMillis != null && publishedMillis >= beforeMillis) return false + } + if (excludedTerms.isNotEmpty()) { + val words = wordsOf("$title $snippet ${host.orEmpty()}") + if (excludedTerms.any { it.lowercase() in words }) return false + } + val haystack = "$title $snippet" + if (excludedPhrases.any { haystack.contains(it, ignoreCase = true) }) return false + return true + } + + /** + * Whether [entry] (a normalized `site:`/`-site:` value) covers [host]. An entry starting with `.` + * (a bare TLD like `.edu`) matches any host ending in it; otherwise the entry must equal the host or + * be one of its parent domains (`example.com` covers `docs.example.com` but not `notexample.com`). + */ + private fun siteMatches( + entry: String, + host: String, + ): Boolean = if (entry.startsWith(".")) host.endsWith(entry) else host == entry || host.endsWith(".$entry") + + /** + * The lowercased extension of [url]'s last path segment (query string and fragment ignored), or + * null when the path has no segment or that segment has no dot. Parsed via [URI] rather than naive + * string-splitting so a bare `https://example.com` never misreads its TLD (the `.com` in the host) + * as a file extension. + */ + private fun extensionOf(url: String): String? { + val path = runCatching { URI(url.trim()).path }.getOrNull() ?: return null + val lastSegment = path.substringAfterLast('/') + if (!lastSegment.contains('.')) return null + return lastSegment.substringAfterLast('.').lowercase().takeIf { it.isNotEmpty() } + } + + /** Lowercased maximal letter/digit runs in [text], for whole-word exclusion matching. */ + private fun wordsOf(text: String): Set { + val out = HashSet() + val current = StringBuilder() + for (ch in text) { + if (ch.isLetterOrDigit()) { + current.append(ch.lowercaseChar()) + } else if (current.isNotEmpty()) { + out.add(current.toString()) + current.setLength(0) + } + } + if (current.isNotEmpty()) out.add(current.toString()) + return out + } +} + +/** + * Parses Google-fu style search operators (`site:`, `-exclude`, `"exact phrase"`, `intitle:`, `inurl:`, + * `filetype:`/`ext:`, `before:`/`after:`, `OR`/`|`) out of a raw query into a [ParsedQuery]. + * + * Pure and deterministic: no I/O, no locale/clock dependence beyond UTC date math, so the same input + * always parses the same way. The server truncates queries at 512 characters before they ever reach + * here, so this has to tolerate garbage (an unterminated quote, a bare `-`, an empty `site:`) without + * throwing, never silently dropping a token it does not understand — anything it cannot make sense of + * falls back to being treated as ordinary query text. + */ +object QueryOperators { + private val RECOGNIZED_OPS = setOf("site", "intitle", "inurl", "filetype", "ext", "before", "after") + private val YEAR = Regex("^(\\d{4})$") + private val YEAR_MONTH = Regex("^(\\d{4})-(\\d{2})$") + private val FULL_DASH = Regex("^(\\d{4})-(\\d{2})-(\\d{2})$") + private val FULL_SLASH = Regex("^(\\d{4})/(\\d{2})/(\\d{2})$") + + fun parse(raw: String): ParsedQuery { + val acc = Accumulator() + for (token in tokenize(raw)) { + if (token == "OR" || token == "|") { + acc.engineParts.add(token) + continue + } + + val negated = token.length > 1 && token.startsWith("-") + val body = if (negated) token.substring(1) else token + + if (body.startsWith("\"")) { + // A blank phrase (a stray `"` or `-"`) is dropped entirely: an empty excluded phrase + // would match every result (`contains("")` is always true) and filter everything out. + val phrase = unquote(body) + if (phrase.isBlank()) continue + if (negated) { + acc.excludedPhrases.add(phrase) + acc.engineParts.add("-\"$phrase\"") + } else { + acc.phrases.add(phrase) + acc.cleanParts.add(phrase) + acc.engineParts.add("\"$phrase\"") + } + continue + } + + val colonIndex = body.indexOf(':') + val opName = if (colonIndex > 0) body.substring(0, colonIndex).lowercase() else null + if (opName != null && opName in RECOGNIZED_OPS) { + val valueRaw = body.substring(colonIndex + 1) + if (applyOperator(opName, negated, token, valueRaw, acc)) continue + } + + if (negated) { + acc.excludedTerms.add(body) + acc.engineParts.add(token) + } else { + acc.cleanParts.add(token) + acc.engineParts.add(token) + } + } + + return ParsedQuery( + raw = raw, + cleanText = acc.cleanParts.joinToString(" "), + engineQuery = acc.engineParts.joinToString(" "), + phrases = acc.phrases, + excludedTerms = acc.excludedTerms, + excludedPhrases = acc.excludedPhrases, + includeSites = acc.includeSites, + excludeSites = acc.excludeSites, + inTitle = acc.inTitle, + notInTitle = acc.notInTitle, + inUrl = acc.inUrl, + notInUrl = acc.notInUrl, + fileTypes = acc.fileTypes, + afterMillis = acc.afterMillis, + beforeMillis = acc.beforeMillis, + ) + } + + /** The mutable accumulators [parse] fills in as it walks the tokens, in original token order. */ + private class Accumulator { + val cleanParts = ArrayList() + val engineParts = ArrayList() + val phrases = ArrayList() + val excludedTerms = ArrayList() + val excludedPhrases = ArrayList() + val includeSites = ArrayList() + val excludeSites = ArrayList() + val inTitle = ArrayList() + val notInTitle = ArrayList() + val inUrl = ArrayList() + val notInUrl = ArrayList() + val fileTypes = ArrayList() + var afterMillis: Long? = null + var beforeMillis: Long? = null + } + + /** + * Handles one `op:value` token, mutating [acc] for the recognized operators. + * + * Returns false to signal that the token was NOT actually consumed as an operator (a `-` negated + * `filetype:`/`ext:`/`before:`/`after:` has no defined negated meaning), so the caller falls back to + * treating it as a plain `-word` exclusion. + */ + private fun applyOperator( + opName: String, + negated: Boolean, + token: String, + valueRaw: String, + acc: Accumulator, + ): Boolean { + when (opName) { + "site" -> { + val value = normalizeSite(unquoteOrRaw(valueRaw)) + if (value.isEmpty()) return true // an empty operator is dropped entirely + if (negated) acc.excludeSites.add(value) else acc.includeSites.add(value) + acc.engineParts.add(if (negated) "-site:$value" else "site:$value") + } + "filetype", "ext" -> { + if (negated) return false // no defined negated meaning; caller treats it as -word + val value = normalizeFileType(unquoteOrRaw(valueRaw)) + if (value.isEmpty()) return true + acc.fileTypes.add(value) + acc.engineParts.add("filetype:$value") + } + "intitle", "inurl" -> { + val value = unquoteOrRaw(valueRaw) + if (value.isEmpty()) return true + if (negated) { + if (opName == "intitle") acc.notInTitle.add(value) else acc.notInUrl.add(value) + // -intitle:/-inurl: are locally-enforced only; dropped from the upstream query. + } else { + if (opName == "intitle") acc.inTitle.add(value) else acc.inUrl.add(value) + acc.cleanParts.add(value) + acc.engineParts.add(value) + } + } + "before", "after" -> { + if (negated) return false // no defined negated meaning; caller treats it as -word + val value = unquoteOrRaw(valueRaw) + if (value.isEmpty()) return true + val millis = parseDate(value) + if (millis == null) { + // Never silently drop something we could not parse: keep the whole token as text. + acc.cleanParts.add(token) + acc.engineParts.add(token) + } else { + if (opName == "after") acc.afterMillis = millis else acc.beforeMillis = millis + // before:/after: are locally-enforced only; dropped from the upstream query. + } + } + } + return true + } + + /** Split [raw] on whitespace, except inside `"..."`; an unterminated quote runs to end of string. */ + private fun tokenize(raw: String): List { + val tokens = ArrayList() + val current = StringBuilder() + var inQuotes = false + for (ch in raw) { + when { + ch == '"' -> { + inQuotes = !inQuotes + current.append(ch) + } + ch.isWhitespace() && !inQuotes -> { + if (current.isNotEmpty()) { + tokens.add(current.toString()) + current.setLength(0) + } + } + else -> current.append(ch) + } + } + if (current.isNotEmpty()) tokens.add(current.toString()) + return tokens + } + + /** Strip the surrounding quotes off [text] (which starts with `"`); an unterminated one runs to the end. */ + private fun unquote(text: String): String { + val closing = text.indexOf('"', 1) + return if (closing >= 0) text.substring(1, closing) else text.substring(1) + } + + private fun unquoteOrRaw(text: String): String = if (text.startsWith("\"")) unquote(text) else text + + /** `*.example.com` / `example.com.` -> `example.com`. */ + private fun normalizeSite(value: String): String { + var v = value.trim() + if (v.startsWith("*.")) v = v.substring(2) + return v.trimEnd('.').lowercase() + } + + /** `.PDF` / `PDF` -> `pdf`. */ + private fun normalizeFileType(value: String): String = value.trim().lowercase().removePrefix(".") + + /** + * `YYYY`, `YYYY-MM`, `YYYY-MM-DD`, or `YYYY/MM/DD` -> UTC epoch millis at the start of that day + * (year/month default to day 1 / January 1). Null for anything that does not match one of those + * shapes or names an impossible calendar date (e.g. month 13). + */ + private fun parseDate(value: String): Long? { + val date = + when { + YEAR.matches(value) -> runCatching { LocalDate.of(value.toInt(), 1, 1) }.getOrNull() + YEAR_MONTH.matches(value) -> { + val (y, m) = YEAR_MONTH.find(value)!!.destructured + runCatching { LocalDate.of(y.toInt(), m.toInt(), 1) }.getOrNull() + } + FULL_DASH.matches(value) -> { + val (y, m, d) = FULL_DASH.find(value)!!.destructured + runCatching { LocalDate.of(y.toInt(), m.toInt(), d.toInt()) }.getOrNull() + } + FULL_SLASH.matches(value) -> { + val (y, m, d) = FULL_SLASH.find(value)!!.destructured + runCatching { LocalDate.of(y.toInt(), m.toInt(), d.toInt()) }.getOrNull() + } + else -> null + } + return date?.atStartOfDay(ZoneOffset.UTC)?.toInstant()?.toEpochMilli() + } +} diff --git a/app/src/test/java/org/searchmob/engine/SearchModifiersProviderTest.kt b/app/src/test/java/org/searchmob/engine/SearchModifiersProviderTest.kt new file mode 100644 index 0000000..212b4a3 --- /dev/null +++ b/app/src/test/java/org/searchmob/engine/SearchModifiersProviderTest.kt @@ -0,0 +1,98 @@ +package org.searchmob.engine + +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * End-to-end coverage of Google-fu style search operators (see + * [org.searchmob.engine.query.QueryOperators]) through [MetaSearchResultProvider]: the query the + * engines actually receive, and the local post-filter applied to the aggregated results. Mirrors the + * fake-adapter pattern in `AggregatorTest`. + */ +class SearchModifiersProviderTest { + private class CapturingEngine( + override val id: String, + private val items: List, + ) : EngineAdapter { + override val displayName = id + override val categories = setOf(SearchCategory.GENERAL) + + /** The last [SearchQuery] this adapter was asked to search, for asserting the transformed query. */ + var lastQuery: SearchQuery? = null + private set + + override suspend fun search( + query: SearchQuery, + ctx: EngineContext, + ): EngineResult { + lastQuery = query + return EngineResult.Success(items) + } + } + + private fun item( + url: String, + title: String = "Title", + snippet: String = "", + position: Int = 0, + ) = EngineResultItem(title = title, url = url, snippet = snippet, engineId = "e", position = position) + + private fun provider(engine: EngineAdapter) = MetaSearchResultProvider(EngineRegistry(listOf(engine))) + + @Test + fun siteOperatorDropsResultsFromOtherHosts() = + runTest { + val engine = + CapturingEngine( + "e", + listOf( + item("https://example.com/a", position = 0), + item("https://other.com/b", position = 1), + ), + ) + val results = provider(engine).search("foo site:example.com") + assertEquals(listOf("https://example.com/a"), results.map { it.url }) + } + + @Test + fun negatedTermExclusionWorksEndToEnd() = + runTest { + val engine = + CapturingEngine( + "e", + listOf( + item("https://a.com/1", title = "About cats", position = 0), + item("https://b.com/1", title = "About dogs", position = 1), + ), + ) + val results = provider(engine).search("pets -cats") + assertEquals(listOf("https://b.com/1"), results.map { it.url }) + } + + @Test + fun engineReceivesTransformedQueryWithIntitleReplacedByBareValue() = + runTest { + val engine = CapturingEngine("e", listOf(item("https://a.com/1"))) + provider(engine).search("mouse intitle:review") + // intitle: is not implemented consistently upstream, so it becomes a bare recall hint; the + // title constraint itself is enforced locally by ParsedQuery.matches. + assertEquals("mouse review", engine.lastQuery?.terms) + assertEquals("mouse review", engine.lastQuery?.rankingTerms) + } + + @Test + fun filetypeOperatorKeepsOnlyMatchingExtension() = + runTest { + val engine = + CapturingEngine( + "e", + listOf( + item("https://a.com/manual.pdf", position = 0), + item("https://a.com/manual.docx", position = 1), + ), + ) + val results = provider(engine).search("manual filetype:pdf") + assertEquals(listOf("https://a.com/manual.pdf"), results.map { it.url }) + } +} diff --git a/app/src/test/java/org/searchmob/engine/query/QueryOperatorsTest.kt b/app/src/test/java/org/searchmob/engine/query/QueryOperatorsTest.kt new file mode 100644 index 0000000..98faf1b --- /dev/null +++ b/app/src/test/java/org/searchmob/engine/query/QueryOperatorsTest.kt @@ -0,0 +1,412 @@ +package org.searchmob.engine.query + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.time.LocalDate +import java.time.ZoneOffset + +class QueryOperatorsTest { + private fun epochMillisOf( + year: Int, + month: Int = 1, + day: Int = 1, + ): Long = LocalDate.of(year, month, day).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli() + + // --- plain terms / phrases ------------------------------------------------------------------- + + @Test + fun plainTermsPassThroughUnchanged() { + val p = QueryOperators.parse("wireless mouse") + assertEquals("wireless mouse", p.cleanText) + assertEquals("wireless mouse", p.engineQuery) + assertFalse(p.hasFilters) + } + + @Test + fun leadingPlusIsNotSpecial() { + // The server's ScopeToken pass runs before this parser, so `+word` and `c++` are ordinary text. + val p = QueryOperators.parse("+word c++") + assertEquals("+word c++", p.cleanText) + assertEquals("+word c++", p.engineQuery) + } + + @Test + fun quotedPhraseFeedsPhrasesCleanTextAndEngineQuery() { + val p = QueryOperators.parse("mouse \"gaming grade\" wireless") + assertEquals(listOf("gaming grade"), p.phrases) + assertEquals("mouse gaming grade wireless", p.cleanText) + assertEquals("mouse \"gaming grade\" wireless", p.engineQuery) + } + + @Test + fun negatedPhraseGoesToExcludedPhrasesOnlyAndIsHiddenFromCleanText() { + val p = QueryOperators.parse("mouse -\"black friday\"") + assertEquals(listOf("black friday"), p.excludedPhrases) + assertTrue(p.phrases.isEmpty()) + assertEquals("mouse", p.cleanText) + assertEquals("mouse -\"black friday\"", p.engineQuery) + } + + @Test + fun negatedTermGoesToExcludedTermsAndIsHiddenFromCleanText() { + val p = QueryOperators.parse("mouse -bluetooth") + assertEquals(listOf("bluetooth"), p.excludedTerms) + assertEquals("mouse", p.cleanText) + assertEquals("mouse -bluetooth", p.engineQuery) + } + + @Test + fun bareDashIsAPlainTerm() { + val p = QueryOperators.parse("mouse -") + assertTrue(p.excludedTerms.isEmpty()) + assertEquals("mouse -", p.cleanText) + assertEquals("mouse -", p.engineQuery) + } + + @Test + fun unrecognizedColonTokenIsAPlainTerm() { + // "10:30" and unknown pseudo-operators must not be mistaken for a recognized op. + val p = QueryOperators.parse("meet at 10:30 badop:xyz") + assertEquals("meet at 10:30 badop:xyz", p.cleanText) + assertEquals("meet at 10:30 badop:xyz", p.engineQuery) + assertFalse(p.hasFilters) + } + + // --- site: ------------------------------------------------------------------------------------- + + @Test + fun siteOperatorParsesAndNormalizes() { + val p = QueryOperators.parse("foo site:*.Example.COM.") + assertEquals(listOf("example.com"), p.includeSites) + assertEquals("foo site:example.com", p.engineQuery) + // site: is a pure operator; it never appears in cleanText. + assertEquals("foo", p.cleanText) + assertTrue(p.hasFilters) + } + + @Test + fun negatedSiteOperatorParses() { + val p = QueryOperators.parse("foo -site:aliexpress.com") + assertEquals(listOf("aliexpress.com"), p.excludeSites) + assertEquals("foo -site:aliexpress.com", p.engineQuery) + } + + @Test + fun emptySiteOperatorIsDroppedEntirely() { + val p = QueryOperators.parse("foo site: bar") + assertTrue(p.includeSites.isEmpty()) + assertEquals("foo bar", p.cleanText) + assertEquals("foo bar", p.engineQuery) + } + + // --- intitle: / inurl: -------------------------------------------------------------------------- + + @Test + fun intitleAndInurlBecomeBareRecallHintsAndLocalFilters() { + val p = QueryOperators.parse("intitle:review inurl:shop") + assertEquals(listOf("review"), p.inTitle) + assertEquals(listOf("shop"), p.inUrl) + assertEquals("review shop", p.cleanText) + assertEquals("review shop", p.engineQuery) + } + + @Test + fun negatedIntitleAndInurlAreLocalOnlyAndDroppedFromEngineQuery() { + val p = QueryOperators.parse("mouse -intitle:sponsored -inurl:ad") + assertEquals(listOf("sponsored"), p.notInTitle) + assertEquals(listOf("ad"), p.notInUrl) + assertEquals("mouse", p.cleanText) + assertEquals("mouse", p.engineQuery) + } + + @Test + fun quotedIntitleValueIsOneEntryWithSpaces() { + val p = QueryOperators.parse("intitle:\"foo bar\"") + assertEquals(listOf("foo bar"), p.inTitle) + assertEquals("foo bar", p.cleanText) + assertEquals("foo bar", p.engineQuery) + } + + // --- filetype: / ext: ---------------------------------------------------------------------------- + + @Test + fun filetypeOperatorParsesAndStripsLeadingDot() { + val p = QueryOperators.parse("manual filetype:.PDF") + assertEquals(listOf("pdf"), p.fileTypes) + assertEquals("manual filetype:pdf", p.engineQuery) + assertEquals("manual", p.cleanText) + } + + @Test + fun extIsAnAliasForFiletype() { + val p = QueryOperators.parse("manual ext:DOC") + assertEquals(listOf("doc"), p.fileTypes) + assertEquals("manual filetype:doc", p.engineQuery) + } + + @Test + fun negatedFiletypeHasNoDefinedMeaningAndFallsBackToExcludedTerm() { + val p = QueryOperators.parse("-filetype:exe") + assertTrue(p.fileTypes.isEmpty()) + assertEquals(listOf("filetype:exe"), p.excludedTerms) + assertEquals("-filetype:exe", p.engineQuery) + } + + // --- before: / after: ----------------------------------------------------------------------------- + + @Test + fun dateFormsParseToUtcStartOfPeriod() { + assertEquals(epochMillisOf(2023), QueryOperators.parse("after:2023").afterMillis) + assertEquals(epochMillisOf(2023, 6), QueryOperators.parse("after:2023-06").afterMillis) + assertEquals(epochMillisOf(2023, 6, 15), QueryOperators.parse("after:2023-06-15").afterMillis) + assertEquals(epochMillisOf(2023, 6, 15), QueryOperators.parse("after:2023/06/15").afterMillis) + assertEquals(epochMillisOf(2024), QueryOperators.parse("before:2024").beforeMillis) + } + + @Test + fun dateOperatorsAreDroppedFromCleanTextAndEngineQuery() { + val p = QueryOperators.parse("news after:2023 before:2024") + assertEquals("news", p.cleanText) + assertEquals("news", p.engineQuery) + } + + @Test + fun repeatedDateOperatorLastOneWins() { + val p = QueryOperators.parse("after:2020 after:2023") + assertEquals(epochMillisOf(2023), p.afterMillis) + } + + @Test + fun invalidDateShapeIsKeptAsPlainTermNeverDroppedSilently() { + val p = QueryOperators.parse("before:notadate") + assertNull(p.beforeMillis) + assertEquals("before:notadate", p.cleanText) + assertEquals("before:notadate", p.engineQuery) + } + + @Test + fun impossibleCalendarDateIsKeptAsPlainTerm() { + val p = QueryOperators.parse("before:2023-13") + assertNull(p.beforeMillis) + assertEquals("before:2023-13", p.cleanText) + assertEquals("before:2023-13", p.engineQuery) + } + + @Test + fun negatedDateOperatorHasNoDefinedMeaningAndFallsBackToExcludedTerm() { + val p = QueryOperators.parse("-after:2023") + assertNull(p.afterMillis) + assertEquals(listOf("after:2023"), p.excludedTerms) + assertEquals("-after:2023", p.engineQuery) + } + + // --- OR / | ---------------------------------------------------------------------------------------- + + @Test + fun orAndPipeAreKeptInEngineQueryButDroppedFromCleanText() { + val p = QueryOperators.parse("cats OR dogs | birds") + assertEquals("cats dogs birds", p.cleanText) + assertEquals("cats OR dogs | birds", p.engineQuery) + } + + @Test + fun lowercaseOrIsAPlainTermNotTheOperator() { + // Only the exact uppercase token "OR" is the operator. + val p = QueryOperators.parse("cats or dogs") + assertEquals("cats or dogs", p.cleanText) + assertEquals("cats or dogs", p.engineQuery) + } + + // --- tokenizer robustness --------------------------------------------------------------------------- + + @Test + fun unterminatedQuoteRunsToEndOfStringWithoutCrashing() { + val p = QueryOperators.parse("foo \"bar baz qux") + assertEquals(listOf("bar baz qux"), p.phrases) + assertEquals("foo bar baz qux", p.cleanText) + assertEquals("foo \"bar baz qux\"", p.engineQuery) + } + + @Test + fun unterminatedQuoteOnANegatedOperatorValueDoesNotCrash() { + val p = QueryOperators.parse("intitle:\"unterminated value") + assertEquals(listOf("unterminated value"), p.inTitle) + } + + // --- engineQuery order preservation & composite -------------------------------------------------- + + @Test + fun engineQueryPreservesOriginalTokenOrderAcrossMixedOperators() { + val p = + QueryOperators.parse( + "wireless mouse \"gaming grade\" -bluetooth site:amazon.com -site:aliexpress.com " + + "intitle:review -intitle:sponsored filetype:pdf before:2024 badtoken:xyz OR keyboards", + ) + assertEquals( + "wireless mouse \"gaming grade\" -bluetooth site:amazon.com -site:aliexpress.com " + + "review filetype:pdf badtoken:xyz OR keyboards", + p.engineQuery, + ) + assertEquals("wireless mouse gaming grade review badtoken:xyz keyboards", p.cleanText) + assertEquals(listOf("bluetooth"), p.excludedTerms) + assertEquals(listOf("amazon.com"), p.includeSites) + assertEquals(listOf("aliexpress.com"), p.excludeSites) + assertEquals(listOf("review"), p.inTitle) + assertEquals(listOf("sponsored"), p.notInTitle) + assertEquals(listOf("pdf"), p.fileTypes) + assertEquals(epochMillisOf(2024), p.beforeMillis) + } + + // --- matches(): site: / -site: ---------------------------------------------------------------------- + + @Test + fun matchesEnforcesSiteAsASuffixOnTheHost() { + val p = QueryOperators.parse("foo site:example.com") + assertTrue(p.matches("T", "https://docs.example.com/page", "", null)) + assertTrue(p.matches("T", "https://example.com/page", "", null)) + assertFalse(p.matches("T", "https://notexample.com/page", "", null)) + } + + @Test + fun matchesSiteSupportsBareTldEntries() { + val p = QueryOperators.parse("research site:.edu") + assertTrue(p.matches("T", "https://mit.edu/x", "", null)) + assertFalse(p.matches("T", "https://mit.com/x", "", null)) + } + + @Test + fun matchesExcludeSiteRejectsMatchingHosts() { + val p = QueryOperators.parse("foo -site:pinterest.com") + assertFalse(p.matches("T", "https://pinterest.com/x", "", null)) + assertFalse(p.matches("T", "https://sub.pinterest.com/x", "", null)) + assertTrue(p.matches("T", "https://other.com/x", "", null)) + } + + @Test + fun matchesIncludeSiteRejectsAnUnparsableHost() { + val p = QueryOperators.parse("foo site:example.com") + assertFalse(p.matches("T", "not a url", "", null)) + } + + // --- matches(): intitle: / inurl: ---------------------------------------------------------------- + + @Test + fun matchesEnforcesIntitleAndNotInTitle() { + val p = QueryOperators.parse("intitle:review -intitle:sponsored") + assertTrue(p.matches("Full Review of X", "https://x.com", "", null)) + assertFalse(p.matches("Just a post", "https://x.com", "", null)) + assertFalse(p.matches("Sponsored Review", "https://x.com", "", null)) + } + + @Test + fun matchesEnforcesInurlAndNotInUrl() { + val p = QueryOperators.parse("inurl:shop -inurl:ad") + assertTrue(p.matches("T", "https://example.com/shop/item", "", null)) + assertFalse(p.matches("T", "https://example.com/other", "", null)) + assertFalse(p.matches("T", "https://example.com/shop/ad/item", "", null)) + } + + // --- matches(): filetype: ------------------------------------------------------------------------- + + @Test + fun matchesExtractsExtensionIgnoringQueryString() { + val p = QueryOperators.parse("filetype:pdf") + assertTrue(p.matches("T", "https://x.y/file.pdf?dl=1", "", null)) + assertFalse(p.matches("T", "https://x.y/file.docx?dl=1", "", null)) + } + + @Test + fun matchesFiletypeRejectsAUrlWithNoExtension() { + val p = QueryOperators.parse("filetype:pdf") + assertFalse(p.matches("T", "https://example.com/", "", null)) + // A dot in the host (its TLD) must not be misread as a file extension. + assertFalse(p.matches("T", "https://example.com", "", null)) + } + + // --- matches(): before: / after: ------------------------------------------------------------------- + + @Test + fun matchesEnforcesDateWindowInclusiveLowerExclusiveUpper() { + val p = QueryOperators.parse("after:2023 before:2024") + val lower = epochMillisOf(2023) + val upper = epochMillisOf(2024) + assertTrue(p.matches("T", "https://x.com", "", lower)) // inclusive lower bound + assertTrue(p.matches("T", "https://x.com", "", upper - 1)) + assertFalse(p.matches("T", "https://x.com", "", upper)) // exclusive upper bound + assertFalse(p.matches("T", "https://x.com", "", lower - 1)) + } + + @Test + fun matchesExcludesAnUndatedResultWhenADateBoundIsSet() { + val p = QueryOperators.parse("after:2023") + assertFalse(p.matches("T", "https://x.com", "", null)) + } + + // --- matches(): -term / -"phrase" ------------------------------------------------------------------ + + @Test + fun matchesEnforcesExcludedTermAsAWholeWord() { + val p = QueryOperators.parse("-cat") + assertTrue(p.matches("Category page", "https://x.com", "listing categories", null)) + assertFalse(p.matches("I love cat food", "https://x.com", "", null)) + // The host is also checked (excluding a term that only appears as the domain name). + assertFalse(p.matches("T", "https://cat.example.com", "", null)) + } + + @Test + fun matchesEnforcesExcludedPhraseAsASubstring() { + val p = QueryOperators.parse("-\"black friday\"") + assertFalse(p.matches("Black Friday Deals", "https://x.com", "", null)) + assertTrue(p.matches("Regular Deals", "https://x.com", "", null)) + } + + // --- matches(): positive terms/phrases are NOT locally enforced -------------------------------------- + + @Test + fun matchesDoesNotEnforcePositiveTermsOrPhrases() { + val p = QueryOperators.parse("mouse \"gaming grade\"") + assertFalse(p.hasFilters) + assertTrue(p.matches("Totally unrelated title", "https://x.com", "no overlap here either", null)) + } + + // --- hasFilters -------------------------------------------------------------------------------------- + + @Test + fun hasFiltersIsFalseForPlainTermsPhrasesAndOr() { + assertFalse(QueryOperators.parse("just plain words").hasFilters) + assertFalse(QueryOperators.parse("\"a phrase\" OR term").hasFilters) + } + + @Test + fun hasFiltersIsTrueForEveryLocallyEnforcedOperator() { + assertTrue(QueryOperators.parse("-excluded").hasFilters) + assertTrue(QueryOperators.parse("-\"excluded phrase\"").hasFilters) + assertTrue(QueryOperators.parse("site:example.com").hasFilters) + assertTrue(QueryOperators.parse("-site:example.com").hasFilters) + assertTrue(QueryOperators.parse("intitle:x").hasFilters) + assertTrue(QueryOperators.parse("-intitle:x").hasFilters) + assertTrue(QueryOperators.parse("inurl:x").hasFilters) + assertTrue(QueryOperators.parse("-inurl:x").hasFilters) + assertTrue(QueryOperators.parse("filetype:pdf").hasFilters) + assertTrue(QueryOperators.parse("after:2023").hasFilters) + assertTrue(QueryOperators.parse("before:2023").hasFilters) + } + + // --- degenerate quotes ------------------------------------------------------------------------------- + + @Test + fun blankPhraseIsDroppedInsteadOfExcludingEverything() { + // A stray `-"` used to add an empty excluded phrase, and `contains("")` is always true, so a + // single stray character filtered out every result. A stray `"` likewise polluted cleanText. + val negated = QueryOperators.parse("mouse -\"") + assertTrue(negated.excludedPhrases.isEmpty()) + assertEquals("mouse", negated.cleanText) + assertTrue(negated.matches("Any title", "https://example.com/a", "any snippet", null)) + val positive = QueryOperators.parse("mouse \"") + assertTrue(positive.phrases.isEmpty()) + assertEquals("mouse", positive.cleanText) + } +} From e971b3d7f1b1ab2993cdde2c2c7eeea6e93996e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Jul 2026 09:50:23 +0000 Subject: [PATCH 2/2] feat(server): Material 3 served UI, operators help, and security hardening Security: reject the forgeable 'Origin: null' on mutation POSTs (a page setting its own no-referrer policy, or a sandboxed iframe, could forge it and bypass the CSRF guard) and serve Referrer-Policy: same-origin so our own form posts carry a real origin; add Content-Security-Policy, Cache-Control: no-store, and Permissions-Policy on every response; make the network-mode token comparison constant-time and accept the token via Authorization: Bearer or X-SearchMob-Token headers in addition to the ?token= query parameter. UI: restyle the served pages to Material 3 (search-bar elevation and focus states, 16px-radius cards, pill buttons, chip and state-layer hovers with plain fallbacks, theme-derived active-tab colors instead of hardcoded hex) and add a collapsible search-operators cheat sheet under the home search box. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016AYAgNYmBkxsEYcgzSAjgT --- README.md | 5 + .../java/org/searchmob/server/SearchServer.kt | 278 ++++++++++++++---- .../server/SearchServerSecurityTest.kt | 132 ++++++++- .../server/WebUiPersonalizationTest.kt | 22 +- 4 files changed, 377 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 652d5cd..7b1f5a5 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,11 @@ so those engines see no cookies/identifier from you, and keeps anything it store Marginalia, Mwmbl, and Wikipedia** (plus optional bring-your-own **Brave**, **Mojeek**, and **Kagi** API keys), then de-duplicated and re-ranked. The app proxies the requests: upstream engines see no cookies, no referrer, no user/device identifier, and a rotated User-Agent. It **never scrapes Google**. +- **Search operators.** The query syntax you already know: `"exact phrase"`, `-term`, `site:` / + `-site:`, `intitle:`, `inurl:`, `filetype:` (or `ext:`), `before:` / `after:` dates, and `OR`. + Operators the upstream engines understand are forwarded to them; all structural filters are also + enforced on-device over the merged results, so they work consistently across every engine. The + served home page has a collapsible cheat sheet. - **Typo and "similar sounding" tolerance.** A misspelled query surfaces a "Did you mean" suggestion, from the engines' own correction when offered, otherwise from a fully on-device corrector (phonetic + edit-distance over a bundled dictionary, enriched by your own history). No new outbound calls. diff --git a/app/src/main/java/org/searchmob/server/SearchServer.kt b/app/src/main/java/org/searchmob/server/SearchServer.kt index f3f79f0..e8eb15e 100644 --- a/app/src/main/java/org/searchmob/server/SearchServer.kt +++ b/app/src/main/java/org/searchmob/server/SearchServer.kt @@ -105,6 +105,7 @@ import java.net.InetSocketAddress import java.net.ServerSocket import java.net.URI import java.net.URLEncoder +import java.security.MessageDigest import java.security.SecureRandom import java.util.Base64 @@ -214,17 +215,38 @@ fun Application.searchModule( // Run before routing on every request: conservative security headers always, plus the Host // allowlist and the token gate for non-loopback clients. intercept(ApplicationCallPipeline.Plugins) { - call.response.headers.append("Referrer-Policy", "no-referrer", safeOnly = false) + // `same-origin`, not `no-referrer`: a same-origin request (including our OWN served forms) + // still carries its Referer/Origin to us, which `sameOrigin()` needs to tell a genuine + // same-origin POST apart from a cross-site one that forces an opaque `Origin: null` (see + // `sameOrigin()`'s doc comment for the attack `no-referrer` used to enable). A cross-origin + // navigation away from us still sends nothing, same as before, and every result/summary/action + // link additionally carries `rel="noreferrer"` as a second, redundant layer. + call.response.headers.append("Referrer-Policy", "same-origin", safeOnly = false) + call.response.headers.append("Content-Security-Policy", CSP, safeOnly = false) call.response.headers.append("X-Content-Type-Options", "nosniff", safeOnly = false) call.response.headers.append("X-Frame-Options", "DENY", safeOnly = false) + // Queries, titles, snippets, and Settings values must never persist in a browser or + // intermediary cache once the page is gone. + call.response.headers.append("Cache-Control", "no-store", safeOnly = false) + // We never touch the camera/location/microphone; deny every embedder (and ourselves) the asks. + call.response.headers.append( + "Permissions-Policy", + "camera=(), geolocation=(), microphone=()", + safeOnly = false, + ) if (!isLoopbackHost(call.request.origin.remoteHost)) { val host = hostnameOnly(call.request.headers["Host"].orEmpty()) if (host.isNotEmpty() && !hostHeaderAllowed(host, allowedHosts)) { call.respondText("Bad Request: host not allowed", status = HttpStatusCode.BadRequest) return@intercept finish() } - val tokenOk = !accessToken.isNullOrEmpty() && call.request.queryParameters["token"] == accessToken - if (call.request.path() in GATED_PATHS && !tokenOk) { + val presented = + presentedToken( + call.request.queryParameters["token"], + call.request.headers["Authorization"], + call.request.headers["X-SearchMob-Token"], + ) + if (call.request.path() in GATED_PATHS && !tokenMatches(presented, accessToken)) { call.respondText("Forbidden", status = HttpStatusCode.Forbidden) return@intercept finish() } @@ -630,6 +652,60 @@ internal fun isLoopbackHost(host: String): Boolean { /** Query routes gated by the access token for non-loopback clients in network mode. */ private val GATED_PATHS = setOf("/search", "/api/search", "/suggest") +/** + * Content-Security-Policy sent on every response. `kotlinx.html` HTML-escapes every piece of dynamic + * content it renders (query text, titles, snippets, domains, settings values, ...), so the small + * inline theme/reveal `