From 39d508119072c495f1d5d6b862b90679407307fa Mon Sep 17 00:00:00 2001 From: codytseng Date: Thu, 23 Apr 2026 16:32:07 +0800 Subject: [PATCH 01/10] feat: add vouch and report API endpoints with NIP-98 auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new POST endpoints (gated by vouch.enabled config) let users submit signed claims directly instead of through a Nostr event. Vouches act as equal-weight follow edges in the ranking graph, deduped against any existing follow from the same source. Reports apply a trust-weighted penalty to the target's final score: final = raw * (1 - R/(R+F)). POST /vouch and POST /report are mutually exclusive per (source, target): posting one atomically removes the opposite side, so there is no DELETE endpoint — toggling sides is the only way to retract. Submissions from pubkeys with no TrustRank (and not in seed_pubkeys) return 200 but are silently dropped to prevent spam-account inflation. The API now opens the DB in read-write mode; WAL + writeMu already coordinate it with the crawler. Migration v4 adds vouches and reports tables. 26 new tests cover NIP-98 middleware, repository mutex/toggle behaviour, and ranking integration (vouch promotes unfollowed users, reports decay scores, untrusted reporters are ignored, follow+vouch edges dedupe). Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 14 +- cmd/api/main.go | 23 +- config.example.yaml | 10 + config/config.go | 11 + internal/api/handler/handler.go | 8 +- internal/api/handler/vouch.go | 89 +++++++ internal/api/middleware/cors.go | 4 +- internal/api/middleware/nip98.go | 177 ++++++++++++++ internal/api/middleware/nip98_test.go | 322 ++++++++++++++++++++++++++ internal/models/user.go | 14 ++ internal/ranking/calculator.go | 81 ++++++- internal/ranking/calculator_test.go | 218 +++++++++++++++++ internal/repository/migration.go | 35 +++ internal/repository/vouch.go | 138 +++++++++++ internal/repository/vouch_test.go | 226 ++++++++++++++++++ 15 files changed, 1358 insertions(+), 12 deletions(-) create mode 100644 internal/api/handler/vouch.go create mode 100644 internal/api/middleware/nip98.go create mode 100644 internal/api/middleware/nip98_test.go create mode 100644 internal/ranking/calculator_test.go create mode 100644 internal/repository/vouch.go create mode 100644 internal/repository/vouch_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 1cd14eb..690c88c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,8 +34,8 @@ docker compose up --build The repository supports two modes via `repository.New(path, mode)`: -- `ModeReadWrite`: For crawler - writes connections and scores -- `ModeReadOnly`: For API - optimized for concurrent reads with `PRAGMA query_only = ON` +- `ModeReadWrite`: For crawler and API (API opens read-write so the NIP-98 endpoints can insert into `vouches`/`reports`). WAL mode + `writeMu` coordinate the two processes. +- `ModeReadOnly`: Reserved for read-only tools; sets `PRAGMA query_only = ON` and a 10-connection pool. ### Key Packages @@ -82,3 +82,13 @@ Copy `config.example.yaml` to `config.yaml` (and `docker-compose.example.yml` to - `crawler.request_interval_ms`: Milliseconds between requests per relay (default: 500) - `crawler.num_contact_processors`: Number of contact event processors (default: 4) - `crawler.num_profile_processors`: Number of profile event processors (default: 4) +- `vouch.enabled`: Enable `POST /vouch` and `POST /report` endpoints (default: false) + +### Vouch & Report Endpoints (when `vouch.enabled: true`) + +Two NIP-98 authenticated endpoints let users contribute to the graph without following: + +- `POST /vouch` with body `{"target": ""}` — registers source→target as a vouch edge in the ranking graph (equal weight to a follow, deduped against follows from the same source). +- `POST /report` with body `{"target": ""}` — reports the target. Reports apply a trust-weighted penalty to the target's final score: `final = raw * (1 - R/(R+F))` where R is the sum of reporter trust_scores and F is the sum of follower/voucher trust_scores. + +No DELETE endpoints — the two actions are mutually exclusive per (source, target). Posting a report implicitly deletes any prior vouch for the same target, and vice versa. Submissions from pubkeys with no TrustRank and not in `seed_pubkeys` return 200 but are silently dropped (prevents spam inflation). Schema stored in `vouches` and `reports` tables (migration v4). diff --git a/cmd/api/main.go b/cmd/api/main.go index 5d3086f..4d1437a 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -27,20 +27,24 @@ func main() { log.Fatalf("Failed to load config: %v", err) } - // Initialize repository in read-only mode - repo, err := repository.New(cfg.Database, repository.ModeReadOnly) + // Initialize repository. Even if vouch endpoints are off, we open in + // read-write mode because it's the single place we centralise migrations + // and keep things consistent with the crawler. WAL + writeMu already + // coordinate concurrent writers across processes. + repo, err := repository.New(cfg.Database, repository.ModeReadWrite) if err != nil { log.Fatalf("Failed to initialize database: %v", err) } defer repo.Close() - log.Println("[API] Database initialized successfully (read-only mode)") + log.Println("[API] Database initialized successfully (read-write mode)") // Initialize cache apiCache := cache.New(10*time.Minute, 10*time.Minute) - // Initialize handler with search config - h := handler.New(repo, apiCache, &cfg.Search) + // Initialize handler with search config and seed pubkeys (seeds always + // qualify for vouch/report submissions regardless of trust_score). + h := handler.New(repo, apiCache, &cfg.Search, cfg.SeedPubkeys) // Setup static file system staticFS, err := fs.Sub(staticFiles, "static") @@ -87,6 +91,15 @@ func main() { http.HandleFunc("/users/", middleware.CORS(h.User)) http.HandleFunc("/search", middleware.CORS(h.Search)) + // Vouch / report endpoints (NIP-98 authenticated). + // When disabled in config, no route is registered and requests fall + // through to the SPA catch-all handler below. + if cfg.Vouch.Enabled { + http.HandleFunc("/vouch", middleware.CORS(middleware.NIP98Auth(h.Vouch))) + http.HandleFunc("/report", middleware.CORS(middleware.NIP98Auth(h.Report))) + log.Println("[API] Vouch/report endpoints enabled") + } + // Serve static assets (js, css, images, etc.) with long cache (1 year for hashed assets) http.HandleFunc("/assets/", func(w http.ResponseWriter, r *http.Request) { path := strings.TrimPrefix(r.URL.Path, "/") diff --git a/config.example.yaml b/config.example.yaml index c774fa3..0de5277 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -36,6 +36,16 @@ crawler: # Number of profile event processors (default: 4) num_profile_processors: 4 +# Vouch / report API endpoints (NIP-98 authenticated). +# When enabled, POST /vouch and POST /report accept signed requests from users +# to vouch for or report another pubkey. Vouches act as equal-weight follow +# edges in the ranking graph (no need to actually follow). Reports shrink the +# target's final score proportional to the trust-weighted report ratio. +# Submissions from pubkeys with no TrustRank (and not in seed_pubkeys) are +# silently ignored. +vouch: + enabled: false + # Default nostr relays to connect to, used for searching uesrs' relay lists relays: - wss://relay.damus.io/ diff --git a/config/config.go b/config/config.go index 9af4d24..38b98f0 100644 --- a/config/config.go +++ b/config/config.go @@ -28,6 +28,12 @@ type CrawlerConfig struct { NumProfileProcessors int `yaml:"num_profile_processors"` // Number of profile event processors (default: 4) } +// VouchConfig enables the /vouch and /report API endpoints. +// Submissions are authenticated via NIP-98. +type VouchConfig struct { + Enabled bool `yaml:"enabled"` // Default: false. When false, the endpoints return 404. +} + // Config represents the application configuration type Config struct { Relays []string `yaml:"relays"` @@ -38,6 +44,7 @@ type Config struct { Search SearchConfig `yaml:"search"` Ranking RankingConfig `yaml:"ranking"` Crawler CrawlerConfig `yaml:"crawler"` + Vouch VouchConfig `yaml:"vouch"` } // Load reads and parses the configuration file @@ -62,6 +69,9 @@ func Load(path string) (*Config, error) { NumContactProcessors: 4, NumProfileProcessors: 4, }, + Vouch: VouchConfig{ + Enabled: false, + }, } if err := yaml.Unmarshal(data, &cfg); err != nil { return nil, err @@ -80,6 +90,7 @@ func Load(path string) (*Config, error) { log.Printf("[CONFIG] - Ranking weights: TrustRank=%.2f, PageRank=%.2f", cfg.Ranking.TrustRankWeight, cfg.Ranking.PageRankWeight) log.Printf("[CONFIG] - Crawler: batch_size=%d, request_interval=%dms, contact_processors=%d, profile_processors=%d", cfg.Crawler.BatchSize, cfg.Crawler.RequestIntervalMs, cfg.Crawler.NumContactProcessors, cfg.Crawler.NumProfileProcessors) + log.Printf("[CONFIG] - Vouch endpoints enabled: %t", cfg.Vouch.Enabled) return &cfg, nil } diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index 3dd58da..77980a4 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -22,14 +22,20 @@ type Handler struct { repo *repository.Repository cache *cache.Cache searchConfig *config.SearchConfig + seedSet map[string]struct{} } // New creates a new Handler instance -func New(repo *repository.Repository, cache *cache.Cache, searchConfig *config.SearchConfig) *Handler { +func New(repo *repository.Repository, cache *cache.Cache, searchConfig *config.SearchConfig, seedPubkeys []string) *Handler { + seedSet := make(map[string]struct{}, len(seedPubkeys)) + for _, pk := range seedPubkeys { + seedSet[pk] = struct{}{} + } return &Handler{ repo: repo, cache: cache, searchConfig: searchConfig, + seedSet: seedSet, } } diff --git a/internal/api/handler/vouch.go b/internal/api/handler/vouch.go new file mode 100644 index 0000000..b8ded87 --- /dev/null +++ b/internal/api/handler/vouch.go @@ -0,0 +1,89 @@ +package handler + +import ( + "encoding/json" + "log" + "net/http" +) + +// relationRequest is the body shape for POST /vouch and POST /report. +type relationRequest struct { + Target string `json:"target"` +} + +// Vouch handles POST /vouch. Expects a NIP-98 authenticated request. +// authorPubkey is injected by the NIP98Auth middleware. +func (h *Handler) Vouch(w http.ResponseWriter, r *http.Request, authorPubkey string) { + h.handleRelation(w, r, authorPubkey, relationKindVouch) +} + +// Report handles POST /report. Expects a NIP-98 authenticated request. +func (h *Handler) Report(w http.ResponseWriter, r *http.Request, authorPubkey string) { + h.handleRelation(w, r, authorPubkey, relationKindReport) +} + +type relationKind int + +const ( + relationKindVouch relationKind = iota + relationKindReport +) + +func (h *Handler) handleRelation(w http.ResponseWriter, r *http.Request, authorPubkey string, kind relationKind) { + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "Method not allowed") + return + } + + var req relationRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "Invalid JSON body") + return + } + target, valid := normalizePubkey(req.Target) + if !valid { + writeError(w, http.StatusBadRequest, "Invalid target pubkey") + return + } + if target == authorPubkey { + writeError(w, http.StatusBadRequest, "Cannot vouch for or report yourself") + return + } + + // Silent-ignore admission rule: author must be a seed or have earned + // TrustRank > 0 in the last ranking round. Respond 200 regardless so + // the client cannot distinguish "not admitted" from "successfully stored". + if !h.authorQualifies(authorPubkey) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return + } + + var err error + switch kind { + case relationKindVouch: + err = h.repo.SetVouch(authorPubkey, target) + case relationKindReport: + err = h.repo.SetReport(authorPubkey, target) + } + if err != nil { + log.Printf("[API] Error setting relation (author=%s target=%s kind=%d): %v", authorPubkey, target, kind, err) + writeError(w, http.StatusInternalServerError, "Failed to store relation") + return + } + + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +// authorQualifies returns true if the author is an explicit seed or has a +// positive last-computed TrustRank score. +func (h *Handler) authorQualifies(pubkey string) bool { + if _, ok := h.seedSet[pubkey]; ok { + return true + } + score, err := h.repo.GetTrustScore(pubkey) + if err != nil { + log.Printf("[API] Error reading trust_score for %s: %v", pubkey, err) + return false + } + return score > 0 +} diff --git a/internal/api/middleware/cors.go b/internal/api/middleware/cors.go index 6ac4c76..38fac4f 100644 --- a/internal/api/middleware/cors.go +++ b/internal/api/middleware/cors.go @@ -6,8 +6,8 @@ import "net/http" func CORS(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") if r.Method == http.MethodOptions { w.WriteHeader(http.StatusOK) diff --git a/internal/api/middleware/nip98.go b/internal/api/middleware/nip98.go new file mode 100644 index 0000000..ef3e832 --- /dev/null +++ b/internal/api/middleware/nip98.go @@ -0,0 +1,177 @@ +package middleware + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "strings" + "time" + + "github.com/nbd-wtf/go-nostr" +) + +// nip98AuthedHandler is invoked after successful NIP-98 validation, with the +// authenticated author pubkey passed in. +type nip98AuthedHandler func(w http.ResponseWriter, r *http.Request, authorPubkey string) + +const nip98Kind = 27235 +const nip98TimeWindow = 60 // seconds + +// NIP98Auth wraps a handler with NIP-98 HTTP Auth validation. +// On success, the authenticated author pubkey (hex) is passed to next. +// On failure, responds 401 with a JSON error and does not invoke next. +func NIP98Auth(next nip98AuthedHandler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Read and replay body so downstream handler can decode it. + // (The payload tag is validated against the exact bytes we read here.) + var body []byte + if r.Body != nil { + b, err := io.ReadAll(r.Body) + if err != nil { + writeJSONError(w, http.StatusBadRequest, "failed to read body") + return + } + body = b + r.Body = io.NopCloser(bytes.NewReader(body)) + } + + ev, err := parseNIP98Header(r.Header.Get("Authorization")) + if err != nil { + writeJSONError(w, http.StatusUnauthorized, err.Error()) + return + } + + if err := validateNIP98Event(ev, r, body); err != nil { + writeJSONError(w, http.StatusUnauthorized, err.Error()) + return + } + + ok, err := ev.CheckSignature() + if err != nil || !ok { + writeJSONError(w, http.StatusUnauthorized, "invalid signature") + return + } + + next(w, r, ev.PubKey) + } +} + +func parseNIP98Header(h string) (*nostr.Event, error) { + if h == "" { + return nil, errNIP98("missing Authorization header") + } + const prefix = "Nostr " + if !strings.HasPrefix(h, prefix) { + return nil, errNIP98("Authorization must start with 'Nostr '") + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(h, prefix)) + if err != nil { + // Also accept URL-safe / unpadded base64 variants for resilience. + if raw2, err2 := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(h, prefix)); err2 == nil { + raw = raw2 + } else { + return nil, errNIP98("bad base64 in Authorization") + } + } + var ev nostr.Event + if err := json.Unmarshal(raw, &ev); err != nil { + return nil, errNIP98("bad event JSON") + } + return &ev, nil +} + +func validateNIP98Event(ev *nostr.Event, r *http.Request, body []byte) error { + if ev.Kind != nip98Kind { + return errNIP98("wrong kind") + } + now := time.Now().Unix() + created := int64(ev.CreatedAt) + delta := now - created + if delta < 0 { + delta = -delta + } + if delta > nip98TimeWindow { + return errNIP98("timestamp out of window") + } + + uTag := tagValue(ev.Tags, "u") + methodTag := tagValue(ev.Tags, "method") + if uTag == "" || methodTag == "" { + return errNIP98("missing u or method tag") + } + + if !strings.EqualFold(methodTag, r.Method) { + return errNIP98("method tag mismatch") + } + if !urlMatches(uTag, r) { + return errNIP98("u tag mismatch") + } + + // Payload tag is required for POST/PUT/PATCH with a non-empty body, + // optional otherwise (per NIP-98). + if len(body) > 0 && requiresPayloadTag(r.Method) { + payloadTag := tagValue(ev.Tags, "payload") + if payloadTag == "" { + return errNIP98("missing payload tag") + } + sum := sha256.Sum256(body) + if !strings.EqualFold(payloadTag, hex.EncodeToString(sum[:])) { + return errNIP98("payload tag mismatch") + } + } + return nil +} + +func requiresPayloadTag(method string) bool { + switch strings.ToUpper(method) { + case http.MethodPost, http.MethodPut, http.MethodPatch: + return true + } + return false +} + +// urlMatches compares the NIP-98 u tag against the actual request URL. +// Supports reverse-proxy deployments via X-Forwarded-Proto / X-Forwarded-Host. +func urlMatches(uTag string, r *http.Request) bool { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" { + scheme = proto + } + host := r.Host + if xfh := r.Header.Get("X-Forwarded-Host"); xfh != "" { + host = xfh + } + reconstructed := scheme + "://" + host + r.URL.RequestURI() + return uTag == reconstructed +} + +func tagValue(tags nostr.Tags, name string) string { + for _, t := range tags { + if len(t) >= 2 && t[0] == name { + return t[1] + } + } + return "" +} + +// errNIP98 wraps a sentinel error to keep the reasons user-visible yet terse. +type nip98Err string + +func (e nip98Err) Error() string { return string(e) } +func errNIP98(msg string) error { return nip98Err(msg) } + +func writeJSONError(w http.ResponseWriter, status int, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]string{ + "error": http.StatusText(status), + "message": message, + }) +} diff --git a/internal/api/middleware/nip98_test.go b/internal/api/middleware/nip98_test.go new file mode 100644 index 0000000..4d1e5de --- /dev/null +++ b/internal/api/middleware/nip98_test.go @@ -0,0 +1,322 @@ +package middleware + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/nbd-wtf/go-nostr" +) + +// signAuthEvent builds and signs a NIP-98 auth event for the given request +// parameters. Test helper. +func signAuthEvent(t *testing.T, sk, method, url string, body []byte, createdAt int64) string { + t.Helper() + tags := nostr.Tags{ + nostr.Tag{"u", url}, + nostr.Tag{"method", method}, + } + if len(body) > 0 { + sum := sha256.Sum256(body) + tags = append(tags, nostr.Tag{"payload", hex.EncodeToString(sum[:])}) + } + ev := nostr.Event{ + Kind: nip98Kind, + CreatedAt: nostr.Timestamp(createdAt), + Tags: tags, + Content: "", + } + if err := ev.Sign(sk); err != nil { + t.Fatalf("sign failed: %v", err) + } + raw, err := json.Marshal(ev) + if err != nil { + t.Fatalf("marshal failed: %v", err) + } + return "Nostr " + base64.StdEncoding.EncodeToString(raw) +} + +func mustGenKey(t *testing.T) (sk, pk string) { + t.Helper() + sk = nostr.GeneratePrivateKey() + pk, err := nostr.GetPublicKey(sk) + if err != nil { + t.Fatalf("derive pubkey: %v", err) + } + return +} + +// runRequest fires a request against a handler wrapped with NIP98Auth, using +// httptest.NewServer so r.Host/Scheme are realistic for the u-tag reconstruct. +func runRequest(t *testing.T, path string, method string, body []byte, authHeader string) (int, string, string) { + t.Helper() + var gotPubkey string + h := NIP98Auth(func(w http.ResponseWriter, r *http.Request, pubkey string) { + gotPubkey = pubkey + // Echo back body to verify it's been replayed correctly. + b, _ := io.ReadAll(r.Body) + _, _ = w.Write(b) + }) + + srv := httptest.NewServer(http.HandlerFunc(h)) + defer srv.Close() + + var rdr io.Reader + if body != nil { + rdr = bytes.NewReader(body) + } + req, err := http.NewRequest(method, srv.URL+path, rdr) + if err != nil { + t.Fatal(err) + } + if authHeader != "" { + req.Header.Set("Authorization", authHeader) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + respBody, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(respBody), gotPubkey +} + +// buildURL returns the URL that NIP98Auth will reconstruct for matching. +func buildURL(serverURL, path string) string { + return serverURL + path +} + +func TestNIP98_HappyPath(t *testing.T) { + sk, pk := mustGenKey(t) + body := []byte(`{"target":"xyz"}`) + + // First set up server so we know its URL for the u tag. + var authHeader string + var gotPubkey string + mux := http.NewServeMux() + mux.Handle("/vouch", NIP98Auth(func(w http.ResponseWriter, r *http.Request, pubkey string) { + gotPubkey = pubkey + b, _ := io.ReadAll(r.Body) + _, _ = w.Write(b) + })) + srv := httptest.NewServer(mux) + defer srv.Close() + + authHeader = signAuthEvent(t, sk, "POST", srv.URL+"/vouch", body, time.Now().Unix()) + + req, _ := http.NewRequest("POST", srv.URL+"/vouch", bytes.NewReader(body)) + req.Header.Set("Authorization", authHeader) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + respBody, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != 200 { + t.Fatalf("expected 200, got %d: %s", resp.StatusCode, respBody) + } + if gotPubkey != pk { + t.Fatalf("expected pubkey %q, got %q", pk, gotPubkey) + } + if string(respBody) != string(body) { + t.Fatalf("body was not replayed to handler; got %q", respBody) + } +} + +func TestNIP98_MissingHeader(t *testing.T) { + status, _, _ := runRequest(t, "/vouch", "POST", []byte(`{}`), "") + if status != 401 { + t.Fatalf("expected 401, got %d", status) + } +} + +func TestNIP98_WrongScheme(t *testing.T) { + status, _, _ := runRequest(t, "/vouch", "POST", []byte(`{}`), "Basic xyz") + if status != 401 { + t.Fatalf("expected 401, got %d", status) + } +} + +func TestNIP98_BadBase64(t *testing.T) { + status, _, _ := runRequest(t, "/vouch", "POST", []byte(`{}`), "Nostr !!!not-base64!!!") + if status != 401 { + t.Fatalf("expected 401, got %d", status) + } +} + +func TestNIP98_WrongKind(t *testing.T) { + sk, _ := mustGenKey(t) + // Build server to get URL. + srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) + defer srv.Close() + + ev := nostr.Event{ + Kind: 1, // wrong kind + CreatedAt: nostr.Timestamp(time.Now().Unix()), + Tags: nostr.Tags{{"u", srv.URL + "/vouch"}, {"method", "POST"}}, + } + if err := ev.Sign(sk); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(ev) + auth := "Nostr " + base64.StdEncoding.EncodeToString(raw) + + req, _ := http.NewRequest("POST", srv.URL+"/vouch", bytes.NewReader(nil)) + req.Header.Set("Authorization", auth) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 401 { + t.Fatalf("expected 401, got %d", resp.StatusCode) + } +} + +func TestNIP98_TimestampTooOld(t *testing.T) { + sk, _ := mustGenKey(t) + srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) + defer srv.Close() + + auth := signAuthEvent(t, sk, "POST", srv.URL+"/vouch", nil, time.Now().Unix()-120) + + req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) + req.Header.Set("Authorization", auth) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 401 { + t.Fatalf("expected 401 for old timestamp, got %d", resp.StatusCode) + } +} + +func TestNIP98_MethodMismatch(t *testing.T) { + sk, _ := mustGenKey(t) + srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) + defer srv.Close() + + // Sign with GET but send POST. + auth := signAuthEvent(t, sk, "GET", srv.URL+"/vouch", nil, time.Now().Unix()) + req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) + req.Header.Set("Authorization", auth) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 401 { + t.Fatalf("expected 401 for method mismatch, got %d", resp.StatusCode) + } +} + +func TestNIP98_URLMismatch(t *testing.T) { + sk, _ := mustGenKey(t) + srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) + defer srv.Close() + + auth := signAuthEvent(t, sk, "POST", "https://other.example/vouch", nil, time.Now().Unix()) + req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) + req.Header.Set("Authorization", auth) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 401 { + t.Fatalf("expected 401 for URL mismatch, got %d", resp.StatusCode) + } +} + +func TestNIP98_PayloadMismatch(t *testing.T) { + sk, _ := mustGenKey(t) + srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) + defer srv.Close() + + signedBody := []byte(`{"target":"a"}`) + actualBody := []byte(`{"target":"b"}`) + + auth := signAuthEvent(t, sk, "POST", srv.URL+"/vouch", signedBody, time.Now().Unix()) + req, _ := http.NewRequest("POST", srv.URL+"/vouch", bytes.NewReader(actualBody)) + req.Header.Set("Authorization", auth) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 401 { + t.Fatalf("expected 401 for payload mismatch, got %d", resp.StatusCode) + } +} + +func TestNIP98_InvalidSignature(t *testing.T) { + sk, _ := mustGenKey(t) + srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) + defer srv.Close() + + auth := signAuthEvent(t, sk, "POST", srv.URL+"/vouch", nil, time.Now().Unix()) + + // Corrupt one byte of the base64-encoded signature at the tail end. + b := []byte(auth) + // Flip a character a few positions from the end (before padding). + idx := len(b) - 10 + if b[idx] == 'A' { + b[idx] = 'B' + } else { + b[idx] = 'A' + } + corrupted := string(b) + + req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) + req.Header.Set("Authorization", corrupted) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + // Could be 401 (sig invalid) or 401 (base64 invalid) — both fine. + if resp.StatusCode != 401 { + t.Fatalf("expected 401 for corrupted auth, got %d", resp.StatusCode) + } +} + +func TestNIP98_URLReconstructWithForwardedProto(t *testing.T) { + sk, pk := mustGenKey(t) + var gotPubkey string + h := NIP98Auth(func(w http.ResponseWriter, r *http.Request, pubkey string) { + gotPubkey = pubkey + }) + srv := httptest.NewServer(http.HandlerFunc(h)) + defer srv.Close() + + // The u tag claims https, but actual srv is http. X-Forwarded-Proto=https + // should make the reconstruction succeed. + host := strings.TrimPrefix(srv.URL, "http://") + auth := signAuthEvent(t, sk, "POST", "https://"+host+"/vouch", nil, time.Now().Unix()) + + req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) + req.Header.Set("Authorization", auth) + req.Header.Set("X-Forwarded-Proto", "https") + + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("expected 200 with X-Forwarded-Proto, got %d", resp.StatusCode) + } + if gotPubkey != pk { + t.Fatalf("expected pk %q, got %q", pk, gotPubkey) + } +} diff --git a/internal/models/user.go b/internal/models/user.go index 2e6611c..791db54 100644 --- a/internal/models/user.go +++ b/internal/models/user.go @@ -24,3 +24,17 @@ type Connection struct { Source string Target string } + +// Vouch represents a user's explicit endorsement of another user, +// authenticated via NIP-98 and submitted through the API. +type Vouch struct { + Source string + Target string +} + +// ReportAggregate summarises reports against a single target pubkey. +// Only reporters with trust_score > 0 contribute. +type ReportAggregate struct { + NumReporters int + TotalReporterTrust float64 +} diff --git a/internal/ranking/calculator.go b/internal/ranking/calculator.go index ce1a98c..580ad0c 100644 --- a/internal/ranking/calculator.go +++ b/internal/ranking/calculator.go @@ -59,12 +59,22 @@ func (c *Calculator) Calculate() error { cutoffTime := time.Now().UTC().AddDate(0, 0, -30) + // edgeSet dedupes (source, target) pairs across follow and vouch edges so + // a user who both follows and vouches for the same target contributes a + // single graph edge (not double flow). + edgeSet := make(map[int64]bool) + encodeEdge := func(s, t int32) int64 { return int64(s)<<32 | int64(uint32(t)) } + err := c.repo.StreamConnectionsInTx(func(conn models.Connection) error { sourceID := getID(conn.Source) targetID := getID(conn.Target) if sourceID != targetID { - edges = append(edges, edge{source: sourceID, target: targetID}) + key := encodeEdge(sourceID, targetID) + if !edgeSet[key] { + edgeSet[key] = true + edges = append(edges, edge{source: sourceID, target: targetID}) + } } connectionCount++ return nil @@ -74,13 +84,47 @@ func (c *Calculator) Calculate() error { return err } + // Vouch edges: admit only those from pubkeys with positive last-round + // TrustRank (seeds always admitted so the feature works on first run when + // no one has trust_score written yet). + vouchAdmitted := 0 + qualifying, qErr := c.repo.GetPubkeysWithPositiveTrust() + if qErr != nil { + log.Printf(" [WARN] Failed to load qualifying pubkeys for vouch admission: %v", qErr) + qualifying = make(map[string]struct{}) + } + for _, s := range c.seedPubkeys { + qualifying[s] = struct{}{} + } + if err := c.repo.StreamVouches(func(v models.Vouch) error { + if _, ok := qualifying[v.Source]; !ok { + return nil + } + sourceID := getID(v.Source) + targetID := getID(v.Target) + if sourceID == targetID { + return nil + } + key := encodeEdge(sourceID, targetID) + if edgeSet[key] { + return nil + } + edgeSet[key] = true + edges = append(edges, edge{source: sourceID, target: targetID}) + vouchAdmitted++ + return nil + }); err != nil { + return err + } + log.Printf(" [INFO] Vouch edges admitted: %d", vouchAdmitted) + numNodes := len(idToPubkey) if numNodes == 0 { log.Println(" [WARN] Graph is empty, skipping calculation") return nil } - log.Printf(" [INFO] Processing %d nodes, %d connections", numNodes, connectionCount) + log.Printf(" [INFO] Processing %d nodes, %d connections (+ %d vouches)", numNodes, connectionCount, vouchAdmitted) // Build seed node set for TrustRank seedSet := make(map[int32]bool) @@ -126,6 +170,39 @@ func (c *Calculator) Calculate() error { scores[i] = c.trustRankWeight*trustScores[i] + c.pageRankWeight*pageScores[i] } + // Apply trust-weighted report penalty: scale each target's scores by + // (1 - penalty) where penalty = R / (R + F + ε). R is the sum of reporter + // trust_scores (only reporters with trust_score > 0 count); F is the sum + // of follower/voucher trust_scores (in-edges). Rank is computed after + // penalty so penalized accounts drop in the final ordering. + if reports, err := c.repo.GetTrustWeightedReports(); err != nil { + log.Printf(" [WARN] Failed to load reports for penalty: %v", err) + } else if len(reports) > 0 { + fTrust := make([]float64, numNodes) + for i := range numNodes { + for _, j := range inLinks[i] { + fTrust[i] += trustScores[j] + } + } + penalized := 0 + for i := range numNodes { + agg, ok := reports[idToPubkey[i]] + if !ok || agg.TotalReporterTrust <= 0 { + continue + } + penalty := agg.TotalReporterTrust / (agg.TotalReporterTrust + fTrust[i] + 1e-9) + if penalty > 1 { + penalty = 1 + } + factor := 1 - penalty + scores[i] *= factor + trustScores[i] *= factor + pageScores[i] *= factor + penalized++ + } + log.Printf(" [INFO] Applied report penalty to %d pubkeys", penalized) + } + // Calculate ranks based on scores rankList := make([]scoreWithID, numNodes) for i := range numNodes { diff --git a/internal/ranking/calculator_test.go b/internal/ranking/calculator_test.go new file mode 100644 index 0000000..870814c --- /dev/null +++ b/internal/ranking/calculator_test.go @@ -0,0 +1,218 @@ +package ranking + +import ( + "path/filepath" + "testing" + "time" + + "fayan/internal/repository" +) + +func newTestRepo(t *testing.T) *repository.Repository { + t.Helper() + dir := t.TempDir() + repo, err := repository.New(filepath.Join(dir, "test.db"), repository.ModeReadWrite) + if err != nil { + t.Fatalf("open repo: %v", err) + } + t.Cleanup(func() { _ = repo.Close() }) + return repo +} + +func insertFollow(t *testing.T, repo *repository.Repository, source, target string) { + t.Helper() + err := repo.BatchUpsertPubkeysAndConnections( + []string{source, target}, + []repository.Connection{{Source: source, Target: target}}, + ) + if err != nil { + t.Fatalf("insert follow %s->%s: %v", source, target, err) + } +} + +func getUser(t *testing.T, repo *repository.Repository, pubkey string) (rank *int, followers, following int, score float64) { + t.Helper() + info, err := repo.GetUserByPubkey(pubkey) + if err != nil { + t.Fatalf("get user %s: %v", pubkey, err) + } + if info == nil { + return nil, 0, 0, 0 + } + return info.Rank, info.Followers, info.Following, info.Score +} + +// TestVouchPromotesUnfollowedUser verifies the core value proposition: after +// two ranking cycles (first establishes seed trust, second admits vouch edge), +// a newbie with no followers but one vouch from a seed receives a rank. +func TestVouchPromotesUnfollowedUser(t *testing.T) { + repo := newTestRepo(t) + + seeds := []string{"seed1", "seed2", "seed3"} + // Make seeds mutually follow so they have outgoing edges; TrustRank + // requires at least some graph structure to propagate. + insertFollow(t, repo, "seed1", "seed2") + insertFollow(t, repo, "seed2", "seed3") + insertFollow(t, repo, "seed3", "seed1") + + calc := NewCalculator(repo, seeds, 0.7, 0.3) + + // First pass: seeds acquire trust_score > 0. + if err := calc.Calculate(); err != nil { + t.Fatal(err) + } + + // Seed vouches for a newbie nobody follows. + if err := repo.SetVouch("seed1", "newbie"); err != nil { + t.Fatal(err) + } + + // Second pass: vouch edge admitted because seed1 has trust_score > 0. + if err := calc.Calculate(); err != nil { + t.Fatal(err) + } + + rank, followers, _, score := getUser(t, repo, "newbie") + if rank == nil { + t.Fatalf("newbie should have a rank after vouch") + } + if followers != 1 { + t.Fatalf("newbie should have 1 follower (the vouch edge), got %d", followers) + } + if score <= 0 { + t.Fatalf("newbie score should be positive, got %v", score) + } +} + +// TestVouchAndFollowDedupe verifies A following AND vouching for B only +// produces one edge (A's following count = 1, not 2). +func TestVouchAndFollowDedupe(t *testing.T) { + repo := newTestRepo(t) + + insertFollow(t, repo, "a", "b") + if err := repo.SetVouch("a", "b"); err != nil { + t.Fatal(err) + } + // Give A trust so vouch edge would be admitted. + repo.BatchUpdatePubkeys([]repository.PubkeyUpdate{{ + Pubkey: "a", + Score: 0.5, + Rank: 1, + TrustScore: 0.5, + PageScore: 0.5, + Followers: 0, + Following: 1, + }}) + + calc := NewCalculator(repo, []string{"a"}, 0.7, 0.3) + if err := calc.Calculate(); err != nil { + t.Fatal(err) + } + + _, _, following, _ := getUser(t, repo, "a") + if following != 1 { + t.Fatalf("A should have following=1 after dedup, got %d", following) + } + _, followers, _, _ := getUser(t, repo, "b") + if followers != 1 { + t.Fatalf("B should have followers=1 after dedup, got %d", followers) + } +} + +// TestReportDecaysScore verifies a well-connected pubkey loses score when +// reported by multiple trusted accounts. +func TestReportDecaysScore(t *testing.T) { + repo := newTestRepo(t) + + // Build a graph where X has several followers (seeds), so X's score starts high. + seeds := []string{"seed1", "seed2", "seed3"} + insertFollow(t, repo, "seed1", "seed2") + insertFollow(t, repo, "seed2", "seed3") + insertFollow(t, repo, "seed3", "seed1") + insertFollow(t, repo, "seed1", "x") + insertFollow(t, repo, "seed2", "x") + insertFollow(t, repo, "seed3", "x") + + calc := NewCalculator(repo, seeds, 0.7, 0.3) + if err := calc.Calculate(); err != nil { + t.Fatal(err) + } + + _, _, _, scoreBefore := getUser(t, repo, "x") + if scoreBefore <= 0 { + t.Fatalf("setup should give X a positive score, got %v", scoreBefore) + } + + // All three seeds report X. + for _, s := range seeds { + if err := repo.SetReport(s, "x"); err != nil { + t.Fatal(err) + } + } + + if err := calc.Calculate(); err != nil { + t.Fatal(err) + } + + _, _, _, scoreAfter := getUser(t, repo, "x") + if scoreAfter >= scoreBefore { + t.Fatalf("score should decay after reports: before=%v after=%v", scoreBefore, scoreAfter) + } + // All followers also report → R == F → penalty = 0.5 → score halves. + // Allow a small tolerance for rounding. + if scoreAfter > scoreBefore*0.55 { + t.Fatalf("expected penalty near 0.5 (R==F), score dropped only from %v to %v", scoreBefore, scoreAfter) + } +} + +// TestReportWithNoTrustIgnored verifies reports from untrusted accounts have +// no effect on the reported user's score. +func TestReportWithNoTrustIgnored(t *testing.T) { + repo := newTestRepo(t) + + seeds := []string{"seed1", "seed2"} + insertFollow(t, repo, "seed1", "seed2") + insertFollow(t, repo, "seed2", "seed1") + insertFollow(t, repo, "seed1", "target") + insertFollow(t, repo, "seed2", "target") + + // Add an untrusted account (no inbound edges, no trust). + if _, err := repo.DB().Exec( + `INSERT INTO pubkeys (pubkey, trust_score, created_at, updated_at) VALUES (?, 0, ?, ?);`, + "troll", time.Now(), time.Now(), + ); err != nil { + t.Fatal(err) + } + if err := repo.SetReport("troll", "target"); err != nil { + t.Fatal(err) + } + + calc := NewCalculator(repo, seeds, 0.7, 0.3) + if err := calc.Calculate(); err != nil { + t.Fatal(err) + } + _, _, _, scoreWithTrollReport := getUser(t, repo, "target") + + // Remove the troll report, recompute. + if _, err := repo.DB().Exec( + `DELETE FROM reports WHERE source_pubkey = ? AND target_pubkey = ?;`, + "troll", "target", + ); err != nil { + t.Fatal(err) + } + if err := calc.Calculate(); err != nil { + t.Fatal(err) + } + _, _, _, scoreWithoutReport := getUser(t, repo, "target") + + if absDiff(scoreWithTrollReport, scoreWithoutReport) > 1e-9 { + t.Fatalf("untrusted report should not change score: with=%v without=%v", scoreWithTrollReport, scoreWithoutReport) + } +} + +func absDiff(a, b float64) float64 { + if a > b { + return a - b + } + return b - a +} diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 24ab0a1..3dc98d8 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -95,6 +95,41 @@ var migrations = []Migration{ return nil }, }, + { + Version: 4, + Name: "add_vouches_and_reports", + Up: func(db *sql.DB) error { + vouchesTable := ` + CREATE TABLE IF NOT EXISTS vouches ( + source_pubkey TEXT NOT NULL, + target_pubkey TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY (source_pubkey, target_pubkey) + );` + + reportsTable := ` + CREATE TABLE IF NOT EXISTS reports ( + source_pubkey TEXT NOT NULL, + target_pubkey TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY (source_pubkey, target_pubkey) + );` + + if _, err := db.Exec(vouchesTable); err != nil { + return fmt.Errorf("failed to create vouches table: %w", err) + } + if _, err := db.Exec(reportsTable); err != nil { + return fmt.Errorf("failed to create reports table: %w", err) + } + if _, err := db.Exec("CREATE INDEX IF NOT EXISTS idx_vouches_target ON vouches(target_pubkey);"); err != nil { + return fmt.Errorf("failed to create idx_vouches_target: %w", err) + } + if _, err := db.Exec("CREATE INDEX IF NOT EXISTS idx_reports_target ON reports(target_pubkey);"); err != nil { + return fmt.Errorf("failed to create idx_reports_target: %w", err) + } + return nil + }, + }, } // RunMigrations executes all pending database migrations diff --git a/internal/repository/vouch.go b/internal/repository/vouch.go new file mode 100644 index 0000000..fb4c825 --- /dev/null +++ b/internal/repository/vouch.go @@ -0,0 +1,138 @@ +package repository + +import ( + "fmt" + "time" + + "fayan/internal/models" +) + +// SetVouch records source→target as a vouch relationship and atomically removes +// any existing report from source to the same target (mutual-exclusion toggle). +// The target pubkey row is upserted so ranking can cover brand-new targets. +func (r *Repository) SetVouch(source, target string) error { + return r.setRelation(source, target, "vouches", "reports") +} + +// SetReport records source→target as a report and atomically removes any +// existing vouch from source to the same target. +func (r *Repository) SetReport(source, target string) error { + return r.setRelation(source, target, "reports", "vouches") +} + +func (r *Repository) setRelation(source, target, insertTable, deleteTable string) error { + r.writeMu.Lock() + defer r.writeMu.Unlock() + + tx, err := r.db.Begin() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + now := time.Now().UTC() + + if _, err := tx.Exec( + `INSERT INTO pubkeys (pubkey, created_at, updated_at) VALUES (?, ?, ?) ON CONFLICT(pubkey) DO NOTHING;`, + target, now, now, + ); err != nil { + return fmt.Errorf("failed to upsert target pubkey: %w", err) + } + + if _, err := tx.Exec( + fmt.Sprintf(`DELETE FROM %s WHERE source_pubkey = ? AND target_pubkey = ?;`, deleteTable), + source, target, + ); err != nil { + return fmt.Errorf("failed to delete opposite relation: %w", err) + } + + if _, err := tx.Exec( + fmt.Sprintf(`INSERT OR REPLACE INTO %s (source_pubkey, target_pubkey, created_at) VALUES (?, ?, ?);`, insertTable), + source, target, now, + ); err != nil { + return fmt.Errorf("failed to insert relation: %w", err) + } + + return tx.Commit() +} + +// GetTrustScore returns the trust_score for a pubkey. Returns 0 if the pubkey +// is not present in the pubkeys table. +func (r *Repository) GetTrustScore(pubkey string) (float64, error) { + var score float64 + err := r.db.QueryRow("SELECT COALESCE(trust_score, 0) FROM pubkeys WHERE pubkey = ?;", pubkey).Scan(&score) + if err != nil { + // sql.ErrNoRows → pubkey not known → trust is zero. + return 0, nil + } + return score, nil +} + +// StreamVouches streams all vouch edges. Shape mirrors StreamConnections. +func (r *Repository) StreamVouches(callback func(models.Vouch) error) error { + rows, err := r.db.Query("SELECT source_pubkey, target_pubkey FROM vouches;") + if err != nil { + return fmt.Errorf("failed to query vouches: %w", err) + } + defer rows.Close() + + for rows.Next() { + var v models.Vouch + if err := rows.Scan(&v.Source, &v.Target); err != nil { + return fmt.Errorf("failed to scan vouch: %w", err) + } + if err := callback(v); err != nil { + return fmt.Errorf("callback error: %w", err) + } + } + return rows.Err() +} + +// GetPubkeysWithPositiveTrust returns the set of pubkeys whose last-computed +// trust_score is > 0. Used as the vouch-edge admission filter in ranking. +func (r *Repository) GetPubkeysWithPositiveTrust() (map[string]struct{}, error) { + rows, err := r.db.Query("SELECT pubkey FROM pubkeys WHERE trust_score > 0;") + if err != nil { + return nil, fmt.Errorf("failed to query pubkeys with positive trust: %w", err) + } + defer rows.Close() + + result := make(map[string]struct{}) + for rows.Next() { + var pk string + if err := rows.Scan(&pk); err != nil { + return nil, fmt.Errorf("failed to scan pubkey: %w", err) + } + result[pk] = struct{}{} + } + return result, rows.Err() +} + +// GetTrustWeightedReports aggregates reports per target, weighting each report +// by the reporter's trust_score. Reporters with trust_score ≤ 0 are excluded — +// the same admission rule that gates vouch edges. +func (r *Repository) GetTrustWeightedReports() (map[string]models.ReportAggregate, error) { + query := ` + SELECT r.target_pubkey, COUNT(*), COALESCE(SUM(p.trust_score), 0) + FROM reports r + JOIN pubkeys p ON p.pubkey = r.source_pubkey + WHERE p.trust_score > 0 + GROUP BY r.target_pubkey; + ` + rows, err := r.db.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query weighted reports: %w", err) + } + defer rows.Close() + + result := make(map[string]models.ReportAggregate) + for rows.Next() { + var target string + var agg models.ReportAggregate + if err := rows.Scan(&target, &agg.NumReporters, &agg.TotalReporterTrust); err != nil { + return nil, fmt.Errorf("failed to scan report aggregate: %w", err) + } + result[target] = agg + } + return result, rows.Err() +} diff --git a/internal/repository/vouch_test.go b/internal/repository/vouch_test.go new file mode 100644 index 0000000..014c568 --- /dev/null +++ b/internal/repository/vouch_test.go @@ -0,0 +1,226 @@ +package repository + +import ( + "path/filepath" + "testing" + "time" + + "fayan/internal/models" +) + +func newTestRepo(t *testing.T) *Repository { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "test.db") + repo, err := New(path, ModeReadWrite) + if err != nil { + t.Fatalf("failed to open repo: %v", err) + } + t.Cleanup(func() { _ = repo.Close() }) + return repo +} + +// seedPubkey inserts a pubkey with the given trust_score so tests can simulate +// an account that has earned reputation in a previous ranking round. +func seedPubkey(t *testing.T, repo *Repository, pubkey string, trustScore float64) { + t.Helper() + now := time.Now().UTC() + if _, err := repo.db.Exec( + `INSERT INTO pubkeys (pubkey, trust_score, created_at, updated_at) VALUES (?, ?, ?, ?) + ON CONFLICT(pubkey) DO UPDATE SET trust_score = excluded.trust_score;`, + pubkey, trustScore, now, now, + ); err != nil { + t.Fatalf("failed to seed pubkey: %v", err) + } +} + +func countRows(t *testing.T, repo *Repository, table, source, target string) int { + t.Helper() + var n int + q := "SELECT COUNT(*) FROM " + table + " WHERE source_pubkey = ? AND target_pubkey = ?;" + if err := repo.db.QueryRow(q, source, target).Scan(&n); err != nil { + t.Fatalf("count query failed: %v", err) + } + return n +} + +func TestSetVouch_NewInsert(t *testing.T) { + repo := newTestRepo(t) + if err := repo.SetVouch("alice", "bob"); err != nil { + t.Fatalf("SetVouch failed: %v", err) + } + if countRows(t, repo, "vouches", "alice", "bob") != 1 { + t.Fatalf("expected one vouch row") + } +} + +func TestSetVouch_Idempotent(t *testing.T) { + repo := newTestRepo(t) + if err := repo.SetVouch("alice", "bob"); err != nil { + t.Fatal(err) + } + if err := repo.SetVouch("alice", "bob"); err != nil { + t.Fatal(err) + } + if countRows(t, repo, "vouches", "alice", "bob") != 1 { + t.Fatalf("expected exactly one vouch after duplicate SetVouch") + } +} + +func TestSetVouch_MutualExclusion_DeletesReport(t *testing.T) { + repo := newTestRepo(t) + if err := repo.SetReport("alice", "bob"); err != nil { + t.Fatal(err) + } + if countRows(t, repo, "reports", "alice", "bob") != 1 { + t.Fatalf("expected report pre-existing") + } + + if err := repo.SetVouch("alice", "bob"); err != nil { + t.Fatal(err) + } + if countRows(t, repo, "reports", "alice", "bob") != 0 { + t.Fatalf("expected prior report to be deleted by SetVouch") + } + if countRows(t, repo, "vouches", "alice", "bob") != 1 { + t.Fatalf("expected vouch to exist") + } +} + +func TestSetReport_MutualExclusion_DeletesVouch(t *testing.T) { + repo := newTestRepo(t) + if err := repo.SetVouch("alice", "bob"); err != nil { + t.Fatal(err) + } + if err := repo.SetReport("alice", "bob"); err != nil { + t.Fatal(err) + } + if countRows(t, repo, "vouches", "alice", "bob") != 0 { + t.Fatalf("expected prior vouch to be deleted by SetReport") + } + if countRows(t, repo, "reports", "alice", "bob") != 1 { + t.Fatalf("expected report to exist") + } +} + +func TestSetVouch_UpsertsTargetPubkey(t *testing.T) { + repo := newTestRepo(t) + if err := repo.SetVouch("alice", "brand-new-target"); err != nil { + t.Fatal(err) + } + var n int + if err := repo.db.QueryRow("SELECT COUNT(*) FROM pubkeys WHERE pubkey = ?;", "brand-new-target").Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("expected target pubkey to be upserted into pubkeys table") + } +} + +func TestGetTrustScore_UnknownPubkey(t *testing.T) { + repo := newTestRepo(t) + score, err := repo.GetTrustScore("who-dis") + if err != nil { + t.Fatal(err) + } + if score != 0 { + t.Fatalf("unknown pubkey should return 0 trust, got %v", score) + } +} + +func TestGetTrustScore_KnownPubkey(t *testing.T) { + repo := newTestRepo(t) + seedPubkey(t, repo, "trusted", 0.42) + score, err := repo.GetTrustScore("trusted") + if err != nil { + t.Fatal(err) + } + if score != 0.42 { + t.Fatalf("expected 0.42, got %v", score) + } +} + +func TestStreamVouches(t *testing.T) { + repo := newTestRepo(t) + if err := repo.SetVouch("alice", "bob"); err != nil { + t.Fatal(err) + } + if err := repo.SetVouch("alice", "charlie"); err != nil { + t.Fatal(err) + } + if err := repo.SetVouch("dave", "bob"); err != nil { + t.Fatal(err) + } + + var got []models.Vouch + if err := repo.StreamVouches(func(v models.Vouch) error { + got = append(got, v) + return nil + }); err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("expected 3 vouches, got %d", len(got)) + } +} + +func TestGetPubkeysWithPositiveTrust(t *testing.T) { + repo := newTestRepo(t) + seedPubkey(t, repo, "high", 0.5) + seedPubkey(t, repo, "zero", 0) + seedPubkey(t, repo, "neg", -0.1) // should be excluded + + set, err := repo.GetPubkeysWithPositiveTrust() + if err != nil { + t.Fatal(err) + } + if _, ok := set["high"]; !ok { + t.Fatalf("expected 'high' in set") + } + if _, ok := set["zero"]; ok { + t.Fatalf("did not expect 'zero' in set") + } + if _, ok := set["neg"]; ok { + t.Fatalf("did not expect 'neg' in set") + } +} + +func TestGetTrustWeightedReports(t *testing.T) { + repo := newTestRepo(t) + seedPubkey(t, repo, "r1", 0.3) + seedPubkey(t, repo, "r2", 0.7) + seedPubkey(t, repo, "r3", 0) // untrusted; should be excluded + + if err := repo.SetReport("r1", "target"); err != nil { + t.Fatal(err) + } + if err := repo.SetReport("r2", "target"); err != nil { + t.Fatal(err) + } + if err := repo.SetReport("r3", "target"); err != nil { + t.Fatal(err) + } + + reports, err := repo.GetTrustWeightedReports() + if err != nil { + t.Fatal(err) + } + agg, ok := reports["target"] + if !ok { + t.Fatalf("expected 'target' in aggregates") + } + if agg.NumReporters != 2 { + t.Fatalf("expected 2 trusted reporters, got %d", agg.NumReporters) + } + expected := 0.3 + 0.7 + if absDiff(agg.TotalReporterTrust, expected) > 1e-9 { + t.Fatalf("expected trust sum %v, got %v", expected, agg.TotalReporterTrust) + } +} + +func absDiff(a, b float64) float64 { + if a > b { + return a - b + } + return b - a +} From e7cac704761aae43c0d63b350f52eafd9ddbc95b Mon Sep 17 00:00:00 2001 From: codytseng Date: Fri, 24 Apr 2026 14:29:13 +0800 Subject: [PATCH 02/10] refactor: drop DBMode parameter from repository.New Both API and crawler now open the database read-write, so the ModeReadOnly branch was dead code. Collapse to a single New(path) entry point. The API previously inherited the write-mode pool size of 1, which would have serialized its reads; restore a 10-connection pool for all callers. Writes are still serialized by writeMu and SQLite's own locks, so extra connections do not cause contention. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 7 ++-- cmd/api/main.go | 8 ++--- cmd/crawler/main.go | 4 +-- internal/ranking/calculator_test.go | 2 +- internal/repository/repository.go | 51 +++++++++-------------------- internal/repository/vouch_test.go | 2 +- 6 files changed, 23 insertions(+), 51 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 690c88c..d8e8a9e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,12 +30,9 @@ docker compose up --build - **Crawler** (`cmd/crawler/main.go`): Crawls the Nostr network, stores follow relationships in SQLite, periodically calculates PageRank/TrustRank scores - **API** (`cmd/api/main.go`): Read-only HTTP server that queries reputation data -### Database Access Modes +### Database Access -The repository supports two modes via `repository.New(path, mode)`: - -- `ModeReadWrite`: For crawler and API (API opens read-write so the NIP-98 endpoints can insert into `vouches`/`reports`). WAL mode + `writeMu` coordinate the two processes. -- `ModeReadOnly`: Reserved for read-only tools; sets `PRAGMA query_only = ON` and a 10-connection pool. +`repository.New(path)` opens a single read-write handle. WAL mode allows multiple concurrent readers alongside a single writer; within a process `writeMu` serializes writers, and SQLite's own locks coordinate across the crawler and API processes. The connection pool is sized for 10 concurrent readers. ### Key Packages diff --git a/cmd/api/main.go b/cmd/api/main.go index 4d1437a..dc965ef 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -27,17 +27,13 @@ func main() { log.Fatalf("Failed to load config: %v", err) } - // Initialize repository. Even if vouch endpoints are off, we open in - // read-write mode because it's the single place we centralise migrations - // and keep things consistent with the crawler. WAL + writeMu already - // coordinate concurrent writers across processes. - repo, err := repository.New(cfg.Database, repository.ModeReadWrite) + repo, err := repository.New(cfg.Database) if err != nil { log.Fatalf("Failed to initialize database: %v", err) } defer repo.Close() - log.Println("[API] Database initialized successfully (read-write mode)") + log.Println("[API] Database initialized successfully") // Initialize cache apiCache := cache.New(10*time.Minute, 10*time.Minute) diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go index bc27bea..a4feb92 100644 --- a/cmd/crawler/main.go +++ b/cmd/crawler/main.go @@ -31,9 +31,9 @@ func main() { log.Fatalf("[CONFIG] Failed to load configuration: %v", err) } - // 2. Initialize Repository in read-write mode + // 2. Initialize Repository log.Println("[DATABASE] Initializing...") - repo, err := repository.New(cfg.Database, repository.ModeReadWrite) + repo, err := repository.New(cfg.Database) if err != nil { log.Fatalf("[DATABASE] Failed to initialize: %v", err) } diff --git a/internal/ranking/calculator_test.go b/internal/ranking/calculator_test.go index 870814c..ac76f1f 100644 --- a/internal/ranking/calculator_test.go +++ b/internal/ranking/calculator_test.go @@ -11,7 +11,7 @@ import ( func newTestRepo(t *testing.T) *repository.Repository { t.Helper() dir := t.TempDir() - repo, err := repository.New(filepath.Join(dir, "test.db"), repository.ModeReadWrite) + repo, err := repository.New(filepath.Join(dir, "test.db")) if err != nil { t.Fatalf("open repo: %v", err) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index cc0c1dd..a68d318 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -11,16 +11,6 @@ import ( _ "github.com/mattn/go-sqlite3" ) -// DBMode specifies the database access mode -type DBMode int - -const ( - // ModeReadWrite is for crawler - optimized for writes - ModeReadWrite DBMode = iota - // ModeReadOnly is for API - optimized for reads - ModeReadOnly -) - // totalUsersCache caches the count of users type totalUsersCache struct { count int @@ -36,11 +26,14 @@ type Repository struct { writeMu sync.Mutex // Serializes all write operations for SQLite } -// New creates a new Repository instance -func New(dataSourceName string, mode DBMode) (*Repository, error) { - // Build DSN with parameters that apply to ALL connections in the pool - // This is critical - PRAGMA statements only affect a single connection, - // but DSN parameters are applied when each connection is created +// New creates a new Repository instance. +// WAL mode allows concurrent readers alongside a single writer; writeMu +// serializes writers within this process, SQLite's own locks coordinate +// across processes. +func New(dataSourceName string) (*Repository, error) { + // Build DSN with parameters that apply to ALL connections in the pool. + // DSN parameters are applied when each connection is created, while + // PRAGMA statements would only affect whichever connection ran them. dsnParams := []string{ "_journal_mode=WAL", "_synchronous=NORMAL", @@ -48,11 +41,7 @@ func New(dataSourceName string, mode DBMode) (*Repository, error) { "_cache_size=-64000", "_txlock=immediate", // Acquire write lock at BEGIN, not at first write } - if mode == ModeReadOnly { - dsnParams = append(dsnParams, "_query_only=true") - } - // Append parameters to DSN separator := "?" if strings.Contains(dataSourceName, "?") { separator = "&" @@ -68,20 +57,13 @@ func New(dataSourceName string, mode DBMode) (*Repository, error) { return nil, fmt.Errorf("could not connect to database: %w", err) } - // Configure connection pool based on mode - if mode == ModeReadOnly { - db.SetMaxOpenConns(10) - db.SetMaxIdleConns(5) - } else { - // For write mode: use single connection to avoid lock contention - // SQLite only allows one writer at a time anyway - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) - } + // Connection pool: multiple connections so reads can run concurrently + // under WAL. Writes serialize on writeMu, so extra connections do not + // cause write contention. + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(5) db.SetConnMaxLifetime(time.Hour) - // These PRAGMAs don't have DSN equivalents, set on initial connection - // With MaxOpenConns=1 for write mode, this is sufficient additionalPragmas := []string{ "PRAGMA temp_store = MEMORY;", "PRAGMA mmap_size = 1073741824;", @@ -102,11 +84,8 @@ func New(dataSourceName string, mode DBMode) (*Repository, error) { }, } - // Run migrations in read-write mode - if mode == ModeReadWrite { - if err := repo.RunMigrations(); err != nil { - return nil, fmt.Errorf("could not run migrations: %w", err) - } + if err := repo.RunMigrations(); err != nil { + return nil, fmt.Errorf("could not run migrations: %w", err) } return repo, nil diff --git a/internal/repository/vouch_test.go b/internal/repository/vouch_test.go index 014c568..9d406dd 100644 --- a/internal/repository/vouch_test.go +++ b/internal/repository/vouch_test.go @@ -12,7 +12,7 @@ func newTestRepo(t *testing.T) *Repository { t.Helper() dir := t.TempDir() path := filepath.Join(dir, "test.db") - repo, err := New(path, ModeReadWrite) + repo, err := New(path) if err != nil { t.Fatalf("failed to open repo: %v", err) } From e3453658fb6f07a507242e422960f748acf6a03b Mon Sep 17 00:00:00 2001 From: codytseng Date: Fri, 24 Apr 2026 17:36:25 +0800 Subject: [PATCH 03/10] feat: make vouch edge weight configurable (default 0.5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A vouch is a weaker signal than an actual follow: the user is asserting non-spam without committing to see the target's posts. Carry a per-edge weight through the graph so vouch-only edges contribute proportionally less flow than a full follow. Config: vouch.weight (default 0.5, range (0, 1]). PageRank and TrustRank inner loops now divide by outWeight (sum of outgoing edge weights) instead of outDegree (count). Each in-edge carries its own weight; a source's score flows to each target in proportion to weight / outWeight. Follow and vouch edges share the same adjacency list — only their weights differ. outDegree is still tracked as an int so the pubkeys table's Following column remains a count. New test verifies that lowering the weight produces a lower score for a vouch-only recipient. Co-Authored-By: Claude Opus 4.7 (1M context) --- cmd/crawler/main.go | 2 +- config.example.yaml | 7 ++- config/config.go | 6 ++- internal/ranking/calculator.go | 67 ++++++++++++++++++----------- internal/ranking/calculator_test.go | 49 +++++++++++++++++++-- 5 files changed, 98 insertions(+), 33 deletions(-) diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go index a4feb92..052bd6d 100644 --- a/cmd/crawler/main.go +++ b/cmd/crawler/main.go @@ -41,7 +41,7 @@ func main() { log.Println("[DATABASE] Ready (read-write mode)") // 3. Create ranking calculator - calculator := ranking.NewCalculator(repo, cfg.SeedPubkeys, cfg.Ranking.TrustRankWeight, cfg.Ranking.PageRankWeight) + calculator := ranking.NewCalculator(repo, cfg.SeedPubkeys, cfg.Ranking.TrustRankWeight, cfg.Ranking.PageRankWeight, cfg.Vouch.Weight) // 4. Perform an initial rank calculation log.Println("[RANK] Performing initial calculation...") diff --git a/config.example.yaml b/config.example.yaml index 0de5277..45fdc02 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -38,13 +38,16 @@ crawler: # Vouch / report API endpoints (NIP-98 authenticated). # When enabled, POST /vouch and POST /report accept signed requests from users -# to vouch for or report another pubkey. Vouches act as equal-weight follow -# edges in the ranking graph (no need to actually follow). Reports shrink the +# to vouch for or report another pubkey. Vouches act as weighted follow edges +# in the ranking graph (no need to actually follow). Reports shrink the # target's final score proportional to the trust-weighted report ratio. # Submissions from pubkeys with no TrustRank (and not in seed_pubkeys) are # silently ignored. vouch: enabled: false + # Weight of a vouch edge relative to a follow edge (1.0). Must be in (0, 1]. + # Lower values make vouches contribute less flow than a follow. Default: 0.5. + weight: 0.5 # Default nostr relays to connect to, used for searching uesrs' relay lists relays: diff --git a/config/config.go b/config/config.go index 38b98f0..636c96e 100644 --- a/config/config.go +++ b/config/config.go @@ -32,6 +32,9 @@ type CrawlerConfig struct { // Submissions are authenticated via NIP-98. type VouchConfig struct { Enabled bool `yaml:"enabled"` // Default: false. When false, the endpoints return 404. + // Weight of a vouch edge relative to a follow edge (1.0). Must be in (0, 1]. + // Lower values make vouches contribute less flow than follows. Default: 0.5. + Weight float64 `yaml:"weight"` } // Config represents the application configuration @@ -71,6 +74,7 @@ func Load(path string) (*Config, error) { }, Vouch: VouchConfig{ Enabled: false, + Weight: 0.5, }, } if err := yaml.Unmarshal(data, &cfg); err != nil { @@ -90,7 +94,7 @@ func Load(path string) (*Config, error) { log.Printf("[CONFIG] - Ranking weights: TrustRank=%.2f, PageRank=%.2f", cfg.Ranking.TrustRankWeight, cfg.Ranking.PageRankWeight) log.Printf("[CONFIG] - Crawler: batch_size=%d, request_interval=%dms, contact_processors=%d, profile_processors=%d", cfg.Crawler.BatchSize, cfg.Crawler.RequestIntervalMs, cfg.Crawler.NumContactProcessors, cfg.Crawler.NumProfileProcessors) - log.Printf("[CONFIG] - Vouch endpoints enabled: %t", cfg.Vouch.Enabled) + log.Printf("[CONFIG] - Vouch endpoints enabled: %t (weight=%.2f)", cfg.Vouch.Enabled, cfg.Vouch.Weight) return &cfg, nil } diff --git a/internal/ranking/calculator.go b/internal/ranking/calculator.go index 580ad0c..48ba192 100644 --- a/internal/ranking/calculator.go +++ b/internal/ranking/calculator.go @@ -15,21 +15,31 @@ type scoreWithID struct { score float64 } +// inLink represents a single weighted in-edge in the adjacency list. +// Weight is 1.0 for follow edges and vouchWeight for vouch-only edges. +type inLink struct { + source int32 + weight float64 +} + // Calculator handles PageRank and TrustRank calculations type Calculator struct { repo *repository.Repository seedPubkeys []string trustRankWeight float64 pageRankWeight float64 + vouchWeight float64 } -// NewCalculator creates a new Calculator instance -func NewCalculator(repo *repository.Repository, seedPubkeys []string, trustRankWeight, pageRankWeight float64) *Calculator { +// NewCalculator creates a new Calculator instance. +// vouchWeight is the relative weight of a vouch-only edge (follow edges are 1.0). +func NewCalculator(repo *repository.Repository, seedPubkeys []string, trustRankWeight, pageRankWeight, vouchWeight float64) *Calculator { return &Calculator{ repo: repo, seedPubkeys: seedPubkeys, trustRankWeight: trustRankWeight, pageRankWeight: pageRankWeight, + vouchWeight: vouchWeight, } } @@ -41,6 +51,7 @@ func (c *Calculator) Calculate() error { type edge struct { source int32 target int32 + weight float64 } edges := make([]edge, 0, 1000) @@ -61,7 +72,7 @@ func (c *Calculator) Calculate() error { // edgeSet dedupes (source, target) pairs across follow and vouch edges so // a user who both follows and vouches for the same target contributes a - // single graph edge (not double flow). + // single follow-weighted edge (not double flow). edgeSet := make(map[int64]bool) encodeEdge := func(s, t int32) int64 { return int64(s)<<32 | int64(uint32(t)) } @@ -73,7 +84,7 @@ func (c *Calculator) Calculate() error { key := encodeEdge(sourceID, targetID) if !edgeSet[key] { edgeSet[key] = true - edges = append(edges, edge{source: sourceID, target: targetID}) + edges = append(edges, edge{source: sourceID, target: targetID, weight: 1.0}) } } connectionCount++ @@ -110,13 +121,13 @@ func (c *Calculator) Calculate() error { return nil } edgeSet[key] = true - edges = append(edges, edge{source: sourceID, target: targetID}) + edges = append(edges, edge{source: sourceID, target: targetID, weight: c.vouchWeight}) vouchAdmitted++ return nil }); err != nil { return err } - log.Printf(" [INFO] Vouch edges admitted: %d", vouchAdmitted) + log.Printf(" [INFO] Vouch edges admitted: %d (weight=%.2f)", vouchAdmitted, c.vouchWeight) numNodes := len(idToPubkey) if numNodes == 0 { @@ -135,12 +146,16 @@ func (c *Calculator) Calculate() error { } log.Printf(" [INFO] Found %d seed nodes in graph (out of %d configured)", len(seedSet), len(c.seedPubkeys)) - // Build the graph using slices (Adjacency List) - inLinks := make([][]int32, numNodes) + // Build the weighted graph. + // outWeight[i] is the sum of outgoing edge weights (used by flow math). + // outDegree[i] is the discrete count (used only for the Following field). + inLinks := make([][]inLink, numNodes) + outWeight := make([]float64, numNodes) outDegree := make([]int32, numNodes) for _, e := range edges { - inLinks[e.target] = append(inLinks[e.target], e.source) + inLinks[e.target] = append(inLinks[e.target], inLink{source: e.source, weight: e.weight}) + outWeight[e.source] += e.weight outDegree[e.source]++ } @@ -152,13 +167,13 @@ func (c *Calculator) Calculate() error { // Run PageRank log.Println(" [INFO] Running PageRank...") - pageScores := c.runPageRank(numNodes, inLinks, outDegree, dampingFactor, tolerance, maxIterations) + pageScores := c.runPageRank(numNodes, inLinks, outWeight, dampingFactor, tolerance, maxIterations) // Run TrustRank var trustScores []float64 if len(seedSet) > 0 { log.Println(" [INFO] Running TrustRank...") - trustScores = c.runTrustRank(numNodes, inLinks, outDegree, seedSet, dampingFactor, tolerance, maxIterations) + trustScores = c.runTrustRank(numNodes, inLinks, outWeight, seedSet, dampingFactor, tolerance, maxIterations) } else { log.Println(" [WARN] No seed nodes found, skipping TrustRank") trustScores = make([]float64, numNodes) @@ -173,15 +188,15 @@ func (c *Calculator) Calculate() error { // Apply trust-weighted report penalty: scale each target's scores by // (1 - penalty) where penalty = R / (R + F + ε). R is the sum of reporter // trust_scores (only reporters with trust_score > 0 count); F is the sum - // of follower/voucher trust_scores (in-edges). Rank is computed after - // penalty so penalized accounts drop in the final ordering. + // of follower/voucher trust_scores weighted by their edge weights. Rank + // is computed after penalty so penalized accounts drop in the ordering. if reports, err := c.repo.GetTrustWeightedReports(); err != nil { log.Printf(" [WARN] Failed to load reports for penalty: %v", err) } else if len(reports) > 0 { fTrust := make([]float64, numNodes) for i := range numNodes { - for _, j := range inLinks[i] { - fTrust[i] += trustScores[j] + for _, link := range inLinks[i] { + fTrust[i] += trustScores[link.source] * link.weight } } penalized := 0 @@ -265,8 +280,10 @@ func (c *Calculator) Calculate() error { return nil } -// runPageRank executes the PageRank algorithm -func (c *Calculator) runPageRank(numNodes int, inLinks [][]int32, outDegree []int32, damping, tolerance float64, maxIterations int) []float64 { +// runPageRank executes the weighted PageRank algorithm. +// Each in-edge carries its own weight; a node's score is distributed among +// its out-neighbors in proportion to edge weight (sum equals outWeight[j]). +func (c *Calculator) runPageRank(numNodes int, inLinks [][]inLink, outWeight []float64, damping, tolerance float64, maxIterations int) []float64 { scores := make([]float64, numNodes) newScores := make([]float64, numNodes) initialScore := 1.0 / float64(numNodes) @@ -278,15 +295,15 @@ func (c *Calculator) runPageRank(numNodes int, inLinks [][]int32, outDegree []in for iter := 0; iter < maxIterations; iter++ { danglingSum := 0.0 for i := range numNodes { - if outDegree[i] == 0 { + if outWeight[i] == 0 { danglingSum += scores[i] } } for i := range numNodes { sum := 0.0 - for _, j := range inLinks[i] { - sum += scores[j] / float64(outDegree[j]) + for _, link := range inLinks[i] { + sum += scores[link.source] * link.weight / outWeight[link.source] } newScores[i] = (1-damping)/float64(numNodes) + damping*(sum+danglingSum/float64(numNodes)) } @@ -308,8 +325,8 @@ func (c *Calculator) runPageRank(numNodes int, inLinks [][]int32, outDegree []in return scores } -// runTrustRank executes the TrustRank algorithm -func (c *Calculator) runTrustRank(numNodes int, inLinks [][]int32, outDegree []int32, seedSet map[int32]bool, damping, tolerance float64, maxIterations int) []float64 { +// runTrustRank executes the weighted TrustRank algorithm. +func (c *Calculator) runTrustRank(numNodes int, inLinks [][]inLink, outWeight []float64, seedSet map[int32]bool, damping, tolerance float64, maxIterations int) []float64 { scores := make([]float64, numNodes) newScores := make([]float64, numNodes) @@ -322,15 +339,15 @@ func (c *Calculator) runTrustRank(numNodes int, inLinks [][]int32, outDegree []i for iter := range maxIterations { danglingSum := 0.0 for i := range numNodes { - if outDegree[i] == 0 { + if outWeight[i] == 0 { danglingSum += scores[i] } } for i := range numNodes { sum := 0.0 - for _, j := range inLinks[i] { - sum += scores[j] / float64(outDegree[j]) + for _, link := range inLinks[i] { + sum += scores[link.source] * link.weight / outWeight[link.source] } // In TrustRank, dangling node scores only flow back to seed nodes diff --git a/internal/ranking/calculator_test.go b/internal/ranking/calculator_test.go index ac76f1f..45dff7f 100644 --- a/internal/ranking/calculator_test.go +++ b/internal/ranking/calculator_test.go @@ -55,7 +55,7 @@ func TestVouchPromotesUnfollowedUser(t *testing.T) { insertFollow(t, repo, "seed2", "seed3") insertFollow(t, repo, "seed3", "seed1") - calc := NewCalculator(repo, seeds, 0.7, 0.3) + calc := NewCalculator(repo, seeds, 0.7, 0.3, 0.5) // First pass: seeds acquire trust_score > 0. if err := calc.Calculate(); err != nil { @@ -84,6 +84,47 @@ func TestVouchPromotesUnfollowedUser(t *testing.T) { } } +// TestVouchWeightShrinksContribution verifies that a lower vouchWeight +// reduces the score a vouch-only edge contributes, relative to a full-weight +// (1.0) follow edge. +func TestVouchWeightShrinksContribution(t *testing.T) { + repo := newTestRepo(t) + + seeds := []string{"seed1", "seed2", "seed3"} + insertFollow(t, repo, "seed1", "seed2") + insertFollow(t, repo, "seed2", "seed3") + insertFollow(t, repo, "seed3", "seed1") + + // Bootstrap seed trust. + calcHigh := NewCalculator(repo, seeds, 0.7, 0.3, 1.0) + if err := calcHigh.Calculate(); err != nil { + t.Fatal(err) + } + if err := repo.SetVouch("seed1", "newbie"); err != nil { + t.Fatal(err) + } + + if err := calcHigh.Calculate(); err != nil { + t.Fatal(err) + } + _, _, _, scoreAtWeight1 := getUser(t, repo, "newbie") + + // Run the same graph again with vouchWeight=0.25. + calcLow := NewCalculator(repo, seeds, 0.7, 0.3, 0.25) + if err := calcLow.Calculate(); err != nil { + t.Fatal(err) + } + _, _, _, scoreAtWeight025 := getUser(t, repo, "newbie") + + if !(scoreAtWeight025 < scoreAtWeight1) { + t.Fatalf("expected lower vouch weight to produce lower score, got %.6g (w=0.25) vs %.6g (w=1.0)", + scoreAtWeight025, scoreAtWeight1) + } + if scoreAtWeight025 <= 0 { + t.Fatalf("score at w=0.25 should still be positive, got %v", scoreAtWeight025) + } +} + // TestVouchAndFollowDedupe verifies A following AND vouching for B only // produces one edge (A's following count = 1, not 2). func TestVouchAndFollowDedupe(t *testing.T) { @@ -104,7 +145,7 @@ func TestVouchAndFollowDedupe(t *testing.T) { Following: 1, }}) - calc := NewCalculator(repo, []string{"a"}, 0.7, 0.3) + calc := NewCalculator(repo, []string{"a"}, 0.7, 0.3, 0.5) if err := calc.Calculate(); err != nil { t.Fatal(err) } @@ -133,7 +174,7 @@ func TestReportDecaysScore(t *testing.T) { insertFollow(t, repo, "seed2", "x") insertFollow(t, repo, "seed3", "x") - calc := NewCalculator(repo, seeds, 0.7, 0.3) + calc := NewCalculator(repo, seeds, 0.7, 0.3, 0.5) if err := calc.Calculate(); err != nil { t.Fatal(err) } @@ -187,7 +228,7 @@ func TestReportWithNoTrustIgnored(t *testing.T) { t.Fatal(err) } - calc := NewCalculator(repo, seeds, 0.7, 0.3) + calc := NewCalculator(repo, seeds, 0.7, 0.3, 0.5) if err := calc.Calculate(); err != nil { t.Fatal(err) } From ca5a55360bfbf61aba8040b053ff9775fb933245 Mon Sep 17 00:00:00 2001 From: codytseng Date: Fri, 24 Apr 2026 17:41:09 +0800 Subject: [PATCH 04/10] refactor: collapse vouch.enabled and vouch.weight into a single knob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having both fields required the user to keep them consistent: enabling with weight=0 (or disabling with weight=0.5) both produced inconsistent states. Use weight alone — 0 means off (endpoints return 404 and the ranking calculator skips streaming vouches entirely), > 0 means on and is the edge weight. Default 0 preserves prior off-by-default behaviour. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 4 +-- cmd/api/main.go | 8 ++--- config.example.yaml | 20 ++++++------- config/config.go | 24 ++++++++++----- internal/ranking/calculator.go | 53 ++++++++++++++++++---------------- 5 files changed, 60 insertions(+), 49 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d8e8a9e..3849a1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,9 +79,9 @@ Copy `config.example.yaml` to `config.yaml` (and `docker-compose.example.yml` to - `crawler.request_interval_ms`: Milliseconds between requests per relay (default: 500) - `crawler.num_contact_processors`: Number of contact event processors (default: 4) - `crawler.num_profile_processors`: Number of profile event processors (default: 4) -- `vouch.enabled`: Enable `POST /vouch` and `POST /report` endpoints (default: false) +- `vouch.weight`: Enables the feature (0 = disabled, default) and sets the weight of a vouch edge relative to a follow edge (1.0). Typical enabled value: `0.5`. -### Vouch & Report Endpoints (when `vouch.enabled: true`) +### Vouch & Report Endpoints (when `vouch.weight > 0`) Two NIP-98 authenticated endpoints let users contribute to the graph without following: diff --git a/cmd/api/main.go b/cmd/api/main.go index dc965ef..5db9269 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -88,12 +88,12 @@ func main() { http.HandleFunc("/search", middleware.CORS(h.Search)) // Vouch / report endpoints (NIP-98 authenticated). - // When disabled in config, no route is registered and requests fall - // through to the SPA catch-all handler below. - if cfg.Vouch.Enabled { + // When vouch.weight <= 0 the feature is disabled: no route is registered + // and requests fall through to the SPA catch-all handler below. + if cfg.Vouch.Enabled() { http.HandleFunc("/vouch", middleware.CORS(middleware.NIP98Auth(h.Vouch))) http.HandleFunc("/report", middleware.CORS(middleware.NIP98Auth(h.Report))) - log.Println("[API] Vouch/report endpoints enabled") + log.Printf("[API] Vouch/report endpoints enabled (weight=%.2f)", cfg.Vouch.Weight) } // Serve static assets (js, css, images, etc.) with long cache (1 year for hashed assets) diff --git a/config.example.yaml b/config.example.yaml index 45fdc02..913a353 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -37,17 +37,17 @@ crawler: num_profile_processors: 4 # Vouch / report API endpoints (NIP-98 authenticated). -# When enabled, POST /vouch and POST /report accept signed requests from users -# to vouch for or report another pubkey. Vouches act as weighted follow edges -# in the ranking graph (no need to actually follow). Reports shrink the -# target's final score proportional to the trust-weighted report ratio. -# Submissions from pubkeys with no TrustRank (and not in seed_pubkeys) are -# silently ignored. +# +# `weight` is the single control: set to 0 (default) to disable the feature +# entirely — endpoints return 404 and vouches are ignored by ranking. +# Set to a value in (0, 1] to enable. The value is the weight of a vouch +# edge relative to a follow edge (1.0); e.g. 0.5 means each vouch +# contributes half the flow of an actual follow. Reports are always active +# when the feature is enabled and apply a trust-weighted penalty to the +# target's final score. Submissions from pubkeys with no TrustRank (and +# not in seed_pubkeys) are silently ignored. vouch: - enabled: false - # Weight of a vouch edge relative to a follow edge (1.0). Must be in (0, 1]. - # Lower values make vouches contribute less flow than a follow. Default: 0.5. - weight: 0.5 + weight: 0 # Default nostr relays to connect to, used for searching uesrs' relay lists relays: diff --git a/config/config.go b/config/config.go index 636c96e..0e3fec8 100644 --- a/config/config.go +++ b/config/config.go @@ -28,15 +28,20 @@ type CrawlerConfig struct { NumProfileProcessors int `yaml:"num_profile_processors"` // Number of profile event processors (default: 4) } -// VouchConfig enables the /vouch and /report API endpoints. -// Submissions are authenticated via NIP-98. +// VouchConfig controls the /vouch and /report API endpoints and the +// weight of vouch edges in the ranking graph. +// +// A single knob: weight == 0 disables the feature entirely (endpoints +// return 404, vouches are not read by the ranking calculator); +// weight > 0 enables the feature and sets the vouch-edge weight relative +// to a follow edge (1.0). Typical values: 0.5. Must be in [0, 1]. type VouchConfig struct { - Enabled bool `yaml:"enabled"` // Default: false. When false, the endpoints return 404. - // Weight of a vouch edge relative to a follow edge (1.0). Must be in (0, 1]. - // Lower values make vouches contribute less flow than follows. Default: 0.5. Weight float64 `yaml:"weight"` } +// Enabled reports whether the vouch feature is active. +func (v VouchConfig) Enabled() bool { return v.Weight > 0 } + // Config represents the application configuration type Config struct { Relays []string `yaml:"relays"` @@ -73,8 +78,7 @@ func Load(path string) (*Config, error) { NumProfileProcessors: 4, }, Vouch: VouchConfig{ - Enabled: false, - Weight: 0.5, + Weight: 0, // disabled by default }, } if err := yaml.Unmarshal(data, &cfg); err != nil { @@ -94,7 +98,11 @@ func Load(path string) (*Config, error) { log.Printf("[CONFIG] - Ranking weights: TrustRank=%.2f, PageRank=%.2f", cfg.Ranking.TrustRankWeight, cfg.Ranking.PageRankWeight) log.Printf("[CONFIG] - Crawler: batch_size=%d, request_interval=%dms, contact_processors=%d, profile_processors=%d", cfg.Crawler.BatchSize, cfg.Crawler.RequestIntervalMs, cfg.Crawler.NumContactProcessors, cfg.Crawler.NumProfileProcessors) - log.Printf("[CONFIG] - Vouch endpoints enabled: %t (weight=%.2f)", cfg.Vouch.Enabled, cfg.Vouch.Weight) + if cfg.Vouch.Enabled() { + log.Printf("[CONFIG] - Vouch endpoints enabled (weight=%.2f)", cfg.Vouch.Weight) + } else { + log.Printf("[CONFIG] - Vouch endpoints disabled (weight=0)") + } return &cfg, nil } diff --git a/internal/ranking/calculator.go b/internal/ranking/calculator.go index 48ba192..5c83d93 100644 --- a/internal/ranking/calculator.go +++ b/internal/ranking/calculator.go @@ -97,37 +97,40 @@ func (c *Calculator) Calculate() error { // Vouch edges: admit only those from pubkeys with positive last-round // TrustRank (seeds always admitted so the feature works on first run when - // no one has trust_score written yet). + // no one has trust_score written yet). Skipped entirely when the feature + // is disabled (vouchWeight <= 0). vouchAdmitted := 0 - qualifying, qErr := c.repo.GetPubkeysWithPositiveTrust() - if qErr != nil { - log.Printf(" [WARN] Failed to load qualifying pubkeys for vouch admission: %v", qErr) - qualifying = make(map[string]struct{}) - } - for _, s := range c.seedPubkeys { - qualifying[s] = struct{}{} - } - if err := c.repo.StreamVouches(func(v models.Vouch) error { - if _, ok := qualifying[v.Source]; !ok { - return nil + if c.vouchWeight > 0 { + qualifying, qErr := c.repo.GetPubkeysWithPositiveTrust() + if qErr != nil { + log.Printf(" [WARN] Failed to load qualifying pubkeys for vouch admission: %v", qErr) + qualifying = make(map[string]struct{}) } - sourceID := getID(v.Source) - targetID := getID(v.Target) - if sourceID == targetID { - return nil + for _, s := range c.seedPubkeys { + qualifying[s] = struct{}{} } - key := encodeEdge(sourceID, targetID) - if edgeSet[key] { + if err := c.repo.StreamVouches(func(v models.Vouch) error { + if _, ok := qualifying[v.Source]; !ok { + return nil + } + sourceID := getID(v.Source) + targetID := getID(v.Target) + if sourceID == targetID { + return nil + } + key := encodeEdge(sourceID, targetID) + if edgeSet[key] { + return nil + } + edgeSet[key] = true + edges = append(edges, edge{source: sourceID, target: targetID, weight: c.vouchWeight}) + vouchAdmitted++ return nil + }); err != nil { + return err } - edgeSet[key] = true - edges = append(edges, edge{source: sourceID, target: targetID, weight: c.vouchWeight}) - vouchAdmitted++ - return nil - }); err != nil { - return err + log.Printf(" [INFO] Vouch edges admitted: %d (weight=%.2f)", vouchAdmitted, c.vouchWeight) } - log.Printf(" [INFO] Vouch edges admitted: %d (weight=%.2f)", vouchAdmitted, c.vouchWeight) numNodes := len(idToPubkey) if numNodes == 0 { From 0df469d51848b52b9a0e37b25277c8d144a6e126 Mon Sep 17 00:00:00 2001 From: codytseng Date: Thu, 18 Jun 2026 19:23:32 +0800 Subject: [PATCH 05/10] feat: ingest vouch/report as nostr events instead of a private API Vouches and reports are now plain signed Nostr events rather than a NIP-98-wrapped private API, so the data is publishable to any relay and shared with the wider network. A new internal/ingest package backs both ingestion paths: - Crawler (pull): alongside kind:3/0, fetches each author's kind:1984 reports and kind:10040 vouch sets. Replaceable kinds (3/0/10040) share one small-limit query; kind:1984 (append-only) gets its own query capped at the newest 50 so it can't crowd out the replaceable events. - POST /event (push): accepts a single signed event for immediate ingestion, keeping the anti-inflation rule (untrusted authors dropped). Both paths verify signatures before storing. The old /vouch and /report endpoints and the NIP-98 middleware are retired. Reports are kind:1984 profile-level (p tag, no e tag) spam/impersonation events. Vouches are membership in a custom replaceable kind:10040 set and follow the same lifecycle as follow edges: refreshed via last_seen, never actively deleted, aged out by the ranking staleness window. Vouch-beats- report precedence is resolved at ranking time (GetTrustWeightedReports excludes a reporter who also vouches for the same target). --- CLAUDE.md | 15 +- cmd/api/main.go | 12 +- cmd/crawler/main.go | 2 +- config.example.yaml | 16 +- config/config.go | 15 +- internal/api/handler/event.go | 72 ++++++ internal/api/handler/handler.go | 3 + internal/api/handler/vouch.go | 89 ------- internal/api/middleware/nip98.go | 177 -------------- internal/api/middleware/nip98_test.go | 322 -------------------------- internal/crawler/crawler.go | 286 ++++++++++++++++++----- internal/ingest/ingest.go | 157 +++++++++++++ internal/ingest/ingest_test.go | 86 +++++++ internal/ranking/calculator.go | 2 +- internal/ranking/calculator_test.go | 10 +- internal/repository/migration.go | 8 +- internal/repository/vouch.go | 98 +++++--- internal/repository/vouch_test.go | 154 +++++++++--- 18 files changed, 785 insertions(+), 739 deletions(-) create mode 100644 internal/api/handler/event.go delete mode 100644 internal/api/handler/vouch.go delete mode 100644 internal/api/middleware/nip98.go delete mode 100644 internal/api/middleware/nip98_test.go create mode 100644 internal/ingest/ingest.go create mode 100644 internal/ingest/ingest_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 3849a1d..2c22f6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,11 +81,16 @@ Copy `config.example.yaml` to `config.yaml` (and `docker-compose.example.yml` to - `crawler.num_profile_processors`: Number of profile event processors (default: 4) - `vouch.weight`: Enables the feature (0 = disabled, default) and sets the weight of a vouch edge relative to a follow edge (1.0). Typical enabled value: `0.5`. -### Vouch & Report Endpoints (when `vouch.weight > 0`) +### Vouch & Report via Nostr Events (when `vouch.weight > 0`) -Two NIP-98 authenticated endpoints let users contribute to the graph without following: +Vouches and reports are plain signed Nostr events, not a private API. They flow in two ways (both verify the event signature before storing): -- `POST /vouch` with body `{"target": ""}` — registers source→target as a vouch edge in the ranking graph (equal weight to a follow, deduped against follows from the same source). -- `POST /report` with body `{"target": ""}` — reports the target. Reports apply a trust-weighted penalty to the target's final score: `final = raw * (1 - R/(R+F))` where R is the sum of reporter trust_scores and F is the sum of follower/voucher trust_scores. +1. **Crawler ingestion (pull)** — alongside kind:3/0, the crawler fetches each crawled author's kind:1984 reports and kind:10040 vouch set from their relays. Replaceable kinds (3/0/10040) share one small-limit query; kind:1984 (append-only, potentially many) gets its own query capped at the newest 50, kept separate so reports can't crowd out the replaceable events. +2. **`POST /event` (push)** — accepts a single signed event (kind 3 / 1984 / 10040) for immediate ingestion. As an open write endpoint it keeps the anti-inflation rule: events from pubkeys with no TrustRank and not in `seed_pubkeys` return 200 but are silently dropped. (The crawler path does not filter this way — ranking already discounts untrusted sources.) -No DELETE endpoints — the two actions are mutually exclusive per (source, target). Posting a report implicitly deletes any prior vouch for the same target, and vice versa. Submissions from pubkeys with no TrustRank and not in `seed_pubkeys` return 200 but are silently dropped (prevents spam inflation). Schema stored in `vouches` and `reports` tables (migration v4). +Shared parsing/storage lives in `internal/ingest` so both paths behave identically. + +- **Vouch** = membership in the author's **kind:10040** vouch set (a custom replaceable event; not in any NIP). Its `p` tags list the vouched pubkeys. Registers source→target as a vouch edge (weight `vouch.weight`, deduped against a follow from the same source). Vouches follow the **same lifecycle as follow edges** (`vouches.last_seen`, not active deletion): each set refreshes its edges' `last_seen`; a pubkey dropped from the set is not deleted but stops being refreshed and ages out via the same staleness window as follows (`StreamVouches` filters on the ranking cutoff). So revoking a vouch takes effect after the window, exactly like unfollowing. +- **Report** = a **kind:1984** (NIP-56) event targeting a **profile** (`p` tag, no `e` tag) with report type `spam` or `impersonation` (other types ignored). Applies a trust-weighted penalty to the target's final score: `final = raw * (1 - R/(R+F))` where R is the sum of reporter trust_scores and F is the sum of follower/voucher trust_scores. + +No mutual exclusion at write time — `vouches` and `reports` rows coexist. Precedence is resolved at ranking time: if a source both vouches for and reports the same target, the report is ignored (vouch beats report). Stored in the `vouches` and `reports` tables (schema from migration v4, unchanged). diff --git a/cmd/api/main.go b/cmd/api/main.go index 5db9269..178a007 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -87,13 +87,13 @@ func main() { http.HandleFunc("/users/", middleware.CORS(h.User)) http.HandleFunc("/search", middleware.CORS(h.Search)) - // Vouch / report endpoints (NIP-98 authenticated). - // When vouch.weight <= 0 the feature is disabled: no route is registered - // and requests fall through to the SPA catch-all handler below. + // Event ingestion endpoint. Accepts signed Nostr events (kind 3 / 1984 / + // 10040) as an immediate push complement to the crawler. When vouch.weight + // <= 0 the feature is disabled: no route is registered and requests fall + // through to the SPA catch-all handler below. if cfg.Vouch.Enabled() { - http.HandleFunc("/vouch", middleware.CORS(middleware.NIP98Auth(h.Vouch))) - http.HandleFunc("/report", middleware.CORS(middleware.NIP98Auth(h.Report))) - log.Printf("[API] Vouch/report endpoints enabled (weight=%.2f)", cfg.Vouch.Weight) + http.HandleFunc("/event", middleware.CORS(h.PostEvent)) + log.Printf("[API] Event ingestion endpoint enabled (weight=%.2f)", cfg.Vouch.Weight) } // Serve static assets (js, css, images, etc.) with long cache (1 year for hashed assets) diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go index 052bd6d..bcf84bc 100644 --- a/cmd/crawler/main.go +++ b/cmd/crawler/main.go @@ -58,7 +58,7 @@ func main() { NumContactProcessors: cfg.Crawler.NumContactProcessors, NumProfileProcessors: cfg.Crawler.NumProfileProcessors, } - c := crawler.NewCrawler(repo, cfg.Relays, cfg.SeedPubkeys, &cfg.Search, crawlerConfig) + c := crawler.NewCrawler(repo, cfg.Relays, cfg.SeedPubkeys, &cfg.Search, crawlerConfig, cfg.Vouch.Enabled()) c.Start() // 6. Periodically Calculate Ranks diff --git a/config.example.yaml b/config.example.yaml index 913a353..b5f873f 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -36,16 +36,16 @@ crawler: # Number of profile event processors (default: 4) num_profile_processors: 4 -# Vouch / report API endpoints (NIP-98 authenticated). +# Vouch / report feature. Fed by standard signed Nostr events — the crawler +# ingests them from relays, and the POST /event endpoint accepts pushes too. # # `weight` is the single control: set to 0 (default) to disable the feature -# entirely — endpoints return 404 and vouches are ignored by ranking. -# Set to a value in (0, 1] to enable. The value is the weight of a vouch -# edge relative to a follow edge (1.0); e.g. 0.5 means each vouch -# contributes half the flow of an actual follow. Reports are always active -# when the feature is enabled and apply a trust-weighted penalty to the -# target's final score. Submissions from pubkeys with no TrustRank (and -# not in seed_pubkeys) are silently ignored. +# entirely — the crawler skips kind:1984 / kind:10040, POST /event returns 404, +# and vouches are ignored by ranking. Set to a value in (0, 1] to enable. The +# value is the weight of a vouch edge relative to a follow edge (1.0); e.g. 0.5 +# means each vouch contributes half the flow of an actual follow. Reports apply +# a trust-weighted penalty to the target's final score. POST /event submissions +# from pubkeys with no TrustRank (and not in seed_pubkeys) are silently ignored. vouch: weight: 0 diff --git a/config/config.go b/config/config.go index 0e3fec8..5614ad9 100644 --- a/config/config.go +++ b/config/config.go @@ -28,12 +28,13 @@ type CrawlerConfig struct { NumProfileProcessors int `yaml:"num_profile_processors"` // Number of profile event processors (default: 4) } -// VouchConfig controls the /vouch and /report API endpoints and the -// weight of vouch edges in the ranking graph. +// VouchConfig controls the vouch/report feature: whether the crawler ingests +// kind:1984 reports and kind:10040 vouch sets, whether POST /event is served, +// and the weight of vouch edges in the ranking graph. // -// A single knob: weight == 0 disables the feature entirely (endpoints -// return 404, vouches are not read by the ranking calculator); -// weight > 0 enables the feature and sets the vouch-edge weight relative +// A single knob: weight == 0 disables the feature entirely (crawler skips +// those kinds, POST /event returns 404, vouches are not read by the ranking +// calculator); weight > 0 enables it and sets the vouch-edge weight relative // to a follow edge (1.0). Typical values: 0.5. Must be in [0, 1]. type VouchConfig struct { Weight float64 `yaml:"weight"` @@ -99,9 +100,9 @@ func Load(path string) (*Config, error) { log.Printf("[CONFIG] - Crawler: batch_size=%d, request_interval=%dms, contact_processors=%d, profile_processors=%d", cfg.Crawler.BatchSize, cfg.Crawler.RequestIntervalMs, cfg.Crawler.NumContactProcessors, cfg.Crawler.NumProfileProcessors) if cfg.Vouch.Enabled() { - log.Printf("[CONFIG] - Vouch endpoints enabled (weight=%.2f)", cfg.Vouch.Weight) + log.Printf("[CONFIG] - Vouch/report feature enabled (weight=%.2f)", cfg.Vouch.Weight) } else { - log.Printf("[CONFIG] - Vouch endpoints disabled (weight=0)") + log.Printf("[CONFIG] - Vouch/report feature disabled (weight=0)") } return &cfg, nil diff --git a/internal/api/handler/event.go b/internal/api/handler/event.go new file mode 100644 index 0000000..5dc7967 --- /dev/null +++ b/internal/api/handler/event.go @@ -0,0 +1,72 @@ +package handler + +import ( + "encoding/json" + "log" + "net/http" + + "github.com/nbd-wtf/go-nostr" +) + +// PostEvent handles POST /event. It accepts a single signed Nostr event, an +// immediate push complement to the crawler's relay subscriptions. The same +// event can (and should) also be published to public relays — Fayan is just one +// of many aggregators. Supported kinds: 3 (contacts), 1984 (reports), 10040 +// (vouch sets); other kinds are rejected. +// +// As an open write endpoint it keeps the anti-inflation admission rule: the +// author must be a seed or have earned TrustRank > 0, otherwise the event is +// silently dropped with 200 so the client cannot probe admission. (The crawler +// path does not filter this way — it aggregates public events as-is, and the +// ranking stage already discounts untrusted sources.) +func (h *Handler) PostEvent(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "Method not allowed") + return + } + + var ev nostr.Event + if err := json.NewDecoder(r.Body).Decode(&ev); err != nil { + writeError(w, http.StatusBadRequest, "Invalid event JSON") + return + } + + ok, err := ev.CheckSignature() + if err != nil || !ok { + writeError(w, http.StatusUnauthorized, "Invalid event signature") + return + } + + // Silent-ignore admission rule (see doc comment). + if !h.authorQualifies(ev.PubKey) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return + } + + handled, err := h.ingester.Apply(&ev) + if err != nil { + log.Printf("[API] Error ingesting event (kind=%d pubkey=%s): %v", ev.Kind, ev.PubKey, err) + writeError(w, http.StatusInternalServerError, "Failed to ingest event") + return + } + if !handled { + writeError(w, http.StatusBadRequest, "Unsupported event kind") + return + } + + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +// authorQualifies returns true if the author is an explicit seed or has a +// positive last-computed TrustRank score. +func (h *Handler) authorQualifies(pubkey string) bool { + if _, ok := h.seedSet[pubkey]; ok { + return true + } + score, err := h.repo.GetTrustScore(pubkey) + if err != nil { + log.Printf("[API] Error reading trust_score for %s: %v", pubkey, err) + return false + } + return score > 0 +} diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index 77980a4..f3f6a7e 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -10,6 +10,7 @@ import ( "fayan/config" "fayan/internal/cache" + "fayan/internal/ingest" "fayan/internal/models" "fayan/internal/repository" @@ -23,6 +24,7 @@ type Handler struct { cache *cache.Cache searchConfig *config.SearchConfig seedSet map[string]struct{} + ingester *ingest.Ingester } // New creates a new Handler instance @@ -36,6 +38,7 @@ func New(repo *repository.Repository, cache *cache.Cache, searchConfig *config.S cache: cache, searchConfig: searchConfig, seedSet: seedSet, + ingester: ingest.New(repo), } } diff --git a/internal/api/handler/vouch.go b/internal/api/handler/vouch.go deleted file mode 100644 index b8ded87..0000000 --- a/internal/api/handler/vouch.go +++ /dev/null @@ -1,89 +0,0 @@ -package handler - -import ( - "encoding/json" - "log" - "net/http" -) - -// relationRequest is the body shape for POST /vouch and POST /report. -type relationRequest struct { - Target string `json:"target"` -} - -// Vouch handles POST /vouch. Expects a NIP-98 authenticated request. -// authorPubkey is injected by the NIP98Auth middleware. -func (h *Handler) Vouch(w http.ResponseWriter, r *http.Request, authorPubkey string) { - h.handleRelation(w, r, authorPubkey, relationKindVouch) -} - -// Report handles POST /report. Expects a NIP-98 authenticated request. -func (h *Handler) Report(w http.ResponseWriter, r *http.Request, authorPubkey string) { - h.handleRelation(w, r, authorPubkey, relationKindReport) -} - -type relationKind int - -const ( - relationKindVouch relationKind = iota - relationKindReport -) - -func (h *Handler) handleRelation(w http.ResponseWriter, r *http.Request, authorPubkey string, kind relationKind) { - if r.Method != http.MethodPost { - writeError(w, http.StatusMethodNotAllowed, "Method not allowed") - return - } - - var req relationRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "Invalid JSON body") - return - } - target, valid := normalizePubkey(req.Target) - if !valid { - writeError(w, http.StatusBadRequest, "Invalid target pubkey") - return - } - if target == authorPubkey { - writeError(w, http.StatusBadRequest, "Cannot vouch for or report yourself") - return - } - - // Silent-ignore admission rule: author must be a seed or have earned - // TrustRank > 0 in the last ranking round. Respond 200 regardless so - // the client cannot distinguish "not admitted" from "successfully stored". - if !h.authorQualifies(authorPubkey) { - writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) - return - } - - var err error - switch kind { - case relationKindVouch: - err = h.repo.SetVouch(authorPubkey, target) - case relationKindReport: - err = h.repo.SetReport(authorPubkey, target) - } - if err != nil { - log.Printf("[API] Error setting relation (author=%s target=%s kind=%d): %v", authorPubkey, target, kind, err) - writeError(w, http.StatusInternalServerError, "Failed to store relation") - return - } - - writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) -} - -// authorQualifies returns true if the author is an explicit seed or has a -// positive last-computed TrustRank score. -func (h *Handler) authorQualifies(pubkey string) bool { - if _, ok := h.seedSet[pubkey]; ok { - return true - } - score, err := h.repo.GetTrustScore(pubkey) - if err != nil { - log.Printf("[API] Error reading trust_score for %s: %v", pubkey, err) - return false - } - return score > 0 -} diff --git a/internal/api/middleware/nip98.go b/internal/api/middleware/nip98.go deleted file mode 100644 index ef3e832..0000000 --- a/internal/api/middleware/nip98.go +++ /dev/null @@ -1,177 +0,0 @@ -package middleware - -import ( - "bytes" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "io" - "net/http" - "strings" - "time" - - "github.com/nbd-wtf/go-nostr" -) - -// nip98AuthedHandler is invoked after successful NIP-98 validation, with the -// authenticated author pubkey passed in. -type nip98AuthedHandler func(w http.ResponseWriter, r *http.Request, authorPubkey string) - -const nip98Kind = 27235 -const nip98TimeWindow = 60 // seconds - -// NIP98Auth wraps a handler with NIP-98 HTTP Auth validation. -// On success, the authenticated author pubkey (hex) is passed to next. -// On failure, responds 401 with a JSON error and does not invoke next. -func NIP98Auth(next nip98AuthedHandler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // Read and replay body so downstream handler can decode it. - // (The payload tag is validated against the exact bytes we read here.) - var body []byte - if r.Body != nil { - b, err := io.ReadAll(r.Body) - if err != nil { - writeJSONError(w, http.StatusBadRequest, "failed to read body") - return - } - body = b - r.Body = io.NopCloser(bytes.NewReader(body)) - } - - ev, err := parseNIP98Header(r.Header.Get("Authorization")) - if err != nil { - writeJSONError(w, http.StatusUnauthorized, err.Error()) - return - } - - if err := validateNIP98Event(ev, r, body); err != nil { - writeJSONError(w, http.StatusUnauthorized, err.Error()) - return - } - - ok, err := ev.CheckSignature() - if err != nil || !ok { - writeJSONError(w, http.StatusUnauthorized, "invalid signature") - return - } - - next(w, r, ev.PubKey) - } -} - -func parseNIP98Header(h string) (*nostr.Event, error) { - if h == "" { - return nil, errNIP98("missing Authorization header") - } - const prefix = "Nostr " - if !strings.HasPrefix(h, prefix) { - return nil, errNIP98("Authorization must start with 'Nostr '") - } - raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(h, prefix)) - if err != nil { - // Also accept URL-safe / unpadded base64 variants for resilience. - if raw2, err2 := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(h, prefix)); err2 == nil { - raw = raw2 - } else { - return nil, errNIP98("bad base64 in Authorization") - } - } - var ev nostr.Event - if err := json.Unmarshal(raw, &ev); err != nil { - return nil, errNIP98("bad event JSON") - } - return &ev, nil -} - -func validateNIP98Event(ev *nostr.Event, r *http.Request, body []byte) error { - if ev.Kind != nip98Kind { - return errNIP98("wrong kind") - } - now := time.Now().Unix() - created := int64(ev.CreatedAt) - delta := now - created - if delta < 0 { - delta = -delta - } - if delta > nip98TimeWindow { - return errNIP98("timestamp out of window") - } - - uTag := tagValue(ev.Tags, "u") - methodTag := tagValue(ev.Tags, "method") - if uTag == "" || methodTag == "" { - return errNIP98("missing u or method tag") - } - - if !strings.EqualFold(methodTag, r.Method) { - return errNIP98("method tag mismatch") - } - if !urlMatches(uTag, r) { - return errNIP98("u tag mismatch") - } - - // Payload tag is required for POST/PUT/PATCH with a non-empty body, - // optional otherwise (per NIP-98). - if len(body) > 0 && requiresPayloadTag(r.Method) { - payloadTag := tagValue(ev.Tags, "payload") - if payloadTag == "" { - return errNIP98("missing payload tag") - } - sum := sha256.Sum256(body) - if !strings.EqualFold(payloadTag, hex.EncodeToString(sum[:])) { - return errNIP98("payload tag mismatch") - } - } - return nil -} - -func requiresPayloadTag(method string) bool { - switch strings.ToUpper(method) { - case http.MethodPost, http.MethodPut, http.MethodPatch: - return true - } - return false -} - -// urlMatches compares the NIP-98 u tag against the actual request URL. -// Supports reverse-proxy deployments via X-Forwarded-Proto / X-Forwarded-Host. -func urlMatches(uTag string, r *http.Request) bool { - scheme := "http" - if r.TLS != nil { - scheme = "https" - } - if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" { - scheme = proto - } - host := r.Host - if xfh := r.Header.Get("X-Forwarded-Host"); xfh != "" { - host = xfh - } - reconstructed := scheme + "://" + host + r.URL.RequestURI() - return uTag == reconstructed -} - -func tagValue(tags nostr.Tags, name string) string { - for _, t := range tags { - if len(t) >= 2 && t[0] == name { - return t[1] - } - } - return "" -} - -// errNIP98 wraps a sentinel error to keep the reasons user-visible yet terse. -type nip98Err string - -func (e nip98Err) Error() string { return string(e) } -func errNIP98(msg string) error { return nip98Err(msg) } - -func writeJSONError(w http.ResponseWriter, status int, message string) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(map[string]string{ - "error": http.StatusText(status), - "message": message, - }) -} diff --git a/internal/api/middleware/nip98_test.go b/internal/api/middleware/nip98_test.go deleted file mode 100644 index 4d1e5de..0000000 --- a/internal/api/middleware/nip98_test.go +++ /dev/null @@ -1,322 +0,0 @@ -package middleware - -import ( - "bytes" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/nbd-wtf/go-nostr" -) - -// signAuthEvent builds and signs a NIP-98 auth event for the given request -// parameters. Test helper. -func signAuthEvent(t *testing.T, sk, method, url string, body []byte, createdAt int64) string { - t.Helper() - tags := nostr.Tags{ - nostr.Tag{"u", url}, - nostr.Tag{"method", method}, - } - if len(body) > 0 { - sum := sha256.Sum256(body) - tags = append(tags, nostr.Tag{"payload", hex.EncodeToString(sum[:])}) - } - ev := nostr.Event{ - Kind: nip98Kind, - CreatedAt: nostr.Timestamp(createdAt), - Tags: tags, - Content: "", - } - if err := ev.Sign(sk); err != nil { - t.Fatalf("sign failed: %v", err) - } - raw, err := json.Marshal(ev) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - return "Nostr " + base64.StdEncoding.EncodeToString(raw) -} - -func mustGenKey(t *testing.T) (sk, pk string) { - t.Helper() - sk = nostr.GeneratePrivateKey() - pk, err := nostr.GetPublicKey(sk) - if err != nil { - t.Fatalf("derive pubkey: %v", err) - } - return -} - -// runRequest fires a request against a handler wrapped with NIP98Auth, using -// httptest.NewServer so r.Host/Scheme are realistic for the u-tag reconstruct. -func runRequest(t *testing.T, path string, method string, body []byte, authHeader string) (int, string, string) { - t.Helper() - var gotPubkey string - h := NIP98Auth(func(w http.ResponseWriter, r *http.Request, pubkey string) { - gotPubkey = pubkey - // Echo back body to verify it's been replayed correctly. - b, _ := io.ReadAll(r.Body) - _, _ = w.Write(b) - }) - - srv := httptest.NewServer(http.HandlerFunc(h)) - defer srv.Close() - - var rdr io.Reader - if body != nil { - rdr = bytes.NewReader(body) - } - req, err := http.NewRequest(method, srv.URL+path, rdr) - if err != nil { - t.Fatal(err) - } - if authHeader != "" { - req.Header.Set("Authorization", authHeader) - } - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - respBody, _ := io.ReadAll(resp.Body) - return resp.StatusCode, string(respBody), gotPubkey -} - -// buildURL returns the URL that NIP98Auth will reconstruct for matching. -func buildURL(serverURL, path string) string { - return serverURL + path -} - -func TestNIP98_HappyPath(t *testing.T) { - sk, pk := mustGenKey(t) - body := []byte(`{"target":"xyz"}`) - - // First set up server so we know its URL for the u tag. - var authHeader string - var gotPubkey string - mux := http.NewServeMux() - mux.Handle("/vouch", NIP98Auth(func(w http.ResponseWriter, r *http.Request, pubkey string) { - gotPubkey = pubkey - b, _ := io.ReadAll(r.Body) - _, _ = w.Write(b) - })) - srv := httptest.NewServer(mux) - defer srv.Close() - - authHeader = signAuthEvent(t, sk, "POST", srv.URL+"/vouch", body, time.Now().Unix()) - - req, _ := http.NewRequest("POST", srv.URL+"/vouch", bytes.NewReader(body)) - req.Header.Set("Authorization", authHeader) - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - respBody, _ := io.ReadAll(resp.Body) - - if resp.StatusCode != 200 { - t.Fatalf("expected 200, got %d: %s", resp.StatusCode, respBody) - } - if gotPubkey != pk { - t.Fatalf("expected pubkey %q, got %q", pk, gotPubkey) - } - if string(respBody) != string(body) { - t.Fatalf("body was not replayed to handler; got %q", respBody) - } -} - -func TestNIP98_MissingHeader(t *testing.T) { - status, _, _ := runRequest(t, "/vouch", "POST", []byte(`{}`), "") - if status != 401 { - t.Fatalf("expected 401, got %d", status) - } -} - -func TestNIP98_WrongScheme(t *testing.T) { - status, _, _ := runRequest(t, "/vouch", "POST", []byte(`{}`), "Basic xyz") - if status != 401 { - t.Fatalf("expected 401, got %d", status) - } -} - -func TestNIP98_BadBase64(t *testing.T) { - status, _, _ := runRequest(t, "/vouch", "POST", []byte(`{}`), "Nostr !!!not-base64!!!") - if status != 401 { - t.Fatalf("expected 401, got %d", status) - } -} - -func TestNIP98_WrongKind(t *testing.T) { - sk, _ := mustGenKey(t) - // Build server to get URL. - srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) - defer srv.Close() - - ev := nostr.Event{ - Kind: 1, // wrong kind - CreatedAt: nostr.Timestamp(time.Now().Unix()), - Tags: nostr.Tags{{"u", srv.URL + "/vouch"}, {"method", "POST"}}, - } - if err := ev.Sign(sk); err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(ev) - auth := "Nostr " + base64.StdEncoding.EncodeToString(raw) - - req, _ := http.NewRequest("POST", srv.URL+"/vouch", bytes.NewReader(nil)) - req.Header.Set("Authorization", auth) - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 401 { - t.Fatalf("expected 401, got %d", resp.StatusCode) - } -} - -func TestNIP98_TimestampTooOld(t *testing.T) { - sk, _ := mustGenKey(t) - srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) - defer srv.Close() - - auth := signAuthEvent(t, sk, "POST", srv.URL+"/vouch", nil, time.Now().Unix()-120) - - req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) - req.Header.Set("Authorization", auth) - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 401 { - t.Fatalf("expected 401 for old timestamp, got %d", resp.StatusCode) - } -} - -func TestNIP98_MethodMismatch(t *testing.T) { - sk, _ := mustGenKey(t) - srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) - defer srv.Close() - - // Sign with GET but send POST. - auth := signAuthEvent(t, sk, "GET", srv.URL+"/vouch", nil, time.Now().Unix()) - req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) - req.Header.Set("Authorization", auth) - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 401 { - t.Fatalf("expected 401 for method mismatch, got %d", resp.StatusCode) - } -} - -func TestNIP98_URLMismatch(t *testing.T) { - sk, _ := mustGenKey(t) - srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) - defer srv.Close() - - auth := signAuthEvent(t, sk, "POST", "https://other.example/vouch", nil, time.Now().Unix()) - req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) - req.Header.Set("Authorization", auth) - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 401 { - t.Fatalf("expected 401 for URL mismatch, got %d", resp.StatusCode) - } -} - -func TestNIP98_PayloadMismatch(t *testing.T) { - sk, _ := mustGenKey(t) - srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) - defer srv.Close() - - signedBody := []byte(`{"target":"a"}`) - actualBody := []byte(`{"target":"b"}`) - - auth := signAuthEvent(t, sk, "POST", srv.URL+"/vouch", signedBody, time.Now().Unix()) - req, _ := http.NewRequest("POST", srv.URL+"/vouch", bytes.NewReader(actualBody)) - req.Header.Set("Authorization", auth) - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 401 { - t.Fatalf("expected 401 for payload mismatch, got %d", resp.StatusCode) - } -} - -func TestNIP98_InvalidSignature(t *testing.T) { - sk, _ := mustGenKey(t) - srv := httptest.NewServer(NIP98Auth(func(w http.ResponseWriter, r *http.Request, pk string) {})) - defer srv.Close() - - auth := signAuthEvent(t, sk, "POST", srv.URL+"/vouch", nil, time.Now().Unix()) - - // Corrupt one byte of the base64-encoded signature at the tail end. - b := []byte(auth) - // Flip a character a few positions from the end (before padding). - idx := len(b) - 10 - if b[idx] == 'A' { - b[idx] = 'B' - } else { - b[idx] = 'A' - } - corrupted := string(b) - - req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) - req.Header.Set("Authorization", corrupted) - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - // Could be 401 (sig invalid) or 401 (base64 invalid) — both fine. - if resp.StatusCode != 401 { - t.Fatalf("expected 401 for corrupted auth, got %d", resp.StatusCode) - } -} - -func TestNIP98_URLReconstructWithForwardedProto(t *testing.T) { - sk, pk := mustGenKey(t) - var gotPubkey string - h := NIP98Auth(func(w http.ResponseWriter, r *http.Request, pubkey string) { - gotPubkey = pubkey - }) - srv := httptest.NewServer(http.HandlerFunc(h)) - defer srv.Close() - - // The u tag claims https, but actual srv is http. X-Forwarded-Proto=https - // should make the reconstruction succeed. - host := strings.TrimPrefix(srv.URL, "http://") - auth := signAuthEvent(t, sk, "POST", "https://"+host+"/vouch", nil, time.Now().Unix()) - - req, _ := http.NewRequest("POST", srv.URL+"/vouch", nil) - req.Header.Set("Authorization", auth) - req.Header.Set("X-Forwarded-Proto", "https") - - resp, err := srv.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("expected 200 with X-Forwarded-Proto, got %d", resp.StatusCode) - } - if gotPubkey != pk { - t.Fatalf("expected pk %q, got %q", pk, gotPubkey) - } -} diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index 852ec98..ec4ebd5 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -10,6 +10,7 @@ import ( "time" "fayan/config" + "fayan/internal/ingest" "fayan/internal/repository" "github.com/nbd-wtf/go-nostr" @@ -17,6 +18,13 @@ import ( "golang.org/x/time/rate" ) +// maxReportsPerQuery caps how many kind:1984 reports a single relay query pulls +// per author batch. Reports are append-only and a busy account can have many, +// so without a cap they would crowd out the replaceable events sharing a fetch. +// We deliberately do not paginate — successive crawl cycles re-fetch the batch, +// and reports are sparse, so the newest 50 per query suffice. +const maxReportsPerQuery = 50 + // CrawlerConfig holds the crawler configuration parameters type CrawlerConfig struct { BatchSize int @@ -33,8 +41,11 @@ type Crawler struct { seedPubkeys []string searchConfig *config.SearchConfig crawlerConfig *CrawlerConfig + vouchEnabled bool + ingester *ingest.Ingester contactsChan chan *nostr.Event profilesChan chan *nostr.Event + reportsChan chan *nostr.Event crawled map[string]bool crawledMu sync.Mutex relayLimiters map[string]*rate.Limiter @@ -56,8 +67,9 @@ type Crawler struct { cancel context.CancelFunc } -// NewCrawler creates a new Crawler instance. -func NewCrawler(repo *repository.Repository, relays []string, seedPubkeys []string, searchConfig *config.SearchConfig, crawlerConfig *CrawlerConfig) *Crawler { +// NewCrawler creates a new Crawler instance. When vouchEnabled is true the +// crawler also fetches kind:1984 reports and kind:10040 vouch sets. +func NewCrawler(repo *repository.Repository, relays []string, seedPubkeys []string, searchConfig *config.SearchConfig, crawlerConfig *CrawlerConfig, vouchEnabled bool) *Crawler { ctx, cancel := context.WithCancel(context.Background()) relayOptions := []nostr.RelayOption{ @@ -69,6 +81,7 @@ func NewCrawler(repo *repository.Repository, relays []string, seedPubkeys []stri // Calculate channel buffer sizes based on batch size contactsChanSize := crawlerConfig.BatchSize * 3 profilesChanSize := crawlerConfig.BatchSize * crawlerConfig.NumProfileProcessors * 2 + reportsChanSize := crawlerConfig.BatchSize * 2 c := &Crawler{ repo: repo, @@ -77,8 +90,11 @@ func NewCrawler(repo *repository.Repository, relays []string, seedPubkeys []stri seedPubkeys: seedPubkeys, searchConfig: searchConfig, crawlerConfig: crawlerConfig, + vouchEnabled: vouchEnabled, + ingester: ingest.New(repo), contactsChan: make(chan *nostr.Event, contactsChanSize), profilesChan: make(chan *nostr.Event, profilesChanSize), + reportsChan: make(chan *nostr.Event, reportsChanSize), crawled: make(map[string]bool), relayLimiters: make(map[string]*rate.Limiter), relayHealth: NewRelayHealthTracker(), @@ -110,6 +126,7 @@ func (c *Crawler) Stop() { // Now it's safe to close channels (no more senders) close(c.contactsChan) close(c.profilesChan) + close(c.reportsChan) // Stop the pool manager (this will close all relay connections) c.poolManager.Stop() @@ -218,6 +235,17 @@ func (c *Crawler) Start() { } } + // Processors for kind:1984 report events (only when the feature is enabled) + if c.vouchEnabled { + for range c.crawlerConfig.NumContactProcessors { + c.wg.Add(1) + go func() { + defer c.wg.Done() + c.reportProcessor() + }() + } + } + // Status reporter (not tracked in wg since it's non-critical) go c.statusReporter() } @@ -264,9 +292,7 @@ func (c *Crawler) contactProcessor() { if !ok { return // Channel closed, exit } - if event != nil { - c.processKind3Event(event) - } + c.dispatchRelationEvent(event) case <-c.ctx.Done(): // Context cancelled, drain remaining events before exiting c.drainContactsChan() @@ -275,6 +301,20 @@ func (c *Crawler) contactProcessor() { } } +// dispatchRelationEvent routes a relation event from the contacts channel to +// its handler by kind (kind:3 follows or kind:10040 vouch sets). +func (c *Crawler) dispatchRelationEvent(event *nostr.Event) { + if event == nil { + return + } + switch event.Kind { + case ingest.KindContacts: + c.processKind3Event(event) + case ingest.KindVouchSet: + c.processVouchSetEvent(event) + } +} + // drainContactsChan processes any remaining events in the contacts channel func (c *Crawler) drainContactsChan() { for { @@ -283,9 +323,7 @@ func (c *Crawler) drainContactsChan() { if !ok { return } - if event != nil { - c.processKind3Event(event) - } + c.dispatchRelationEvent(event) default: return } @@ -324,10 +362,12 @@ func (c *Crawler) fetchBatch(pubkeys []string) { } } - // Step 3: Fetch contacts (and profiles if search is enabled) from each relay concurrently + // Step 3: Fetch contacts (and profiles/vouch sets) from each relay concurrently // Collect results from all relays contactEvents := make(map[string]*nostr.Event) profileEvents := make(map[string]*nostr.Event) + vouchSetEvents := make(map[string]*nostr.Event) + var reportEvents []*nostr.Event var wg sync.WaitGroup var eventsMu sync.Mutex // Protect concurrent map writes @@ -337,7 +377,12 @@ func (c *Crawler) fetchBatch(pubkeys []string) { wg.Add(1) go func(r string, u []string) { defer wg.Done() - contacts, profiles := c.fetchEventsFromRelay(r, u, fetchProfiles) + contacts, profiles, vouchSets := c.fetchEventsFromRelay(r, u, fetchProfiles) + + var reports []*nostr.Event + if c.vouchEnabled { + reports = c.fetchReportsFromRelay(r, u) + } // Use mutex to protect map access eventsMu.Lock() @@ -351,6 +396,12 @@ func (c *Crawler) fetchBatch(pubkeys []string) { profileEvents[pubkey] = event } } + for pubkey, event := range vouchSets { + if existing, exists := vouchSetEvents[pubkey]; !exists || event.CreatedAt > existing.CreatedAt { + vouchSetEvents[pubkey] = event + } + } + reportEvents = append(reportEvents, reports...) eventsMu.Unlock() }(relay, users) } @@ -358,7 +409,8 @@ func (c *Crawler) fetchBatch(pubkeys []string) { // Wait for all relays to finish wg.Wait() - // Step 5: Check against global timestamps and send to processors + // Step 5: Check against global timestamps and send to processors. + // Contacts and vouch sets share the contacts channel (dispatched by kind). for _, event := range contactEvents { select { case c.contactsChan <- event: @@ -367,6 +419,14 @@ func (c *Crawler) fetchBatch(pubkeys []string) { } } + for _, event := range vouchSetEvents { + select { + case c.contactsChan <- event: + case <-c.ctx.Done(): + return + } + } + // Send profile events to profile processor if fetchProfiles { for _, event := range profileEvents { @@ -377,41 +437,57 @@ func (c *Crawler) fetchBatch(pubkeys []string) { } } } + + // Send report events to report processor + for _, event := range reportEvents { + select { + case c.reportsChan <- event: + case <-c.ctx.Done(): + return + } + } } -// fetchEventsFromRelay fetches contacts (kind 3) and optionally profiles (kind 0) for multiple users from a single relay -// Returns maps of pubkey -> latest event for contacts and profiles -func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProfiles bool) (map[string]*nostr.Event, map[string]*nostr.Event) { +// fetchEventsFromRelay fetches the replaceable events for multiple users from a +// single relay: kind 3 (contacts), optionally kind 0 (profiles), and — when the +// vouch feature is enabled — kind 10040 (vouch sets). All are one-per-author, so +// they share a single small-limit query. kind:1984 reports are NOT fetched here; +// being append-only and numerous, they get their own query (fetchReportsFromRelay). +// Returns maps of pubkey -> latest event for contacts, profiles, and vouch sets. +func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProfiles bool) (map[string]*nostr.Event, map[string]*nostr.Event, map[string]*nostr.Event) { if len(pubkeys) == 0 { - return nil, nil + return nil, nil, nil } // Check if context is cancelled select { case <-c.ctx.Done(): - return nil, nil + return nil, nil, nil default: } // Skip if relay is banned if c.relayHealth.IsRelayBanned(relay) { - return nil, nil + return nil, nil, nil } // Apply rate limiting for this specific relay limiter := c.getRelayLimiter(relay) if err := limiter.Wait(c.ctx); err != nil { - return nil, nil + return nil, nil, nil } ctx, cancel := context.WithTimeout(c.ctx, 15*time.Second) defer cancel() - // Build filter with kinds 3 (contacts) and optionally 0 (profiles) + // Build filter with kinds 3 (contacts), optionally 0 (profiles) and 10040 (vouch sets) kinds := []int{3} if fetchProfiles { kinds = append(kinds, 0) } + if c.vouchEnabled { + kinds = append(kinds, ingest.KindVouchSet) + } filter := nostr.Filter{ Kinds: kinds, @@ -430,6 +506,7 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf // Collect events and keep only the latest for each pubkey contacts := make(map[string]*nostr.Event) profiles := make(map[string]*nostr.Event) + vouchSets := make(map[string]*nostr.Event) timer := time.NewTimer(10 * time.Second) // Slightly less than context timeout defer timer.Stop() channelClosed := false @@ -445,7 +522,7 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf channelClosed = true c.relayHealth.RecordSuccess(relay) } - return contacts, profiles + return contacts, profiles, vouchSets } ev := relayEvent.Event @@ -460,19 +537,80 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf if existing, exists := profiles[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { profiles[ev.PubKey] = ev } + case ingest.KindVouchSet: + if existing, exists := vouchSets[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { + vouchSets[ev.PubKey] = ev + } } case <-timer.C: // Timeout - this could indicate connection issues if !channelClosed { c.relayHealth.RecordFailure(relay, "timeout - no response") } - return contacts, profiles + return contacts, profiles, vouchSets case <-ctx.Done(): // Context cancelled if !channelClosed { c.relayHealth.RecordFailure(relay, "context cancelled") } - return contacts, profiles + return contacts, profiles, vouchSets + } + } +} + +// fetchReportsFromRelay fetches kind:1984 reports for multiple users from a +// single relay in their own capped query, kept separate from the replaceable +// events so a flood of reports cannot crowd them out. Returns all matching +// events (an author may have many); de-duplication happens at ingest time. +func (c *Crawler) fetchReportsFromRelay(relay string, pubkeys []string) []*nostr.Event { + if len(pubkeys) == 0 { + return nil + } + + select { + case <-c.ctx.Done(): + return nil + default: + } + + if c.relayHealth.IsRelayBanned(relay) { + return nil + } + + limiter := c.getRelayLimiter(relay) + if err := limiter.Wait(c.ctx); err != nil { + return nil + } + + ctx, cancel := context.WithTimeout(c.ctx, 15*time.Second) + defer cancel() + + limit := maxReportsPerQuery + filter := nostr.Filter{ + Kinds: []int{ingest.KindReport}, + Authors: pubkeys, + Limit: limit, + } + + pool := c.poolManager.GetPool() + eventsChan := pool.FetchMany(ctx, []string{relay}, filter) + c.poolManager.TrackRelayUsage(relay) + + var reports []*nostr.Event + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + + for { + select { + case relayEvent, ok := <-eventsChan: + if !ok { + return reports + } + reports = append(reports, relayEvent.Event) + case <-timer.C: + return reports + case <-ctx.Done(): + return reports } } } @@ -584,39 +722,13 @@ func (c *Crawler) isValidRelay(url string) bool { // processKind3Event parses a kind:3 event and updates the database and work queue. // Uses batch writes to reduce lock contention. func (c *Crawler) processKind3Event(ev *nostr.Event) { - if ev.Kind != 3 { + if ev.Kind != ingest.KindContacts { return } - // Collect all pubkeys and connections for batch write - pubkeySet := make(map[string]bool) - pubkeySet[ev.PubKey] = true - - var connections []repository.Connection - - for _, tag := range ev.Tags { - if len(tag) >= 2 && tag[0] == "p" { - targetPubkey := tag[1] - if !nostr.IsValidPublicKey(targetPubkey) { - continue - } - if targetPubkey == ev.PubKey { - continue - } - - pubkeySet[targetPubkey] = true - connections = append(connections, repository.Connection{ - Source: ev.PubKey, - Target: targetPubkey, - }) - } - } - - // Convert set to slice - pubkeys := make([]string, 0, len(pubkeySet)) - for pk := range pubkeySet { - pubkeys = append(pubkeys, pk) - } + // Parse with the shared ingest parser so the crawler and POST /event paths + // produce identical follow edges. + pubkeys, connections := ingest.ParseContacts(ev) // Batch write all pubkeys and connections in a single transaction if err := c.repo.BatchUpsertPubkeysAndConnections(pubkeys, connections); err != nil { @@ -625,12 +737,80 @@ func (c *Crawler) processKind3Event(ev *nostr.Event) { // Update crawled map c.crawledMu.Lock() - for pk := range pubkeySet { + for _, pk := range pubkeys { c.crawled[pk] = true } c.crawledMu.Unlock() } +// processVouchSetEvent verifies a kind:10040 event's signature and replaces the +// author's vouch edges with the listed pubkeys. +func (c *Crawler) processVouchSetEvent(ev *nostr.Event) { + if ev.Kind != ingest.KindVouchSet { + return + } + if ok, err := ev.CheckSignature(); err != nil || !ok { + return + } + if err := c.ingester.ApplyVouchSet(ev); err != nil { + log.Printf("[CRAWLER] Error applying vouch set for %s: %v", ev.PubKey, err) + } +} + +// processReportEvent verifies a kind:1984 event's signature and stores it as a +// report edge when it is a profile-level spam/impersonation report. +func (c *Crawler) processReportEvent(ev *nostr.Event) { + if ev.Kind != ingest.KindReport { + return + } + if ok, err := ev.CheckSignature(); err != nil || !ok { + return + } + if _, err := c.ingester.ApplyReport(ev); err != nil { + log.Printf("[CRAWLER] Error applying report from %s: %v", ev.PubKey, err) + } +} + +// reportProcessor handles processing of report events (kind 1984) +func (c *Crawler) reportProcessor() { + for { + if c.waitIfPaused() { + c.drainReportsChan() + return + } + + select { + case event, ok := <-c.reportsChan: + if !ok { + return + } + if event != nil { + c.processReportEvent(event) + } + case <-c.ctx.Done(): + c.drainReportsChan() + return + } + } +} + +// drainReportsChan processes any remaining events in the reports channel +func (c *Crawler) drainReportsChan() { + for { + select { + case event, ok := <-c.reportsChan: + if !ok { + return + } + if event != nil { + c.processReportEvent(event) + } + default: + return + } + } +} + // profileProcessor handles processing of profile events (kind 0) func (c *Crawler) profileProcessor() { for { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go new file mode 100644 index 0000000..508385e --- /dev/null +++ b/internal/ingest/ingest.go @@ -0,0 +1,157 @@ +// Package ingest turns signed Nostr events into reputation-graph data. The same +// logic backs both the crawler (events pulled from relays) and the API's +// POST /event endpoint (events pushed by clients), so the two paths stay in sync. +package ingest + +import ( + "fayan/internal/repository" + + "github.com/nbd-wtf/go-nostr" +) + +// Supported event kinds. +const ( + KindContacts = 3 // NIP-02 contact list → follow edges + KindReport = 1984 // NIP-56 report → report edge (profile-level only) + KindVouchSet = 10040 // Fayan vouch set: a replaceable list of vouched pubkeys +) + +// acceptedReportTypes are the NIP-56 report types that affect reputation. Fayan +// is a spam-detection system, so only these two carry weight; other types +// (nudity, profanity, …) are ignored. +var acceptedReportTypes = map[string]bool{ + "spam": true, + "impersonation": true, +} + +// Ingester applies signature-verified Nostr events to the repository. +type Ingester struct { + repo *repository.Repository +} + +// New creates an Ingester backed by repo. +func New(repo *repository.Repository) *Ingester { + return &Ingester{repo: repo} +} + +// Apply dispatches a verified event by kind, reporting whether the kind is one +// Fayan ingests. The caller is responsible for verifying ev's signature first. +func (in *Ingester) Apply(ev *nostr.Event) (handled bool, err error) { + switch ev.Kind { + case KindContacts: + return true, in.ApplyContacts(ev) + case KindReport: + _, err := in.ApplyReport(ev) + return true, err + case KindVouchSet: + return true, in.ApplyVouchSet(ev) + } + return false, nil +} + +// ApplyContacts parses a kind:3 event into follow connections and persists them. +func (in *Ingester) ApplyContacts(ev *nostr.Event) error { + if ev.Kind != KindContacts { + return nil + } + pubkeys, connections := ParseContacts(ev) + return in.repo.BatchUpsertPubkeysAndConnections(pubkeys, connections) +} + +// ParseContacts extracts the author plus followed pubkeys and the follow edges +// from a kind:3 event. Exported so the crawler shares one parser with the API. +func ParseContacts(ev *nostr.Event) ([]string, []repository.Connection) { + pubkeySet := make(map[string]bool) + pubkeySet[ev.PubKey] = true + + var connections []repository.Connection + for _, tag := range ev.Tags { + if len(tag) >= 2 && tag[0] == "p" { + target := tag[1] + if !nostr.IsValidPublicKey(target) || target == ev.PubKey { + continue + } + pubkeySet[target] = true + connections = append(connections, repository.Connection{Source: ev.PubKey, Target: target}) + } + } + + pubkeys := make([]string, 0, len(pubkeySet)) + for pk := range pubkeySet { + pubkeys = append(pubkeys, pk) + } + return pubkeys, connections +} + +// ApplyReport stores a profile-level spam/impersonation report as a report edge. +// It returns false (without error) when the event is not an accepted profile +// report: it targets a specific event (has an e tag), lacks a usable p target, +// or carries a report type Fayan does not weigh. +func (in *Ingester) ApplyReport(ev *nostr.Event) (bool, error) { + if ev.Kind != KindReport { + return false, nil + } + target, ok := profileReportTarget(ev) + if !ok { + return false, nil + } + return true, in.repo.UpsertReport(ev.PubKey, target, ev.CreatedAt.Time()) +} + +// profileReportTarget returns the reported pubkey when ev is a NIP-56 report +// that targets a profile (not a specific event) with an accepted report type. +// Any e tag disqualifies the event — that is a note-level report, out of scope. +func profileReportTarget(ev *nostr.Event) (string, bool) { + target := "" + for _, tag := range ev.Tags { + if len(tag) == 0 { + continue + } + switch tag[0] { + case "e": + return "", false + case "p": + if target != "" || len(tag) < 2 { + continue + } + pk := tag[1] + if !nostr.IsValidPublicKey(pk) || pk == ev.PubKey { + continue + } + // NIP-56 carries the report type in the p tag's third element. + if len(tag) < 3 || !acceptedReportTypes[tag[2]] { + continue + } + target = pk + } + } + return target, target != "" +} + +// ApplyVouchSet replaces the author's vouch edges with the pubkeys listed in a +// kind:10040 event, honouring the replaceable semantics of the set. +func (in *Ingester) ApplyVouchSet(ev *nostr.Event) error { + if ev.Kind != KindVouchSet { + return nil + } + targets := ParseVouchTargets(ev) + return in.repo.UpsertVouches(ev.PubKey, targets) +} + +// ParseVouchTargets extracts the valid, de-duplicated pubkeys an author vouches +// for from a kind:10040 event's p tags (excluding the author themselves). +func ParseVouchTargets(ev *nostr.Event) []string { + seen := make(map[string]bool) + var targets []string + for _, tag := range ev.Tags { + if len(tag) >= 2 && tag[0] == "p" { + pk := tag[1] + if !nostr.IsValidPublicKey(pk) || pk == ev.PubKey || seen[pk] { + continue + } + seen[pk] = true + targets = append(targets, pk) + } + } + return targets +} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go new file mode 100644 index 0000000..b7c42a9 --- /dev/null +++ b/internal/ingest/ingest_test.go @@ -0,0 +1,86 @@ +package ingest + +import ( + "testing" + + "github.com/nbd-wtf/go-nostr" +) + +// mustPubkey returns a real, curve-valid x-only pubkey (IsValidPublicKey parses +// the secp256k1 point, so arbitrary hex will not do). +func mustPubkey(t *testing.T) string { + t.Helper() + pk, err := nostr.GetPublicKey(nostr.GeneratePrivateKey()) + if err != nil { + t.Fatalf("derive pubkey: %v", err) + } + return pk +} + +func TestParseContacts(t *testing.T) { + author, bob, carol := mustPubkey(t), mustPubkey(t), mustPubkey(t) + ev := &nostr.Event{ + PubKey: author, + Kind: KindContacts, + Tags: nostr.Tags{ + {"p", bob}, + {"p", carol}, + {"p", author}, // self — excluded + {"p", "not-hex"}, // invalid — excluded + {"e", "whatever"}, // non-p — ignored + }, + } + pubkeys, conns := ParseContacts(ev) + if len(conns) != 2 { + t.Fatalf("expected 2 connections, got %d", len(conns)) + } + // author + bob + carol = 3 distinct pubkeys + if len(pubkeys) != 3 { + t.Fatalf("expected 3 pubkeys, got %d", len(pubkeys)) + } +} + +func TestParseVouchTargets(t *testing.T) { + author, bob, carol := mustPubkey(t), mustPubkey(t), mustPubkey(t) + ev := &nostr.Event{ + PubKey: author, + Kind: KindVouchSet, + Tags: nostr.Tags{ + {"p", bob}, + {"p", bob}, // duplicate — collapsed + {"p", carol}, + {"p", author}, // self — excluded + }, + } + targets := ParseVouchTargets(ev) + if len(targets) != 2 { + t.Fatalf("expected 2 deduped targets, got %d: %v", len(targets), targets) + } +} + +func TestProfileReportTarget(t *testing.T) { + author, bob := mustPubkey(t), mustPubkey(t) + cases := []struct { + name string + tags nostr.Tags + want string + wantOK bool + }{ + {"spam profile report", nostr.Tags{{"p", bob, "spam"}}, bob, true}, + {"impersonation profile report", nostr.Tags{{"p", bob, "impersonation"}}, bob, true}, + {"unweighted type (nudity)", nostr.Tags{{"p", bob, "nudity"}}, "", false}, + {"missing report type", nostr.Tags{{"p", bob}}, "", false}, + {"event-level report (has e tag)", nostr.Tags{{"e", "evt", "spam"}, {"p", bob, "spam"}}, "", false}, + {"self report", nostr.Tags{{"p", author, "spam"}}, "", false}, + {"no p tag", nostr.Tags{{"e", "evt", "spam"}}, "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ev := &nostr.Event{PubKey: author, Kind: KindReport, Tags: tc.tags} + got, ok := profileReportTarget(ev) + if ok != tc.wantOK || got != tc.want { + t.Fatalf("got (%q, %v), want (%q, %v)", got, ok, tc.want, tc.wantOK) + } + }) + } +} diff --git a/internal/ranking/calculator.go b/internal/ranking/calculator.go index 5c83d93..c491010 100644 --- a/internal/ranking/calculator.go +++ b/internal/ranking/calculator.go @@ -126,7 +126,7 @@ func (c *Calculator) Calculate() error { edges = append(edges, edge{source: sourceID, target: targetID, weight: c.vouchWeight}) vouchAdmitted++ return nil - }); err != nil { + }, &cutoffTime); err != nil { return err } log.Printf(" [INFO] Vouch edges admitted: %d (weight=%.2f)", vouchAdmitted, c.vouchWeight) diff --git a/internal/ranking/calculator_test.go b/internal/ranking/calculator_test.go index 45dff7f..522d2de 100644 --- a/internal/ranking/calculator_test.go +++ b/internal/ranking/calculator_test.go @@ -63,7 +63,7 @@ func TestVouchPromotesUnfollowedUser(t *testing.T) { } // Seed vouches for a newbie nobody follows. - if err := repo.SetVouch("seed1", "newbie"); err != nil { + if err := repo.UpsertVouches("seed1", []string{"newbie"}); err != nil { t.Fatal(err) } @@ -100,7 +100,7 @@ func TestVouchWeightShrinksContribution(t *testing.T) { if err := calcHigh.Calculate(); err != nil { t.Fatal(err) } - if err := repo.SetVouch("seed1", "newbie"); err != nil { + if err := repo.UpsertVouches("seed1", []string{"newbie"}); err != nil { t.Fatal(err) } @@ -131,7 +131,7 @@ func TestVouchAndFollowDedupe(t *testing.T) { repo := newTestRepo(t) insertFollow(t, repo, "a", "b") - if err := repo.SetVouch("a", "b"); err != nil { + if err := repo.UpsertVouches("a", []string{"b"}); err != nil { t.Fatal(err) } // Give A trust so vouch edge would be admitted. @@ -186,7 +186,7 @@ func TestReportDecaysScore(t *testing.T) { // All three seeds report X. for _, s := range seeds { - if err := repo.SetReport(s, "x"); err != nil { + if err := repo.UpsertReport(s, "x", time.Now()); err != nil { t.Fatal(err) } } @@ -224,7 +224,7 @@ func TestReportWithNoTrustIgnored(t *testing.T) { ); err != nil { t.Fatal(err) } - if err := repo.SetReport("troll", "target"); err != nil { + if err := repo.UpsertReport("troll", "target", time.Now()); err != nil { t.Fatal(err) } diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 3dc98d8..ef9e039 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -99,11 +99,14 @@ var migrations = []Migration{ Version: 4, Name: "add_vouches_and_reports", Up: func(db *sql.DB) error { + // Vouches share the follow-edge lifecycle: refreshed on each + // kind:10040 set, never actively deleted, aged out by a staleness + // window — hence last_seen (cf. connections), not created_at. vouchesTable := ` CREATE TABLE IF NOT EXISTS vouches ( source_pubkey TEXT NOT NULL, target_pubkey TEXT NOT NULL, - created_at TIMESTAMP NOT NULL, + last_seen TIMESTAMP NOT NULL, PRIMARY KEY (source_pubkey, target_pubkey) );` @@ -124,6 +127,9 @@ var migrations = []Migration{ if _, err := db.Exec("CREATE INDEX IF NOT EXISTS idx_vouches_target ON vouches(target_pubkey);"); err != nil { return fmt.Errorf("failed to create idx_vouches_target: %w", err) } + if _, err := db.Exec("CREATE INDEX IF NOT EXISTS idx_vouches_last_seen ON vouches(last_seen);"); err != nil { + return fmt.Errorf("failed to create idx_vouches_last_seen: %w", err) + } if _, err := db.Exec("CREATE INDEX IF NOT EXISTS idx_reports_target ON reports(target_pubkey);"); err != nil { return fmt.Errorf("failed to create idx_reports_target: %w", err) } diff --git a/internal/repository/vouch.go b/internal/repository/vouch.go index fb4c825..5b61d6b 100644 --- a/internal/repository/vouch.go +++ b/internal/repository/vouch.go @@ -1,26 +1,24 @@ package repository import ( + "database/sql" "fmt" "time" "fayan/internal/models" ) -// SetVouch records source→target as a vouch relationship and atomically removes -// any existing report from source to the same target (mutual-exclusion toggle). -// The target pubkey row is upserted so ranking can cover brand-new targets. -func (r *Repository) SetVouch(source, target string) error { - return r.setRelation(source, target, "vouches", "reports") -} - -// SetReport records source→target as a report and atomically removes any -// existing vouch from source to the same target. -func (r *Repository) SetReport(source, target string) error { - return r.setRelation(source, target, "reports", "vouches") -} +// UpsertVouches refreshes the vouch edges from source for the pubkeys in the +// latest kind:10040 set, mirroring how kind:3 contacts are stored: each edge is +// upserted with last_seen = now and never actively deleted. A vouch dropped +// from the set simply stops being refreshed and ages out of the ranking graph +// via the same staleness window as follows (see StreamVouches). Targets are +// upserted into pubkeys so ranking can cover brand-new accounts. +func (r *Repository) UpsertVouches(source string, targets []string) error { + if len(targets) == 0 { + return nil + } -func (r *Repository) setRelation(source, target, insertTable, deleteTable string) error { r.writeMu.Lock() defer r.writeMu.Unlock() @@ -32,6 +30,46 @@ func (r *Repository) setRelation(source, target, insertTable, deleteTable string now := time.Now().UTC() + pkStmt, err := tx.Prepare(`INSERT INTO pubkeys (pubkey, created_at, updated_at) VALUES (?, ?, ?) ON CONFLICT(pubkey) DO NOTHING;`) + if err != nil { + return fmt.Errorf("failed to prepare pubkey statement: %w", err) + } + defer pkStmt.Close() + + vStmt, err := tx.Prepare(`REPLACE INTO vouches (source_pubkey, target_pubkey, last_seen) VALUES (?, ?, ?);`) + if err != nil { + return fmt.Errorf("failed to prepare vouch statement: %w", err) + } + defer vStmt.Close() + + for _, target := range targets { + if _, err := pkStmt.Exec(target, now, now); err != nil { + return fmt.Errorf("failed to upsert target pubkey %s: %w", target, err) + } + if _, err := vStmt.Exec(source, target, now); err != nil { + return fmt.Errorf("failed to insert vouch %s -> %s: %w", source, target, err) + } + } + + return tx.Commit() +} + +// UpsertReport records source→target as a report edge. Reports are additive +// (kind:1984 events are not replaceable); re-reporting the same target just +// refreshes the timestamp. The target is upserted into pubkeys so ranking can +// cover brand-new accounts. No vouch is touched — the vouch-beats-report +// precedence is resolved at ranking time (see GetTrustWeightedReports). +func (r *Repository) UpsertReport(source, target string, createdAt time.Time) error { + r.writeMu.Lock() + defer r.writeMu.Unlock() + + tx, err := r.db.Begin() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + + now := createdAt.UTC() if _, err := tx.Exec( `INSERT INTO pubkeys (pubkey, created_at, updated_at) VALUES (?, ?, ?) ON CONFLICT(pubkey) DO NOTHING;`, target, now, now, @@ -40,17 +78,10 @@ func (r *Repository) setRelation(source, target, insertTable, deleteTable string } if _, err := tx.Exec( - fmt.Sprintf(`DELETE FROM %s WHERE source_pubkey = ? AND target_pubkey = ?;`, deleteTable), - source, target, - ); err != nil { - return fmt.Errorf("failed to delete opposite relation: %w", err) - } - - if _, err := tx.Exec( - fmt.Sprintf(`INSERT OR REPLACE INTO %s (source_pubkey, target_pubkey, created_at) VALUES (?, ?, ?);`, insertTable), + `INSERT OR REPLACE INTO reports (source_pubkey, target_pubkey, created_at) VALUES (?, ?, ?);`, source, target, now, ); err != nil { - return fmt.Errorf("failed to insert relation: %w", err) + return fmt.Errorf("failed to insert report: %w", err) } return tx.Commit() @@ -68,9 +99,17 @@ func (r *Repository) GetTrustScore(pubkey string) (float64, error) { return score, nil } -// StreamVouches streams all vouch edges. Shape mirrors StreamConnections. -func (r *Repository) StreamVouches(callback func(models.Vouch) error) error { - rows, err := r.db.Query("SELECT source_pubkey, target_pubkey FROM vouches;") +// StreamVouches streams vouch edges. When afterTime is non-nil, only edges +// refreshed at or after it are returned — the same staleness window that ages +// out follow edges, so a vouch dropped from a set eventually stops counting. +func (r *Repository) StreamVouches(callback func(models.Vouch) error, afterTime *time.Time) error { + var rows *sql.Rows + var err error + if afterTime != nil { + rows, err = r.db.Query("SELECT source_pubkey, target_pubkey FROM vouches WHERE last_seen >= ?;", afterTime) + } else { + rows, err = r.db.Query("SELECT source_pubkey, target_pubkey FROM vouches;") + } if err != nil { return fmt.Errorf("failed to query vouches: %w", err) } @@ -110,13 +149,20 @@ func (r *Repository) GetPubkeysWithPositiveTrust() (map[string]struct{}, error) // GetTrustWeightedReports aggregates reports per target, weighting each report // by the reporter's trust_score. Reporters with trust_score ≤ 0 are excluded — -// the same admission rule that gates vouch edges. +// the same admission rule that gates vouch edges. A report is also ignored when +// the same source vouches for the same target: vouch beats report, resolved +// here at ranking time rather than by mutual exclusion at write time. func (r *Repository) GetTrustWeightedReports() (map[string]models.ReportAggregate, error) { query := ` SELECT r.target_pubkey, COUNT(*), COALESCE(SUM(p.trust_score), 0) FROM reports r JOIN pubkeys p ON p.pubkey = r.source_pubkey WHERE p.trust_score > 0 + AND NOT EXISTS ( + SELECT 1 FROM vouches v + WHERE v.source_pubkey = r.source_pubkey + AND v.target_pubkey = r.target_pubkey + ) GROUP BY r.target_pubkey; ` rows, err := r.db.Query(query) diff --git a/internal/repository/vouch_test.go b/internal/repository/vouch_test.go index 9d406dd..053718f 100644 --- a/internal/repository/vouch_test.go +++ b/internal/repository/vouch_test.go @@ -44,76 +44,100 @@ func countRows(t *testing.T, repo *Repository, table, source, target string) int return n } -func TestSetVouch_NewInsert(t *testing.T) { +var ( + t1 = time.Unix(1_700_000_000, 0).UTC() + t2 = time.Unix(1_700_000_100, 0).UTC() +) + +func TestUpsertVouches_NewInsert(t *testing.T) { repo := newTestRepo(t) - if err := repo.SetVouch("alice", "bob"); err != nil { - t.Fatalf("SetVouch failed: %v", err) + if err := repo.UpsertVouches("alice", []string{"bob"}); err != nil { + t.Fatalf("UpsertVouches failed: %v", err) } if countRows(t, repo, "vouches", "alice", "bob") != 1 { t.Fatalf("expected one vouch row") } } -func TestSetVouch_Idempotent(t *testing.T) { +// TestUpsertVouches_DoesNotDelete verifies vouches follow the follow-edge +// lifecycle: a target dropped from a later set is NOT actively removed — it +// lingers (to be aged out by the staleness window at ranking time). +func TestUpsertVouches_DoesNotDelete(t *testing.T) { repo := newTestRepo(t) - if err := repo.SetVouch("alice", "bob"); err != nil { + if err := repo.UpsertVouches("alice", []string{"bob", "charlie"}); err != nil { t.Fatal(err) } - if err := repo.SetVouch("alice", "bob"); err != nil { + // A later set without bob must not delete bob's edge. + if err := repo.UpsertVouches("alice", []string{"charlie", "dave"}); err != nil { t.Fatal(err) } if countRows(t, repo, "vouches", "alice", "bob") != 1 { - t.Fatalf("expected exactly one vouch after duplicate SetVouch") + t.Fatalf("expected bob to linger (not actively deleted)") + } + if countRows(t, repo, "vouches", "alice", "charlie") != 1 { + t.Fatalf("expected charlie to remain") + } + if countRows(t, repo, "vouches", "alice", "dave") != 1 { + t.Fatalf("expected dave to be added") } } -func TestSetVouch_MutualExclusion_DeletesReport(t *testing.T) { +func TestUpsertVouches_EmptyNoop(t *testing.T) { repo := newTestRepo(t) - if err := repo.SetReport("alice", "bob"); err != nil { + if err := repo.UpsertVouches("alice", []string{"bob"}); err != nil { t.Fatal(err) } - if countRows(t, repo, "reports", "alice", "bob") != 1 { - t.Fatalf("expected report pre-existing") + // An empty set is a no-op: nothing is refreshed, nothing is deleted. + if err := repo.UpsertVouches("alice", nil); err != nil { + t.Fatal(err) + } + if countRows(t, repo, "vouches", "alice", "bob") != 1 { + t.Fatalf("expected bob to remain after empty set (no active delete)") } +} - if err := repo.SetVouch("alice", "bob"); err != nil { +func TestUpsertVouches_UpsertsTargetPubkey(t *testing.T) { + repo := newTestRepo(t) + if err := repo.UpsertVouches("alice", []string{"brand-new-target"}); err != nil { t.Fatal(err) } - if countRows(t, repo, "reports", "alice", "bob") != 0 { - t.Fatalf("expected prior report to be deleted by SetVouch") + var n int + if err := repo.db.QueryRow("SELECT COUNT(*) FROM pubkeys WHERE pubkey = ?;", "brand-new-target").Scan(&n); err != nil { + t.Fatal(err) } - if countRows(t, repo, "vouches", "alice", "bob") != 1 { - t.Fatalf("expected vouch to exist") + if n != 1 { + t.Fatalf("expected target pubkey to be upserted into pubkeys table") } } -func TestSetReport_MutualExclusion_DeletesVouch(t *testing.T) { +func TestUpsertReport_NewAndIdempotent(t *testing.T) { repo := newTestRepo(t) - if err := repo.SetVouch("alice", "bob"); err != nil { + if err := repo.UpsertReport("alice", "bob", t1); err != nil { t.Fatal(err) } - if err := repo.SetReport("alice", "bob"); err != nil { + if err := repo.UpsertReport("alice", "bob", t2); err != nil { t.Fatal(err) } - if countRows(t, repo, "vouches", "alice", "bob") != 0 { - t.Fatalf("expected prior vouch to be deleted by SetReport") - } if countRows(t, repo, "reports", "alice", "bob") != 1 { - t.Fatalf("expected report to exist") + t.Fatalf("expected exactly one report row after re-report") } } -func TestSetVouch_UpsertsTargetPubkey(t *testing.T) { +func TestVouchAndReportCoexist(t *testing.T) { repo := newTestRepo(t) - if err := repo.SetVouch("alice", "brand-new-target"); err != nil { + // No mutual exclusion at write time: both rows persist; precedence is + // resolved at ranking time (vouch beats report). + if err := repo.UpsertVouches("alice", []string{"bob"}); err != nil { t.Fatal(err) } - var n int - if err := repo.db.QueryRow("SELECT COUNT(*) FROM pubkeys WHERE pubkey = ?;", "brand-new-target").Scan(&n); err != nil { + if err := repo.UpsertReport("alice", "bob", t1); err != nil { t.Fatal(err) } - if n != 1 { - t.Fatalf("expected target pubkey to be upserted into pubkeys table") + if countRows(t, repo, "vouches", "alice", "bob") != 1 { + t.Fatalf("expected vouch to persist alongside report") + } + if countRows(t, repo, "reports", "alice", "bob") != 1 { + t.Fatalf("expected report to persist alongside vouch") } } @@ -142,13 +166,10 @@ func TestGetTrustScore_KnownPubkey(t *testing.T) { func TestStreamVouches(t *testing.T) { repo := newTestRepo(t) - if err := repo.SetVouch("alice", "bob"); err != nil { + if err := repo.UpsertVouches("alice", []string{"bob", "charlie"}); err != nil { t.Fatal(err) } - if err := repo.SetVouch("alice", "charlie"); err != nil { - t.Fatal(err) - } - if err := repo.SetVouch("dave", "bob"); err != nil { + if err := repo.UpsertVouches("dave", []string{"bob"}); err != nil { t.Fatal(err) } @@ -156,7 +177,7 @@ func TestStreamVouches(t *testing.T) { if err := repo.StreamVouches(func(v models.Vouch) error { got = append(got, v) return nil - }); err != nil { + }, nil); err != nil { t.Fatal(err) } if len(got) != 3 { @@ -164,6 +185,32 @@ func TestStreamVouches(t *testing.T) { } } +// TestStreamVouches_StaleFiltered verifies the staleness window: edges last +// seen before the cutoff are excluded, the same way stale follow edges are. +func TestStreamVouches_StaleFiltered(t *testing.T) { + repo := newTestRepo(t) + if err := repo.UpsertVouches("alice", []string{"bob"}); err != nil { + t.Fatal(err) + } + + count := func(after *time.Time) int { + n := 0 + if err := repo.StreamVouches(func(models.Vouch) error { n++; return nil }, after); err != nil { + t.Fatal(err) + } + return n + } + + past := time.Now().UTC().Add(-time.Hour) + if count(&past) != 1 { + t.Fatalf("expected the fresh vouch to pass a past cutoff") + } + future := time.Now().UTC().Add(time.Hour) + if count(&future) != 0 { + t.Fatalf("expected the vouch to be filtered out by a future cutoff") + } +} + func TestGetPubkeysWithPositiveTrust(t *testing.T) { repo := newTestRepo(t) seedPubkey(t, repo, "high", 0.5) @@ -191,13 +238,13 @@ func TestGetTrustWeightedReports(t *testing.T) { seedPubkey(t, repo, "r2", 0.7) seedPubkey(t, repo, "r3", 0) // untrusted; should be excluded - if err := repo.SetReport("r1", "target"); err != nil { + if err := repo.UpsertReport("r1", "target", t1); err != nil { t.Fatal(err) } - if err := repo.SetReport("r2", "target"); err != nil { + if err := repo.UpsertReport("r2", "target", t1); err != nil { t.Fatal(err) } - if err := repo.SetReport("r3", "target"); err != nil { + if err := repo.UpsertReport("r3", "target", t1); err != nil { t.Fatal(err) } @@ -218,6 +265,37 @@ func TestGetTrustWeightedReports(t *testing.T) { } } +// TestGetTrustWeightedReports_VouchBeatsReport verifies a reporter who also +// vouches for the same target is excluded from the report aggregate. +func TestGetTrustWeightedReports_VouchBeatsReport(t *testing.T) { + repo := newTestRepo(t) + seedPubkey(t, repo, "r1", 0.3) + seedPubkey(t, repo, "r2", 0.7) + + // Both report target; r1 also vouches for target → r1's report is ignored. + if err := repo.UpsertReport("r1", "target", t1); err != nil { + t.Fatal(err) + } + if err := repo.UpsertReport("r2", "target", t1); err != nil { + t.Fatal(err) + } + if err := repo.UpsertVouches("r1", []string{"target"}); err != nil { + t.Fatal(err) + } + + reports, err := repo.GetTrustWeightedReports() + if err != nil { + t.Fatal(err) + } + agg := reports["target"] + if agg.NumReporters != 1 { + t.Fatalf("expected 1 effective reporter (r1's vouch beats its report), got %d", agg.NumReporters) + } + if absDiff(agg.TotalReporterTrust, 0.7) > 1e-9 { + t.Fatalf("expected trust sum 0.7 (only r2), got %v", agg.TotalReporterTrust) + } +} + func absDiff(a, b float64) float64 { if a > b { return a - b From 437703a3b2fd79f476dcb2154dab9d9259f14558 Mon Sep 17 00:00:00 2001 From: codytseng Date: Thu, 18 Jun 2026 19:43:32 +0800 Subject: [PATCH 06/10] refactor: drop vouch-beats-report precedence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source that both vouches for and reports the same target no longer gets special handling. The vouch adds flow and the report subtracts it at ranking time, which roughly cancels out on its own — so the extra NOT EXISTS subquery in GetTrustWeightedReports was redundant complexity. --- CLAUDE.md | 2 +- internal/repository/vouch.go | 11 +++------- internal/repository/vouch_test.go | 35 ++----------------------------- 3 files changed, 6 insertions(+), 42 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2c22f6e..862dc3f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,4 +93,4 @@ Shared parsing/storage lives in `internal/ingest` so both paths behave identical - **Vouch** = membership in the author's **kind:10040** vouch set (a custom replaceable event; not in any NIP). Its `p` tags list the vouched pubkeys. Registers source→target as a vouch edge (weight `vouch.weight`, deduped against a follow from the same source). Vouches follow the **same lifecycle as follow edges** (`vouches.last_seen`, not active deletion): each set refreshes its edges' `last_seen`; a pubkey dropped from the set is not deleted but stops being refreshed and ages out via the same staleness window as follows (`StreamVouches` filters on the ranking cutoff). So revoking a vouch takes effect after the window, exactly like unfollowing. - **Report** = a **kind:1984** (NIP-56) event targeting a **profile** (`p` tag, no `e` tag) with report type `spam` or `impersonation` (other types ignored). Applies a trust-weighted penalty to the target's final score: `final = raw * (1 - R/(R+F))` where R is the sum of reporter trust_scores and F is the sum of follower/voucher trust_scores. -No mutual exclusion at write time — `vouches` and `reports` rows coexist. Precedence is resolved at ranking time: if a source both vouches for and reports the same target, the report is ignored (vouch beats report). Stored in the `vouches` and `reports` tables (schema from migration v4, unchanged). +No mutual exclusion — `vouches` and `reports` rows coexist independently. A source that both vouches for and reports the same target is not specially handled: the vouch adds flow and the report subtracts it at ranking time, which roughly cancels out. Stored in the `vouches` and `reports` tables (schema from migration v4). diff --git a/internal/repository/vouch.go b/internal/repository/vouch.go index 5b61d6b..9f07b7f 100644 --- a/internal/repository/vouch.go +++ b/internal/repository/vouch.go @@ -149,20 +149,15 @@ func (r *Repository) GetPubkeysWithPositiveTrust() (map[string]struct{}, error) // GetTrustWeightedReports aggregates reports per target, weighting each report // by the reporter's trust_score. Reporters with trust_score ≤ 0 are excluded — -// the same admission rule that gates vouch edges. A report is also ignored when -// the same source vouches for the same target: vouch beats report, resolved -// here at ranking time rather than by mutual exclusion at write time. +// the same admission rule that gates vouch edges. A source that both vouches +// for and reports the same target is not specially handled: the vouch adds +// flow and the report subtracts it, which roughly cancels out on its own. func (r *Repository) GetTrustWeightedReports() (map[string]models.ReportAggregate, error) { query := ` SELECT r.target_pubkey, COUNT(*), COALESCE(SUM(p.trust_score), 0) FROM reports r JOIN pubkeys p ON p.pubkey = r.source_pubkey WHERE p.trust_score > 0 - AND NOT EXISTS ( - SELECT 1 FROM vouches v - WHERE v.source_pubkey = r.source_pubkey - AND v.target_pubkey = r.target_pubkey - ) GROUP BY r.target_pubkey; ` rows, err := r.db.Query(query) diff --git a/internal/repository/vouch_test.go b/internal/repository/vouch_test.go index 053718f..12bd77b 100644 --- a/internal/repository/vouch_test.go +++ b/internal/repository/vouch_test.go @@ -125,8 +125,8 @@ func TestUpsertReport_NewAndIdempotent(t *testing.T) { func TestVouchAndReportCoexist(t *testing.T) { repo := newTestRepo(t) - // No mutual exclusion at write time: both rows persist; precedence is - // resolved at ranking time (vouch beats report). + // No mutual exclusion at write time: both rows persist independently. The + // vouch adds flow and the report subtracts it at ranking time. if err := repo.UpsertVouches("alice", []string{"bob"}); err != nil { t.Fatal(err) } @@ -265,37 +265,6 @@ func TestGetTrustWeightedReports(t *testing.T) { } } -// TestGetTrustWeightedReports_VouchBeatsReport verifies a reporter who also -// vouches for the same target is excluded from the report aggregate. -func TestGetTrustWeightedReports_VouchBeatsReport(t *testing.T) { - repo := newTestRepo(t) - seedPubkey(t, repo, "r1", 0.3) - seedPubkey(t, repo, "r2", 0.7) - - // Both report target; r1 also vouches for target → r1's report is ignored. - if err := repo.UpsertReport("r1", "target", t1); err != nil { - t.Fatal(err) - } - if err := repo.UpsertReport("r2", "target", t1); err != nil { - t.Fatal(err) - } - if err := repo.UpsertVouches("r1", []string{"target"}); err != nil { - t.Fatal(err) - } - - reports, err := repo.GetTrustWeightedReports() - if err != nil { - t.Fatal(err) - } - agg := reports["target"] - if agg.NumReporters != 1 { - t.Fatalf("expected 1 effective reporter (r1's vouch beats its report), got %d", agg.NumReporters) - } - if absDiff(agg.TotalReporterTrust, 0.7) > 1e-9 { - t.Fatalf("expected trust sum 0.7 (only r2), got %v", agg.TotalReporterTrust) - } -} - func absDiff(a, b float64) float64 { if a > b { return a - b From 2dddb41063b0080bde5d774dfdbef974b80a337e Mon Sep 17 00:00:00 2001 From: codytseng Date: Thu, 18 Jun 2026 19:49:30 +0800 Subject: [PATCH 07/10] refactor: use NIP-51 kind:30000 follow set for vouches Vouches now live in a standard NIP-51 follow set (kind:30000) tagged d=vouch, instead of a custom kind:10040. Other clients can render it as a people list, and the generic identifier (no project prefix) leaves room for a shared convention. Other follow sets are ignored. Because kind:30000 is addressable by d, it can't share the no-d query with kind:3/0; it gets its own #d-filtered fetch (fetchVouchSetsFromRelay), and ingest gates it via IsVouchSet. --- CLAUDE.md | 6 +- cmd/api/main.go | 2 +- config.example.yaml | 2 +- config/config.go | 2 +- internal/api/handler/event.go | 2 +- internal/crawler/crawler.go | 107 +++++++++++++++++++++++-------- internal/ingest/ingest.go | 36 +++++++++-- internal/ingest/ingest_test.go | 27 ++++++++ internal/repository/migration.go | 2 +- internal/repository/vouch.go | 2 +- 10 files changed, 147 insertions(+), 41 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 862dc3f..a27c82b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -85,12 +85,12 @@ Copy `config.example.yaml` to `config.yaml` (and `docker-compose.example.yml` to Vouches and reports are plain signed Nostr events, not a private API. They flow in two ways (both verify the event signature before storing): -1. **Crawler ingestion (pull)** — alongside kind:3/0, the crawler fetches each crawled author's kind:1984 reports and kind:10040 vouch set from their relays. Replaceable kinds (3/0/10040) share one small-limit query; kind:1984 (append-only, potentially many) gets its own query capped at the newest 50, kept separate so reports can't crowd out the replaceable events. -2. **`POST /event` (push)** — accepts a single signed event (kind 3 / 1984 / 10040) for immediate ingestion. As an open write endpoint it keeps the anti-inflation rule: events from pubkeys with no TrustRank and not in `seed_pubkeys` return 200 but are silently dropped. (The crawler path does not filter this way — ranking already discounts untrusted sources.) +1. **Crawler ingestion (pull)** — alongside kind:3/0 (one shared query), the crawler fetches each crawled author's kind:1984 reports and kind:30000 vouch set in their own queries. The vouch set is fetched with a `#d` filter on the vouch identifier (so only it, not the user's other follow sets, comes back); kind:1984 (append-only, potentially many) is capped at the newest 50 and kept separate so reports can't crowd out the replaceable events. +2. **`POST /event` (push)** — accepts a single signed event (kind 3 / 1984 / 30000) for immediate ingestion. As an open write endpoint it keeps the anti-inflation rule: events from pubkeys with no TrustRank and not in `seed_pubkeys` return 200 but are silently dropped. (The crawler path does not filter this way — ranking already discounts untrusted sources.) Shared parsing/storage lives in `internal/ingest` so both paths behave identically. -- **Vouch** = membership in the author's **kind:10040** vouch set (a custom replaceable event; not in any NIP). Its `p` tags list the vouched pubkeys. Registers source→target as a vouch edge (weight `vouch.weight`, deduped against a follow from the same source). Vouches follow the **same lifecycle as follow edges** (`vouches.last_seen`, not active deletion): each set refreshes its edges' `last_seen`; a pubkey dropped from the set is not deleted but stops being refreshed and ages out via the same staleness window as follows (`StreamVouches` filters on the ranking cutoff). So revoking a vouch takes effect after the window, exactly like unfollowing. +- **Vouch** = membership in the author's vouch set: a **NIP-51 follow set (kind:30000)** tagged `d=vouch` (regular follow sets with any other `d` are ignored; the identifier is deliberately generic so it can become a shared convention). Its `p` tags list the vouched pubkeys. Registers source→target as a vouch edge (weight `vouch.weight`, deduped against a follow from the same source). Vouches follow the **same lifecycle as follow edges** (`vouches.last_seen`, not active deletion): each set refreshes its edges' `last_seen`; a pubkey dropped from the set is not deleted but stops being refreshed and ages out via the same staleness window as follows (`StreamVouches` filters on the ranking cutoff). So revoking a vouch takes effect after the window, exactly like unfollowing. - **Report** = a **kind:1984** (NIP-56) event targeting a **profile** (`p` tag, no `e` tag) with report type `spam` or `impersonation` (other types ignored). Applies a trust-weighted penalty to the target's final score: `final = raw * (1 - R/(R+F))` where R is the sum of reporter trust_scores and F is the sum of follower/voucher trust_scores. No mutual exclusion — `vouches` and `reports` rows coexist independently. A source that both vouches for and reports the same target is not specially handled: the vouch adds flow and the report subtracts it at ranking time, which roughly cancels out. Stored in the `vouches` and `reports` tables (schema from migration v4). diff --git a/cmd/api/main.go b/cmd/api/main.go index 178a007..1137092 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -88,7 +88,7 @@ func main() { http.HandleFunc("/search", middleware.CORS(h.Search)) // Event ingestion endpoint. Accepts signed Nostr events (kind 3 / 1984 / - // 10040) as an immediate push complement to the crawler. When vouch.weight + // 30000) as an immediate push complement to the crawler. When vouch.weight // <= 0 the feature is disabled: no route is registered and requests fall // through to the SPA catch-all handler below. if cfg.Vouch.Enabled() { diff --git a/config.example.yaml b/config.example.yaml index b5f873f..d549922 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -40,7 +40,7 @@ crawler: # ingests them from relays, and the POST /event endpoint accepts pushes too. # # `weight` is the single control: set to 0 (default) to disable the feature -# entirely — the crawler skips kind:1984 / kind:10040, POST /event returns 404, +# entirely — the crawler skips kind:1984 / kind:30000, POST /event returns 404, # and vouches are ignored by ranking. Set to a value in (0, 1] to enable. The # value is the weight of a vouch edge relative to a follow edge (1.0); e.g. 0.5 # means each vouch contributes half the flow of an actual follow. Reports apply diff --git a/config/config.go b/config/config.go index 5614ad9..85de88e 100644 --- a/config/config.go +++ b/config/config.go @@ -29,7 +29,7 @@ type CrawlerConfig struct { } // VouchConfig controls the vouch/report feature: whether the crawler ingests -// kind:1984 reports and kind:10040 vouch sets, whether POST /event is served, +// kind:1984 reports and kind:30000 vouch sets, whether POST /event is served, // and the weight of vouch edges in the ranking graph. // // A single knob: weight == 0 disables the feature entirely (crawler skips diff --git a/internal/api/handler/event.go b/internal/api/handler/event.go index 5dc7967..b7b41d7 100644 --- a/internal/api/handler/event.go +++ b/internal/api/handler/event.go @@ -11,7 +11,7 @@ import ( // PostEvent handles POST /event. It accepts a single signed Nostr event, an // immediate push complement to the crawler's relay subscriptions. The same // event can (and should) also be published to public relays — Fayan is just one -// of many aggregators. Supported kinds: 3 (contacts), 1984 (reports), 10040 +// of many aggregators. Supported kinds: 3 (contacts), 1984 (reports), 30000 // (vouch sets); other kinds are rejected. // // As an open write endpoint it keeps the anti-inflation admission rule: the diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index ec4ebd5..1423571 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -68,7 +68,7 @@ type Crawler struct { } // NewCrawler creates a new Crawler instance. When vouchEnabled is true the -// crawler also fetches kind:1984 reports and kind:10040 vouch sets. +// crawler also fetches kind:1984 reports and kind:30000 vouch sets. func NewCrawler(repo *repository.Repository, relays []string, seedPubkeys []string, searchConfig *config.SearchConfig, crawlerConfig *CrawlerConfig, vouchEnabled bool) *Crawler { ctx, cancel := context.WithCancel(context.Background()) @@ -302,7 +302,7 @@ func (c *Crawler) contactProcessor() { } // dispatchRelationEvent routes a relation event from the contacts channel to -// its handler by kind (kind:3 follows or kind:10040 vouch sets). +// its handler by kind (kind:3 follows or kind:30000 vouch sets). func (c *Crawler) dispatchRelationEvent(event *nostr.Event) { if event == nil { return @@ -377,10 +377,12 @@ func (c *Crawler) fetchBatch(pubkeys []string) { wg.Add(1) go func(r string, u []string) { defer wg.Done() - contacts, profiles, vouchSets := c.fetchEventsFromRelay(r, u, fetchProfiles) + contacts, profiles := c.fetchEventsFromRelay(r, u, fetchProfiles) + var vouchSets map[string]*nostr.Event var reports []*nostr.Event if c.vouchEnabled { + vouchSets = c.fetchVouchSetsFromRelay(r, u) reports = c.fetchReportsFromRelay(r, u) } @@ -448,46 +450,43 @@ func (c *Crawler) fetchBatch(pubkeys []string) { } } -// fetchEventsFromRelay fetches the replaceable events for multiple users from a -// single relay: kind 3 (contacts), optionally kind 0 (profiles), and — when the -// vouch feature is enabled — kind 10040 (vouch sets). All are one-per-author, so -// they share a single small-limit query. kind:1984 reports are NOT fetched here; -// being append-only and numerous, they get their own query (fetchReportsFromRelay). -// Returns maps of pubkey -> latest event for contacts, profiles, and vouch sets. -func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProfiles bool) (map[string]*nostr.Event, map[string]*nostr.Event, map[string]*nostr.Event) { +// fetchEventsFromRelay fetches the shared replaceable events for multiple users +// from a single relay: kind 3 (contacts) and optionally kind 0 (profiles). Both +// are one-per-author and carry no `d` tag, so they share a single query. Vouch +// sets (kind:30000, addressable by `d`) and reports (kind:1984, append-only) +// each need different filters and are fetched separately. +// Returns maps of pubkey -> latest event for contacts and profiles. +func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProfiles bool) (map[string]*nostr.Event, map[string]*nostr.Event) { if len(pubkeys) == 0 { - return nil, nil, nil + return nil, nil } // Check if context is cancelled select { case <-c.ctx.Done(): - return nil, nil, nil + return nil, nil default: } // Skip if relay is banned if c.relayHealth.IsRelayBanned(relay) { - return nil, nil, nil + return nil, nil } // Apply rate limiting for this specific relay limiter := c.getRelayLimiter(relay) if err := limiter.Wait(c.ctx); err != nil { - return nil, nil, nil + return nil, nil } ctx, cancel := context.WithTimeout(c.ctx, 15*time.Second) defer cancel() - // Build filter with kinds 3 (contacts), optionally 0 (profiles) and 10040 (vouch sets) + // Build filter with kinds 3 (contacts) and optionally 0 (profiles) kinds := []int{3} if fetchProfiles { kinds = append(kinds, 0) } - if c.vouchEnabled { - kinds = append(kinds, ingest.KindVouchSet) - } filter := nostr.Filter{ Kinds: kinds, @@ -506,7 +505,6 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf // Collect events and keep only the latest for each pubkey contacts := make(map[string]*nostr.Event) profiles := make(map[string]*nostr.Event) - vouchSets := make(map[string]*nostr.Event) timer := time.NewTimer(10 * time.Second) // Slightly less than context timeout defer timer.Stop() channelClosed := false @@ -522,7 +520,7 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf channelClosed = true c.relayHealth.RecordSuccess(relay) } - return contacts, profiles, vouchSets + return contacts, profiles } ev := relayEvent.Event @@ -537,23 +535,78 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf if existing, exists := profiles[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { profiles[ev.PubKey] = ev } - case ingest.KindVouchSet: - if existing, exists := vouchSets[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { - vouchSets[ev.PubKey] = ev - } } case <-timer.C: // Timeout - this could indicate connection issues if !channelClosed { c.relayHealth.RecordFailure(relay, "timeout - no response") } - return contacts, profiles, vouchSets + return contacts, profiles case <-ctx.Done(): // Context cancelled if !channelClosed { c.relayHealth.RecordFailure(relay, "context cancelled") } - return contacts, profiles, vouchSets + return contacts, profiles + } + } +} + +// fetchVouchSetsFromRelay fetches the vouch set (NIP-51 kind:30000 tagged with +// VouchSetIdentifier) for multiple users from a single relay. The #d filter +// returns only the vouch set, not the users' other follow sets; being an +// addressable event there is at most one per author. Returns pubkey -> latest. +func (c *Crawler) fetchVouchSetsFromRelay(relay string, pubkeys []string) map[string]*nostr.Event { + if len(pubkeys) == 0 { + return nil + } + + select { + case <-c.ctx.Done(): + return nil + default: + } + + if c.relayHealth.IsRelayBanned(relay) { + return nil + } + + limiter := c.getRelayLimiter(relay) + if err := limiter.Wait(c.ctx); err != nil { + return nil + } + + ctx, cancel := context.WithTimeout(c.ctx, 15*time.Second) + defer cancel() + + filter := nostr.Filter{ + Kinds: []int{ingest.KindVouchSet}, + Authors: pubkeys, + Tags: nostr.TagMap{"d": []string{ingest.VouchSetIdentifier}}, + } + + pool := c.poolManager.GetPool() + eventsChan := pool.FetchMany(ctx, []string{relay}, filter) + c.poolManager.TrackRelayUsage(relay) + + vouchSets := make(map[string]*nostr.Event) + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + + for { + select { + case relayEvent, ok := <-eventsChan: + if !ok { + return vouchSets + } + ev := relayEvent.Event + if existing, exists := vouchSets[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { + vouchSets[ev.PubKey] = ev + } + case <-timer.C: + return vouchSets + case <-ctx.Done(): + return vouchSets } } } @@ -743,7 +796,7 @@ func (c *Crawler) processKind3Event(ev *nostr.Event) { c.crawledMu.Unlock() } -// processVouchSetEvent verifies a kind:10040 event's signature and replaces the +// processVouchSetEvent verifies a kind:30000 event's signature and replaces the // author's vouch edges with the listed pubkeys. func (c *Crawler) processVouchSetEvent(ev *nostr.Event) { if ev.Kind != ingest.KindVouchSet { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 508385e..d7af852 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -13,9 +13,15 @@ import ( const ( KindContacts = 3 // NIP-02 contact list → follow edges KindReport = 1984 // NIP-56 report → report edge (profile-level only) - KindVouchSet = 10040 // Fayan vouch set: a replaceable list of vouched pubkeys + KindVouchSet = 30000 // NIP-51 follow set; the vouch set is the one tagged d=VouchSetIdentifier ) +// VouchSetIdentifier is the NIP-51 `d` tag value identifying the follow set +// used as a vouch set. A kind:30000 with any other `d` is a regular follow set +// and is ignored. The value is intentionally generic (no project prefix) so it +// can serve as a shared convention if one emerges. +const VouchSetIdentifier = "vouch" + // acceptedReportTypes are the NIP-56 report types that affect reputation. Fayan // is a spam-detection system, so only these two carry weight; other types // (nudity, profanity, …) are ignored. @@ -44,11 +50,30 @@ func (in *Ingester) Apply(ev *nostr.Event) (handled bool, err error) { _, err := in.ApplyReport(ev) return true, err case KindVouchSet: + if !IsVouchSet(ev) { + return false, nil // some other follow set — not ours + } return true, in.ApplyVouchSet(ev) } return false, nil } +// IsVouchSet reports whether ev is the NIP-51 follow set (kind:30000) Fayan +// uses as a vouch set, i.e. tagged with VouchSetIdentifier. +func IsVouchSet(ev *nostr.Event) bool { + return ev.Kind == KindVouchSet && dTagValue(ev.Tags) == VouchSetIdentifier +} + +// dTagValue returns the value of the first `d` tag, or "" if absent. +func dTagValue(tags nostr.Tags) string { + for _, tag := range tags { + if len(tag) >= 2 && tag[0] == "d" { + return tag[1] + } + } + return "" +} + // ApplyContacts parses a kind:3 event into follow connections and persists them. func (in *Ingester) ApplyContacts(ev *nostr.Event) error { if ev.Kind != KindContacts { @@ -128,10 +153,11 @@ func profileReportTarget(ev *nostr.Event) (string, bool) { return target, target != "" } -// ApplyVouchSet replaces the author's vouch edges with the pubkeys listed in a -// kind:10040 event, honouring the replaceable semantics of the set. +// ApplyVouchSet refreshes the author's vouch edges from the pubkeys listed in +// their vouch set (the NIP-51 kind:30000 follow set tagged with +// VouchSetIdentifier). A no-op for any other event. func (in *Ingester) ApplyVouchSet(ev *nostr.Event) error { - if ev.Kind != KindVouchSet { + if !IsVouchSet(ev) { return nil } targets := ParseVouchTargets(ev) @@ -139,7 +165,7 @@ func (in *Ingester) ApplyVouchSet(ev *nostr.Event) error { } // ParseVouchTargets extracts the valid, de-duplicated pubkeys an author vouches -// for from a kind:10040 event's p tags (excluding the author themselves). +// for from a follow set's p tags (excluding the author themselves). func ParseVouchTargets(ev *nostr.Event) []string { seen := make(map[string]bool) var targets []string diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index b7c42a9..68215ea 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -46,6 +46,7 @@ func TestParseVouchTargets(t *testing.T) { PubKey: author, Kind: KindVouchSet, Tags: nostr.Tags{ + {"d", VouchSetIdentifier}, {"p", bob}, {"p", bob}, // duplicate — collapsed {"p", carol}, @@ -58,6 +59,32 @@ func TestParseVouchTargets(t *testing.T) { } } +func TestIsVouchSet(t *testing.T) { + author := mustPubkey(t) + cases := []struct { + name string + kind int + d string + want bool + }{ + {"vouch set", KindVouchSet, VouchSetIdentifier, true}, + {"other follow set", KindVouchSet, "friends", false}, + {"follow set without d", KindVouchSet, "", false}, + {"wrong kind", KindContacts, VouchSetIdentifier, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ev := &nostr.Event{PubKey: author, Kind: tc.kind} + if tc.d != "" { + ev.Tags = nostr.Tags{{"d", tc.d}} + } + if got := IsVouchSet(ev); got != tc.want { + t.Fatalf("IsVouchSet = %v, want %v", got, tc.want) + } + }) + } +} + func TestProfileReportTarget(t *testing.T) { author, bob := mustPubkey(t), mustPubkey(t) cases := []struct { diff --git a/internal/repository/migration.go b/internal/repository/migration.go index ef9e039..1a8eda6 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -100,7 +100,7 @@ var migrations = []Migration{ Name: "add_vouches_and_reports", Up: func(db *sql.DB) error { // Vouches share the follow-edge lifecycle: refreshed on each - // kind:10040 set, never actively deleted, aged out by a staleness + // kind:30000 set, never actively deleted, aged out by a staleness // window — hence last_seen (cf. connections), not created_at. vouchesTable := ` CREATE TABLE IF NOT EXISTS vouches ( diff --git a/internal/repository/vouch.go b/internal/repository/vouch.go index 9f07b7f..2b60d8b 100644 --- a/internal/repository/vouch.go +++ b/internal/repository/vouch.go @@ -9,7 +9,7 @@ import ( ) // UpsertVouches refreshes the vouch edges from source for the pubkeys in the -// latest kind:10040 set, mirroring how kind:3 contacts are stored: each edge is +// latest kind:30000 set, mirroring how kind:3 contacts are stored: each edge is // upserted with last_seen = now and never actively deleted. A vouch dropped // from the set simply stops being refreshed and ages out of the ranking graph // via the same staleness window as follows (see StreamVouches). Targets are From 5f3663deb4a325da8f369ea25a1adacd1680ed53 Mon Sep 17 00:00:00 2001 From: codytseng Date: Thu, 18 Jun 2026 20:08:13 +0800 Subject: [PATCH 08/10] refactor: fetch contacts and vouch set in one subscription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A REQ carries multiple filters (OR'd), so the kind:3/0 filter and the kind:30000 vouch-set filter (constrained by #d, scoped to its own filter) now ride in a single SubManyEose call per relay — one connection, one round-trip — instead of two separate fetches. kind:1984 reports stay in their own subscription to keep their per-query limit clean. --- CLAUDE.md | 2 +- internal/crawler/crawler.go | 118 +++++++++++------------------------- 2 files changed, 37 insertions(+), 83 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a27c82b..b81f0e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -85,7 +85,7 @@ Copy `config.example.yaml` to `config.yaml` (and `docker-compose.example.yml` to Vouches and reports are plain signed Nostr events, not a private API. They flow in two ways (both verify the event signature before storing): -1. **Crawler ingestion (pull)** — alongside kind:3/0 (one shared query), the crawler fetches each crawled author's kind:1984 reports and kind:30000 vouch set in their own queries. The vouch set is fetched with a `#d` filter on the vouch identifier (so only it, not the user's other follow sets, comes back); kind:1984 (append-only, potentially many) is capped at the newest 50 and kept separate so reports can't crowd out the replaceable events. +1. **Crawler ingestion (pull)** — one subscription per relay carries two filters (a REQ OR's multiple filters): filter 1 is kind:3 (+kind:0 when search is on); filter 2 is the kind:30000 vouch set constrained by `#d` (scoped to that filter, so it doesn't affect 3/0). kind:1984 reports are fetched in a separate subscription, capped at the newest 50, so an append-only flood can't crowd out the replaceable events. 2. **`POST /event` (push)** — accepts a single signed event (kind 3 / 1984 / 30000) for immediate ingestion. As an open write endpoint it keeps the anti-inflation rule: events from pubkeys with no TrustRank and not in `seed_pubkeys` return 200 but are silently dropped. (The crawler path does not filter this way — ranking already discounts untrusted sources.) Shared parsing/storage lives in `internal/ingest` so both paths behave identically. diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index 1423571..6df7a59 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -377,12 +377,10 @@ func (c *Crawler) fetchBatch(pubkeys []string) { wg.Add(1) go func(r string, u []string) { defer wg.Done() - contacts, profiles := c.fetchEventsFromRelay(r, u, fetchProfiles) + contacts, profiles, vouchSets := c.fetchEventsFromRelay(r, u, fetchProfiles) - var vouchSets map[string]*nostr.Event var reports []*nostr.Event if c.vouchEnabled { - vouchSets = c.fetchVouchSetsFromRelay(r, u) reports = c.fetchReportsFromRelay(r, u) } @@ -450,54 +448,64 @@ func (c *Crawler) fetchBatch(pubkeys []string) { } } -// fetchEventsFromRelay fetches the shared replaceable events for multiple users -// from a single relay: kind 3 (contacts) and optionally kind 0 (profiles). Both -// are one-per-author and carry no `d` tag, so they share a single query. Vouch -// sets (kind:30000, addressable by `d`) and reports (kind:1984, append-only) -// each need different filters and are fetched separately. -// Returns maps of pubkey -> latest event for contacts and profiles. -func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProfiles bool) (map[string]*nostr.Event, map[string]*nostr.Event) { +// fetchEventsFromRelay fetches the replaceable events for multiple users from a +// single relay in ONE subscription. A REQ carries multiple filters (OR'd), so: +// - filter 1: kind 3 (contacts) + optional kind 0 (profiles) — one-per-author, +// no `d` tag; +// - filter 2 (when vouch is enabled): the vouch set (kind:30000) constrained +// by `#d` — that constraint applies only to its own filter, not to 3/0. +// +// One connection, one round-trip. kind:1984 reports are fetched separately +// (fetchReportsFromRelay): being append-only they carry a per-query limit that +// is cleaner to reason about in its own subscription. +// Returns maps of pubkey -> latest event for contacts, profiles and vouch sets. +func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProfiles bool) (map[string]*nostr.Event, map[string]*nostr.Event, map[string]*nostr.Event) { if len(pubkeys) == 0 { - return nil, nil + return nil, nil, nil } // Check if context is cancelled select { case <-c.ctx.Done(): - return nil, nil + return nil, nil, nil default: } // Skip if relay is banned if c.relayHealth.IsRelayBanned(relay) { - return nil, nil + return nil, nil, nil } // Apply rate limiting for this specific relay limiter := c.getRelayLimiter(relay) if err := limiter.Wait(c.ctx); err != nil { - return nil, nil + return nil, nil, nil } ctx, cancel := context.WithTimeout(c.ctx, 15*time.Second) defer cancel() - // Build filter with kinds 3 (contacts) and optionally 0 (profiles) + // Filter 1: contacts (+ profiles). kinds := []int{3} if fetchProfiles { kinds = append(kinds, 0) } + filters := nostr.Filters{{Kinds: kinds, Authors: pubkeys}} - filter := nostr.Filter{ - Kinds: kinds, - Authors: pubkeys, + // Filter 2: the vouch set; its #d constraint is scoped to this filter only. + if c.vouchEnabled { + filters = append(filters, nostr.Filter{ + Kinds: []int{ingest.KindVouchSet}, + Authors: pubkeys, + Tags: nostr.TagMap{"d": []string{ingest.VouchSetIdentifier}}, + }) } // Get the current pool from pool manager pool := c.poolManager.GetPool() - // SubscribeMany returns a channel of RelayEvent - eventsChan := pool.FetchMany(ctx, []string{relay}, filter) + // SubManyEose is like FetchMany but takes multiple filters; ends on EOSE. + eventsChan := pool.SubManyEose(ctx, []string{relay}, filters) // Track relay usage c.poolManager.TrackRelayUsage(relay) @@ -505,6 +513,7 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf // Collect events and keep only the latest for each pubkey contacts := make(map[string]*nostr.Event) profiles := make(map[string]*nostr.Event) + vouchSets := make(map[string]*nostr.Event) timer := time.NewTimer(10 * time.Second) // Slightly less than context timeout defer timer.Stop() channelClosed := false @@ -520,7 +529,7 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf channelClosed = true c.relayHealth.RecordSuccess(relay) } - return contacts, profiles + return contacts, profiles, vouchSets } ev := relayEvent.Event @@ -535,78 +544,23 @@ func (c *Crawler) fetchEventsFromRelay(relay string, pubkeys []string, fetchProf if existing, exists := profiles[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { profiles[ev.PubKey] = ev } + case ingest.KindVouchSet: + if existing, exists := vouchSets[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { + vouchSets[ev.PubKey] = ev + } } case <-timer.C: // Timeout - this could indicate connection issues if !channelClosed { c.relayHealth.RecordFailure(relay, "timeout - no response") } - return contacts, profiles + return contacts, profiles, vouchSets case <-ctx.Done(): // Context cancelled if !channelClosed { c.relayHealth.RecordFailure(relay, "context cancelled") } - return contacts, profiles - } - } -} - -// fetchVouchSetsFromRelay fetches the vouch set (NIP-51 kind:30000 tagged with -// VouchSetIdentifier) for multiple users from a single relay. The #d filter -// returns only the vouch set, not the users' other follow sets; being an -// addressable event there is at most one per author. Returns pubkey -> latest. -func (c *Crawler) fetchVouchSetsFromRelay(relay string, pubkeys []string) map[string]*nostr.Event { - if len(pubkeys) == 0 { - return nil - } - - select { - case <-c.ctx.Done(): - return nil - default: - } - - if c.relayHealth.IsRelayBanned(relay) { - return nil - } - - limiter := c.getRelayLimiter(relay) - if err := limiter.Wait(c.ctx); err != nil { - return nil - } - - ctx, cancel := context.WithTimeout(c.ctx, 15*time.Second) - defer cancel() - - filter := nostr.Filter{ - Kinds: []int{ingest.KindVouchSet}, - Authors: pubkeys, - Tags: nostr.TagMap{"d": []string{ingest.VouchSetIdentifier}}, - } - - pool := c.poolManager.GetPool() - eventsChan := pool.FetchMany(ctx, []string{relay}, filter) - c.poolManager.TrackRelayUsage(relay) - - vouchSets := make(map[string]*nostr.Event) - timer := time.NewTimer(10 * time.Second) - defer timer.Stop() - - for { - select { - case relayEvent, ok := <-eventsChan: - if !ok { - return vouchSets - } - ev := relayEvent.Event - if existing, exists := vouchSets[ev.PubKey]; !exists || ev.CreatedAt > existing.CreatedAt { - vouchSets[ev.PubKey] = ev - } - case <-timer.C: - return vouchSets - case <-ctx.Done(): - return vouchSets + return contacts, profiles, vouchSets } } } From e8eace43b3bbf9819541bdc58d004c5847898b75 Mon Sep 17 00:00:00 2001 From: codytseng Date: Thu, 18 Jun 2026 20:43:39 +0800 Subject: [PATCH 09/10] refactor: cleanups from /simplify review - ingest: drop hand-written dTagValue, use go-nostr's Tags.GetD() - ranking: compute report-penalty F only for reported targets instead of scanning the whole graph (drops an O(numNodes) alloc + full edge scan) - repository/models: remove unused ReportAggregate.NumReporters field and its COUNT(*) (only TotalReporterTrust is consumed) --- internal/ingest/ingest.go | 12 +----------- internal/models/user.go | 1 - internal/ranking/calculator.go | 29 ++++++++++++++++------------- internal/repository/vouch.go | 4 ++-- internal/repository/vouch_test.go | 3 --- 5 files changed, 19 insertions(+), 30 deletions(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index d7af852..faf76d2 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -61,17 +61,7 @@ func (in *Ingester) Apply(ev *nostr.Event) (handled bool, err error) { // IsVouchSet reports whether ev is the NIP-51 follow set (kind:30000) Fayan // uses as a vouch set, i.e. tagged with VouchSetIdentifier. func IsVouchSet(ev *nostr.Event) bool { - return ev.Kind == KindVouchSet && dTagValue(ev.Tags) == VouchSetIdentifier -} - -// dTagValue returns the value of the first `d` tag, or "" if absent. -func dTagValue(tags nostr.Tags) string { - for _, tag := range tags { - if len(tag) >= 2 && tag[0] == "d" { - return tag[1] - } - } - return "" + return ev.Kind == KindVouchSet && ev.Tags.GetD() == VouchSetIdentifier } // ApplyContacts parses a kind:3 event into follow connections and persists them. diff --git a/internal/models/user.go b/internal/models/user.go index 791db54..9da4394 100644 --- a/internal/models/user.go +++ b/internal/models/user.go @@ -35,6 +35,5 @@ type Vouch struct { // ReportAggregate summarises reports against a single target pubkey. // Only reporters with trust_score > 0 contribute. type ReportAggregate struct { - NumReporters int TotalReporterTrust float64 } diff --git a/internal/ranking/calculator.go b/internal/ranking/calculator.go index c491010..8629efd 100644 --- a/internal/ranking/calculator.go +++ b/internal/ranking/calculator.go @@ -196,26 +196,29 @@ func (c *Calculator) Calculate() error { if reports, err := c.repo.GetTrustWeightedReports(); err != nil { log.Printf(" [WARN] Failed to load reports for penalty: %v", err) } else if len(reports) > 0 { - fTrust := make([]float64, numNodes) - for i := range numNodes { - for _, link := range inLinks[i] { - fTrust[i] += trustScores[link.source] * link.weight - } - } + // Reported targets are sparse, so compute F (weighted follower/voucher + // trust) only for them rather than scanning the whole graph. penalized := 0 - for i := range numNodes { - agg, ok := reports[idToPubkey[i]] - if !ok || agg.TotalReporterTrust <= 0 { + for target, agg := range reports { + if agg.TotalReporterTrust <= 0 { + continue + } + id, ok := pubkeyToID[target] + if !ok { continue } - penalty := agg.TotalReporterTrust / (agg.TotalReporterTrust + fTrust[i] + 1e-9) + fTrust := 0.0 + for _, link := range inLinks[id] { + fTrust += trustScores[link.source] * link.weight + } + penalty := agg.TotalReporterTrust / (agg.TotalReporterTrust + fTrust + 1e-9) if penalty > 1 { penalty = 1 } factor := 1 - penalty - scores[i] *= factor - trustScores[i] *= factor - pageScores[i] *= factor + scores[id] *= factor + trustScores[id] *= factor + pageScores[id] *= factor penalized++ } log.Printf(" [INFO] Applied report penalty to %d pubkeys", penalized) diff --git a/internal/repository/vouch.go b/internal/repository/vouch.go index 2b60d8b..fef63a9 100644 --- a/internal/repository/vouch.go +++ b/internal/repository/vouch.go @@ -154,7 +154,7 @@ func (r *Repository) GetPubkeysWithPositiveTrust() (map[string]struct{}, error) // flow and the report subtracts it, which roughly cancels out on its own. func (r *Repository) GetTrustWeightedReports() (map[string]models.ReportAggregate, error) { query := ` - SELECT r.target_pubkey, COUNT(*), COALESCE(SUM(p.trust_score), 0) + SELECT r.target_pubkey, COALESCE(SUM(p.trust_score), 0) FROM reports r JOIN pubkeys p ON p.pubkey = r.source_pubkey WHERE p.trust_score > 0 @@ -170,7 +170,7 @@ func (r *Repository) GetTrustWeightedReports() (map[string]models.ReportAggregat for rows.Next() { var target string var agg models.ReportAggregate - if err := rows.Scan(&target, &agg.NumReporters, &agg.TotalReporterTrust); err != nil { + if err := rows.Scan(&target, &agg.TotalReporterTrust); err != nil { return nil, fmt.Errorf("failed to scan report aggregate: %w", err) } result[target] = agg diff --git a/internal/repository/vouch_test.go b/internal/repository/vouch_test.go index 12bd77b..bc1ee6e 100644 --- a/internal/repository/vouch_test.go +++ b/internal/repository/vouch_test.go @@ -256,9 +256,6 @@ func TestGetTrustWeightedReports(t *testing.T) { if !ok { t.Fatalf("expected 'target' in aggregates") } - if agg.NumReporters != 2 { - t.Fatalf("expected 2 trusted reporters, got %d", agg.NumReporters) - } expected := 0.3 + 0.7 if absDiff(agg.TotalReporterTrust, expected) > 1e-9 { t.Fatalf("expected trust sum %v, got %v", expected, agg.TotalReporterTrust) From 4b628a1089472d83dc5fc1113daa3ae414e49f0c Mon Sep 17 00:00:00 2001 From: codytseng Date: Fri, 19 Jun 2026 14:16:02 +0800 Subject: [PATCH 10/10] feat: process POST /event asynchronously --- CLAUDE.md | 2 +- cmd/api/main.go | 3 +- internal/api/handler/event.go | 72 +++++++++++++++++++-------------- internal/api/handler/handler.go | 11 ++++- 4 files changed, 53 insertions(+), 35 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b81f0e0..6fa249c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,7 +86,7 @@ Copy `config.example.yaml` to `config.yaml` (and `docker-compose.example.yml` to Vouches and reports are plain signed Nostr events, not a private API. They flow in two ways (both verify the event signature before storing): 1. **Crawler ingestion (pull)** — one subscription per relay carries two filters (a REQ OR's multiple filters): filter 1 is kind:3 (+kind:0 when search is on); filter 2 is the kind:30000 vouch set constrained by `#d` (scoped to that filter, so it doesn't affect 3/0). kind:1984 reports are fetched in a separate subscription, capped at the newest 50, so an append-only flood can't crowd out the replaceable events. -2. **`POST /event` (push)** — accepts a single signed event (kind 3 / 1984 / 30000) for immediate ingestion. As an open write endpoint it keeps the anti-inflation rule: events from pubkeys with no TrustRank and not in `seed_pubkeys` return 200 but are silently dropped. (The crawler path does not filter this way — ranking already discounts untrusted sources.) +2. **`POST /event` (push)** — accepts a single signed event (kind 3 / 1984 / 30000). It is fire-and-forget: the event is queued and the request returns 202 immediately and unconditionally. Background workers then verify the signature, apply the anti-inflation admission rule (events from pubkeys with no TrustRank and not in `seed_pubkeys` are dropped), and persist. A full queue simply drops the event — the crawler ingests it from relays anyway. (The crawler path does not apply the admission rule — ranking already discounts untrusted sources.) Shared parsing/storage lives in `internal/ingest` so both paths behave identically. diff --git a/cmd/api/main.go b/cmd/api/main.go index 1137092..4b3319b 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -88,7 +88,8 @@ func main() { http.HandleFunc("/search", middleware.CORS(h.Search)) // Event ingestion endpoint. Accepts signed Nostr events (kind 3 / 1984 / - // 30000) as an immediate push complement to the crawler. When vouch.weight + // 30000) as a push complement to the crawler; it queues the event and + // returns 202 immediately, processing it asynchronously. When vouch.weight // <= 0 the feature is disabled: no route is registered and requests fall // through to the SPA catch-all handler below. if cfg.Vouch.Enabled() { diff --git a/internal/api/handler/event.go b/internal/api/handler/event.go index b7b41d7..f03eb6f 100644 --- a/internal/api/handler/event.go +++ b/internal/api/handler/event.go @@ -2,59 +2,69 @@ package handler import ( "encoding/json" + "io" "log" "net/http" "github.com/nbd-wtf/go-nostr" ) -// PostEvent handles POST /event. It accepts a single signed Nostr event, an -// immediate push complement to the crawler's relay subscriptions. The same -// event can (and should) also be published to public relays — Fayan is just one -// of many aggregators. Supported kinds: 3 (contacts), 1984 (reports), 30000 -// (vouch sets); other kinds are rejected. -// -// As an open write endpoint it keeps the anti-inflation admission rule: the -// author must be a seed or have earned TrustRank > 0, otherwise the event is -// silently dropped with 200 so the client cannot probe admission. (The crawler -// path does not filter this way — it aggregates public events as-is, and the -// ranking stage already discounts untrusted sources.) +// Async ingest tuning for POST /event. +const ( + maxEventBytes = 1 << 20 // 1 MiB cap on a request body; Nostr events are small + ingestQueueSize = 1024 // buffered events awaiting background processing + ingestWorkers = 4 // background workers draining the queue +) + +// PostEvent handles POST /event as a fire-and-forget intake: it reads the body, +// queues the event, and returns 202 immediately and unconditionally. Signature +// verification, the anti-inflation admission check, and persistence all happen +// on a background worker (processEvent) — the client learns nothing about the +// outcome. This is just a push accelerator: the same event is published to +// public relays, and the crawler ingests it from there regardless, so a full +// queue can simply drop the event. func (h *Handler) PostEvent(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeError(w, http.StatusMethodNotAllowed, "Method not allowed") return } + // Decode synchronously — the body is gone once we return — then hand off. var ev nostr.Event - if err := json.NewDecoder(r.Body).Decode(&ev); err != nil { - writeError(w, http.StatusBadRequest, "Invalid event JSON") - return + if err := json.NewDecoder(io.LimitReader(r.Body, maxEventBytes)).Decode(&ev); err == nil { + select { + case h.ingestCh <- &ev: + default: + // Queue full: drop. The crawler will pick the event up from relays. + log.Printf("[API] Ingest queue full, dropped event kind=%d", ev.Kind) + } } - ok, err := ev.CheckSignature() - if err != nil || !ok { - writeError(w, http.StatusUnauthorized, "Invalid event signature") - return - } + writeJSON(w, http.StatusAccepted, map[string]string{"status": "accepted"}) +} - // Silent-ignore admission rule (see doc comment). - if !h.authorQualifies(ev.PubKey) { - writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) - return +// ingestWorker drains the async ingest queue, processing one event at a time. +func (h *Handler) ingestWorker() { + for ev := range h.ingestCh { + h.processEvent(ev) } +} - handled, err := h.ingester.Apply(&ev) - if err != nil { - log.Printf("[API] Error ingesting event (kind=%d pubkey=%s): %v", ev.Kind, ev.PubKey, err) - writeError(w, http.StatusInternalServerError, "Failed to ingest event") +// processEvent verifies, admits, and persists a single queued event off the +// request path. Every failure is silently dropped (logged at most) — there is +// no caller to report back to. +func (h *Handler) processEvent(ev *nostr.Event) { + if ok, err := ev.CheckSignature(); err != nil || !ok { return } - if !handled { - writeError(w, http.StatusBadRequest, "Unsupported event kind") + // Anti-inflation admission: only seeds or pubkeys with positive TrustRank + // contribute, so an untrusted author cannot inflate the graph by pushing. + if !h.authorQualifies(ev.PubKey) { return } - - writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + if _, err := h.ingester.Apply(ev); err != nil { + log.Printf("[API] Error ingesting event (kind=%d pubkey=%s): %v", ev.Kind, ev.PubKey, err) + } } // authorQualifies returns true if the author is an explicit seed or has a diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index f3f6a7e..8c1a2fc 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -25,21 +25,28 @@ type Handler struct { searchConfig *config.SearchConfig seedSet map[string]struct{} ingester *ingest.Ingester + ingestCh chan *nostr.Event } -// New creates a new Handler instance +// New creates a new Handler instance and starts the background workers that +// process events posted to /event asynchronously. func New(repo *repository.Repository, cache *cache.Cache, searchConfig *config.SearchConfig, seedPubkeys []string) *Handler { seedSet := make(map[string]struct{}, len(seedPubkeys)) for _, pk := range seedPubkeys { seedSet[pk] = struct{}{} } - return &Handler{ + h := &Handler{ repo: repo, cache: cache, searchConfig: searchConfig, seedSet: seedSet, ingester: ingest.New(repo), + ingestCh: make(chan *nostr.Event, ingestQueueSize), } + for range ingestWorkers { + go h.ingestWorker() + } + return h } // HealthResponse represents the health check response