diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..6d444d2 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# wiki-trace is free, MIT, self-hostable. If you'd like to support +# development, sponsoring on GitHub keeps the project independent. +github: [OmkarRayAI] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..92ef9cf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,63 @@ +name: Bug report +description: Something works incorrectly or crashes. +title: "[bug] " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for filing. Smaller repros get fixed faster. + wiki-trace stores its state as JSONL on disk — pasting the + first few lines of `.wikitrace/spans.jsonl` is usually all + the diagnostic data we need. + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: What did you expect, what did you observe? + validations: + required: true + + - type: textarea + id: repro + attributes: + label: Repro + description: Smallest possible script + commands. + render: shell + validations: + required: true + + - type: input + id: version + attributes: + label: wikitrace version + description: e.g. `0.2.1` or commit SHA + placeholder: "0.2.1" + validations: + required: true + + - type: input + id: python + attributes: + label: Python version + placeholder: "3.12.3" + validations: + required: false + + - type: input + id: provider + attributes: + label: Provider / framework (if relevant) + placeholder: "openai 1.40 / langchain-core 0.3.5 / crewai 0.70" + validations: + required: false + + - type: textarea + id: span-snippet + attributes: + label: Relevant spans (if relevant) + description: A few lines from `.wikitrace/spans.jsonl` near the issue. + render: json + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..a7d9d98 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Security report + url: https://github.com/OmkarRayAI/wiki-trace/blob/main/SECURITY.md + about: Don't open a public issue. See SECURITY.md for the disclosure path. + - name: Question or discussion + url: https://github.com/OmkarRayAI/wiki-trace/discussions + about: For "how do I..." questions, design discussions, or feedback. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..9237b08 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,41 @@ +name: Feature request +description: Suggest a new capability, integration, or API change. +title: "[feat] " +labels: ["enhancement"] +body: + - type: textarea + id: problem + attributes: + label: What problem are you trying to solve? + description: Describe the use case, not the solution. + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposal + description: | + Optional. If you've thought about the API or implementation, + share it. Otherwise we'll figure it out together. + validations: + required: false + + - type: input + id: alternatives + attributes: + label: Alternatives considered + description: Other tools or approaches you've tried. + validations: + required: false + + - type: dropdown + id: priority + attributes: + label: How blocking is this? + options: + - "Nice to have" + - "Would unblock a current project" + - "Critical — we can't ship without it" + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/integration_request.yml b/.github/ISSUE_TEMPLATE/integration_request.yml new file mode 100644 index 0000000..3b85c57 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/integration_request.yml @@ -0,0 +1,41 @@ +name: Integration request +description: A framework, provider, or tool you want wiki-trace to support. +title: "[integration] " +labels: ["integration"] +body: + - type: input + id: name + attributes: + label: What framework / provider / tool? + placeholder: "LlamaIndex, OpenAI Assistants, Haystack, Bedrock, ..." + validations: + required: true + + - type: input + id: link + attributes: + label: Link to its docs / SDK + placeholder: "https://..." + validations: + required: true + + - type: textarea + id: context + attributes: + label: How are you using it today? + description: | + What does your stack look like? Are you running this in + production, eval, or just exploring? + validations: + required: true + + - type: dropdown + id: would-you-help + attributes: + label: Would you be willing to help test? + options: + - "Yes — I can run a draft adapter against my real stack" + - "Maybe — depends on the time commitment" + - "No — just filing the request" + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..de8faa4 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ + + +## What + +A line or two on what this changes. + +## Why + +The motivation: what problem does this solve, what was broken, what +new capability does it unlock. + +## How to test + +```bash +# the commands a reviewer should run to verify +``` + +## Checklist + +- [ ] Tests added or updated (or `n/a` with reason) +- [ ] CI is green on this branch +- [ ] Docs updated if the public API changed (`README.md`, `CHANGELOG.md`) +- [ ] No secrets in the diff (run `git diff | grep -i 'sk-\|key=\|token='`) + +🤖 Generated with [Claude Code](https://claude.com/claude-code) — only if true. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..3ffcbe2 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,95 @@ +name: publish + +# Publishes to PyPI when a `v*` tag is pushed. +# +# Setup (one-time): +# 1. Create a PyPI account, generate a project-scoped API token at +# https://pypi.org/manage/account/token/ +# 2. In repo Settings → Secrets → Actions, add PYPI_API_TOKEN +# 3. Tag a release: `git tag v0.2.1 && git push origin v0.2.1` + +on: + push: + tags: + - "v*" + +jobs: + pypi: + name: PyPI + runs-on: ubuntu-latest + permissions: + contents: read + # Required for PyPI Trusted Publishing (no API token needed once + # configured at https://pypi.org/manage/account/publishing/). + id-token: write + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Build sdist + wheel + run: | + python -m pip install --upgrade pip build + python -m build + + - name: Verify dist + run: | + python -m pip install twine + python -m twine check dist/* + + - name: Publish to PyPI (trusted publishing) + # If you've configured Trusted Publishing on PyPI, no token is + # needed — id-token: write above is enough. + # Otherwise, fall back to API token: comment out this step and + # uncomment the one below. + uses: pypa/gh-action-pypi-publish@release/v1 + + # Fallback: API token. Uncomment if Trusted Publishing isn't set up. + # - name: Publish to PyPI (API token) + # env: + # TWINE_USERNAME: __token__ + # TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # run: python -m twine upload dist/* + + npm: + name: npm (sdk-js) + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk-js + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + registry-url: "https://registry.npmjs.org" + cache: npm + cache-dependency-path: sdk-js/package-lock.json + + - name: Install + run: npm ci + + - name: Build + run: npm run build + + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + if [ -z "$NODE_AUTH_TOKEN" ]; then + echo "NPM_TOKEN secret not set — skipping JS publish." + exit 0 + fi + # Only publish if the tag's version matches package.json + PKG_VERSION=$(node -p "require('./package.json').version") + TAG_VERSION="${GITHUB_REF_NAME#v}" + if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then + echo "package.json version ($PKG_VERSION) != tag ($TAG_VERSION) — skipping." + exit 0 + fi + npm publish --access public diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5c1a70e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,84 @@ +# Changelog + +All notable changes to wiki-trace are documented here. The format is +loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +Versions follow [Semantic Versioning](https://semver.org/) — `0.x` is +stable enough to use, may break between minor versions until 1.0. + +## [Unreleased] + +### Added +- Launch-ready repo polish (see `README.md` for the full pitch) +- `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, `SECURITY.md`, issue/PR + templates under `.github/` +- `examples/` reorganized with one runnable script per integration + +## [0.2.1] - 2026-06-05 + +### Added +- **Real-API verification via OpenRouter** — first live end-to-end check + of `wikitrace.openai.patch` against a real network endpoint. 2 tests + pass in <4s on the free tier (#9). +- OpenRouter integration test path (`tests/integration/test_openrouter_real.py`) + and `ci-real-api.yml` GitHub Actions workflow gated on repo secrets (#7, #8). +- `wikitrace.replay_trace(trace_id, agent=, new_model=)` — re-drive a + recorded trace through a different agent and diff outcomes via the + existing `RunDiff` infrastructure (#3). +- `/contribution` dashboard route — surfaces page-contribution as a + first-class metric (#3). +- **Cloud Postgres backend** via `DATABASE_URL` — same `Database` class, + asyncpg pool, JSONB round-trip, schema versioning. SQLite remains + the zero-dep default (#4). +- **Self-service cloud signup** at `POST /v1/signup` (no admin key); + per-tenant usage metering with daily counters; admin overview at + `/v1/admin/usage`; one-command `docker-compose up` deploy (#6). +- **Cloud-mode dashboard** with `/sign-in` + `/sign-up` flows. Same + Next.js binary, multi-tenant when `WIKITRACE_BACKEND=cloud` (#1). +- **Tests + CI**: 88-test pytest suite, GitHub Actions matrix on Python + 3.11/3.12 with a Postgres service container, dashboard typecheck job, + lint job (#5). + +### Changed +- README: `Tests` section added; pricing table now reflects "self-hosted + cloud" tier; integrations table in progress. + +### Fixed +- OpenRouter `/` price-prefix lookup + ([commit 38aa1db](https://github.com/OmkarRayAI/wiki-trace/commit/38aa1db)). + +## [0.2.0] - 2026-06-03 + +### Added +- **Self-service SaaS surface** — multi-tenant cloud server with API-key + auth, tenant isolation, admin CLI, Helicone-compat passthrough. +- **JS/TS SDK alpha** (Node + browser) — first non-Python language; + feature parity at the SDK level. +- **HTTP ingest server** — multi-language entry point at + `python -m wikitrace.ingest_serve`. Helicone async-log compatible. +- **Provider patches** — `wikitrace.openai.patch()` and + `.anthropic.patch()` for sync/async/streaming. +- **Production runtime** — async batched JSONL writer (~23k spans/sec), + rate-limit-aware retry with exponential backoff + jitter, cost + budgeting (`wikitrace.budget(usd=10)`). +- **Eval primitives** — `Dataset`, `run_eval`, `compare_runs`, + `load_run`, 16 built-in judges (deterministic + LLM-as-judge), + `@wikitrace.eval` decorator. +- **OpenTelemetry export** — `wikitrace.otel.install()` pipes spans + into Phoenix / Datadog / Honeycomb / any OTLP collector. +- **Multi-step planner traces** — nested span trees via + `wikitrace.session()` + `step()`; LangChain handler emits + `tool_call` / `agent_action` / streaming `llm_call` spans. +- **Decorator API** — `@wikitrace.trace`, `@wikitrace.tool` (sync + async). +- **Sessions / users / tags** — `wikitrace.session(id=, user=, tags=[])` + ambient attribution; contextvars-based, async-safe. +- **Five framework adapters** — LangChain (production-tested), CrewAI / + Google ADK / Agno / OpenAI / Anthropic (mocked-verified). +- **Helicone-style observability dashboard** — `/requests`, `/sessions`, + `/users`, `/properties`, `/evaluators` routes. + +## [0.1.0] - 2026-05-31 + +### Added +- Initial release — Python SDK (~750 LOC, stdlib-only), Next.js + dashboard, LangChain integration, JSONL contract, citation tracking, + curated wiki page flow. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3535e55 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,26 @@ +# Code of Conduct + +We follow the [Contributor Covenant v2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). + +## Short version + +- Be kind. Engineers come in all shapes; assume good faith. +- Critique code, not people. "This is wrong because X" not "you're wrong." +- No harassment, slurs, sexual attention, doxxing, sustained disruption, + or political flame wars in project spaces. +- If something feels off, flag it. The maintainer is reachable at + **omkarrayai@gmail.com**. + +## Scope + +Applies to all project spaces (repo, issues, PRs, discussions, any +official chat) and to public spaces where someone is representing the +project. + +## Enforcement + +Reports go to **omkarrayai@gmail.com**. We respond within 7 days with +the response we believe is appropriate to the situation, in line with +the Contributor Covenant's enforcement guidelines. + +We will respect the privacy and safety of the reporter. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..51dc5ab --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,94 @@ +# Contributing + +Thanks for considering a contribution. wiki-trace is small, opinionated, +and happy to grow — just keep the principles in `README.md` (JSONL is +the contract; findings are spans; no telemetry exfiltration) intact. + +## Quick path + +```bash +git clone https://github.com/OmkarRayAI/wiki-trace.git +cd wiki-trace +pip install -e '.[cloud,langchain,dev]' +pytest -q tests/ +``` + +If `pytest -q tests/` exits 0, you're set up. + +## Branch and PR conventions + +- One topic per branch: `feat/foo`, `fix/bar`, `docs/baz`, `test/quux`. +- Open the PR against `main`. CI must be green before merge. +- Squash-merge unless the branch genuinely benefits from preserved + per-commit history (rare). +- Reference any related issues in the body. + +## What CI checks + +Every push and PR runs `.github/workflows/ci.yml`: + +- `pytest` matrix on Python 3.11 + 3.12, with a Postgres 16 service container +- `npx tsc --noEmit` on `app/` (Next.js dashboard) +- `python -m compileall` on `wikitrace/` and `tests/` + +A separate `.github/workflows/ci-real-api.yml` runs once a week (and +on manual dispatch) when repo secrets are set, exercising +`wikitrace.openai.patch` and `.anthropic.patch` against live endpoints. +The default suite never spends money. + +## What goes where + +| Surface | Path | +|---|---| +| Python SDK | `wikitrace/` | +| JS/TS SDK | `sdk-js/` | +| Cloud server | `wikitrace/cloud/` | +| Dashboard | `app/` | +| Examples | `examples/` | +| Tests | `tests/` | +| Docs | `README.md`, `PRD.md`, this file | + +## Adding a new framework adapter + +The pattern is `wikitrace/integrations//`: + +``` +wikitrace/integrations// + __init__.py # public surface (lazy import + raise ImportError if not installed) + .py # the actual handler / patch / wrapper +wikitrace//__init__.py # short alias: `from wikitrace. import …` +``` + +Add an extra in `pyproject.toml` so `pip install 'wikitrace[]'` +pulls the right deps: + +```toml +[project.optional-dependencies] + = ["whatever-the-framework-calls-itself>=X.Y.Z"] +``` + +Mock-test it in `tests/test_.py`. If you have a real key for +the framework's LLM, also add `tests/integration/test__real.py` +gated on its env var. See `tests/integration/test_openrouter_real.py` +as the canonical pattern. + +## Style + +- Python: stdlib-only in `wikitrace/sdk.py`. Anything heavier goes + behind an `[extra]`. +- TypeScript: zero runtime deps in `sdk-js/`. Node 18+. +- Comments: only for the *why*, not the *what*. The codebase is small + enough to read. + +## Reporting issues + +Bug reports: include the smallest repro you can. wiki-trace stores +its state as JSONL on disk; `cat .wikitrace/spans.jsonl | head -3` +in your repro is usually all the diagnostic data we need. + +Security issues: see `SECURITY.md`. Don't open a public issue. + +## License + +By contributing you agree your work is licensed under the project's +MIT license. diff --git a/README.md b/README.md index c101144..14d3d98 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,51 @@ +
+ # wiki-trace -> The observability platform for Generative AI. +**The open-source observability platform for LLM applications.** + +Trace every request, cost, and agent step in production — +self-hosted, zero telemetry exfiltration, one line of code. + +[![CI](https://github.com/OmkarRayAI/wiki-trace/actions/workflows/ci.yml/badge.svg)](https://github.com/OmkarRayAI/wiki-trace/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/wikitrace.svg)](https://pypi.org/project/wikitrace/) +[![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg)](https://pypi.org/project/wikitrace/) +[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) +[![GitHub stars](https://img.shields.io/github/stars/OmkarRayAI/wiki-trace?style=social)](https://github.com/OmkarRayAI/wiki-trace/stargazers) + +[Quick start](#quick-start) · +[Integrations](#integrations) · +[Self-hosted cloud](#self-hosted-cloud) · +[Roadmap](ROADMAP.md) · +[Discussions](https://github.com/OmkarRayAI/wiki-trace/discussions) -**wiki-trace** is the open-source platform developers use to monitor, -debug, and improve production-ready LLM applications. Get instant -insights into latency, costs, and quality — without changing how you -build. +
-One line of code. Every request logged. Every cost tracked. Every -agent step replayable. +--- -[Quick start](#quick-start) · [Integrations](#integrations) · -[Sessions](#sessions) · [Custom Properties](#custom-properties) · -[Evaluators](#evaluators) · [Datasets & Experiments](#datasets--experiments) +## What it is + +wiki-trace is the open-source telemetry layer the OpenAI and Anthropic +SDKs don't ship with. Drop it next to whatever you're already running: + +- 🐍 **Python SDK** — one-line `patch()` for OpenAI / Anthropic / + OpenRouter (sync, async, streaming). Decorators for any function. + Stdlib core; ~750 LOC. +- 🟦 **JS/TS SDK** — same API surface in Node + browsers. +- 🌐 **HTTP ingest** — POST JSON from any language; speaks the + Helicone async-log protocol natively (drop-in compatible). +- 📊 **Next.js dashboard** — Helicone-style requests, sessions, users, + properties, evaluators, page-contribution. +- 🧪 **16 built-in evaluators** — exact match, contains, JSON / SQL / + schema / PII / safety, plus LLM-as-judge (Phoenix-style). +- 🔁 **Multi-step replay** — re-drive a recorded trace through a new + model, diff outcomes per question. +- ☁️ **Self-hosted multi-tenant cloud** — FastAPI + Postgres or SQLite, + API-key auth, per-tenant isolation, one-command `docker compose up`. +- 📡 **OpenTelemetry export** — pipe spans into Phoenix, Datadog, + Honeycomb, Grafana, or any OTLP collector. + +**Open-source. Apache-2.0 / MIT. Your data never leaves your machine.** --- @@ -627,8 +660,11 @@ PRD.md product requirements / pitch ## Community -- **GitHub Issues** — report bugs, request integrations. -- **Pull requests** — welcome. The codebase is small enough that you can read the whole thing in an afternoon. +- **[GitHub Discussions](https://github.com/OmkarRayAI/wiki-trace/discussions)** — questions, design discussion, feedback. +- **[GitHub Issues](https://github.com/OmkarRayAI/wiki-trace/issues)** — bugs, integration requests, feature requests. Templates included. +- **[Roadmap](ROADMAP.md)** — what's next, with reasoning. +- **[Contributing guide](CONTRIBUTING.md)** — small repo; you can read the whole thing in an afternoon. +- **[Security policy](SECURITY.md)** — coordinated disclosure. The core principles, in order: @@ -637,6 +673,12 @@ The core principles, in order: 3. **Findings are spans.** Detection rules write `finding:` spans rather than a separate table. 4. **No telemetry exfiltration.** Customer data never leaves the user's machine. +If wiki-trace saves you time, **a [star on GitHub](https://github.com/OmkarRayAI/wiki-trace) is the cheapest way to say thanks** — it directly shapes how many other developers find this project. + +## Star history + +[![Star History Chart](https://api.star-history.com/svg?repos=OmkarRayAI/wiki-trace&type=Date)](https://star-history.com/#OmkarRayAI/wiki-trace&Date) + --- ## License diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..8f89730 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,60 @@ +# Roadmap + +Direction, in rough priority order. Items at the top are the next +session's work; items at the bottom are strategic bets that need +either user demand or external resources to start. + +## Now + +- **Real-API verification on every alpha adapter.** OpenRouter is + green; Anthropic is scaffolded and waiting on a key; CrewAI / ADK / + Agno need live tests against actual chains. The README's "alpha + (mocked)" labels stay until we've run real traffic through each. +- **Native LlamaIndex / OpenAI Assistants / Haystack adapters.** Same + pattern as the existing five. File an issue with your stack and we'll + prioritize. +- **Anthropic free-tier real-API test** — once Anthropic offers a free + endpoint or we settle on a paid budget for CI. + +## Next + +- **JS/TS SDK to GA** — the alpha is feature-complete at the SDK + surface; promotion needs production hardening (edge runtime + verification on Cloudflare Workers + Vercel Edge, Playwright tests + for AsyncLocalStorage in Node, browser SSE handling). +- **Sweeps / hyperparameter search** — Weave's pillar. The eval + infrastructure (`Dataset`, `run_eval`, `compare_runs`) is the + foundation; we need a `sweep()` API that runs N agent configs over + one dataset and produces an aggregate diff. +- **Native Go SDK.** HTTP ingest server already lets Go talk to + wikitrace, but a native SDK with span context propagation would + match what Helicone offers. + +## Later + +- **Hosted SaaS** ("we run it for you") — multi-week buildout: deployed + Postgres, Stripe billing, sign-up flow, marketing site, SOC 2 path, + RBAC. Self-hosted cloud (`docker-compose up`) is the open-source + alternative and remains free forever. +- **Native LangSmith protocol compat** so anyone pointed at LangSmith + can swap base URL → wikitrace, same way the Helicone-compat + endpoints already work. +- **Browser dashboard for streaming traces** — currently the + `/contribution` and `/requests` routes paint on page load. A live + `EventSource`-fed view over `spans-live.jsonl` would close the + "watch your agent run" gap. +- **Compliance certifications** — SOC 2 Type 1, HIPAA. Customer-driven; + not on the path for self-hosted users. + +## Won't do + +- A vector database. Embeddings live in your retriever; we attach + metrics to whatever ID it returns. +- A model gateway. We use OpenRouter as the default but never proxy + inference for billing; the proxy mode is for telemetry only. +- Telemetry exfiltration. Customer data never leaves their machine. + +--- + +Have an opinion on order? Open an issue or comment on an existing one. +The roadmap is a draft, not a contract. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..d3d2b50 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,48 @@ +# Security policy + +## Supported versions + +`main` is the only branch we maintain. Releases are tagged from `main`; +security fixes land there first. If you're on a tagged release older +than the latest by more than two minor versions, please upgrade before +filing a report — the issue may already be fixed. + +## Reporting a vulnerability + +Please do **not** open a public GitHub issue for security reports. +Instead, email **omkarrayai@gmail.com** with: + +- A description of the issue and its impact (what an attacker can do) +- The version / commit affected +- A minimal repro if you have one +- Whether you'd like credit in the eventual disclosure + +We aim to acknowledge within 72 hours and ship a fix or mitigation +within 14 days for high-severity issues. Lower-severity issues are +queued normally. + +## Scope + +In scope: + +- The Python SDK (`wikitrace/`) +- The cloud server (`wikitrace.cloud`) including auth, tenant + isolation, and the admin endpoints +- The HTTP ingest server (`wikitrace.ingest_server`) including + Helicone-compat endpoints and proxy mode +- The Next.js dashboard (`app/`) sign-in / cookie / session paths +- The JS/TS SDK (`sdk-js/`) + +Out of scope: + +- Theoretical attacks requiring the operator to deliberately + misconfigure the server (e.g. running with `WIKITRACE_CLOUD_ADMIN_KEY` + unset and relying on `/v1/admin/*` for security) +- Issues in dependencies — please report those upstream +- DoS via uncapped ingestion (the writer is bounded, but the underlying + filesystem isn't; that's an operator-side capacity concern) + +## Disclosure + +We prefer coordinated disclosure: fix lands first, public CVE / advisory +follows. We'll credit you in the advisory unless you ask us not to. diff --git a/docs/LAUNCH.md b/docs/LAUNCH.md new file mode 100644 index 0000000..58512e2 --- /dev/null +++ b/docs/LAUNCH.md @@ -0,0 +1,245 @@ +# Launch playbook + +A coordinated launch is the only way a small open-source project gets +discovered. This playbook is the actual posts to make and the order +to make them in. **You** post these — I cannot post on your behalf. + +> **Honest expectations.** 1k stars in a month is achievable with a +> good launch. 100k stars in a year is moonshot territory — no +> open-source LLM observability project has crossed that bar. Plan +> for the realistic curve, hope for the long tail. + +## T-1 day: pre-launch checks + +- [ ] Repo description is current (`gh repo edit --description '...'`) +- [ ] Topics set: `llm`, `observability`, `tracing`, `evaluation`, + `langchain`, `openai`, `anthropic`, `helicone`, `phoenix`, `weave`, + `rag`, `agents`, `python`, `typescript`, `self-hosted`, `open-source` +- [ ] Social preview image set (Settings → Social preview) +- [ ] CI green on `main` +- [ ] PyPI release tagged: `git tag v0.2.1 && git push origin v0.2.1` +- [ ] `pip install wikitrace` works from a clean env +- [ ] Examples actually run end-to-end with `OPENROUTER_API_KEY` +- [ ] CHANGELOG up to date + +## Hacker News + +**Title** (the single highest-leverage decision): + +> Show HN: wiki-trace — open-source LLM observability you self-host + +Or, if you want the comparison framing: + +> Show HN: Self-hosted alternative to Helicone/Phoenix/Weave for LLM tracing + +**Body** (paste into the URL field with a link to the repo, or use +"Show HN" with text): + +``` +wiki-trace is the observability layer the OpenAI and Anthropic SDKs +don't ship with. One line of code, every request traced, every cost +tracked, every agent step replayable. + +What's there today (all MIT, all self-hosted): + + • One-line patch() for OpenAI / Anthropic / OpenRouter (sync + async + streaming) + • Multi-step planner traces — tool calls, agent actions, token-level streaming + • 16 built-in evaluators incl. RAG-faithfulness, hallucination, JSON/SQL valid, PII + • Replay any recorded trace through a different model, diff outcomes + • Multi-tenant cloud server (FastAPI + Postgres or SQLite, docker-compose up) + • Helicone-compat ingest — point any Helicone client at us, no code changes + • OpenTelemetry export — pipe spans into Phoenix, Datadog, Honeycomb + • JS/TS SDK alpha (Node + browsers) + • 88 tests, CI on every push, real-API verification via OpenRouter + +Verified working: `pip install wikitrace`; `OPENROUTER_API_KEY=... +python examples/openai_quickstart.py` produces a traced span in <4s +on the free tier (no credit card needed). + +Honest scope: + + • SDK + cloud are stable. Five framework adapters are alpha (LangChain + is production-tested; CrewAI/ADK/Agno mocked-verified pending real chains). + • No "we run it for you" SaaS. Self-host today; that's the bet. + +What I'd love feedback on: + + • The patch surface — are there providers you'd want this to cover? + • Cost-budget API — is `wikitrace.budget(usd=10)` the right shape? + • Replay — useful or gimmicky? + +Repo: https://github.com/OmkarRayAI/wiki-trace +``` + +**When to post**: Tuesday or Wednesday, 8–10am Pacific. Avoid weekends. + +**After posting**: Reply quickly to every comment within the first +hour. The first 5 comments determine the trajectory. + +## ProductHunt + +**Tagline**: "Open-source LLM observability you self-host" + +**Description**: + +``` +wiki-trace is the open-source telemetry layer the OpenAI and Anthropic +SDKs don't ship with. Trace every request, cost, and agent step in +production — without sending your data to a third party. + +ONE LINE OF CODE + import wikitrace.openai + wikitrace.openai.patch() + +Every chat.completions call now produces a span with model, tokens, +cost (built-in price table for 100+ models), latency, retry count, +and per-token streaming events. + +WHY SELF-HOSTED MATTERS +- Your conversations never leave your infra +- No SaaS vendor lock-in +- MIT license — fork and modify freely +- One-command docker-compose deploy + +WHAT'S IN THE BOX +- Python + JS/TS SDKs +- LangChain, CrewAI, Google ADK, Agno adapters +- 16 evaluators (deterministic + LLM-as-judge) +- Multi-tenant cloud server (FastAPI + Postgres) +- Helicone-compat ingest (drop-in) +- OpenTelemetry export + +Try it: pip install wikitrace + +Free verification path with OpenRouter (no credit card): + OPENROUTER_API_KEY=sk-or-... python examples/openai_quickstart.py +``` + +**First-comment**: post a more personal "why I built this" note as the +maker. People upvote the maker's energy as much as the product. + +## X / Twitter thread + +``` +1/ Shipped wiki-trace — open-source observability for LLM apps you +self-host. + +The OpenAI/Anthropic SDKs don't ship with telemetry. wiki-trace fills +that gap in one line of code. MIT license, your data never leaves +your machine. + +[link] + +2/ Why this exists: I needed Helicone-grade tracing without sending my +data to a third party. Building it locally turned out to be a small +SDK + a JSONL contract + a Next.js dashboard. + +The whole core SDK is ~750 LOC, stdlib-only. + +3/ What's in the box: +• 1-line patch() for OpenAI / Anthropic / OpenRouter +• 16 built-in evaluators (incl. RAG-faithfulness, hallucination, PII) +• Replay any trace through a different model, diff outcomes +• Multi-tenant cloud (FastAPI + Postgres, `docker compose up`) +• OpenTelemetry export + +4/ The honest pitch: self-hosted is the entire bet. + +If you want a SaaS dashboard, Helicone/Phoenix/Weave are mature. + +If you want to keep your conversations on your infra, fork and modify +the source, and pay nothing — wiki-trace is for you. + +5/ Free verification path: +- Sign up at openrouter.ai (no card) +- pip install wikitrace +- OPENROUTER_API_KEY=... python examples/openai_quickstart.py + +A real-API span lands in <4s. + +6/ What I'd love feedback on: +- Provider coverage you want next +- Whether the cost-budget API (wikitrace.budget(usd=10)) makes sense +- Whether replay (re-run a recorded trace through a new model) is + useful or gimmicky + +Star and watch if you want to follow along: [link] +``` + +**Pin the first tweet of the thread** to your profile. + +## Reddit + +**r/LocalLLaMA**: +- Title: "wiki-trace: open-source LLM tracing you self-host (MIT)" +- Lead with the privacy / self-hosted angle. This community values it. + +**r/MachineLearning**: +- Title: "[P] wiki-trace: open-source observability for LLM apps" +- Lead with the technical depth — span shape, replay, evaluator library. + +**r/programming**: +- Probably skip unless your launch is going viral elsewhere. + +## Newsletters / blogs to email + +- **TLDR AI** (tldr.tech/ai) +- **The Pragmatic Engineer** (pragmaticengineer.com — hard to land but high-value) +- **AlphaSignal** (alphasignal.ai) +- **Latent Space** (latent.space — has a mailing list submission form) +- **swyx newsletter** +- **Hugging Face newsletter** (huggingface.co) +- **Awesome-Generative-AI** lists on GitHub + +Pitch template (90 words): + +``` +Subject: wiki-trace — open-source LLM observability for self-hosted teams + +Hi [name], + +I built wiki-trace, an open-source alternative to Helicone/Phoenix/Weave +that runs entirely on your own infrastructure. One line of code adds +tracing, cost tracking, and replay to OpenAI/Anthropic/OpenRouter +calls. Multi-tenant cloud is `docker compose up`. + +Verified working: free tier via OpenRouter, no card. 88 tests, full CI. + +Repo: https://github.com/OmkarRayAI/wiki-trace +Quickstart: https://github.com/OmkarRayAI/wiki-trace#quick-start + +Happy to send a 30-line repro if it'd help. Thanks for considering. +``` + +## Distribution channels (long-tail) + +- **awesome-generative-ai-apps** GitHub list — open a PR +- **awesome-llm-observability** GitHub list — open a PR +- **dev.to** — write a "How I built wiki-trace" deep-dive +- **HackerNoon**, **Substack** — adapt the dev.to post +- **Show in your network** — direct DMs to ML engineers you respect +- **Discord communities**: LangChain, LlamaIndex, OpenAI dev, AI Engineer + +## After-launch checklist + +- [ ] Reply to every HN comment within the first 6 hours +- [ ] Reply to every issue / discussion within 24 hours for the + first week +- [ ] Ship one user-requested feature in week 1 — visible momentum + matters more than which feature +- [ ] Tweet a "what I learned" thread day 7 with star count + insights +- [ ] Add new contributors as collaborators if they ship 2+ PRs + +## Realistic numbers + +- **Launch day**: 50–500 stars if HN front-pages, 0–50 otherwise +- **Week 1**: 200–2,000 stars total in a good launch +- **Month 1**: 500–5,000 stars depending on follow-through +- **Year 1 (1k goal)**: ~realistic with sustained shipping +- **Year 1 (100k goal)**: not realistic without a category-defining + moment (think Ollama-on-launch, LangChain-circa-2023). Plan for + the realistic curve. + +The single best predictor of star growth: **shipping visible, useful +PRs every week for the first 3 months.** Star count tracks momentum, +not feature count. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..5b95344 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,27 @@ +# Examples + +Each script is self-contained and runnable in under 30 seconds. + +| Script | What it shows | +|---|---| +| [`openai_quickstart.py`](openai_quickstart.py) | One-line OpenAI / OpenRouter auto-patching with cost + streaming | +| [`anthropic_quickstart.py`](anthropic_quickstart.py) | One-line Anthropic auto-patching | +| [`decorators_quickstart.py`](decorators_quickstart.py) | `@trace` and `@tool` for arbitrary Python (sync + async) | +| [`eval_quickstart.py`](eval_quickstart.py) | Dataset, judges, `run_eval`, `compare_runs` — no LLM required | +| [`budget_quickstart.py`](budget_quickstart.py) | Cost budgeting with `wikitrace.budget(usd=...)` | +| [`langchain_rag.py`](langchain_rag.py) | LangChain `WikitraceCallbackHandler` end-to-end | +| [`byo_rag.py`](byo_rag.py) | Bring-your-own RAG — manual `wikitrace.span()` calls | + +## Free verification path + +Most examples need an LLM key. **OpenRouter offers free-tier models** +that work with the OpenAI-compatible patch: + +```bash +# Sign up at openrouter.ai, no card needed +export OPENROUTER_API_KEY=sk-or-... +python examples/openai_quickstart.py +``` + +The script auto-detects `OPENROUTER_API_KEY` and uses +`liquid/lfm-2.5-1.2b-instruct:free` — verified working in CI. diff --git a/examples/anthropic_quickstart.py b/examples/anthropic_quickstart.py new file mode 100644 index 0000000..2106ee1 --- /dev/null +++ b/examples/anthropic_quickstart.py @@ -0,0 +1,30 @@ +"""60-second quickstart: Anthropic + wiki-trace. + + pip install 'wikitrace' anthropic + ANTHROPIC_API_KEY=sk-ant-... python examples/anthropic_quickstart.py +""" + +import anthropic +import wikitrace +import wikitrace.anthropic + + +def main(): + wikitrace.anthropic.patch() + + client = anthropic.Anthropic() + wikitrace.init(pipeline="quickstart-anthropic") + with wikitrace.session(id="demo-anthropic", user="alice"): + msg = client.messages.create( + model="claude-haiku-4-5", + max_tokens=20, + messages=[{"role": "user", "content": "what color is the sky?"}], + ) + print("Answer:", msg.content[0].text) + wikitrace.end() + + print("\n✓ Spans written to .wikitrace/spans.jsonl") + + +if __name__ == "__main__": + main() diff --git a/examples/budget_quickstart.py b/examples/budget_quickstart.py new file mode 100644 index 0000000..6ad1a05 --- /dev/null +++ b/examples/budget_quickstart.py @@ -0,0 +1,29 @@ +"""Cost budgeting — hard-cap LLM spend in CI / demos / batch jobs. + + python examples/budget_quickstart.py +""" + +import wikitrace +from wikitrace.budget import budget, BudgetExceeded, current_cost, check + + +def main(): + wikitrace.init(pipeline="budget-demo") + print("Running with $0.05 cap, on_exceed='raise'...\n") + + try: + with budget(usd=0.05, name="demo"): + for i in range(20): + # Simulate a patched LLM call carrying cost_usd + with wikitrace.span("llm_call", model="gpt-4o", cost_usd=0.01): + pass + print(f" iteration {i}: spent ${current_cost():.4f}") + check() # short-circuit cleanly between iterations + except BudgetExceeded as e: + print(f"\n✓ Budget caught: {e}") + + wikitrace.end() + + +if __name__ == "__main__": + main() diff --git a/examples/decorators_quickstart.py b/examples/decorators_quickstart.py new file mode 100644 index 0000000..8cdcce2 --- /dev/null +++ b/examples/decorators_quickstart.py @@ -0,0 +1,33 @@ +"""@trace and @tool decorators — instrument arbitrary functions. + + python examples/decorators_quickstart.py +""" + +import asyncio +import wikitrace + + +@wikitrace.tool(name="search") +def search(query: str) -> list[str]: + """Pretend retrieval — anything that takes a query and returns chunks.""" + return [f"chunk for {query}", "second chunk"] + + +@wikitrace.trace(name="answer") +async def answer(question: str) -> str: + chunks = search(question) + await asyncio.sleep(0.01) # pretend the LLM is busy + return f"Based on {len(chunks)} sources: blue." + + +async def main(): + wikitrace.init(pipeline="decorators-demo") + with wikitrace.session(id="run-1"): + result = await answer("what color is the sky?") + print(result) + wikitrace.end() + print("\n✓ Spans written to .wikitrace/spans.jsonl") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/eval_quickstart.py b/examples/eval_quickstart.py new file mode 100644 index 0000000..62b8456 --- /dev/null +++ b/examples/eval_quickstart.py @@ -0,0 +1,46 @@ +"""Eval suite + run comparison — no LLM required. + + python examples/eval_quickstart.py +""" + +import wikitrace +from wikitrace.evals import Dataset, run_eval, compare_runs +from wikitrace import judges + + +def main(): + ds = Dataset([ + {"qid": "q1", "input": "what color is the sky?", "expected": ["blue"]}, + {"qid": "q2", "input": "2+2?", "expected": "4"}, + {"qid": "q3", "input": "name two penguins", "expected": ["emperor", "king"]}, + ]) + + def agent_v1(q): + if "sky" in q: return "the sky is blue" + if "2+2" in q: return "5" # wrong + if "penguin" in q: return "emperor" # missing king + + def agent_v2(q): + if "sky" in q: return "the sky is blue" + if "2+2" in q: return "4" # fixed + if "penguin" in q: return "emperor and king" # fixed + + print("Running v1...") + r1 = run_eval(agent_v1, dataset=ds, + judges=[judges.contains_all], + name="agent_v1", model="gpt-4o") + print(f" pass_rate = {r1.summary['pass_rate']:.2f}") + + print("Running v2...") + r2 = run_eval(agent_v2, dataset=ds, + judges=[judges.contains_all], + name="agent_v2", model="gpt-4o-mini") + print(f" pass_rate = {r2.summary['pass_rate']:.2f}") + + print("\nDiff:") + diff = compare_runs(r1, r2) + diff.print_table() + + +if __name__ == "__main__": + main() diff --git a/examples/openai_quickstart.py b/examples/openai_quickstart.py new file mode 100644 index 0000000..a8f3ee2 --- /dev/null +++ b/examples/openai_quickstart.py @@ -0,0 +1,70 @@ +"""60-second quickstart: OpenAI + wiki-trace. + +Every chat.completions call gets traced — cost, tokens, latency, +streaming events. No code changes besides one patch() line. + + pip install 'wikitrace' openai + OPENAI_API_KEY=sk-... python examples/openai_quickstart.py + +Or run it for free against OpenRouter: + + pip install 'wikitrace' openai + OPENROUTER_API_KEY=sk-or-... python examples/openai_quickstart.py + +After the run, inspect ``.wikitrace/spans.jsonl``: + + cat .wikitrace/spans.jsonl | python -m json.tool +""" + +import os +import openai +import wikitrace +import wikitrace.openai + + +def main(): + # One line: every OpenAI call is now a wiki-trace span. + wikitrace.openai.patch() + + # Pick OpenRouter (free) or OpenAI based on which key is set. + if os.getenv("OPENROUTER_API_KEY"): + client = openai.OpenAI( + api_key=os.environ["OPENROUTER_API_KEY"], + base_url="https://openrouter.ai/api/v1", + ) + model = "liquid/lfm-2.5-1.2b-instruct:free" + else: + client = openai.OpenAI() + model = "gpt-4o-mini" + + wikitrace.init(pipeline="quickstart") + with wikitrace.session(id="demo-1", user="alice", tags=["example"]): + # Non-streaming + resp = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": "what color is the sky?"}], + max_tokens=20, + ) + print("Answer:", resp.choices[0].message.content) + + # Streaming — every token becomes a span event + print("\nStreaming: ", end="", flush=True) + stream = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": "count to 3"}], + max_tokens=20, + stream=True, + ) + for chunk in stream: + delta = chunk.choices[0].delta.content if chunk.choices else None + if delta: + print(delta, end="", flush=True) + print() + + wikitrace.end() + print("\n✓ Spans written to .wikitrace/spans.jsonl") + print(" Run `python -m wikitrace serve` to see the dashboard.") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 09e7391..d50c2fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "wikitrace" version = "0.2.0" -description = "Knowledge quality tracing for LLM features. Drop a PDF, ask a question, see every citation traced." +description = "Open-source observability for LLM apps: trace every request, cost, and agent step. Self-hosted, MIT." readme = "README.md" requires-python = ">=3.9" license = { text = "MIT" }