feat: add anonymize core crate

.ai/manifest.json

@@ -6,6 +6,7 @@
       "stella-public-repo",
       "engineering",
       "typescript",
+      "rust",
       "testing",
       "linting"
     ],

.cargo/config.toml

@@ -0,0 +1,9 @@
+# Use sparse index for faster resolution.
+[registries.crates-io]
+protocol = "sparse"
+
+[alias]
+ci-fmt = "fmt --all -- --check"
+ci-clippy = "clippy --workspace --all-targets --all-features --locked -- -D warnings"
+ci-dylint = "dylint --workspace --all"
+ci-test = "test --workspace --all-features --locked"

.github/tools/check-brand-case.mjs

@@ -0,0 +1,75 @@
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import process from "node:process";
+
+const DISALLOWED = ["S", "tella"].join("");
+const EXPECTED = DISALLOWED.toLowerCase();
+const IGNORED_PATHS = new Set(["AGENTS.md", "CLAUDE.md", "GEMINI.md"]);
+const IGNORED_PREFIXES = [
+  ".ai/",
+  ".agents/",
+  ".claude/",
+  ".github/assets/",
+  "packages/data/dictionaries/",
+  "packages/*/dist/",
+  "packages/anonymize/wasm/dist/",
+  "target/",
+];
+
+const trackedFiles = execFileSync("git", ["ls-files", "-z"], {
+  encoding: "utf8",
+})
+  .split("\0")
+  .filter(Boolean)
+  .filter((file) => !isIgnored(file));
+
+let hasFailure = false;
+
+for (const file of trackedFiles) {
+  const content = readFileSync(file);
+  if (content.includes(0)) {
+    continue;
+  }
+
+  const text = content.toString("utf8");
+  let index = text.indexOf(DISALLOWED);
+  while (index !== -1) {
+    const { line, column } = lineColumnFor(text, index);
+    console.error(
+      `${file}:${line}:${column} uses disallowed brand casing; use "${EXPECTED}"`,
+    );
+    hasFailure = true;
+    index = text.indexOf(DISALLOWED, index + DISALLOWED.length);
+  }
+}
+
+if (hasFailure) {
+  process.exit(1);
+}
+
+function isIgnored(file) {
+  if (IGNORED_PATHS.has(file)) {
+    return true;
+  }
+  return IGNORED_PREFIXES.some((pattern) => {
+    if (!pattern.includes("*")) {
+      return file.startsWith(pattern);
+    }
+    const [prefix, suffix] = pattern.split("*");
+    return file.startsWith(prefix) && file.includes(suffix);
+  });
+}
+
+function lineColumnFor(text, index) {
+  let line = 1;
+  let column = 1;
+  for (let i = 0; i < index; i += 1) {
+    if (text.charCodeAt(i) === 10) {
+      line += 1;
+      column = 1;
+      continue;
+    }
+    column += 1;
+  }
+  return { line, column };
+}

.github/tools/check-packlist.mjs

@@ -7,6 +7,13 @@ const PACKAGES = [
     expected: [
       "dist/index.d.mts",
       "dist/index.mjs",
+      "dist/native.d.mts",
+      "dist/native.mjs",
+      "dist/native-node.d.mts",
+      "dist/native-node.mjs",
+      "index.cjs",
+      "stella_anonymize_napi.node",
+      "native-pipeline.stlanonpkg",
       // Dynamically imported corpus chunk; missing means the
       // bundler stopped resolving the non-Western name imports.
       "dist/names-nw-in.mjs",

.github/tools/check-python-wheel.mjs

@@ -0,0 +1,132 @@
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import process from "node:process";
+
+const outDir = mkdtempSync(join(tmpdir(), "stella-anonymize-wheel-"));
+const profile = process.env.ANONYMIZE_PYTHON_WHEEL_PROFILE ?? "ci";
+
+try {
+  execFileSync(
+    "uvx",
+    [
+      "--from",
+      "maturin>=1.14,<2",
+      "maturin",
+      "build",
+      "--manifest-path",
+      "crates/anonymize-py/Cargo.toml",
+      "--locked",
+      "--profile",
+      profile,
+      "--out",
+      outDir,
+    ],
+    { stdio: "inherit" },
+  );
+
+  const wheel = readdirSync(outDir).find((file) => file.endsWith(".whl"));
+  if (wheel === undefined) {
+    throw new Error("maturin did not emit a wheel");
+  }
+
+  const wheelPath = join(outDir, wheel);
+  const files = new Set(JSON.parse(readWheelFiles(wheelPath)));
+  const required = [
+    "stella_anonymize/__init__.py",
+    "stella_anonymize/__init__.pyi",
+    "stella_anonymize/_native.pyi",
+    "stella_anonymize/py.typed",
+  ];
+  const missing = required.filter((file) => !files.has(file));
+  if (missing.length > 0) {
+    throw new Error(`wheel is missing files: ${missing.join(", ")}`);
+  }
+  if (![...files].some(isNativeExtension)) {
+    throw new Error("wheel is missing the native _native extension");
+  }
+  smokeInstalledWheel(wheelPath);
+
+  console.log(
+    JSON.stringify({
+      event: "python-wheel-check",
+      wheel,
+      profile,
+    }),
+  );
+} finally {
+  rmSync(outDir, { force: true, recursive: true });
+}
+
+function readWheelFiles(wheelPath) {
+  return execFileSync(
+    "python3",
+    [
+      "-c",
+      [
+        "import json, sys, zipfile",
+        "with zipfile.ZipFile(sys.argv[1]) as wheel:",
+        "    print(json.dumps(wheel.namelist()))",
+      ].join("\n"),
+      wheelPath,
+    ],
+    { encoding: "utf8" },
+  );
+}
+
+function smokeInstalledWheel(wheelPath) {
+  execFileSync(
+    "uv",
+    [
+      "run",
+      "--isolated",
+      "--no-project",
+      "--python",
+      "3.11",
+      "--with",
+      wheelPath,
+      "python",
+      "-c",
+      [
+        "import json",
+        "import stella_anonymize as anonymize",
+        "required = [",
+        "    'PreparedAnonymizer',",
+        "    'PreparedSearch',",
+        "    'load_prepared_package',",
+        "    'prepare_search_package',",
+        "    'redact_text',",
+        "]",
+        "missing = [name for name in required if not hasattr(anonymize, name)]",
+        "if missing:",
+        "    raise SystemExit(f'missing exports: {missing}')",
+        "config_json = json.dumps({",
+        "    'regex_patterns': [{'kind': 'regex', 'pattern': r'\\b[A-Z]{2}\\d{4}\\b'}],",
+        "    'slices': {'regex': {'start': 0, 'end': 1}},",
+        "    'regex_meta': [{'label': 'registration number', 'score': 1.0}],",
+        "})",
+        "package_bytes = anonymize.prepare_search_package(config_json)",
+        "prepared = anonymize.load_prepared_package(package_bytes)",
+        "result = prepared.redact_text('Reference AB1234')",
+        "if result.redaction.entity_count != 1:",
+        "    raise SystemExit(f'unexpected entity count: {result.redaction.entity_count}')",
+        "if result.redaction.redacted_text == 'Reference AB1234':",
+        "    raise SystemExit('redaction did not change text')",
+        "print(json.dumps({",
+        "    'event': 'python-wheel-import-smoke',",
+        "    'version': anonymize.native_package_version(),",
+        "    'entity_count': result.redaction.entity_count,",
+        "}))",
+      ].join("\n"),
+    ],
+    { stdio: "inherit" },
+  );
+}
+
+function isNativeExtension(file) {
+  return (
+    file.startsWith("stella_anonymize/_native.") &&
+    [".so", ".pyd", ".dll", ".dylib"].some((suffix) => file.endsWith(suffix))
+  );
+}

.github/tools/sync-runtime-version.mjs

@@ -10,7 +10,16 @@ const PACKAGE_FILES = [
   "packages/cli/package.json",
 ];
 
+const CARGO_WORKSPACE_MANIFEST = "Cargo.toml";
+const CARGO_LOCKED_PACKAGES = [
+  "stella-anonymize-adapter-contract",
+  "stella-anonymize-core",
+  "stella-anonymize-napi",
+  "stella-anonymize-py",
+];
+const PYPROJECT_FILES = ["crates/anonymize-py/pyproject.toml"];
 const LOCK_FILE = "bun.lock";
+const CARGO_LOCK_FILE = "Cargo.lock";
 const checkOnly = process.argv.includes("--check");
 const version = readFileSync("VERSION", "utf8").trim();
 
@@ -30,6 +39,29 @@ const SYNCED_DEPENDENCY_RANGE_RE = /("@stll\/anonymize": "\^)([^"]+)(")/g;
 
 const escapeRegExp = (value) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
 
+const syncTextVersion = ({ file, label, re }) => {
+  const text = readFileSync(file, "utf8");
+  const match = text.match(re);
+  if (!match) {
+    console.error(`${file} has no ${label} version entry`);
+    hasMismatch = true;
+    return;
+  }
+  const current = match[2];
+  if (current === version) {
+    return;
+  }
+  if (checkOnly) {
+    console.error(
+      `${file} has ${label} version ${current}; expected ${version}`,
+    );
+    hasMismatch = true;
+    return;
+  }
+  writeFileSync(file, text.replace(re, `$1${version}$3`));
+  console.log(`Updated ${file} ${label} version to ${version}`);
+};
+
 for (const file of PACKAGE_FILES) {
   const pkg = JSON.parse(readFileSync(file, "utf8"));
   const wantedRange = `^${version}`;
@@ -61,6 +93,34 @@ for (const file of PACKAGE_FILES) {
   console.log(`Updated ${file} to ${version}`);
 }
 
+syncTextVersion({
+  file: CARGO_WORKSPACE_MANIFEST,
+  label: "Cargo workspace",
+  re: /(\[workspace\.package\][\s\S]*?\nversion\s*=\s*")([^"]+)(")/,
+});
+
+for (const file of PYPROJECT_FILES) {
+  const text = readFileSync(file, "utf8");
+  const explicitVersion = text.match(/^version\s*=\s*"([^"]+)"/m);
+  if (explicitVersion) {
+    syncTextVersion({
+      file,
+      label: "Python project",
+      re: /(^version\s*=\s*")([^"]+)(")/m,
+    });
+    continue;
+  }
+
+  if (/\bdynamic\s*=\s*\[[^\]]*"version"[^\]]*\]/m.test(text)) {
+    continue;
+  }
+
+  console.error(
+    `${file} must either derive version dynamically from Cargo or match VERSION`,
+  );
+  hasMismatch = true;
+}
+
 const lockText = readFileSync(LOCK_FILE, "utf8");
 let lockChanged = false;
 let syncedLockText = lockText.replaceAll(
@@ -116,6 +176,45 @@ if (lockChanged) {
   );
 }
 
+const cargoLockText = readFileSync(CARGO_LOCK_FILE, "utf8");
+let cargoLockChanged = false;
+let syncedCargoLockText = cargoLockText;
+
+for (const packageName of CARGO_LOCKED_PACKAGES) {
+  const packageVersionRe = new RegExp(
+    `(\\[\\[package\\]\\]\\nname = "${escapeRegExp(packageName)}"\\nversion = ")([^"]+)(")`,
+  );
+  const match = syncedCargoLockText.match(packageVersionRe);
+  if (!match) {
+    console.error(`${CARGO_LOCK_FILE} has no package entry for ${packageName}`);
+    hasMismatch = true;
+    continue;
+  }
+  const lockedVersion = match[2];
+  if (lockedVersion === version) {
+    continue;
+  }
+  if (checkOnly) {
+    console.error(
+      `${CARGO_LOCK_FILE} package ${packageName} has version ${lockedVersion}; expected ${version}`,
+    );
+    hasMismatch = true;
+    continue;
+  }
+  syncedCargoLockText = syncedCargoLockText.replace(
+    packageVersionRe,
+    `$1${version}$3`,
+  );
+  cargoLockChanged = true;
+}
+
+if (cargoLockChanged) {
+  writeFileSync(CARGO_LOCK_FILE, syncedCargoLockText);
+  console.log(
+    `Updated ${CARGO_LOCK_FILE} local package versions to ${version}`,
+  );
+}
+
 if (hasMismatch) {
   process.exit(1);
 }