feat: add feedback loop (#76) (#85)

Author: dancer

.github/scripts/validate.ts

@@ -4,47 +4,15 @@ import { readFileSync, existsSync } from 'fs';
 
 const schema = z.object({
   confidence: z.number().min(0).max(1).optional(),
-  theme: z.string().optional(),
-  themes: z.record(z.string(), z.record(z.string(), z.string())).optional(),
-  labels: z.record(z.string(), z.string()).optional(),
-  rules: z
+  model: z.string().optional(),
+  examples: z
     .array(
       z.object({
-        match: z.string(),
-        add: z.array(z.string()),
+        title: z.string(),
+        labels: z.array(z.string()),
       }),
     )
     .optional(),
-  reactions: z
-    .object({
-      start: z.string().optional(),
-      complete: z.string().optional(),
-    })
-    .optional(),
-  ignore: z
-    .object({
-      users: z.array(z.string()).optional(),
-      labels: z.array(z.string()).optional(),
-    })
-    .optional(),
-  duplicates: z
-    .object({
-      enabled: z.boolean().optional(),
-      threshold: z.number().min(0).max(1).optional(),
-      label: z.string().optional(),
-      comment: z.boolean().optional(),
-      close: z.boolean().optional(),
-    })
-    .optional(),
-  autorespond: z
-    .object({
-      enabled: z.boolean().optional(),
-      label: z.string().optional(),
-      context: z.string().optional(),
-      requirements: z.record(z.string(), z.array(z.string())).optional(),
-      message: z.string().optional(),
-    })
-    .optional(),
 });
 
 const configpath = '.github/tigent.yml';

.github/tigent.yml

@@ -19,15 +19,15 @@ labels:
   help-wanted: muted
 
 rules:
-  - match: "crash|broken|not working"
+  - match: 'crash|broken|not working'
     add: [bug, p1]
-  - match: "security|vulnerability|cve"
+  - match: 'security|vulnerability|cve'
     add: [security, p0]
-  - match: "docs|readme|typo"
+  - match: 'docs|readme|typo'
     add: [documentation]
-  - match: "config|yaml|yml"
+  - match: 'config|yaml|yml'
     add: [config]
-  - match: "webhook|payload"
+  - match: 'webhook|payload'
     add: [webhook]
 
 reactions:

.husky/pre-commit

@@ -0,0 +1 @@
+pnpm prettier-fix && git add -u

app/api/webhook/feedback.ts

@@ -0,0 +1,126 @@
+import type { Gh, Config } from './triage';
+import { classify, fetchlabels, addlabels } from './triage';
+import { createpr } from './learn';
+
+const allowed = ['OWNER', 'MEMBER', 'COLLABORATOR'];
+
+export async function handlecomment(gh: Gh, config: Config, payload: any) {
+  const comment = payload.comment;
+  const body: string = comment.body?.trim() || '';
+  const association: string = comment.author_association || '';
+
+  if (!allowed.includes(association)) return;
+  if (!body.toLowerCase().startsWith('@tigent')) return;
+
+  const command = body.slice(7).trim().toLowerCase();
+  const issue = payload.issue;
+
+  if (command === 'why') {
+    await handlewhy(gh, config, issue, comment.id);
+  } else if (command.startsWith('wrong')) {
+    const rest = body.slice(7).trim().slice(5).trim();
+    const labels = parselabels(rest);
+    if (labels.length > 0) {
+      await handlewrong(gh, config, issue, comment.id, labels);
+    }
+  }
+}
+
+async function handlewhy(
+  gh: Gh,
+  config: Config,
+  issue: any,
+  commentid: number,
+) {
+  await reactcomment(gh, commentid);
+
+  const labels = await fetchlabels(gh);
+  const result = await classify(config, labels, issue.title, issue.body || '');
+
+  const labelstr = result.labels.join(', ');
+  const body = `**labels:** ${labelstr}\n**confidence:** ${result.confidence}\n\n${result.reasoning}`;
+
+  await gh.octokit.rest.issues.createComment({
+    owner: gh.owner,
+    repo: gh.repo,
+    issue_number: issue.number,
+    body,
+  });
+}
+
+async function handlewrong(
+  gh: Gh,
+  config: Config,
+  issue: any,
+  commentid: number,
+  correctlabels: string[],
+) {
+  await reactcomment(gh, commentid);
+
+  const [repolabels, current] = await Promise.all([
+    fetchlabels(gh),
+    gh.octokit.rest.issues.listLabelsOnIssue({
+      owner: gh.owner,
+      repo: gh.repo,
+      issue_number: issue.number,
+    }),
+  ]);
+
+  const result = await classify(
+    config,
+    repolabels,
+    issue.title,
+    issue.body || '',
+  );
+
+  const ailabels = result.labels;
+  const existing = current.data.map(l => l.name);
+
+  for (const label of ailabels) {
+    if (existing.includes(label)) {
+      await gh.octokit.rest.issues.removeLabel({
+        owner: gh.owner,
+        repo: gh.repo,
+        issue_number: issue.number,
+        name: label,
+      });
+    }
+  }
+
+  const validcorrect = correctlabels.filter(l =>
+    repolabels.some(x => x.name.toLowerCase() === l.toLowerCase()),
+  );
+  const matchedlabels = validcorrect.map(l => {
+    const match = repolabels.find(
+      x => x.name.toLowerCase() === l.toLowerCase(),
+    );
+    return match!.name;
+  });
+
+  if (matchedlabels.length > 0) {
+    await addlabels(gh, issue.number, matchedlabels);
+  }
+
+  await createpr(gh, issue.number, issue.title, matchedlabels);
+}
+
+function parselabels(input: string): string[] {
+  const cleaned = input
+    .replace(/^,/, '')
+    .replace(/^should be/i, '')
+    .trim();
+  if (!cleaned) return [];
+  return cleaned
+    .split(',')
+    .map(s => s.trim())
+    .filter(Boolean);
+}
+
+async function reactcomment(gh: Gh, commentid: number) {
+  await gh.octokit.rest.reactions.createForIssueComment({
+    owner: gh.owner,
+    repo: gh.repo,
+    comment_id: commentid,
+    content: 'eyes',
+  });
+}

app/api/webhook/learn.ts

@@ -0,0 +1,78 @@
+import { Octokit } from 'octokit';
+import { parse, stringify } from 'yaml';
+import type { Gh, Config } from './triage';
+
+const dancer = process.env.DANCER_PAT
+  ? new Octokit({ auth: process.env.DANCER_PAT })
+  : null;
+
+export async function createpr(
+  gh: Gh,
+  issue: number,
+  title: string,
+  labels: string[],
+) {
+  if (!dancer) return;
+
+  const { data: repo } = await dancer.rest.repos.get({
+    owner: gh.owner,
+    repo: gh.repo,
+  });
+  const branch = `tigent/learn-${issue}`;
+  const defaultbranch = repo.default_branch;
+
+  const { data: ref } = await dancer.rest.git.getRef({
+    owner: gh.owner,
+    repo: gh.repo,
+    ref: `heads/${defaultbranch}`,
+  });
+  const sha = ref.object.sha;
+
+  await dancer.rest.git.createRef({
+    owner: gh.owner,
+    repo: gh.repo,
+    ref: `refs/heads/${branch}`,
+    sha,
+  });
+
+  let config: Partial<Config> = {};
+  let filesha: string | undefined;
+
+  try {
+    const { data } = await dancer.rest.repos.getContent({
+      owner: gh.owner,
+      repo: gh.repo,
+      path: '.github/tigent.yml',
+      ref: defaultbranch,
+    });
+    if ('content' in data) {
+      const content = Buffer.from(data.content, 'base64').toString();
+      config = (parse(content) as Partial<Config>) || {};
+      filesha = data.sha;
+    }
+  } catch {}
+
+  if (!config.examples) config.examples = [];
+  config.examples.push({ title, labels });
+
+  const yaml = stringify(config);
+
+  await dancer.rest.repos.createOrUpdateFileContents({
+    owner: gh.owner,
+    repo: gh.repo,
+    path: '.github/tigent.yml',
+    message: `fix: add learning example from #${issue}`,
+    content: Buffer.from(yaml).toString('base64'),
+    branch,
+    ...(filesha ? { sha: filesha } : {}),
+  });
+
+  await dancer.rest.pulls.create({
+    owner: gh.owner,
+    repo: gh.repo,
+    title: `fix: learn from #${issue} correction`,
+    body: `adds example to \`.github/tigent.yml\` from issue #${issue} correction.\n\n\`\`\`yaml\n- title: "${title}"\n  labels: [${labels.join(', ')}]\n\`\`\``,
+    head: branch,
+    base: defaultbranch,
+  });
+}