From 09ee70ee0e6b59a356516ffc1391f8993c26b958 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Mon, 9 Feb 2026 14:57:57 -0500 Subject: [PATCH 01/27] wip --- checkie.gemspec | 7 ++-- lib/checkie.rb | 8 +++++ lib/checkie/matcher.rb | 37 ++++++++++++++++++++ lib/checkie/parser.rb | 44 +++++++++++++++++------- lib/checkie/poster.rb | 45 +++++++++++++++++++++++++ lib/checkie/runner.rb | 76 ++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 199 insertions(+), 18 deletions(-) diff --git a/checkie.gemspec b/checkie.gemspec index 97109dd..cd8b649 100644 --- a/checkie.gemspec +++ b/checkie.gemspec @@ -1,8 +1,8 @@ Gem::Specification.new do |s| s.name = %q{checkie} - s.version = "0.0.1" + s.version = "0.0.3" s.authors = %q{Alignable} - s.date = %q{2018-05-23} + s.date = %q{2026-02-04} s.summary = %q{Hi, It's Checkie! Looks like you're trying to write some code} s.files = [ "lib/checkie.rb" @@ -11,9 +11,10 @@ Gem::Specification.new do |s| s.license = 'MIT' s.add_runtime_dependency 'dotenv' - s.add_runtime_dependency "octokit", "~>4.22.0" + s.add_runtime_dependency "octokit" s.add_runtime_dependency "git_diff_parser" s.add_runtime_dependency "activesupport" + s.add_runtime_dependency "pry" # to write specs for rules.rb s.add_runtime_dependency "rspec" diff --git a/lib/checkie.rb b/lib/checkie.rb index e34539a..124d5ae 100644 --- a/lib/checkie.rb +++ b/lib/checkie.rb @@ -19,11 +19,19 @@ def matching(file_pattern, &block) Checkie::Parser.instance.add_matching_file(file_pattern,&block) end +def matching_ai(file_pattern, rules:, exclude: nil, &block) + Checkie::Parser.instance.add_matching_file(file_pattern, ai: true, rules: rules, exclude: exclude, &block) +end + def file_rule(name,description,references:[]) Checkie::Parser.instance.add_file_rule(name, description, references: references) end +def file_rule_ai(name, description) + Checkie::Parser.instance.add_file_rule(name, description, ai: true) +end + def run(url, action) Checkie::Runner.new.run(url, action) end diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index a7a3609..6af0541 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -16,6 +16,43 @@ def check(rule,references = nil ) @rules[rule.to_s] += references if references end + def check_rule(rule) + @rules[rule.to_s] ||= [] + rule + end + + + def match_ai + ruleset_to_files = {} + @pr.each do |file| + rules = gather_rules_ai(file[:filename]) + next if rules.empty? + changeset = {name: file[:filename], patch: file[:patch]} + if ruleset_to_files.key?(rules) + ruleset_to_files[rules].append(changeset) + else + ruleset_to_files[rules] = [changeset] + end + end + ruleset_to_files.map do |k,v| + [k.join("\n"), v] + end + end + + def gather_rules_ai(path) + applicable = Set[] + parser.matches_ai.each do |rule| + if File.fnmatch(rule[:pattern], path, File::FNM_EXTGLOB) + next if rule[:exclude] && File.fnmatch(rule[:exclude],path, File::Constants::FNM_EXTGLOB) + # rule[:text] = parser.rules_ai[rule[]] + rule[:rules].each do |r| + applicable.add(r[:description]) + end + end + end + return applicable + end + def match all_files = Checkie::FileMatchSet.new(self) diff --git a/lib/checkie/parser.rb b/lib/checkie/parser.rb index 5d64162..6c1e136 100644 --- a/lib/checkie/parser.rb +++ b/lib/checkie/parser.rb @@ -2,7 +2,7 @@ class Checkie::Parser include Singleton - attr_reader :matches, :rules + attr_reader :matches, :rules, :matches_ai, :rules_ai def initialize reinitialize @@ -10,23 +10,43 @@ def initialize def reinitialize @rules = {} + @rules_ai = {} @matches = [] + @matches_ai = [] end - def add_matching_file(glob_pattern, matching_proc=nil, &block) + def add_matching_file(glob_pattern, matching_proc=nil, **opts, &block) matching_proc ||= block - @matches << { pattern: glob_pattern, - matching_proc: matching_proc - } + obj = { pattern: glob_pattern, + matching_proc: matching_proc + } + if opts[:ai] + obj[:rules] = opts[:rules].map do |r| + @rules_ai[r.to_s] + end + if opts[:exclude] + obj[:exclude] = opts[:exclude] + end + @matches_ai << obj + else + @matches << obj + end end - def add_file_rule(name, description, references: []) - raise "Duplicate rule #{name}" if @rules[name.to_s].present? - @rules[name.to_s] = { - name: name.to_s, - description: description, - references: references - } + def add_file_rule(name, description, references: [], ai: false) + raise "Duplicate rule #{name}" if @rules_ai[name.to_s].present? || @rules_ai[name.to_s].present? + if ai + @rules_ai[name.to_s] = { + name: name.to_s, + description: description + } + else + @rules[name.to_s] = { + name: name.to_s, + description: description, + references: references + } + end end end diff --git a/lib/checkie/poster.rb b/lib/checkie/poster.rb index 59ed564..04e848b 100644 --- a/lib/checkie/poster.rb +++ b/lib/checkie/poster.rb @@ -10,6 +10,31 @@ def initialize(details, dry_run: true) @dry_run = dry_run end + def post_ai_annotations!(rules) + if rules.length > 0 + annotations = annotations_ai(rules) + repo_name = @details[:base][:repo][:full_name] + sha = @details[:head][:sha] + + annotations.each do |a| + if @dry_run + pp a + else + client.create_pull_request_comment( + repo_name, + @details[:number], + a[:message], + sha, + a[:path], + a[:start_line] + ) + end + rescue Octokit::UnprocessableEntity + pp "Bad request sent! Probably a line number issue: #{a[:path]}:#{a[:start_line]}" + end + end + end + # Post file rules as annotations def post_annotations!(rules) if rules.length > 0 @@ -32,6 +57,26 @@ def post_annotations!(rules) private + def annotations_ai(rules) + arr = [] + rules.each do |r| + r = r["violations"] + next if r.empty? + r.each do |annotation| + # Because Claude can be stupid. + next if annotation["suggestion"].downcase.include?("no violation") + arr << { + path: annotation["file"], + start_line: annotation["line_number"], + end_line: annotation["line_number"], + title: annotation["rule"], + message: "#{annotation["issue"]}:\n#{annotation["suggestion"]}" + } + end + end + return arr + end + def annotations(rules) arr = [] diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index c2feb7f..5c54a10 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -1,16 +1,86 @@ +require "open3" +require 'pry' class Checkie::Runner # Action is the pull request action type def run(url, action) fetcher = Checkie::Fetcher.new(url) - poster = Checkie::Poster.new(fetcher.details, dry_run: false) + poster = Checkie::Poster.new(fetcher.details, dry_run: true) if action == "synchronize" || action == "opened" data = fetcher.fetch_files matcher = Checkie::Matcher.new(data) - rules = matcher.match + rules = matcher.match_ai + to_post = call_claude(rules) + poster.post_ai_annotations!(to_post) + # poster.post_annotations!(rules) + end + end + + def call_claude(rule_mapping) + rule_mapping.map do |mapping| + + # mappping == [rule strings joined by \n, arr of patch diffs] + prompt = create_prompt(mapping[0], mapping[1]) + + res = Open3.capture3("claude", "--dangerously-skip-permissions", "-p", prompt) + puts res[0] + prefix = res[0].index("```json") + postfix = res[0].index("```", prefix + 7) + unless prefix && postfix + puts "Bad response from Claude!!", res[0] + next + end + parsed = res[0][prefix+7...postfix] + JSON.parse(parsed) + end + end - poster.post_annotations!(rules) + def create_prompt(rules, diffs) + formatted = diffs.map do |d| + <<-PATCH + File: #{d[:name]} + #{d[:patch]} + PATCH end + <<-PROMPT + You are an AI code reviewer, going through PRs and identifying any changes that + don't follow the teams established best practices. + You are running in the directory: `AlignableWeb/.github/checkie/`. For any file searches ensure you are starting in + `../../`, or in the top-level `AlignableWeb` + Code style rules: + #{rules} + + Git PR diff to evaluate: + #{formatted} + + IMPORTANT: The diff above only shows files matching the rules being checked. Before flagging + any missing files (like spec files), you MUST use the Read tool or Glob tool to verify the file + does not exist in the repository. Do not assume a file is missing just because it's not in the + diff above - it may have been added elsewhere in the PR. + + Instructions: + 1. For each changed file, check if modified lines violate any rules + 1.1 Do not make your own assumptions about how to interpret the rules + 2. line_number MUST be a line that appears in the diff (lines starting with + or -) + 3. If a rule violation exists but no lines in the diff can be commented on, skip that violation + 4. You may read all related files for context + 5. IF THERE ARE NO VIOLATIONS DO NOT RETURN ANYTHING. + + Return valid JSON only: + { + "violations": [ + { + "file": "path/to/file.rb", + "rule": "rule name", + "line_number": 10, + "issue": "what's wrong", + "suggestion": "how to fix" + } + ] + } + + Return empty violations array if no issues found: {"violations": []} + PROMPT end end From e867d5912540d9e7dc538298bab13a6b7d27d5fe Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Mon, 9 Feb 2026 15:25:50 -0500 Subject: [PATCH 02/27] gemfile --- lib/checkie/fetcher.rb | 1 + lib/checkie/matcher.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/checkie/fetcher.rb b/lib/checkie/fetcher.rb index 842c219..445a20f 100644 --- a/lib/checkie/fetcher.rb +++ b/lib/checkie/fetcher.rb @@ -18,6 +18,7 @@ def fetch_files repo_id = @details[:base][:repo][:id] pr_number = @details[:number] + pp "fetching" client.pull_request_files(repo_id, pr_number) end diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index 6af0541..3bef635 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -26,6 +26,7 @@ def match_ai ruleset_to_files = {} @pr.each do |file| rules = gather_rules_ai(file[:filename]) + pp "Gathered rules: #{rules}" next if rules.empty? changeset = {name: file[:filename], patch: file[:patch]} if ruleset_to_files.key?(rules) From c17677c9ada2e9f1f252acded21c2c7d8e061be5 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Mon, 9 Feb 2026 15:30:05 -0500 Subject: [PATCH 03/27] gemfile --- lib/checkie/matcher.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index 3bef635..cf8b21c 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -24,6 +24,7 @@ def check_rule(rule) def match_ai ruleset_to_files = {} + pp parser.matches_ai @pr.each do |file| rules = gather_rules_ai(file[:filename]) pp "Gathered rules: #{rules}" From 07f4627e2d3074df93469842ec785b6d8e445e3e Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Mon, 9 Feb 2026 16:20:27 -0500 Subject: [PATCH 04/27] cleanup --- lib/checkie/matcher.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index cf8b21c..6af0541 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -24,10 +24,8 @@ def check_rule(rule) def match_ai ruleset_to_files = {} - pp parser.matches_ai @pr.each do |file| rules = gather_rules_ai(file[:filename]) - pp "Gathered rules: #{rules}" next if rules.empty? changeset = {name: file[:filename], patch: file[:patch]} if ruleset_to_files.key?(rules) From 7bcdf1eff9df5896f7531ee8f3321a785d188f4d Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 10 Feb 2026 09:13:08 -0500 Subject: [PATCH 05/27] enable non dry run --- lib/checkie/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 5c54a10..f53daf9 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -4,7 +4,7 @@ class Checkie::Runner # Action is the pull request action type def run(url, action) fetcher = Checkie::Fetcher.new(url) - poster = Checkie::Poster.new(fetcher.details, dry_run: true) + poster = Checkie::Poster.new(fetcher.details, dry_run: false) if action == "synchronize" || action == "opened" data = fetcher.fetch_files From ccf2aac70525ff696666516b5a610d185d59bb96 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 10 Feb 2026 09:48:26 -0500 Subject: [PATCH 06/27] run in top level dir --- lib/checkie/runner.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index f53daf9..7481d38 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -4,7 +4,7 @@ class Checkie::Runner # Action is the pull request action type def run(url, action) fetcher = Checkie::Fetcher.new(url) - poster = Checkie::Poster.new(fetcher.details, dry_run: false) + poster = Checkie::Poster.new(fetcher.details, dry_run: true) if action == "synchronize" || action == "opened" data = fetcher.fetch_files @@ -23,7 +23,8 @@ def call_claude(rule_mapping) # mappping == [rule strings joined by \n, arr of patch diffs] prompt = create_prompt(mapping[0], mapping[1]) - res = Open3.capture3("claude", "--dangerously-skip-permissions", "-p", prompt) + repo_dir = File.expand_path("../../", __dir__) + res = Open3.capture3("claude", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) puts res[0] prefix = res[0].index("```json") postfix = res[0].index("```", prefix + 7) @@ -46,8 +47,7 @@ def create_prompt(rules, diffs) <<-PROMPT You are an AI code reviewer, going through PRs and identifying any changes that don't follow the teams established best practices. - You are running in the directory: `AlignableWeb/.github/checkie/`. For any file searches ensure you are starting in - `../../`, or in the top-level `AlignableWeb` + Code style rules: #{rules} From 8092b0494ed90392cae6f1bb36460a49daeb757a Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 10 Feb 2026 09:48:50 -0500 Subject: [PATCH 07/27] enable non dry run --- lib/checkie/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 7481d38..7cf56a8 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -4,7 +4,7 @@ class Checkie::Runner # Action is the pull request action type def run(url, action) fetcher = Checkie::Fetcher.new(url) - poster = Checkie::Poster.new(fetcher.details, dry_run: true) + poster = Checkie::Poster.new(fetcher.details, dry_run: false) if action == "synchronize" || action == "opened" data = fetcher.fetch_files From f09276abd5b3786e9d1e1b8d5677e940d7cd5056 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 10 Feb 2026 10:06:47 -0500 Subject: [PATCH 08/27] try --- lib/checkie/runner.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 7cf56a8..7f21837 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -24,6 +24,7 @@ def call_claude(rule_mapping) prompt = create_prompt(mapping[0], mapping[1]) repo_dir = File.expand_path("../../", __dir__) + pp Open3.capture3("pwd", chdir: repo_dir) res = Open3.capture3("claude", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) puts res[0] prefix = res[0].index("```json") @@ -47,6 +48,7 @@ def create_prompt(rules, diffs) <<-PROMPT You are an AI code reviewer, going through PRs and identifying any changes that don't follow the teams established best practices. + Ensure that any glob or grep tool uses have the `path` parameter set to the top-level of the repo, `AlignableWeb/` Code style rules: #{rules} From 86de31443bf262fdd0f5c381ca61ea332e9cc94f Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 10 Feb 2026 10:22:44 -0500 Subject: [PATCH 09/27] Dir.pwd --- lib/checkie/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 7f21837..ee20565 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -23,7 +23,7 @@ def call_claude(rule_mapping) # mappping == [rule strings joined by \n, arr of patch diffs] prompt = create_prompt(mapping[0], mapping[1]) - repo_dir = File.expand_path("../../", __dir__) + repo_dir = File.expand_path("../../", Dir.pwd) pp Open3.capture3("pwd", chdir: repo_dir) res = Open3.capture3("claude", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) puts res[0] From cdd017ec7754f66a0f41daeb75f41fc4c2b97992 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 11 Feb 2026 09:13:58 -0500 Subject: [PATCH 10/27] cleanup --- checkie.gemspec | 1 - lib/checkie/runner.rb | 3 --- 2 files changed, 4 deletions(-) diff --git a/checkie.gemspec b/checkie.gemspec index cd8b649..4742fad 100644 --- a/checkie.gemspec +++ b/checkie.gemspec @@ -14,7 +14,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency "octokit" s.add_runtime_dependency "git_diff_parser" s.add_runtime_dependency "activesupport" - s.add_runtime_dependency "pry" # to write specs for rules.rb s.add_runtime_dependency "rspec" diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index ee20565..f59b11e 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -19,12 +19,10 @@ def run(url, action) def call_claude(rule_mapping) rule_mapping.map do |mapping| - # mappping == [rule strings joined by \n, arr of patch diffs] prompt = create_prompt(mapping[0], mapping[1]) repo_dir = File.expand_path("../../", Dir.pwd) - pp Open3.capture3("pwd", chdir: repo_dir) res = Open3.capture3("claude", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) puts res[0] prefix = res[0].index("```json") @@ -48,7 +46,6 @@ def create_prompt(rules, diffs) <<-PROMPT You are an AI code reviewer, going through PRs and identifying any changes that don't follow the teams established best practices. - Ensure that any glob or grep tool uses have the `path` parameter set to the top-level of the repo, `AlignableWeb/` Code style rules: #{rules} From 15d73c0967984599074998bc3228e827317a218e Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 11 Feb 2026 09:15:31 -0500 Subject: [PATCH 11/27] cleanup --- lib/checkie/fetcher.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/checkie/fetcher.rb b/lib/checkie/fetcher.rb index 445a20f..842c219 100644 --- a/lib/checkie/fetcher.rb +++ b/lib/checkie/fetcher.rb @@ -18,7 +18,6 @@ def fetch_files repo_id = @details[:base][:repo][:id] pr_number = @details[:number] - pp "fetching" client.pull_request_files(repo_id, pr_number) end From 06575d506217695153a9ae12427993eefaf84d17 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 11 Feb 2026 10:56:01 -0500 Subject: [PATCH 12/27] down model; --- lib/checkie/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index f59b11e..fa0dc8e 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -23,7 +23,7 @@ def call_claude(rule_mapping) prompt = create_prompt(mapping[0], mapping[1]) repo_dir = File.expand_path("../../", Dir.pwd) - res = Open3.capture3("claude", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) + res = Open3.capture3("claude", "--model", "haiku", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) puts res[0] prefix = res[0].index("```json") postfix = res[0].index("```", prefix + 7) From c356cfa29e5174af319b1a1ad4afa27071ec5099 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 11 Feb 2026 11:15:35 -0500 Subject: [PATCH 13/27] rm pry --- lib/checkie/runner.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index fa0dc8e..75339e7 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -1,5 +1,4 @@ require "open3" -require 'pry' class Checkie::Runner # Action is the pull request action type def run(url, action) From 3fa6bb33541cebb974c106362ab723a69c75d34e Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 11 Feb 2026 11:56:34 -0500 Subject: [PATCH 14/27] prompt tweak --- lib/checkie/runner.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 75339e7..6f0cc9d 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -65,6 +65,7 @@ def create_prompt(rules, diffs) 4. You may read all related files for context 5. IF THERE ARE NO VIOLATIONS DO NOT RETURN ANYTHING. + DO NOT INCLUDE ANY OF YOUR REASONING IN THE OUTPUT. JUST JSON OUTPUT. Return valid JSON only: { "violations": [ From 997c6f74d9dcca5b291c9d76eee5a30a25ead5b7 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 11 Feb 2026 15:04:04 -0500 Subject: [PATCH 15/27] wip --- checkie.gemspec | 1 + lib/checkie.rb | 4 +- lib/checkie/matcher.rb | 57 +++++++++++++++------- lib/checkie/parser.rb | 5 +- lib/checkie/poster.rb | 14 +++--- lib/checkie/runner.rb | 106 ++++++++++++++++++++++++++++++++++++----- 6 files changed, 148 insertions(+), 39 deletions(-) diff --git a/checkie.gemspec b/checkie.gemspec index 4742fad..e7a91ff 100644 --- a/checkie.gemspec +++ b/checkie.gemspec @@ -14,6 +14,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency "octokit" s.add_runtime_dependency "git_diff_parser" s.add_runtime_dependency "activesupport" + s.add_runtime_dependency "anthropic" # to write specs for rules.rb s.add_runtime_dependency "rspec" diff --git a/lib/checkie.rb b/lib/checkie.rb index 124d5ae..731b410 100644 --- a/lib/checkie.rb +++ b/lib/checkie.rb @@ -28,8 +28,8 @@ def file_rule(name,description,references:[]) Checkie::Parser.instance.add_file_rule(name, description, references: references) end -def file_rule_ai(name, description) - Checkie::Parser.instance.add_file_rule(name, description, ai: true) +def file_rule_ai(name, description, exploration: false) + Checkie::Parser.instance.add_file_rule(name, description, exploration, ai: true) end def run(url, action) diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index 6af0541..1bf1dd4 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -23,34 +23,59 @@ def check_rule(rule) def match_ai - ruleset_to_files = {} + ruleset_to_files = {exploration: {}, standard: {}} @pr.each do |file| + # std_rules, exp_rules = gather_rules_ai(file[:filename]) rules = gather_rules_ai(file[:filename]) next if rules.empty? + + subset_key = if rules.include?("exploration") + rules.delete("exploration") + :exploration + else + :standard + end + changeset = {name: file[:filename], patch: file[:patch]} - if ruleset_to_files.key?(rules) - ruleset_to_files[rules].append(changeset) - else - ruleset_to_files[rules] = [changeset] - end - end - ruleset_to_files.map do |k,v| - [k.join("\n"), v] + ruleset_to_files[subset_key][rules] ||= [] + ruleset_to_files[subset_key][rules] << changeset + # unless exp_rules.empty? + # subset_key = :exploration + # ruleset_to_files[subset_key][exp_rules] ||= [] + # ruleset_to_files[subset_key][exp_rules] << changeset + # end + # unless std_rules.empty? + # subset_key = :standard + # ruleset_to_files[subset_key][std_rules] ||= [] + # ruleset_to_files[subset_key][std_rules] << changeset + # end end + + { + exploration: ruleset_to_files[:exploration].map { |k, v| [k.join("\n"), v] }, + standard: ruleset_to_files[:standard].map { |k, v| [k.join("\n"), v] } + } end def gather_rules_ai(path) applicable = Set[] + exploration = Set[] parser.matches_ai.each do |rule| - if File.fnmatch(rule[:pattern], path, File::FNM_EXTGLOB) - next if rule[:exclude] && File.fnmatch(rule[:exclude],path, File::Constants::FNM_EXTGLOB) - # rule[:text] = parser.rules_ai[rule[]] - rule[:rules].each do |r| - applicable.add(r[:description]) - end + next unless File.fnmatch(rule[:pattern], path, File::FNM_EXTGLOB) + next if rule[:exclude] && File.fnmatch(rule[:exclude], path, File::FNM_EXTGLOB) + + rule[:rules].each do |r| + # if r[:exploration] + # exploration.add(r[:description]) + # else + # applicable.add(r[:description]) + # end + applicable.add("exploration") if r[:exploration] + applicable.add(r[:description]) end end - return applicable + # [applicable, exploration] + applicable end def match diff --git a/lib/checkie/parser.rb b/lib/checkie/parser.rb index 6c1e136..a07884b 100644 --- a/lib/checkie/parser.rb +++ b/lib/checkie/parser.rb @@ -33,12 +33,13 @@ def add_matching_file(glob_pattern, matching_proc=nil, **opts, &block) end end - def add_file_rule(name, description, references: [], ai: false) + def add_file_rule(name, description, exploration=false, references: [], ai: false) raise "Duplicate rule #{name}" if @rules_ai[name.to_s].present? || @rules_ai[name.to_s].present? if ai @rules_ai[name.to_s] = { name: name.to_s, - description: description + description: description, + exploration: exploration } else @rules[name.to_s] = { diff --git a/lib/checkie/poster.rb b/lib/checkie/poster.rb index 04e848b..6f1bdc1 100644 --- a/lib/checkie/poster.rb +++ b/lib/checkie/poster.rb @@ -60,17 +60,17 @@ def post_annotations!(rules) def annotations_ai(rules) arr = [] rules.each do |r| - r = r["violations"] + r = r[:violations] next if r.empty? r.each do |annotation| # Because Claude can be stupid. - next if annotation["suggestion"].downcase.include?("no violation") + next if annotation[:suggestion].downcase.include?("no violation") arr << { - path: annotation["file"], - start_line: annotation["line_number"], - end_line: annotation["line_number"], - title: annotation["rule"], - message: "#{annotation["issue"]}:\n#{annotation["suggestion"]}" + path: annotation[:file], + start_line: annotation[:line_number], + end_line: annotation[:line_number], + title: annotation[:rule], + message: "#{annotation[:issue]}:\n#{annotation[:suggestion]}" } end end diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 6f0cc9d..2782b48 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -1,27 +1,33 @@ require "open3" +require "anthropic" class Checkie::Runner # Action is the pull request action type def run(url, action) fetcher = Checkie::Fetcher.new(url) - poster = Checkie::Poster.new(fetcher.details, dry_run: false) + poster = Checkie::Poster.new(fetcher.details, dry_run: true) if action == "synchronize" || action == "opened" data = fetcher.fetch_files matcher = Checkie::Matcher.new(data) rules = matcher.match_ai - to_post = call_claude(rules) - poster.post_ai_annotations!(to_post) + to_post = call_claude(rules[:exploration]) + call_claude_api(rules[:standard]) + pp to_post + # poster.post_ai_annotations!(to_post) # poster.post_annotations!(rules) end end def call_claude(rule_mapping) - rule_mapping.map do |mapping| + total_calls = 0 + pp "Beginning checks with Claude Code..." + # assumes that non-exploratory rules have been filtered out + res = rule_mapping.map do |mapping| # mappping == [rule strings joined by \n, arr of patch diffs] prompt = create_prompt(mapping[0], mapping[1]) repo_dir = File.expand_path("../../", Dir.pwd) + total_calls += 1 res = Open3.capture3("claude", "--model", "haiku", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) puts res[0] prefix = res[0].index("```json") @@ -33,6 +39,83 @@ def call_claude(rule_mapping) parsed = res[0][prefix+7...postfix] JSON.parse(parsed) end + pp "Total CC calls: #{total_calls}" + res + end + + def call_claude_api(rule_mapping) + pp "Beginning checks with API..." + # Assumes that exploratory rules have been filtered out + client = Anthropic::Client.new(api_key: ENV["ANTHROPIC_API_KEY"]) + + total_calls = 0 + res = rule_mapping.map do |mapping| + # mapping == [rule strings joined by \n, arr of patch diffs] + prompt = create_prompt(mapping[0], mapping[1]) + + begin + response = client.messages.create( + model: "claude-haiku-4-5-20251001", + max_tokens: 4096, + messages: [ + { + role: "user", + content: prompt + } + ], + tools: [ + { + name: "report_violations", + description: "Report code style violations found in the PR diff", + input_schema: structured_schema + } + ], + tool_choice: { + type: "tool", + name: "report_violations" + } + ) + total_calls += 1 + + # Extract the tool use result + tool_use = response.content&.find { |block| block.type == :tool_use } + pp tool_use + if tool_use && tool_use.input + tool_use.input + else + puts "No tool use found in response" + { "violations" => [] } + end + rescue => e + puts "Error calling Claude API: #{e.message}" + { "violations" => [] } + end + end + pp "Total API calls: #{total_calls}" + res + end + + def structured_schema + { + type: "object", + properties: { + violations: { + type: "array", + items: { + type: "object", + properties: { + file: { type: "string", description: "Path to the file" }, + rule: { type: "string", description: "Name of the rule violated" }, + line_number: { type: "integer", description: "Line number in the diff (must be a + or - line)" }, + issue: { type: "string", description: "Description of what's wrong" }, + suggestion: { type: "string", description: "How to fix the issue" } + }, + required: ["file", "rule", "line_number", "issue", "suggestion"] + } + } + }, + required: ["violations"] + } end def create_prompt(rules, diffs) @@ -58,14 +141,13 @@ def create_prompt(rules, diffs) diff above - it may have been added elsewhere in the PR. Instructions: - 1. For each changed file, check if modified lines violate any rules - 1.1 Do not make your own assumptions about how to interpret the rules - 2. line_number MUST be a line that appears in the diff (lines starting with + or -) - 3. If a rule violation exists but no lines in the diff can be commented on, skip that violation - 4. You may read all related files for context - 5. IF THERE ARE NO VIOLATIONS DO NOT RETURN ANYTHING. - - DO NOT INCLUDE ANY OF YOUR REASONING IN THE OUTPUT. JUST JSON OUTPUT. + 1. For each changed file, check if modified lines ONLY WITHIN THE DIFF violate any rules. + 1.1 DO NOT make your own assumptions about how to interpret the rules + 1.2 DO NOT make up your own rules, such as grammar violations + 1.3 DO NOT RESPOND WITH VIOLATIONS ON UNCHANGED CODE + 2. You may read all related files for context + + Ensure all parts of your response are succinct and to the point. DO NOT ramble. Return valid JSON only: { "violations": [ From f5c24d7306377cf6f1322a534f7b7c91ebab449e Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 17 Feb 2026 10:40:36 -0500 Subject: [PATCH 16/27] add_specs --- lib/checkie.rb | 4 +- lib/checkie/matcher.rb | 51 ++++++--------- lib/checkie/parser.rb | 10 +-- lib/checkie/runner.rb | 8 +-- spec/checkie/matcher_spec.rb | 117 ++++++++++++++++++++++++++++++++++- spec/checkie/parser_spec.rb | 7 +++ 6 files changed, 153 insertions(+), 44 deletions(-) diff --git a/lib/checkie.rb b/lib/checkie.rb index 731b410..b870e92 100644 --- a/lib/checkie.rb +++ b/lib/checkie.rb @@ -28,8 +28,8 @@ def file_rule(name,description,references:[]) Checkie::Parser.instance.add_file_rule(name, description, references: references) end -def file_rule_ai(name, description, exploration: false) - Checkie::Parser.instance.add_file_rule(name, description, exploration, ai: true) +def file_rule_ai(name, description, opts: {}) + Checkie::Parser.instance.add_file_rule(name, description, opts) end def run(url, action) diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index 1bf1dd4..cc94d07 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -25,30 +25,20 @@ def check_rule(rule) def match_ai ruleset_to_files = {exploration: {}, standard: {}} @pr.each do |file| - # std_rules, exp_rules = gather_rules_ai(file[:filename]) - rules = gather_rules_ai(file[:filename]) - next if rules.empty? - - subset_key = if rules.include?("exploration") - rules.delete("exploration") - :exploration - else - :standard - end + std_rules, exp_rules = gather_rules_ai(file[:filename]) + next if std_rules.empty? && exp_rules.empty? changeset = {name: file[:filename], patch: file[:patch]} - ruleset_to_files[subset_key][rules] ||= [] - ruleset_to_files[subset_key][rules] << changeset - # unless exp_rules.empty? - # subset_key = :exploration - # ruleset_to_files[subset_key][exp_rules] ||= [] - # ruleset_to_files[subset_key][exp_rules] << changeset - # end - # unless std_rules.empty? - # subset_key = :standard - # ruleset_to_files[subset_key][std_rules] ||= [] - # ruleset_to_files[subset_key][std_rules] << changeset - # end + unless exp_rules.empty? + subset_key = :exploration + ruleset_to_files[subset_key][exp_rules] ||= [] + ruleset_to_files[subset_key][exp_rules] << changeset + end + unless std_rules.empty? + subset_key = :standard + ruleset_to_files[subset_key][std_rules] ||= [] + ruleset_to_files[subset_key][std_rules] << changeset + end end { @@ -62,20 +52,17 @@ def gather_rules_ai(path) exploration = Set[] parser.matches_ai.each do |rule| next unless File.fnmatch(rule[:pattern], path, File::FNM_EXTGLOB) - next if rule[:exclude] && File.fnmatch(rule[:exclude], path, File::FNM_EXTGLOB) + next if rule[:exclude] && rule[:exclude].any? { |p| File.fnmatch(p, path, File::FNM_EXTGLOB) } rule[:rules].each do |r| - # if r[:exploration] - # exploration.add(r[:description]) - # else - # applicable.add(r[:description]) - # end - applicable.add("exploration") if r[:exploration] - applicable.add(r[:description]) + if r[:exploration] + exploration.add(r[:description]) + else + applicable.add(r[:description]) + end end end - # [applicable, exploration] - applicable + [applicable, exploration] end def match diff --git a/lib/checkie/parser.rb b/lib/checkie/parser.rb index a07884b..f646873 100644 --- a/lib/checkie/parser.rb +++ b/lib/checkie/parser.rb @@ -25,7 +25,7 @@ def add_matching_file(glob_pattern, matching_proc=nil, **opts, &block) @rules_ai[r.to_s] end if opts[:exclude] - obj[:exclude] = opts[:exclude] + obj[:exclude] = Array(opts[:exclude]) end @matches_ai << obj else @@ -33,19 +33,19 @@ def add_matching_file(glob_pattern, matching_proc=nil, **opts, &block) end end - def add_file_rule(name, description, exploration=false, references: [], ai: false) + def add_file_rule(name, description, opts={}) raise "Duplicate rule #{name}" if @rules_ai[name.to_s].present? || @rules_ai[name.to_s].present? - if ai + if opts[:ai] @rules_ai[name.to_s] = { name: name.to_s, description: description, - exploration: exploration + exploration: opts[:exploration], } else @rules[name.to_s] = { name: name.to_s, description: description, - references: references + references: opts[:references] } end end diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 2782b48..d098d36 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -11,10 +11,10 @@ def run(url, action) matcher = Checkie::Matcher.new(data) rules = matcher.match_ai - to_post = call_claude(rules[:exploration]) + call_claude_api(rules[:standard]) - pp to_post - # poster.post_ai_annotations!(to_post) - # poster.post_annotations!(rules) + ai_rules = call_claude(rules[:exploration]) + call_claude_api(rules[:standard]) + reg_rules = matcher.match + poster.post_ai_annotations!(ai_rules) + poster.post_annotations!(reg_rules) end end diff --git a/spec/checkie/matcher_spec.rb b/spec/checkie/matcher_spec.rb index 1b273b7..53b8af7 100644 --- a/spec/checkie/matcher_spec.rb +++ b/spec/checkie/matcher_spec.rb @@ -32,7 +32,7 @@ expect(matcher.match).to eq({"readme"=>[["README.md", {additions: 45, deletions: 0, :url=>"https://github.com/Alignable/checkie/blob/dd52731e1f894f53e5972e57255405961ca4ab38/README.md"}]] }) end - + it 'supports extglob' do instance.add_matching_file("{README,abc}*") do |changes,files| files.added do @@ -43,4 +43,119 @@ end end + describe "#match_ai" do + it "returns empty rulesets without any AI rules" do + expect(matcher.match_ai).to eq({exploration: [], standard: []}) + end + + it "returns standard rules when AI rules are added" do + instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_matching_file("README*", ai: true, rules: [:readme_check]) + + result = matcher.match_ai + expect(result[:standard].length).to eq(1) + expect(result[:standard][0][0]).to eq("You edited readme") + expect(result[:standard][0][1].length).to eq(1) + expect(result[:standard][0][1][0][:name]).to eq("README.md") + expect(result[:exploration]).to eq([]) + end + + it "returns exploration rules when marked as exploration" do + instance.add_file_rule(:readme_explore, "Consider readme impact", {ai: true, exploration: true}) + instance.add_matching_file("README*", ai: true, rules: [:readme_explore]) + + result = matcher.match_ai + expect(result[:exploration].length).to eq(1) + expect(result[:exploration][0][0]).to eq("Consider readme impact") + expect(result[:exploration][0][1].length).to eq(1) + expect(result[:exploration][0][1][0][:name]).to eq("README.md") + expect(result[:standard]).to eq([]) + end + + it "separates standard and exploration rules" do + instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_file_rule(:readme_explore, "Consider readme impact", {ai: true, exploration: true}) + instance.add_matching_file("README*", ai: true, rules: [:readme_check, :readme_explore]) + + result = matcher.match_ai + expect(result[:standard].length).to eq(1) + expect(result[:exploration].length).to eq(1) + expect(result[:standard][0][0]).to eq("You edited readme") + expect(result[:exploration][0][0]).to eq("Consider readme impact") + end + + it "supports exclude patterns" do + instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: "README*") + + result = matcher.match_ai + expect(result[:standard]).to eq([]) + expect(result[:exploration]).to eq([]) + end + + it "supports array of exclude patterns" do + instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: ["README*", "LICENSE*"]) + + result = matcher.match_ai + expect(result[:standard]).to eq([]) + expect(result[:exploration]).to eq([]) + end + end + + describe "#gather_rules_ai" do + it "returns empty sets for non-matching paths" do + instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_matching_file("README*", ai: true, rules: [:readme_check]) + + standard, exploration = matcher.gather_rules_ai("other_file.txt") + expect(standard.to_a).to eq([]) + expect(exploration.to_a).to eq([]) + end + + it "returns standard rules for matching paths" do + instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_matching_file("README*", ai: true, rules: [:readme_check]) + + standard, exploration = matcher.gather_rules_ai("README.md") + expect(standard.to_a).to eq(["You edited readme"]) + expect(exploration.to_a).to eq([]) + end + + it "returns exploration rules for matching paths" do + instance.add_file_rule(:readme_explore, "Consider readme impact", {ai: true, exploration: true}) + instance.add_matching_file("README*", ai: true, rules: [:readme_explore]) + + standard, exploration = matcher.gather_rules_ai("README.md") + expect(standard.to_a).to eq([]) + expect(exploration.to_a).to eq(["Consider readme impact"]) + end + + it "excludes paths matching exclude pattern" do + instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: ["README*"]) + + standard, exploration = matcher.gather_rules_ai("README.md") + expect(standard.to_a).to eq([]) + expect(exploration.to_a).to eq([]) + end + + it "excludes paths matching any pattern in exclude array" do + instance.add_file_rule(:doc_check, "You edited documentation", {ai: true}) + instance.add_matching_file("*", ai: true, rules: [:doc_check], exclude: ["README*", "LICENSE*"]) + + standard_readme, exploration_readme = matcher.gather_rules_ai("README.md") + expect(standard_readme.to_a).to eq([]) + expect(exploration_readme.to_a).to eq([]) + + standard_license, exploration_license = matcher.gather_rules_ai("LICENSE.txt") + expect(standard_license.to_a).to eq([]) + expect(exploration_license.to_a).to eq([]) + + standard_other, exploration_other = matcher.gather_rules_ai("other_file.rb") + expect(standard_other.to_a).to eq(["You edited documentation"]) + expect(exploration_other.to_a).to eq([]) + end + end + end diff --git a/spec/checkie/parser_spec.rb b/spec/checkie/parser_spec.rb index 2afc9f1..b33171e 100644 --- a/spec/checkie/parser_spec.rb +++ b/spec/checkie/parser_spec.rb @@ -16,6 +16,13 @@ expect(instance.rules["something"]).to be_present end + + it "adds an ai rule to the ai list" do + expect do + instance.add_file_rule(:something, "Because", ai: true) + end.to change { instance.rules_ai.length }.by(1) + expect(instance.rules_ai["something"]).to be_present + end end describe "add_matching_file" do From b9a760681fc5b3ffe30f8cf72c93c7eea90d5ae0 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 17 Feb 2026 11:26:41 -0500 Subject: [PATCH 17/27] cleanyup --- lib/checkie/matcher.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index cc94d07..8d01855 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -16,12 +16,6 @@ def check(rule,references = nil ) @rules[rule.to_s] += references if references end - def check_rule(rule) - @rules[rule.to_s] ||= [] - rule - end - - def match_ai ruleset_to_files = {exploration: {}, standard: {}} @pr.each do |file| From 3cfe527c5e12e70e1eb7ec3c482b1575343fe5f5 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 17 Feb 2026 11:27:34 -0500 Subject: [PATCH 18/27] cleanyup --- lib/checkie/poster.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/poster.rb b/lib/checkie/poster.rb index 6f1bdc1..01dbcaa 100644 --- a/lib/checkie/poster.rb +++ b/lib/checkie/poster.rb @@ -70,7 +70,7 @@ def annotations_ai(rules) start_line: annotation[:line_number], end_line: annotation[:line_number], title: annotation[:rule], - message: "#{annotation[:issue]}:\n#{annotation[:suggestion]}" + message: "#{annotation[:issue]}\n#{annotation[:suggestion]}" } end end From 780eadd31933100eea9445673b53edec4508a29e Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 17 Feb 2026 15:21:14 -0500 Subject: [PATCH 19/27] clean up --- lib/checkie.rb | 2 +- lib/checkie/parser.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/checkie.rb b/lib/checkie.rb index b870e92..7608753 100644 --- a/lib/checkie.rb +++ b/lib/checkie.rb @@ -28,7 +28,7 @@ def file_rule(name,description,references:[]) Checkie::Parser.instance.add_file_rule(name, description, references: references) end -def file_rule_ai(name, description, opts: {}) +def file_rule_ai(name, description, **opts) Checkie::Parser.instance.add_file_rule(name, description, opts) end diff --git a/lib/checkie/parser.rb b/lib/checkie/parser.rb index f646873..ad23c4e 100644 --- a/lib/checkie/parser.rb +++ b/lib/checkie/parser.rb @@ -33,7 +33,7 @@ def add_matching_file(glob_pattern, matching_proc=nil, **opts, &block) end end - def add_file_rule(name, description, opts={}) + def add_file_rule(name, description, **opts) raise "Duplicate rule #{name}" if @rules_ai[name.to_s].present? || @rules_ai[name.to_s].present? if opts[:ai] @rules_ai[name.to_s] = { From 331b27bdfbd571d0d8eec70ed4183fd6d3dbfb08 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 17 Feb 2026 15:29:47 -0500 Subject: [PATCH 20/27] fix splat --- lib/checkie.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie.rb b/lib/checkie.rb index 7608753..735e725 100644 --- a/lib/checkie.rb +++ b/lib/checkie.rb @@ -29,7 +29,7 @@ def file_rule(name,description,references:[]) end def file_rule_ai(name, description, **opts) - Checkie::Parser.instance.add_file_rule(name, description, opts) + Checkie::Parser.instance.add_file_rule(name, description, **opts) end def run(url, action) From fb694f960674d81d38ce15c9c06cc88cad6caa96 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 17 Feb 2026 16:31:15 -0500 Subject: [PATCH 21/27] compact rules --- lib/checkie/matcher.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index 8d01855..8ec28a2 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -48,7 +48,7 @@ def gather_rules_ai(path) next unless File.fnmatch(rule[:pattern], path, File::FNM_EXTGLOB) next if rule[:exclude] && rule[:exclude].any? { |p| File.fnmatch(p, path, File::FNM_EXTGLOB) } - rule[:rules].each do |r| + rule[:rules].compact.each do |r| if r[:exploration] exploration.add(r[:description]) else From 4eb3653d7470704897bdfc2a955db4312988bb7f Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Tue, 17 Feb 2026 17:03:37 -0500 Subject: [PATCH 22/27] cleanup --- lib/checkie.rb | 2 +- lib/checkie/runner.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/checkie.rb b/lib/checkie.rb index 735e725..743b436 100644 --- a/lib/checkie.rb +++ b/lib/checkie.rb @@ -29,7 +29,7 @@ def file_rule(name,description,references:[]) end def file_rule_ai(name, description, **opts) - Checkie::Parser.instance.add_file_rule(name, description, **opts) + Checkie::Parser.instance.add_file_rule(name, description, **opts.merge({ai: true})) end def run(url, action) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index d098d36..bd4f720 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -139,6 +139,7 @@ def create_prompt(rules, diffs) any missing files (like spec files), you MUST use the Read tool or Glob tool to verify the file does not exist in the repository. Do not assume a file is missing just because it's not in the diff above - it may have been added elsewhere in the PR. + One block / line of code may violate multiple rules. Include any and all violations. Instructions: 1. For each changed file, check if modified lines ONLY WITHIN THE DIFF violate any rules. From 935307d6b0c555f43ad41e1be66dee072e14acb9 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 18 Feb 2026 10:18:29 -0500 Subject: [PATCH 23/27] annotations not comments --- lib/checkie/poster.rb | 18 ++++++++---------- lib/checkie/runner.rb | 16 +++------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/lib/checkie/poster.rb b/lib/checkie/poster.rb index 01dbcaa..305c32c 100644 --- a/lib/checkie/poster.rb +++ b/lib/checkie/poster.rb @@ -13,21 +13,18 @@ def initialize(details, dry_run: true) def post_ai_annotations!(rules) if rules.length > 0 annotations = annotations_ai(rules) - repo_name = @details[:base][:repo][:full_name] sha = @details[:head][:sha] annotations.each do |a| if @dry_run pp a else - client.create_pull_request_comment( - repo_name, - @details[:number], - a[:message], - sha, - a[:path], - a[:start_line] - ) + repo_id = @details[:base][:repo][:id] + sha = @details[:head][:sha] + check_run_id = client.check_runs_for_ref(repo_id, sha, check_name: "checkie")&.check_runs&.first&.id + annotations.each_slice(GITHUB_ANNOTATION_BATCH) do |a| + client.update_check_run(repo_id, check_run_id, output: { annotations: a, title: "CheckieAI", summary: "#{annotations.length} annotations"}) + end end rescue Octokit::UnprocessableEntity pp "Bad request sent! Probably a line number issue: #{a[:path]}:#{a[:start_line]}" @@ -70,7 +67,8 @@ def annotations_ai(rules) start_line: annotation[:line_number], end_line: annotation[:line_number], title: annotation[:rule], - message: "#{annotation[:issue]}\n#{annotation[:suggestion]}" + message: "#{annotation[:issue]}\n#{annotation[:suggestion]}", + annotation_level: "warning" } end end diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index bd4f720..7def49c 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -13,21 +13,19 @@ def run(url, action) rules = matcher.match_ai ai_rules = call_claude(rules[:exploration]) + call_claude_api(rules[:standard]) reg_rules = matcher.match + pp reg_rules poster.post_ai_annotations!(ai_rules) poster.post_annotations!(reg_rules) end end def call_claude(rule_mapping) - total_calls = 0 - pp "Beginning checks with Claude Code..." # assumes that non-exploratory rules have been filtered out - res = rule_mapping.map do |mapping| + rule_mapping.map do |mapping| # mappping == [rule strings joined by \n, arr of patch diffs] prompt = create_prompt(mapping[0], mapping[1]) repo_dir = File.expand_path("../../", Dir.pwd) - total_calls += 1 res = Open3.capture3("claude", "--model", "haiku", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) puts res[0] prefix = res[0].index("```json") @@ -39,17 +37,13 @@ def call_claude(rule_mapping) parsed = res[0][prefix+7...postfix] JSON.parse(parsed) end - pp "Total CC calls: #{total_calls}" - res end def call_claude_api(rule_mapping) - pp "Beginning checks with API..." # Assumes that exploratory rules have been filtered out client = Anthropic::Client.new(api_key: ENV["ANTHROPIC_API_KEY"]) - total_calls = 0 - res = rule_mapping.map do |mapping| + rule_mapping.map do |mapping| # mapping == [rule strings joined by \n, arr of patch diffs] prompt = create_prompt(mapping[0], mapping[1]) @@ -75,11 +69,9 @@ def call_claude_api(rule_mapping) name: "report_violations" } ) - total_calls += 1 # Extract the tool use result tool_use = response.content&.find { |block| block.type == :tool_use } - pp tool_use if tool_use && tool_use.input tool_use.input else @@ -91,8 +83,6 @@ def call_claude_api(rule_mapping) { "violations" => [] } end end - pp "Total API calls: #{total_calls}" - res end def structured_schema From 36f2f732846dc9b60e24687f1aa9c431f35d2e33 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 18 Feb 2026 10:26:32 -0500 Subject: [PATCH 24/27] cleanup --- lib/checkie/poster.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/poster.rb b/lib/checkie/poster.rb index 305c32c..84ff21d 100644 --- a/lib/checkie/poster.rb +++ b/lib/checkie/poster.rb @@ -5,7 +5,7 @@ class Checkie::Poster LINE_WIDTH = 70 GITHUB_ANNOTATION_BATCH = 50 - def initialize(details, dry_run: true) + def initialize(details, dry_run: false) @details = details @dry_run = dry_run end From d37d7db3d1be22014ca759b7967849848ca26ad9 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 18 Feb 2026 10:31:13 -0500 Subject: [PATCH 25/27] not dry run --- lib/checkie/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 7def49c..52af76f 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -4,7 +4,7 @@ class Checkie::Runner # Action is the pull request action type def run(url, action) fetcher = Checkie::Fetcher.new(url) - poster = Checkie::Poster.new(fetcher.details, dry_run: true) + poster = Checkie::Poster.new(fetcher.details, dry_run: false) if action == "synchronize" || action == "opened" data = fetcher.fetch_files From e80f682af22e5a98888b8297e72e1ad50c19c1f0 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 18 Feb 2026 11:19:40 -0500 Subject: [PATCH 26/27] specfix --- spec/checkie/matcher_spec.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/spec/checkie/matcher_spec.rb b/spec/checkie/matcher_spec.rb index 53b8af7..bd83792 100644 --- a/spec/checkie/matcher_spec.rb +++ b/spec/checkie/matcher_spec.rb @@ -49,7 +49,7 @@ end it "returns standard rules when AI rules are added" do - instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_file_rule(:readme_check, "You edited readme", ai: true) instance.add_matching_file("README*", ai: true, rules: [:readme_check]) result = matcher.match_ai @@ -61,7 +61,7 @@ end it "returns exploration rules when marked as exploration" do - instance.add_file_rule(:readme_explore, "Consider readme impact", {ai: true, exploration: true}) + instance.add_file_rule(:readme_explore, "Consider readme impact", ai: true, exploration: true) instance.add_matching_file("README*", ai: true, rules: [:readme_explore]) result = matcher.match_ai @@ -73,8 +73,8 @@ end it "separates standard and exploration rules" do - instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) - instance.add_file_rule(:readme_explore, "Consider readme impact", {ai: true, exploration: true}) + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_file_rule(:readme_explore, "Consider readme impact", ai: true, exploration: true) instance.add_matching_file("README*", ai: true, rules: [:readme_check, :readme_explore]) result = matcher.match_ai @@ -85,7 +85,7 @@ end it "supports exclude patterns" do - instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_file_rule(:readme_check, "You edited readme", ai: true) instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: "README*") result = matcher.match_ai @@ -94,7 +94,7 @@ end it "supports array of exclude patterns" do - instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_file_rule(:readme_check, "You edited readme", ai: true) instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: ["README*", "LICENSE*"]) result = matcher.match_ai @@ -105,7 +105,7 @@ describe "#gather_rules_ai" do it "returns empty sets for non-matching paths" do - instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_file_rule(:readme_check, "You edited readme", ai: true) instance.add_matching_file("README*", ai: true, rules: [:readme_check]) standard, exploration = matcher.gather_rules_ai("other_file.txt") @@ -114,7 +114,7 @@ end it "returns standard rules for matching paths" do - instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_file_rule(:readme_check, "You edited readme", ai: true) instance.add_matching_file("README*", ai: true, rules: [:readme_check]) standard, exploration = matcher.gather_rules_ai("README.md") @@ -123,7 +123,7 @@ end it "returns exploration rules for matching paths" do - instance.add_file_rule(:readme_explore, "Consider readme impact", {ai: true, exploration: true}) + instance.add_file_rule(:readme_explore, "Consider readme impact", ai: true, exploration: true) instance.add_matching_file("README*", ai: true, rules: [:readme_explore]) standard, exploration = matcher.gather_rules_ai("README.md") @@ -132,7 +132,7 @@ end it "excludes paths matching exclude pattern" do - instance.add_file_rule(:readme_check, "You edited readme", {ai: true}) + instance.add_file_rule(:readme_check, "You edited readme", ai: true) instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: ["README*"]) standard, exploration = matcher.gather_rules_ai("README.md") @@ -141,7 +141,7 @@ end it "excludes paths matching any pattern in exclude array" do - instance.add_file_rule(:doc_check, "You edited documentation", {ai: true}) + instance.add_file_rule(:doc_check, "You edited documentation", ai: true) instance.add_matching_file("*", ai: true, rules: [:doc_check], exclude: ["README*", "LICENSE*"]) standard_readme, exploration_readme = matcher.gather_rules_ai("README.md") From 7d48bf1f6e06c3a2bb351e8bf056c6d5c1887155 Mon Sep 17 00:00:00 2001 From: Aiden Cahill Date: Wed, 18 Feb 2026 11:20:38 -0500 Subject: [PATCH 27/27] specfix --- lib/checkie/poster.rb | 2 +- lib/checkie/runner.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/checkie/poster.rb b/lib/checkie/poster.rb index 84ff21d..305c32c 100644 --- a/lib/checkie/poster.rb +++ b/lib/checkie/poster.rb @@ -5,7 +5,7 @@ class Checkie::Poster LINE_WIDTH = 70 GITHUB_ANNOTATION_BATCH = 50 - def initialize(details, dry_run: false) + def initialize(details, dry_run: true) @details = details @dry_run = dry_run end diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index 52af76f..d744b7e 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -13,7 +13,6 @@ def run(url, action) rules = matcher.match_ai ai_rules = call_claude(rules[:exploration]) + call_claude_api(rules[:standard]) reg_rules = matcher.match - pp reg_rules poster.post_ai_annotations!(ai_rules) poster.post_annotations!(reg_rules) end