diff --git a/checkie.gemspec b/checkie.gemspec index 97109dd..e7a91ff 100644 --- a/checkie.gemspec +++ b/checkie.gemspec @@ -1,8 +1,8 @@ Gem::Specification.new do |s| s.name = %q{checkie} - s.version = "0.0.1" + s.version = "0.0.3" s.authors = %q{Alignable} - s.date = %q{2018-05-23} + s.date = %q{2026-02-04} s.summary = %q{Hi, It's Checkie! Looks like you're trying to write some code} s.files = [ "lib/checkie.rb" @@ -11,9 +11,10 @@ Gem::Specification.new do |s| s.license = 'MIT' s.add_runtime_dependency 'dotenv' - s.add_runtime_dependency "octokit", "~>4.22.0" + s.add_runtime_dependency "octokit" s.add_runtime_dependency "git_diff_parser" s.add_runtime_dependency "activesupport" + s.add_runtime_dependency "anthropic" # to write specs for rules.rb s.add_runtime_dependency "rspec" diff --git a/lib/checkie.rb b/lib/checkie.rb index e34539a..743b436 100644 --- a/lib/checkie.rb +++ b/lib/checkie.rb @@ -19,11 +19,19 @@ def matching(file_pattern, &block) Checkie::Parser.instance.add_matching_file(file_pattern,&block) end +def matching_ai(file_pattern, rules:, exclude: nil, &block) + Checkie::Parser.instance.add_matching_file(file_pattern, ai: true, rules: rules, exclude: exclude, &block) +end + def file_rule(name,description,references:[]) Checkie::Parser.instance.add_file_rule(name, description, references: references) end +def file_rule_ai(name, description, **opts) + Checkie::Parser.instance.add_file_rule(name, description, **opts.merge({ai: true})) +end + def run(url, action) Checkie::Runner.new.run(url, action) end diff --git a/lib/checkie/matcher.rb b/lib/checkie/matcher.rb index a7a3609..8ec28a2 100644 --- a/lib/checkie/matcher.rb +++ b/lib/checkie/matcher.rb @@ -16,6 +16,49 @@ def check(rule,references = nil ) @rules[rule.to_s] += references if references end + def match_ai + ruleset_to_files = {exploration: {}, standard: {}} + @pr.each do |file| + std_rules, exp_rules = gather_rules_ai(file[:filename]) + next if std_rules.empty? && exp_rules.empty? + + changeset = {name: file[:filename], patch: file[:patch]} + unless exp_rules.empty? + subset_key = :exploration + ruleset_to_files[subset_key][exp_rules] ||= [] + ruleset_to_files[subset_key][exp_rules] << changeset + end + unless std_rules.empty? + subset_key = :standard + ruleset_to_files[subset_key][std_rules] ||= [] + ruleset_to_files[subset_key][std_rules] << changeset + end + end + + { + exploration: ruleset_to_files[:exploration].map { |k, v| [k.join("\n"), v] }, + standard: ruleset_to_files[:standard].map { |k, v| [k.join("\n"), v] } + } + end + + def gather_rules_ai(path) + applicable = Set[] + exploration = Set[] + parser.matches_ai.each do |rule| + next unless File.fnmatch(rule[:pattern], path, File::FNM_EXTGLOB) + next if rule[:exclude] && rule[:exclude].any? { |p| File.fnmatch(p, path, File::FNM_EXTGLOB) } + + rule[:rules].compact.each do |r| + if r[:exploration] + exploration.add(r[:description]) + else + applicable.add(r[:description]) + end + end + end + [applicable, exploration] + end + def match all_files = Checkie::FileMatchSet.new(self) diff --git a/lib/checkie/parser.rb b/lib/checkie/parser.rb index 5d64162..ad23c4e 100644 --- a/lib/checkie/parser.rb +++ b/lib/checkie/parser.rb @@ -2,7 +2,7 @@ class Checkie::Parser include Singleton - attr_reader :matches, :rules + attr_reader :matches, :rules, :matches_ai, :rules_ai def initialize reinitialize @@ -10,23 +10,44 @@ def initialize def reinitialize @rules = {} + @rules_ai = {} @matches = [] + @matches_ai = [] end - def add_matching_file(glob_pattern, matching_proc=nil, &block) + def add_matching_file(glob_pattern, matching_proc=nil, **opts, &block) matching_proc ||= block - @matches << { pattern: glob_pattern, - matching_proc: matching_proc - } + obj = { pattern: glob_pattern, + matching_proc: matching_proc + } + if opts[:ai] + obj[:rules] = opts[:rules].map do |r| + @rules_ai[r.to_s] + end + if opts[:exclude] + obj[:exclude] = Array(opts[:exclude]) + end + @matches_ai << obj + else + @matches << obj + end end - def add_file_rule(name, description, references: []) - raise "Duplicate rule #{name}" if @rules[name.to_s].present? - @rules[name.to_s] = { - name: name.to_s, - description: description, - references: references - } + def add_file_rule(name, description, **opts) + raise "Duplicate rule #{name}" if @rules_ai[name.to_s].present? || @rules_ai[name.to_s].present? + if opts[:ai] + @rules_ai[name.to_s] = { + name: name.to_s, + description: description, + exploration: opts[:exploration], + } + else + @rules[name.to_s] = { + name: name.to_s, + description: description, + references: opts[:references] + } + end end end diff --git a/lib/checkie/poster.rb b/lib/checkie/poster.rb index 59ed564..305c32c 100644 --- a/lib/checkie/poster.rb +++ b/lib/checkie/poster.rb @@ -10,6 +10,28 @@ def initialize(details, dry_run: true) @dry_run = dry_run end + def post_ai_annotations!(rules) + if rules.length > 0 + annotations = annotations_ai(rules) + sha = @details[:head][:sha] + + annotations.each do |a| + if @dry_run + pp a + else + repo_id = @details[:base][:repo][:id] + sha = @details[:head][:sha] + check_run_id = client.check_runs_for_ref(repo_id, sha, check_name: "checkie")&.check_runs&.first&.id + annotations.each_slice(GITHUB_ANNOTATION_BATCH) do |a| + client.update_check_run(repo_id, check_run_id, output: { annotations: a, title: "CheckieAI", summary: "#{annotations.length} annotations"}) + end + end + rescue Octokit::UnprocessableEntity + pp "Bad request sent! Probably a line number issue: #{a[:path]}:#{a[:start_line]}" + end + end + end + # Post file rules as annotations def post_annotations!(rules) if rules.length > 0 @@ -32,6 +54,27 @@ def post_annotations!(rules) private + def annotations_ai(rules) + arr = [] + rules.each do |r| + r = r[:violations] + next if r.empty? + r.each do |annotation| + # Because Claude can be stupid. + next if annotation[:suggestion].downcase.include?("no violation") + arr << { + path: annotation[:file], + start_line: annotation[:line_number], + end_line: annotation[:line_number], + title: annotation[:rule], + message: "#{annotation[:issue]}\n#{annotation[:suggestion]}", + annotation_level: "warning" + } + end + end + return arr + end + def annotations(rules) arr = [] diff --git a/lib/checkie/runner.rb b/lib/checkie/runner.rb index c2feb7f..d744b7e 100644 --- a/lib/checkie/runner.rb +++ b/lib/checkie/runner.rb @@ -1,3 +1,5 @@ +require "open3" +require "anthropic" class Checkie::Runner # Action is the pull request action type def run(url, action) @@ -8,9 +10,148 @@ def run(url, action) data = fetcher.fetch_files matcher = Checkie::Matcher.new(data) - rules = matcher.match + rules = matcher.match_ai + ai_rules = call_claude(rules[:exploration]) + call_claude_api(rules[:standard]) + reg_rules = matcher.match + poster.post_ai_annotations!(ai_rules) + poster.post_annotations!(reg_rules) + end + end + + def call_claude(rule_mapping) + # assumes that non-exploratory rules have been filtered out + rule_mapping.map do |mapping| + # mappping == [rule strings joined by \n, arr of patch diffs] + prompt = create_prompt(mapping[0], mapping[1]) + + repo_dir = File.expand_path("../../", Dir.pwd) + res = Open3.capture3("claude", "--model", "haiku", "--dangerously-skip-permissions", "-p", prompt, chdir: repo_dir) + puts res[0] + prefix = res[0].index("```json") + postfix = res[0].index("```", prefix + 7) + unless prefix && postfix + puts "Bad response from Claude!!", res[0] + next + end + parsed = res[0][prefix+7...postfix] + JSON.parse(parsed) + end + end + + def call_claude_api(rule_mapping) + # Assumes that exploratory rules have been filtered out + client = Anthropic::Client.new(api_key: ENV["ANTHROPIC_API_KEY"]) + + rule_mapping.map do |mapping| + # mapping == [rule strings joined by \n, arr of patch diffs] + prompt = create_prompt(mapping[0], mapping[1]) + + begin + response = client.messages.create( + model: "claude-haiku-4-5-20251001", + max_tokens: 4096, + messages: [ + { + role: "user", + content: prompt + } + ], + tools: [ + { + name: "report_violations", + description: "Report code style violations found in the PR diff", + input_schema: structured_schema + } + ], + tool_choice: { + type: "tool", + name: "report_violations" + } + ) + + # Extract the tool use result + tool_use = response.content&.find { |block| block.type == :tool_use } + if tool_use && tool_use.input + tool_use.input + else + puts "No tool use found in response" + { "violations" => [] } + end + rescue => e + puts "Error calling Claude API: #{e.message}" + { "violations" => [] } + end + end + end - poster.post_annotations!(rules) + def structured_schema + { + type: "object", + properties: { + violations: { + type: "array", + items: { + type: "object", + properties: { + file: { type: "string", description: "Path to the file" }, + rule: { type: "string", description: "Name of the rule violated" }, + line_number: { type: "integer", description: "Line number in the diff (must be a + or - line)" }, + issue: { type: "string", description: "Description of what's wrong" }, + suggestion: { type: "string", description: "How to fix the issue" } + }, + required: ["file", "rule", "line_number", "issue", "suggestion"] + } + } + }, + required: ["violations"] + } + end + + def create_prompt(rules, diffs) + formatted = diffs.map do |d| + <<-PATCH + File: #{d[:name]} + #{d[:patch]} + PATCH end + <<-PROMPT + You are an AI code reviewer, going through PRs and identifying any changes that + don't follow the teams established best practices. + + Code style rules: + #{rules} + + Git PR diff to evaluate: + #{formatted} + + IMPORTANT: The diff above only shows files matching the rules being checked. Before flagging + any missing files (like spec files), you MUST use the Read tool or Glob tool to verify the file + does not exist in the repository. Do not assume a file is missing just because it's not in the + diff above - it may have been added elsewhere in the PR. + One block / line of code may violate multiple rules. Include any and all violations. + + Instructions: + 1. For each changed file, check if modified lines ONLY WITHIN THE DIFF violate any rules. + 1.1 DO NOT make your own assumptions about how to interpret the rules + 1.2 DO NOT make up your own rules, such as grammar violations + 1.3 DO NOT RESPOND WITH VIOLATIONS ON UNCHANGED CODE + 2. You may read all related files for context + + Ensure all parts of your response are succinct and to the point. DO NOT ramble. + Return valid JSON only: + { + "violations": [ + { + "file": "path/to/file.rb", + "rule": "rule name", + "line_number": 10, + "issue": "what's wrong", + "suggestion": "how to fix" + } + ] + } + + Return empty violations array if no issues found: {"violations": []} + PROMPT end end diff --git a/spec/checkie/matcher_spec.rb b/spec/checkie/matcher_spec.rb index 1b273b7..bd83792 100644 --- a/spec/checkie/matcher_spec.rb +++ b/spec/checkie/matcher_spec.rb @@ -32,7 +32,7 @@ expect(matcher.match).to eq({"readme"=>[["README.md", {additions: 45, deletions: 0, :url=>"https://github.com/Alignable/checkie/blob/dd52731e1f894f53e5972e57255405961ca4ab38/README.md"}]] }) end - + it 'supports extglob' do instance.add_matching_file("{README,abc}*") do |changes,files| files.added do @@ -43,4 +43,119 @@ end end + describe "#match_ai" do + it "returns empty rulesets without any AI rules" do + expect(matcher.match_ai).to eq({exploration: [], standard: []}) + end + + it "returns standard rules when AI rules are added" do + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_matching_file("README*", ai: true, rules: [:readme_check]) + + result = matcher.match_ai + expect(result[:standard].length).to eq(1) + expect(result[:standard][0][0]).to eq("You edited readme") + expect(result[:standard][0][1].length).to eq(1) + expect(result[:standard][0][1][0][:name]).to eq("README.md") + expect(result[:exploration]).to eq([]) + end + + it "returns exploration rules when marked as exploration" do + instance.add_file_rule(:readme_explore, "Consider readme impact", ai: true, exploration: true) + instance.add_matching_file("README*", ai: true, rules: [:readme_explore]) + + result = matcher.match_ai + expect(result[:exploration].length).to eq(1) + expect(result[:exploration][0][0]).to eq("Consider readme impact") + expect(result[:exploration][0][1].length).to eq(1) + expect(result[:exploration][0][1][0][:name]).to eq("README.md") + expect(result[:standard]).to eq([]) + end + + it "separates standard and exploration rules" do + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_file_rule(:readme_explore, "Consider readme impact", ai: true, exploration: true) + instance.add_matching_file("README*", ai: true, rules: [:readme_check, :readme_explore]) + + result = matcher.match_ai + expect(result[:standard].length).to eq(1) + expect(result[:exploration].length).to eq(1) + expect(result[:standard][0][0]).to eq("You edited readme") + expect(result[:exploration][0][0]).to eq("Consider readme impact") + end + + it "supports exclude patterns" do + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: "README*") + + result = matcher.match_ai + expect(result[:standard]).to eq([]) + expect(result[:exploration]).to eq([]) + end + + it "supports array of exclude patterns" do + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: ["README*", "LICENSE*"]) + + result = matcher.match_ai + expect(result[:standard]).to eq([]) + expect(result[:exploration]).to eq([]) + end + end + + describe "#gather_rules_ai" do + it "returns empty sets for non-matching paths" do + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_matching_file("README*", ai: true, rules: [:readme_check]) + + standard, exploration = matcher.gather_rules_ai("other_file.txt") + expect(standard.to_a).to eq([]) + expect(exploration.to_a).to eq([]) + end + + it "returns standard rules for matching paths" do + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_matching_file("README*", ai: true, rules: [:readme_check]) + + standard, exploration = matcher.gather_rules_ai("README.md") + expect(standard.to_a).to eq(["You edited readme"]) + expect(exploration.to_a).to eq([]) + end + + it "returns exploration rules for matching paths" do + instance.add_file_rule(:readme_explore, "Consider readme impact", ai: true, exploration: true) + instance.add_matching_file("README*", ai: true, rules: [:readme_explore]) + + standard, exploration = matcher.gather_rules_ai("README.md") + expect(standard.to_a).to eq([]) + expect(exploration.to_a).to eq(["Consider readme impact"]) + end + + it "excludes paths matching exclude pattern" do + instance.add_file_rule(:readme_check, "You edited readme", ai: true) + instance.add_matching_file("*", ai: true, rules: [:readme_check], exclude: ["README*"]) + + standard, exploration = matcher.gather_rules_ai("README.md") + expect(standard.to_a).to eq([]) + expect(exploration.to_a).to eq([]) + end + + it "excludes paths matching any pattern in exclude array" do + instance.add_file_rule(:doc_check, "You edited documentation", ai: true) + instance.add_matching_file("*", ai: true, rules: [:doc_check], exclude: ["README*", "LICENSE*"]) + + standard_readme, exploration_readme = matcher.gather_rules_ai("README.md") + expect(standard_readme.to_a).to eq([]) + expect(exploration_readme.to_a).to eq([]) + + standard_license, exploration_license = matcher.gather_rules_ai("LICENSE.txt") + expect(standard_license.to_a).to eq([]) + expect(exploration_license.to_a).to eq([]) + + standard_other, exploration_other = matcher.gather_rules_ai("other_file.rb") + expect(standard_other.to_a).to eq(["You edited documentation"]) + expect(exploration_other.to_a).to eq([]) + end + end + end diff --git a/spec/checkie/parser_spec.rb b/spec/checkie/parser_spec.rb index 2afc9f1..b33171e 100644 --- a/spec/checkie/parser_spec.rb +++ b/spec/checkie/parser_spec.rb @@ -16,6 +16,13 @@ expect(instance.rules["something"]).to be_present end + + it "adds an ai rule to the ai list" do + expect do + instance.add_file_rule(:something, "Because", ai: true) + end.to change { instance.rules_ai.length }.by(1) + expect(instance.rules_ai["something"]).to be_present + end end describe "add_matching_file" do