|
| 1 | +name: Autograding Tests |
| 2 | +on: |
| 3 | + push: |
| 4 | + branches: |
| 5 | + - main |
| 6 | + pull_request: |
| 7 | + branches: |
| 8 | + - main |
| 9 | + repository_dispatch: |
| 10 | + |
| 11 | +concurrency: |
| 12 | + group: autograding-${{ github.ref }} |
| 13 | + cancel-in-progress: true |
| 14 | + |
| 15 | +permissions: |
| 16 | + checks: write |
| 17 | + actions: read |
| 18 | + contents: read |
| 19 | + pull-requests: write |
| 20 | + |
| 21 | +jobs: |
| 22 | + run-autograding-tests: |
| 23 | + name: AI-Powered Feedback and Autograding |
| 24 | + runs-on: ubuntu-latest |
| 25 | + env: |
| 26 | + OPENROUTER_MODEL: ${{ vars.OPENROUTER_MODEL }} |
| 27 | + SYSTEM_PROMPT: ${{ vars.SYSTEM_PROMPT }} |
| 28 | + steps: |
| 29 | + - name: Checkout repository |
| 30 | + uses: actions/checkout@v5 |
| 31 | + with: |
| 32 | + fetch-depth: 0 |
| 33 | + |
| 34 | + - name: Read assignment instructions |
| 35 | + id: instructions |
| 36 | + if: ${{ vars.ENABLE_AI_FEEDBACK == 'true' }} |
| 37 | + run: | |
| 38 | + # Reads the content of the README.md file into an output variable. |
| 39 | + # The `EOF` marker is used to handle multi-line file content. |
| 40 | + echo "instructions=$(cat README.md | sed 's/\"/\\\"/g' | sed 's/$/\\n/' | tr -d '\n' | sed 's/\\n/\\\\n/g')" >> $GITHUB_OUTPUT |
| 41 | +
|
| 42 | + - name: Read source code |
| 43 | + id: source_code |
| 44 | + if: ${{ vars.ENABLE_AI_FEEDBACK == 'true' }} |
| 45 | + run: | |
| 46 | + { |
| 47 | + echo 'source_code<<EOF' |
| 48 | + find src/main/java -type f -name "*.java" | while read -r file; do |
| 49 | + echo "=== File: $file ===" |
| 50 | + cat "$file" |
| 51 | + echo |
| 52 | + done |
| 53 | + echo 'EOF' |
| 54 | + } >> "$GITHUB_OUTPUT" |
| 55 | +
|
| 56 | + - name: Read test code |
| 57 | + id: test_code |
| 58 | + if: ${{ vars.ENABLE_AI_FEEDBACK == 'true' }} |
| 59 | + run: | |
| 60 | + { |
| 61 | + echo 'test_code<<EOF' |
| 62 | + if [ -d "src/test/java" ]; then |
| 63 | + find src/test/java -type f -name "*.java" | while read -r file; do |
| 64 | + echo "=== File: $file ===" |
| 65 | + cat "$file" |
| 66 | + echo |
| 67 | + done |
| 68 | + else |
| 69 | + echo "No test code found." |
| 70 | + fi |
| 71 | + echo 'EOF' |
| 72 | + } >> "$GITHUB_OUTPUT" |
| 73 | + - name: Generate AI Feedback |
| 74 | + id: ai_feedback |
| 75 | + if: ${{ vars.ENABLE_AI_FEEDBACK == 'true' }} |
| 76 | + run: | |
| 77 | + # This step sends the collected data to the OpenRouter API. |
| 78 | + INSTRUCTIONS=$(jq -Rs . <<'EOF' |
| 79 | + ${{ steps.instructions.outputs.instructions }} |
| 80 | + EOF |
| 81 | + ) |
| 82 | + SOURCE_CODE=$(jq -Rs . <<'EOF' |
| 83 | + ${{ steps.source_code.outputs.source_code }} |
| 84 | + EOF |
| 85 | + ) |
| 86 | + TEST_CODE=$(jq -Rs . <<'EOF' |
| 87 | + ${{ steps.test_code.outputs.test_code }} |
| 88 | + EOF |
| 89 | + ) |
| 90 | +
|
| 91 | + if [ -z "$INSTRUCTIONS" ] || [ -z "$SOURCE_CODE" ] || [ -z "$TEST_CODE" ]; then |
| 92 | + echo "Error: One or more required variables are not set." |
| 93 | + exit 1 |
| 94 | + fi |
| 95 | +
|
| 96 | + # Assigning to USER_CONTENT with variable expansion |
| 97 | + PAYLOAD="Please provide feedback on the following Java assignment. |
| 98 | +
|
| 99 | + --- Assignment Instructions --- |
| 100 | + ${INSTRUCTIONS} |
| 101 | +
|
| 102 | + --- Source files --- |
| 103 | + ${SOURCE_CODE} |
| 104 | +
|
| 105 | + --- Test files --- |
| 106 | + ${TEST_CODE}" |
| 107 | +
|
| 108 | + JSON_CONTENT=$(jq -n \ |
| 109 | + --argjson model "$OPENROUTER_MODEL" \ |
| 110 | + --arg system_prompt "$SYSTEM_PROMPT" \ |
| 111 | + --arg payload "$PAYLOAD" \ |
| 112 | + '{ |
| 113 | + models: $model, |
| 114 | + messages: [ |
| 115 | + {role: "system", content: $system_prompt}, |
| 116 | + {role: "user", content: $payload} |
| 117 | + ] |
| 118 | + }') |
| 119 | +
|
| 120 | + echo "$JSON_CONTENT" |
| 121 | +
|
| 122 | + API_RESPONSE=$(echo "$JSON_CONTENT" | curl https://openrouter.ai/api/v1/chat/completions \ |
| 123 | + -H "Authorization: Bearer ${{ secrets.OPENROUTER_API_KEY }}" \ |
| 124 | + -H "Content-Type: application/json" \ |
| 125 | + -d @-) |
| 126 | +
|
| 127 | + echo "$API_RESPONSE" |
| 128 | +
|
| 129 | + FEEDBACK_CONTENT=$(echo "$API_RESPONSE" | jq -r '.choices[0].message.content') |
| 130 | + echo "feedback<<EOF" >> $GITHUB_OUTPUT |
| 131 | + echo "$FEEDBACK_CONTENT" >> $GITHUB_OUTPUT |
| 132 | + echo "EOF" >> $GITHUB_OUTPUT |
| 133 | + - name: Post Feedback as PR Comment ✍️ |
| 134 | + if: ${{ vars.ENABLE_AI_FEEDBACK == 'true' && github.event_name == 'pull_request' }} |
| 135 | + uses: actions/github-script@v8 |
| 136 | + env: |
| 137 | + FEEDBACK_BODY: ${{ steps.ai_feedback.outputs.feedback }} |
| 138 | + with: |
| 139 | + github-token: ${{ secrets.GITHUB_TOKEN }} |
| 140 | + script: | |
| 141 | + const prNumber = context.payload.pull_request.number; |
| 142 | + const { owner, repo } = context.repo; |
| 143 | + const signature = "🤖 AI Feedback"; |
| 144 | + const timestamp = new Date().toISOString(); |
| 145 | + const newEntry = `🕒 _Posted on ${timestamp}_\n\n${process.env.FEEDBACK_BODY}\n\n---\n`; |
| 146 | +
|
| 147 | + const { data: comments } = await github.rest.issues.listComments({ |
| 148 | + owner, |
| 149 | + repo, |
| 150 | + issue_number: prNumber, |
| 151 | + per_page: 100 |
| 152 | + }); |
| 153 | +
|
| 154 | + const existing = comments.find(c => |
| 155 | + c.user?.login === "github-actions[bot]" && |
| 156 | + c.body?.includes(signature) |
| 157 | + ); |
| 158 | +
|
| 159 | + if (existing) { |
| 160 | + const previousContent = existing.body.replace(/^### 🤖 AI Feedback\s*/, '').trim(); |
| 161 | + const collapsed = `<details><summary>Previous Feedback</summary>\n\n${previousContent}\n</details>`; |
| 162 | + const updatedBody = `### ${signature}\n\n${newEntry}${collapsed}`; |
| 163 | + await github.rest.issues.updateComment({ |
| 164 | + owner, |
| 165 | + repo, |
| 166 | + comment_id: existing.id, |
| 167 | + body: updatedBody |
| 168 | + }); |
| 169 | + } else { |
| 170 | + const body = `### ${signature}\n\n${newEntry}`; |
| 171 | + await github.rest.issues.createComment({ |
| 172 | + owner, |
| 173 | + repo, |
| 174 | + issue_number: prNumber, |
| 175 | + body |
| 176 | + }); |
| 177 | + } |
| 178 | + - name: Set up Java 25 |
| 179 | + uses: actions/setup-java@v5 |
| 180 | + with: |
| 181 | + distribution: 'temurin' |
| 182 | + java-version: '25' |
| 183 | + |
| 184 | + - name: Check for modified files and test presence |
| 185 | + id: code-check |
| 186 | + run: | |
| 187 | + # Ensure full history |
| 188 | + git fetch origin main |
| 189 | + |
| 190 | + # Determine branch name safely |
| 191 | + BRANCH_NAME="${{ github.head_ref || github.ref_name }}" |
| 192 | + echo "Using branch: $BRANCH_NAME" |
| 193 | + git checkout "$BRANCH_NAME" |
| 194 | + git branch --set-upstream-to=origin/main "$BRANCH_NAME" |
| 195 | + |
| 196 | + # Check for test files |
| 197 | + has_tests=$(find src/test/java -type f -name "*.java" | grep -q . && echo "true" || echo "false") |
| 198 | + |
| 199 | + # Check for modified files |
| 200 | + changed_files=$(git diff --name-only origin/main...HEAD | grep -E '^src/(main|test)/java/.*\.java$' || true) |
| 201 | + has_changes=$(test -n "$changed_files" && echo "true" || echo "false") |
| 202 | + |
| 203 | + echo "has_tests=$has_tests" >> $GITHUB_OUTPUT |
| 204 | + echo "has_changes=$has_changes" >> $GITHUB_OUTPUT |
| 205 | + echo "should_grade=$([[ $has_tests == 'true' && $has_changes == 'true' ]] && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT |
| 206 | + |
| 207 | + - name: Compilation Check |
| 208 | + id: compilation-check |
| 209 | + if: ${{ steps.code-check.outputs.has_tests == 'true' && steps.code-check.outputs.has_changes == 'true' }} |
| 210 | + uses: classroom-resources/autograding-command-grader@v1 |
| 211 | + with: |
| 212 | + test-name: Compilation Check |
| 213 | + command: mvn -ntp compile |
| 214 | + timeout: 10 |
| 215 | + max-score: 1 |
| 216 | + |
| 217 | + - name: Tests |
| 218 | + id: basic-tests |
| 219 | + if: ${{ steps.code-check.outputs.has_tests == 'true' && steps.code-check.outputs.has_changes == 'true' }} |
| 220 | + uses: classroom-resources/autograding-command-grader@v1 |
| 221 | + with: |
| 222 | + test-name: Tests |
| 223 | + command: mvn -ntp test |
| 224 | + timeout: 10 |
| 225 | + max-score: 1 |
| 226 | + |
| 227 | + - name: Autograding Reporter |
| 228 | + if: ${{ steps.code-check.outputs.should_grade == 'true' }} |
| 229 | + uses: classroom-resources/autograding-grading-reporter@v1 |
| 230 | + env: |
| 231 | + COMPILATION-CHECK_RESULTS: "${{steps.compilation-check.outputs.result}}" |
| 232 | + BASIC-TESTS_RESULTS: "${{steps.basic-tests.outputs.result}}" |
| 233 | + with: |
| 234 | + runners: compilation-check,basic-tests |
| 235 | + |
| 236 | + - name: Reporter Skipped Notice |
| 237 | + if: ${{ steps.code-check.outputs.should_grade != 'true' }} |
| 238 | + run: | |
| 239 | + echo "🛑 Skipping reporter: No grading results available due to missing tests or unchanged code." |
0 commit comments