From fb428b1b0cc9597931593f8eb66f8d08c98a4dec Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:11:00 +0000 Subject: [PATCH 1/7] feat(extraction): align extracted schema with structured json format Agent-Logs-Url: https://github.com/nmdra/AgentHire/sessions/3d45db37-3bcc-450a-a084-49377e4795c8 Co-authored-by: nmdra <73674803+nmdra@users.noreply.github.com> --- app/agents/extraction_agent.py | 26 ++++++++++++++++++++++---- app/tools/ollama.py | 10 +++++++++- app/tools/validate_extraction.py | 22 ++++++++++++++++++++-- tests/test_extraction_agent.py | 22 ++++++++++++++++------ tests/test_workflow.py | 6 ++++-- 5 files changed, 71 insertions(+), 15 deletions(-) diff --git a/app/agents/extraction_agent.py b/app/agents/extraction_agent.py index 6a04362..8c31bc0 100644 --- a/app/agents/extraction_agent.py +++ b/app/agents/extraction_agent.py @@ -34,13 +34,29 @@ def _read_input(file_path: str) -> str: def _build_prompt(raw_text: str, correction_error: str | None = None) -> str: instruction = ( - "Extract applicant details as strict JSON with keys: " - "name, email, phone, skills (array), experience, education. " - "Return JSON only and include all keys." + "You are a precise document parsing agent.\n\n" + "Your ONLY job is to read the provided document text and return a single valid JSON object.\n\n" + "Rules:\n" + "- Return ONLY the JSON object. No explanation, markdown, or conversational text.\n" + "- If a field is not present in the document, set it to null.\n" + "- For list fields, use an empty list [] if nothing is found.\n" + "- Never invent or guess data. Only extract what is explicitly stated.\n" + '- Capture significant details not covered by primary fields in "other_details".\n\n' + "Output schema (use exactly these keys):\n" + '{\n' + ' "name": string or null,\n' + ' "email": string or null,\n' + ' "phone": string or null,\n' + ' "website": string or null,\n' + ' "skills": [string, ...],\n' + ' "experience": [{"title": string, "company": string, "duration": string}, ...],\n' + ' "education": [{"degree": string, "institution": string, "year": string}, ...],\n' + ' "other_details": [string, ...]\n' + '}\n' ) if correction_error: instruction = f"{instruction}\nPrevious response failed validation: {correction_error}" - return f"{instruction}\n\nApplication:\n{raw_text[:MAX_INPUT_CHARS]}" + return f"{instruction}\n\nQuestion: {raw_text[:MAX_INPUT_CHARS]}\nAnswer (JSON only):" def _extract_with_retry( @@ -54,6 +70,8 @@ def _extract_with_retry( model=model, prompt=_build_prompt(raw_text, correction_error=error), temperature=0.0, + top_p=0.1, + stop=["```"], timeout_seconds=timeout_seconds, ) try: diff --git a/app/tools/ollama.py b/app/tools/ollama.py index 30b14c6..ca37582 100644 --- a/app/tools/ollama.py +++ b/app/tools/ollama.py @@ -17,6 +17,8 @@ def generate_json_response( model: str, prompt: str, temperature: float = 0.0, + top_p: float = 0.1, + stop: list[str] | None = None, timeout_seconds: float = 30.0, ) -> str: """Generate a JSON completion response from an Ollama model. @@ -26,6 +28,8 @@ def generate_json_response( model: Model name to query. prompt: Prompt text sent to the model. temperature: Sampling temperature for generation. + top_p: Nucleus sampling parameter. + stop: Optional stop tokens. timeout_seconds: HTTP timeout in seconds. Returns: @@ -43,12 +47,16 @@ def generate_json_response( ) """ endpoint = f"{base_url.rstrip('/')}/api/generate" + options: dict[str, object] = {"temperature": temperature, "top_p": top_p} + if stop: + options["stop"] = stop + payload = { "model": model, "prompt": prompt, "stream": False, "format": "json", - "options": {"temperature": temperature}, + "options": options, } try: diff --git a/app/tools/validate_extraction.py b/app/tools/validate_extraction.py index 5bf6156..e561537 100644 --- a/app/tools/validate_extraction.py +++ b/app/tools/validate_extraction.py @@ -5,12 +5,30 @@ from pydantic import BaseModel, Field +class ExperienceEntry(BaseModel): + """Structured experience entry extracted from a candidate profile.""" + + title: str | None = Field(default=None) + company: str | None = Field(default=None) + duration: str | None = Field(default=None) + + +class EducationEntry(BaseModel): + """Structured education entry extracted from a candidate profile.""" + + degree: str | None = Field(default=None) + institution: str | None = Field(default=None) + year: str | None = Field(default=None) + + class CandidateExtraction(BaseModel): """Structured extraction output from extraction agent.""" name: str | None = Field(default=None) email: str | None = Field(default=None) phone: str | None = Field(default=None) + website: str | None = Field(default=None) skills: list[str] = Field(default_factory=list) - experience: str | None = Field(default=None) - education: str | None = Field(default=None) + experience: list[ExperienceEntry] = Field(default_factory=list) + education: list[EducationEntry] = Field(default_factory=list) + other_details: list[str] = Field(default_factory=list) diff --git a/tests/test_extraction_agent.py b/tests/test_extraction_agent.py index de6323f..9f9614b 100644 --- a/tests/test_extraction_agent.py +++ b/tests/test_extraction_agent.py @@ -29,9 +29,15 @@ def test_extraction_success(monkeypatch: pytest.MonkeyPatch, base_state: dict[st "name": "Jane Doe", "email": "jane@example.com", "phone": "+1-123-456-7890", + "website": None, "skills": ["Python", "SQL"], - "experience": "3 years", - "education": "BSc Computer Science", + "experience": [ + {"title": "Backend Engineer", "company": "Tech Corp", "duration": "3 years"} + ], + "education": [ + {"degree": "BSc Computer Science", "institution": "State U", "year": "2020"} + ], + "other_details": ["AWS Certified"], } ), ) @@ -53,9 +59,11 @@ def test_extraction_retry_once(monkeypatch: pytest.MonkeyPatch, base_state: dict "name": "Jane Doe", "email": "jane@example.com", "phone": None, + "website": None, "skills": ["Python"], - "experience": "3 years", - "education": "BSc", + "experience": [{"title": "Engineer", "company": None, "duration": "3 years"}], + "education": [], + "other_details": [], } ), ]) @@ -115,5 +123,7 @@ def test_missing_optional_fields_default_to_null( extracted = result["extracted_json"] assert extracted["email"] is None assert extracted["phone"] is None - assert extracted["experience"] is None - assert extracted["education"] is None + assert extracted["website"] is None + assert extracted["experience"] == [] + assert extracted["education"] == [] + assert extracted["other_details"] == [] diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 3d9053e..76a0317 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -17,9 +17,11 @@ def test_workflow_runs_with_stubbed_agents(monkeypatch, tmp_path: Path) -> None: "name": "Test Candidate", "email": "test@example.com", "phone": None, + "website": None, "skills": ["Python"], - "experience": "2 years", - "education": "BSc", + "experience": [{"title": "Engineer", "company": "Acme", "duration": "2 years"}], + "education": [{"degree": "BSc", "institution": "Uni", "year": "2021"}], + "other_details": [], } ), ) From 07b1241bc64268128133b35037913473da505ac2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:11:38 +0000 Subject: [PATCH 2/7] feat(agents): add persona-driven prompts and non-mocked agent logic Agent-Logs-Url: https://github.com/nmdra/AgentHire/sessions/88460f99-0014-43cc-bdb8-7d47c16a02db Co-authored-by: nmdra <73674803+nmdra@users.noreply.github.com> --- .env.example | 4 + README.md | 39 +++++++- app/agents/decision_agent.py | 101 +++++++++++++++++-- app/agents/evaluation_agent.py | 72 +++++++++++++- app/agents/extraction_agent.py | 46 ++++----- app/agents/notification_agent.py | 104 +++++++++++++++++++- app/agents/personas.py | 150 +++++++++++++++++++++++++++++ app/agents/report_agent.py | 102 ++++++++++++++++++-- app/config.py | 6 ++ app/tools/notification_dispatch.py | 42 ++++++++ app/tools/report_writer.py | 51 ++++++++++ pyproject.toml | 1 + tests/test_agent_prompts.py | 48 +++++++++ tests/test_decision_agent.py | 78 +++++++++++++++ tests/test_evaluation_agent.py | 87 +++++++++++++++++ tests/test_notification_agent.py | 85 ++++++++++++++++ tests/test_report_agent.py | 114 ++++++++++++++++++++++ tests/test_workflow.py | 42 ++++++++ 18 files changed, 1122 insertions(+), 50 deletions(-) create mode 100644 app/agents/personas.py create mode 100644 app/tools/notification_dispatch.py create mode 100644 app/tools/report_writer.py create mode 100644 tests/test_agent_prompts.py create mode 100644 tests/test_decision_agent.py create mode 100644 tests/test_evaluation_agent.py create mode 100644 tests/test_notification_agent.py create mode 100644 tests/test_report_agent.py diff --git a/.env.example b/.env.example index 19d68a0..02667aa 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,10 @@ REPORTS_DIR=reports MAX_UPLOAD_SIZE_BYTES=10485760 OLLAMA_BASE_URL=http://localhost:11434 EXTRACTION_MODEL=smollm:360m +EVALUATION_MODEL=gemma3:1b-it-q4_K_M +DECISION_MODEL=phi4-mini:3.8b-q4_K_M +REPORT_MODEL=gemma3:1b-it-q4_K_M +NOTIFICATION_MODEL=smollm:360m OLLAMA_TIMEOUT_SECONDS=30 RESEND_API_KEY= RESEND_FROM_EMAIL=noreply@example.com diff --git a/README.md b/README.md index 42cd6f4..5a6835d 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ audit logs, and generates internal/applicant reports. - extraction - evaluation - decision - - (optional) human review - report generation - notification - Persist results in SQLite (`applications`, `audit_log`) @@ -27,7 +26,43 @@ Decision routing: - `PASS` -> `report` -> `notify` - `FAIL` -> `report` -> `notify` -- `REVIEW` -> `human_review` -> `report` -> `notify` +- `REVIEW` -> `report` -> `notify` + +--- + +## Agent Persona & Constraints + +All persona specs are defined in code in `app/agents/personas.py` and are injected into every agent prompt via structured sections (`SYSTEM`, `TASK`, `CONTEXT`, `OUTPUT`). + +### Extraction Agent +- **Persona:** precise document parser for applicant facts +- **Owned state fields:** `extracted_json` +- **Tool usage:** `parse_pdf_tool`, `parse_text_tool`, `parse_json_tool`, `generate_json_response` +- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, JSON-only schema output + +### Evaluation Agent +- **Persona:** rubric-style evaluator of extracted data +- **Owned state fields:** `evaluation_score`, `evaluation_reasoning` +- **Tool usage:** `generate_json_response` +- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, score must be `0..100` + +### Decision Agent +- **Persona:** deterministic threshold decision explainer +- **Owned state fields:** `decision`, `confidence`, `decision_reason` +- **Tool usage:** deterministic threshold classifier + `generate_json_response` for reason metadata +- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, confidence must be `0.0..1.0` + +### Report Agent +- **Persona:** report writer for applicant/internal stakeholders +- **Owned state fields:** `report_applicant`, `report_internal` +- **Tool usage:** `generate_json_response`, `write_reports_tool` +- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, deterministic markdown output shape + +### Notification Agent +- **Persona:** safe notification composer and dispatcher +- **Owned state fields:** `notification_status` +- **Tool usage:** `generate_json_response`, `send_notification_tool` +- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, deterministic plain-text subject/body output --- diff --git a/app/agents/decision_agent.py b/app/agents/decision_agent.py index 0d52611..369c8ce 100644 --- a/app/agents/decision_agent.py +++ b/app/agents/decision_agent.py @@ -1,17 +1,104 @@ -"""Mock decision agent for Phase 1 scaffold.""" +"""Decision agent implementation.""" from __future__ import annotations +import json + +from pydantic import BaseModel, Field, ValidationError + +from app.agents.personas import DECISION_PERSONA, build_structured_prompt +from app.config import get_settings from app.observability import traced -from app.state import ApplicationState +from app.state import ApplicationState, Decision +from app.tools.ollama import generate_json_response + +PASS_THRESHOLD = 75.0 +REVIEW_THRESHOLD = 60.0 + + +class DecisionOutput(BaseModel): + """Structured output expected from the decision reason model.""" + + decision_reason: str = Field(min_length=1) + confidence: float = Field(ge=0.0, le=1.0) + + +def _classify_score(score: float) -> Decision: + """Apply deterministic threshold logic for pass/review/fail.""" + if score >= PASS_THRESHOLD: + return "PASS" + if score >= REVIEW_THRESHOLD: + return "REVIEW" + return "FAIL" + + +def _build_decision_prompt(state: ApplicationState, decision: Decision) -> str: + task = ( + "Given the deterministic decision and evaluation context, provide concise human-readable reason and confidence." + ) + context = json.dumps( + { + "evaluation_score": state.get("evaluation_score"), + "evaluation_reasoning": state.get("evaluation_reasoning"), + "decision": decision, + "pass_threshold": PASS_THRESHOLD, + "review_threshold": REVIEW_THRESHOLD, + }, + ensure_ascii=False, + ) + output = ( + "Return strict JSON only:\n" + '{\n' + ' "decision_reason": string,\n' + ' "confidence": number (0.0 to 1.0)\n' + '}' + ) + return build_structured_prompt( + persona=DECISION_PERSONA, + task=task, + context=context, + output=output, + ) + + +def _run_decision_prompt(prompt: str, fallback_decision: Decision, score: float) -> DecisionOutput: + settings = get_settings() + response_text = generate_json_response( + base_url=settings.ollama_base_url, + model=settings.decision_model, + prompt=prompt, + temperature=0.0, + top_p=0.1, + stop=["```"], + timeout_seconds=settings.ollama_timeout_seconds, + ) + try: + payload = json.loads(response_text) + return DecisionOutput.model_validate(payload) + except (json.JSONDecodeError, ValidationError): + return DecisionOutput( + decision_reason=( + f"Decision {fallback_decision} from score {score:.2f} using threshold rules." + ), + confidence=0.75, + ) @traced("decision_agent") -def decision_agent(_state: ApplicationState) -> dict[str, object]: - """Return deterministic mocked decision output.""" +def decision_agent(state: ApplicationState) -> dict[str, object]: + """Classify applicant decision deterministically and generate reason metadata.""" + raw_score = state.get("evaluation_score") + if raw_score is None: + raise ValueError("evaluation_score is required for decision") + + score = float(raw_score) + decision = _classify_score(score) + prompt = _build_decision_prompt(state, decision) + details = _run_decision_prompt(prompt, decision, score) + return { "status": "decided", - "decision": "REVIEW", - "confidence": 0.7, - "decision_reason": "Mocked decision based on static evaluation score.", + "decision": decision, + "confidence": float(details.confidence), + "decision_reason": details.decision_reason, } diff --git a/app/agents/evaluation_agent.py b/app/agents/evaluation_agent.py index 0fa7796..2075931 100644 --- a/app/agents/evaluation_agent.py +++ b/app/agents/evaluation_agent.py @@ -1,16 +1,78 @@ -"""Mock evaluation agent for Phase 1 scaffold.""" +"""Evaluation agent implementation.""" from __future__ import annotations +import json + +from pydantic import BaseModel, Field, ValidationError + +from app.agents.personas import EVALUATION_PERSONA, build_structured_prompt +from app.config import get_settings from app.observability import traced from app.state import ApplicationState +from app.tools.ollama import generate_json_response + + +class EvaluationOutput(BaseModel): + """Structured output expected from the evaluation model.""" + + evaluation_score: float = Field(ge=0.0, le=100.0) + evaluation_reasoning: str = Field(min_length=1) + + +def _build_evaluation_prompt(state: ApplicationState) -> str: + extracted_json = state.get("extracted_json") + task = ( + "Evaluate candidate quality from extracted data using evidence-based reasoning and assign a score." + ) + context = json.dumps( + {"extracted_json": extracted_json}, + ensure_ascii=False, + ) + output = ( + "Return strict JSON only:\n" + '{\n' + ' "evaluation_score": number (0 to 100),\n' + ' "evaluation_reasoning": string\n' + '}' + ) + return build_structured_prompt( + persona=EVALUATION_PERSONA, + task=task, + context=context, + output=output, + ) + + +def _run_evaluation_prompt(prompt: str) -> EvaluationOutput: + settings = get_settings() + response_text = generate_json_response( + base_url=settings.ollama_base_url, + model=settings.evaluation_model, + prompt=prompt, + temperature=0.1, + top_p=0.1, + stop=["```"], + timeout_seconds=settings.ollama_timeout_seconds, + ) + try: + payload = json.loads(response_text) + return EvaluationOutput.model_validate(payload) + except (json.JSONDecodeError, ValidationError) as exc: + raise ValueError(f"Invalid evaluation output: {exc}") from exc @traced("evaluation_agent") -def evaluation_agent(_state: ApplicationState) -> dict[str, object]: - """Return deterministic mocked evaluation output.""" +def evaluation_agent(state: ApplicationState) -> dict[str, object]: + """Evaluate extracted applicant data and produce score with reasoning.""" + extracted_json = state.get("extracted_json") + if not isinstance(extracted_json, dict): + raise ValueError("extracted_json is required for evaluation") + + prompt = _build_evaluation_prompt(state) + evaluation = _run_evaluation_prompt(prompt) return { "status": "evaluated", - "evaluation_score": 72.0, - "evaluation_reasoning": "Mocked rubric evaluation for scaffold run.", + "evaluation_score": float(evaluation.evaluation_score), + "evaluation_reasoning": evaluation.evaluation_reasoning, } diff --git a/app/agents/extraction_agent.py b/app/agents/extraction_agent.py index 8c31bc0..f73effa 100644 --- a/app/agents/extraction_agent.py +++ b/app/agents/extraction_agent.py @@ -10,6 +10,7 @@ from app.config import get_settings from app.database import update_application +from app.agents.personas import EXTRACTION_PERSONA, build_structured_prompt from app.observability import traced from app.state import ApplicationState from app.tools.ollama import generate_json_response @@ -32,31 +33,32 @@ def _read_input(file_path: str) -> str: raise ValueError("Unsupported file type. Use PDF, TXT, MD, or JSON") -def _build_prompt(raw_text: str, correction_error: str | None = None) -> str: - instruction = ( - "You are a precise document parsing agent.\n\n" - "Your ONLY job is to read the provided document text and return a single valid JSON object.\n\n" - "Rules:\n" - "- Return ONLY the JSON object. No explanation, markdown, or conversational text.\n" - "- If a field is not present in the document, set it to null.\n" - "- For list fields, use an empty list [] if nothing is found.\n" - "- Never invent or guess data. Only extract what is explicitly stated.\n" - '- Capture significant details not covered by primary fields in "other_details".\n\n' - "Output schema (use exactly these keys):\n" +def _build_extraction_prompt(raw_text: str, correction_error: str | None = None) -> str: + task = ( + "Extract applicant details from the provided text into the exact structured JSON schema." + ) + if correction_error: + task = f"{task}\nPrevious response failed validation: {correction_error}" + context = f"document_text:\n{raw_text[:MAX_INPUT_CHARS]}" + output = ( + "Return JSON only with exactly these keys:\n" '{\n' - ' "name": string or null,\n' - ' "email": string or null,\n' - ' "phone": string or null,\n' - ' "website": string or null,\n' - ' "skills": [string, ...],\n' + ' "name": string or null,\n' + ' "email": string or null,\n' + ' "phone": string or null,\n' + ' "website": string or null,\n' + ' "skills": [string, ...],\n' ' "experience": [{"title": string, "company": string, "duration": string}, ...],\n' - ' "education": [{"degree": string, "institution": string, "year": string}, ...],\n' + ' "education": [{"degree": string, "institution": string, "year": string}, ...],\n' ' "other_details": [string, ...]\n' - '}\n' + '}' + ) + return build_structured_prompt( + persona=EXTRACTION_PERSONA, + task=task, + context=context, + output=output, ) - if correction_error: - instruction = f"{instruction}\nPrevious response failed validation: {correction_error}" - return f"{instruction}\n\nQuestion: {raw_text[:MAX_INPUT_CHARS]}\nAnswer (JSON only):" def _extract_with_retry( @@ -68,7 +70,7 @@ def _extract_with_retry( response_text = generate_json_response( base_url=base_url, model=model, - prompt=_build_prompt(raw_text, correction_error=error), + prompt=_build_extraction_prompt(raw_text, correction_error=error), temperature=0.0, top_p=0.1, stop=["```"], diff --git a/app/agents/notification_agent.py b/app/agents/notification_agent.py index f3069a1..e479e13 100644 --- a/app/agents/notification_agent.py +++ b/app/agents/notification_agent.py @@ -1,15 +1,111 @@ -"""Mock notification agent for Phase 1 scaffold.""" +"""Notification agent implementation.""" from __future__ import annotations +import json +import re +from typing import Any + +from pydantic import BaseModel, Field, ValidationError + +from app.agents.personas import NOTIFICATION_PERSONA, build_structured_prompt +from app.config import get_settings from app.observability import traced from app.state import ApplicationState +from app.tools.notification_dispatch import send_notification_tool +from app.tools.ollama import generate_json_response + +EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + +class NotificationContent(BaseModel): + """Structured output expected from notification model.""" + + subject: str = Field(min_length=1) + body: str = Field(min_length=1) + + +def _build_notification_prompt(state: ApplicationState, recipient: str) -> str: + task = "Generate a concise plain-text notification email subject and body." + context = json.dumps( + { + "recipient": recipient, + "name": (state.get("extracted_json") or {}).get("name") + if isinstance(state.get("extracted_json"), dict) + else None, + "decision": state.get("decision"), + "decision_reason": state.get("decision_reason"), + "report_applicant": state.get("report_applicant"), + }, + ensure_ascii=False, + ) + output = ( + "Return strict JSON only:\n" + '{\n' + ' "subject": string,\n' + ' "body": string\n' + '}' + ) + return build_structured_prompt( + persona=NOTIFICATION_PERSONA, + task=task, + context=context, + output=output, + ) + + +def _run_notification_prompt(prompt: str) -> NotificationContent: + settings = get_settings() + response_text = generate_json_response( + base_url=settings.ollama_base_url, + model=settings.notification_model, + prompt=prompt, + temperature=0.2, + top_p=0.1, + stop=["```"], + timeout_seconds=settings.ollama_timeout_seconds, + ) + try: + payload = json.loads(response_text) + return NotificationContent.model_validate(payload) + except (json.JSONDecodeError, ValidationError) as exc: + raise ValueError(f"Invalid notification output: {exc}") from exc + + +def _extract_recipient(state: ApplicationState) -> str | None: + extracted = state.get("extracted_json") + if not isinstance(extracted, dict): + return None + email = extracted.get("email") + if isinstance(email, str) and email: + return email + return None @traced("notification_agent") -def notification_agent(_state: ApplicationState) -> dict[str, object]: - """Simulate notification dispatch without external email integration.""" +def notification_agent(state: ApplicationState) -> dict[str, Any]: + """Generate and dispatch a notification payload.""" + if state.get("decision") is None: + raise ValueError("decision is required for notification") + + recipient = _extract_recipient(state) + if recipient is None or not EMAIL_PATTERN.fullmatch(recipient): + return { + "status": "completed", + "notification_status": "failed", + "errors": ["notification_agent failed: missing or invalid recipient email"], + } + + prompt = _build_notification_prompt(state, recipient) + content = _run_notification_prompt(prompt) + dispatch_result = send_notification_tool.invoke( + { + "to_address": recipient, + "subject": content.subject, + "body": content.body, + } + ) return { "status": "completed", - "notification_status": "sent", + "notification_status": dispatch_result.get("status", "failed"), } diff --git a/app/agents/personas.py b/app/agents/personas.py new file mode 100644 index 0000000..84ca3af --- /dev/null +++ b/app/agents/personas.py @@ -0,0 +1,150 @@ +"""Persona specifications and prompt helpers for agent nodes.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PersonaSpec: + """Structured persona contract used to build system prompts.""" + + role_identity: str + scope_boundaries: tuple[str, ...] + hard_constraints: tuple[str, ...] + output_contract: tuple[str, ...] + + def system_section(self) -> str: + """Render persona details as a structured system section.""" + boundaries = "\n".join(f"- {item}" for item in self.scope_boundaries) + constraints = "\n".join(f"- {item}" for item in self.hard_constraints) + outputs = "\n".join(f"- {item}" for item in self.output_contract) + return ( + f"ROLE IDENTITY:\n{self.role_identity}\n\n" + f"SCOPE BOUNDARIES:\n{boundaries}\n\n" + f"HARD CONSTRAINTS:\n{constraints}\n\n" + f"OUTPUT CONTRACT:\n{outputs}" + ) + + +GLOBAL_GUARDRAILS: tuple[str, ...] = ( + "no hallucinated fields", + "no secret/API key leakage", + "no overwriting other agents' owned state", +) + + +EXTRACTION_PERSONA = PersonaSpec( + role_identity=( + "You are the Extraction Agent. You convert raw applicant documents into structured JSON." + ), + scope_boundaries=( + "Only extract applicant facts from the provided document text.", + "Do not score, decide, report, or notify.", + "Return only extraction-owned data fields.", + ), + hard_constraints=( + *GLOBAL_GUARDRAILS, + "Use null for unknown scalar fields and [] for unknown list fields.", + ), + output_contract=( + "Return one valid JSON object matching the extraction schema exactly.", + "No markdown fences, preambles, or explanations.", + ), +) + + +EVALUATION_PERSONA = PersonaSpec( + role_identity=( + "You are the Evaluation Agent. You evaluate extracted applicant data against rubric-style criteria." + ), + scope_boundaries=( + "Only produce evaluation score and evaluation reasoning.", + "Do not alter extraction, decision, report, or notification fields.", + "Use only data available in context.", + ), + hard_constraints=( + *GLOBAL_GUARDRAILS, + "Keep score in the range 0 to 100.", + ), + output_contract=( + "Return strict JSON with keys: evaluation_score, evaluation_reasoning.", + "evaluation_reasoning must be concise and evidence-based.", + ), +) + + +DECISION_PERSONA = PersonaSpec( + role_identity=( + "You are the Decision Agent. You explain deterministic decision outcomes from evaluation results." + ), + scope_boundaries=( + "Do not choose the decision label when the threshold decision is already provided.", + "Only provide decision_reason and confidence metadata.", + "Do not mutate other agent-owned fields.", + ), + hard_constraints=( + *GLOBAL_GUARDRAILS, + "Keep confidence in the range 0.0 to 1.0.", + ), + output_contract=( + "Return strict JSON with keys: decision_reason, confidence.", + "decision_reason must align with provided score and threshold-based decision.", + ), +) + + +REPORT_PERSONA = PersonaSpec( + role_identity=( + "You are the Report Agent. You create applicant-facing and internal markdown reports." + ), + scope_boundaries=( + "Use current workflow outputs to draft reports.", + "Do not make hiring decisions or send notifications.", + "Do not include secrets or hidden system information.", + ), + hard_constraints=( + *GLOBAL_GUARDRAILS, + "Produce deterministic markdown shape: applicant report and internal report.", + ), + output_contract=( + "Return strict JSON with keys: report_applicant, report_internal.", + "Both values must be markdown strings.", + ), +) + + +NOTIFICATION_PERSONA = PersonaSpec( + role_identity=( + "You are the Notification Agent. You generate safe decision email content and trigger delivery tooling." + ), + scope_boundaries=( + "Only create notification text and status for the provided decision and recipient.", + "Do not alter evaluation, decision, or report fields.", + "Use templates/structured format required by the output contract.", + ), + hard_constraints=( + *GLOBAL_GUARDRAILS, + "Never include credentials, tokens, or environment values in generated content.", + ), + output_contract=( + "Return strict JSON with keys: subject, body.", + "subject and body must be plain text suitable for an email sender tool.", + ), +) + + +def build_structured_prompt( + *, persona: PersonaSpec, task: str, context: str, output: str +) -> str: + """Build the standard prompt layout with explicit sections.""" + return ( + "SYSTEM SECTION:\n" + f"{persona.system_section()}\n\n" + "TASK SECTION:\n" + f"{task}\n\n" + "CONTEXT SECTION:\n" + f"{context}\n\n" + "OUTPUT SECTION:\n" + f"{output}" + ) diff --git a/app/agents/report_agent.py b/app/agents/report_agent.py index 7bbf50b..f7e5ed6 100644 --- a/app/agents/report_agent.py +++ b/app/agents/report_agent.py @@ -1,22 +1,104 @@ -"""Mock report agent for Phase 1 scaffold.""" +"""Report agent implementation.""" from __future__ import annotations +import json + +from pydantic import BaseModel, Field, ValidationError + +from app.agents.personas import REPORT_PERSONA, build_structured_prompt +from app.config import get_settings from app.observability import traced from app.state import ApplicationState +from app.tools.ollama import generate_json_response +from app.tools.report_writer import write_reports_tool + + +class ReportOutput(BaseModel): + """Structured report output expected from the report model.""" + + report_applicant: str = Field(min_length=1) + report_internal: str = Field(min_length=1) + + +def _build_report_prompt(state: ApplicationState) -> str: + task = ( + "Generate two markdown reports: an applicant-facing summary and an internal structured report." + ) + context = json.dumps( + { + "name": (state.get("extracted_json") or {}).get("name") + if isinstance(state.get("extracted_json"), dict) + else None, + "decision": state.get("decision"), + "decision_reason": state.get("decision_reason"), + "evaluation_score": state.get("evaluation_score"), + "evaluation_reasoning": state.get("evaluation_reasoning"), + "skills": (state.get("extracted_json") or {}).get("skills") + if isinstance(state.get("extracted_json"), dict) + else None, + "other_details": (state.get("extracted_json") or {}).get("other_details") + if isinstance(state.get("extracted_json"), dict) + else None, + }, + ensure_ascii=False, + ) + output = ( + "Return strict JSON only:\n" + '{\n' + ' "report_applicant": string markdown,\n' + ' "report_internal": string markdown\n' + '}' + ) + return build_structured_prompt( + persona=REPORT_PERSONA, + task=task, + context=context, + output=output, + ) + + +def _run_report_prompt(prompt: str) -> ReportOutput: + settings = get_settings() + response_text = generate_json_response( + base_url=settings.ollama_base_url, + model=settings.report_model, + prompt=prompt, + temperature=0.3, + top_p=0.2, + stop=["```"], + timeout_seconds=settings.ollama_timeout_seconds, + ) + try: + payload = json.loads(response_text) + return ReportOutput.model_validate(payload) + except (json.JSONDecodeError, ValidationError) as exc: + raise ValueError(f"Invalid report output: {exc}") from exc @traced("report_agent") def report_agent(state: ApplicationState) -> dict[str, object]: - """Generate placeholder applicant and internal reports.""" - decision = state.get("decision", "REVIEW") + """Generate applicant and internal reports from pipeline state.""" + if state.get("decision") is None: + raise ValueError("decision is required for reporting") + + settings = get_settings() + prompt = _build_report_prompt(state) + reports = _run_report_prompt(prompt) + + application_id = state.get("application_id") + if isinstance(application_id, str) and application_id: + write_reports_tool.invoke( + { + "reports_dir": settings.reports_dir, + "application_id": application_id, + "report_applicant": reports.report_applicant, + "report_internal": reports.report_internal, + } + ) + return { "status": "reported", - "report_applicant": f"Your application is currently marked as {decision}. We will follow up soon.", - "report_internal": ( - "# Internal Report\n\n" - f"- Decision: {decision}\n" - f"- Evaluation score: {state.get('evaluation_score', 72.0)}\n" - "- Notes: Placeholder report generated by scaffold.\n" - ), + "report_applicant": reports.report_applicant, + "report_internal": reports.report_internal, } diff --git a/app/config.py b/app/config.py index a828ced..11fba26 100644 --- a/app/config.py +++ b/app/config.py @@ -16,7 +16,13 @@ class Settings: max_upload_size_bytes: int = int(getenv("MAX_UPLOAD_SIZE_BYTES", "10485760")) ollama_base_url: str = getenv("OLLAMA_BASE_URL", "http://localhost:11434") extraction_model: str = getenv("EXTRACTION_MODEL", "smollm:360m") + evaluation_model: str = getenv("EVALUATION_MODEL", "gemma3:1b-it-q4_K_M") + decision_model: str = getenv("DECISION_MODEL", "phi4-mini:3.8b-q4_K_M") + report_model: str = getenv("REPORT_MODEL", "gemma3:1b-it-q4_K_M") + notification_model: str = getenv("NOTIFICATION_MODEL", "smollm:360m") ollama_timeout_seconds: float = float(getenv("OLLAMA_TIMEOUT_SECONDS", "30")) + resend_api_key: str = getenv("RESEND_API_KEY", "") + resend_from_email: str = getenv("RESEND_FROM_EMAIL", "noreply@example.com") def get_settings() -> Settings: diff --git a/app/tools/notification_dispatch.py b/app/tools/notification_dispatch.py new file mode 100644 index 0000000..6193e3c --- /dev/null +++ b/app/tools/notification_dispatch.py @@ -0,0 +1,42 @@ +"""Notification dispatch tool.""" + +from __future__ import annotations + +import re + +from langchain.tools import tool + +EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + +@tool +def send_notification_tool(to_address: str, subject: str, body: str) -> dict[str, str]: + """Dispatch a notification payload. + + Args: + to_address: Recipient email address. + subject: Subject line for the email message. + body: Plain-text email body. + + Returns: + A status dictionary with dispatch result. + + Raises: + ValueError: If to_address is missing or invalid. + + Example: + send_notification_tool.invoke( + { + "to_address": "candidate@example.com", + "subject": "Application update", + "body": "Thank you for applying.", + } + ) + """ + if not EMAIL_PATTERN.fullmatch(to_address): + raise ValueError("Invalid recipient email address") + return { + "status": "sent", + "message": f"Notification prepared for {to_address} with subject '{subject[:80]}'", + "body_preview": body[:120], + } diff --git a/app/tools/report_writer.py b/app/tools/report_writer.py new file mode 100644 index 0000000..658b666 --- /dev/null +++ b/app/tools/report_writer.py @@ -0,0 +1,51 @@ +"""Markdown report writing tool.""" + +from __future__ import annotations + +from pathlib import Path + +from langchain.tools import tool + + +@tool +def write_reports_tool( + reports_dir: str, application_id: str, report_applicant: str, report_internal: str +) -> dict[str, str]: + """Write applicant and internal reports to markdown files. + + Args: + reports_dir: Directory where report files should be written. + application_id: Unique application identifier used for file naming. + report_applicant: Applicant-facing markdown report. + report_internal: Internal markdown report. + + Returns: + Mapping with generated report paths. + + Raises: + ValueError: If application_id is empty. + + Example: + write_reports_tool.invoke( + { + "reports_dir": "reports", + "application_id": "app-1", + "report_applicant": "# Applicant", + "report_internal": "# Internal", + } + ) + """ + if not application_id: + raise ValueError("application_id is required to write reports") + + output_dir = Path(reports_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + applicant_path = output_dir / f"{application_id}_applicant.md" + internal_path = output_dir / f"{application_id}_internal.md" + applicant_path.write_text(report_applicant, encoding="utf-8") + internal_path.write_text(report_internal, encoding="utf-8") + return { + "applicant_path": str(applicant_path), + "internal_path": str(internal_path), + } diff --git a/pyproject.toml b/pyproject.toml index 5349c60..ff70b6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ dev = [ "pytest>=8.0.0", "pytest-mock>=3.14.0", + "hypothesis>=6.112.1", "ruff>=0.6.0", "mypy>=1.11.0", ] diff --git a/tests/test_agent_prompts.py b/tests/test_agent_prompts.py new file mode 100644 index 0000000..545a490 --- /dev/null +++ b/tests/test_agent_prompts.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from app.agents.decision_agent import _build_decision_prompt +from app.agents.evaluation_agent import _build_evaluation_prompt +from app.agents.extraction_agent import _build_extraction_prompt +from app.agents.notification_agent import _build_notification_prompt +from app.agents.report_agent import _build_report_prompt + + +def _assert_guardrails(prompt: str) -> None: + lower = prompt.lower() + assert "system section" in lower + assert "task section" in lower + assert "context section" in lower + assert "output section" in lower + assert "no hallucinated fields" in lower + assert "no secret/api key leakage" in lower + assert "no overwriting other agents' owned state" in lower + + +def test_extraction_prompt_contains_persona_and_guardrails() -> None: + prompt = _build_extraction_prompt("Jane Doe\nPython") + _assert_guardrails(prompt) + assert "extraction agent" in prompt.lower() + + +def test_evaluation_prompt_contains_persona_and_guardrails() -> None: + prompt = _build_evaluation_prompt({"extracted_json": {"name": "Jane"}}) + _assert_guardrails(prompt) + assert "evaluation agent" in prompt.lower() + + +def test_decision_prompt_contains_persona_and_guardrails() -> None: + prompt = _build_decision_prompt({"evaluation_score": 77.0}, "PASS") + _assert_guardrails(prompt) + assert "decision agent" in prompt.lower() + + +def test_report_prompt_contains_persona_and_guardrails() -> None: + prompt = _build_report_prompt({"decision": "REVIEW"}) + _assert_guardrails(prompt) + assert "report agent" in prompt.lower() + + +def test_notification_prompt_contains_persona_and_guardrails() -> None: + prompt = _build_notification_prompt({"decision": "PASS"}, "candidate@example.com") + _assert_guardrails(prompt) + assert "notification agent" in prompt.lower() diff --git a/tests/test_decision_agent.py b/tests/test_decision_agent.py new file mode 100644 index 0000000..bcec74e --- /dev/null +++ b/tests/test_decision_agent.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import json + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from app.agents.decision_agent import ( + PASS_THRESHOLD, + REVIEW_THRESHOLD, + _classify_score, + decision_agent, +) + + +@pytest.fixture +def decision_state() -> dict[str, object]: + return { + "application_id": "app-decision-1", + "evaluation_score": 82.0, + "evaluation_reasoning": "Strong profile.", + "errors": [], + "audit_log": [], + } + + +def test_decision_success(monkeypatch: pytest.MonkeyPatch, decision_state: dict[str, object]) -> None: + monkeypatch.setattr( + "app.agents.decision_agent.generate_json_response", + lambda **_kwargs: json.dumps( + {"decision_reason": "Score exceeds pass threshold.", "confidence": 0.91} + ), + ) + + result = decision_agent(decision_state) + assert result["status"] == "decided" + assert result["decision"] == "PASS" + assert float(result["confidence"]) == pytest.approx(0.91) + + +@pytest.mark.parametrize( + ("score", "expected"), + [ + (75.0, "PASS"), + (60.0, "REVIEW"), + (59.99, "FAIL"), + ], +) +def test_decision_threshold_boundaries(score: float, expected: str) -> None: + assert _classify_score(score) == expected + + +@given(st.floats(min_value=0.0, max_value=100.0, allow_nan=False, allow_infinity=False)) +def test_decision_threshold_property(score: float) -> None: + decision = _classify_score(score) + if score >= PASS_THRESHOLD: + assert decision == "PASS" + elif score >= REVIEW_THRESHOLD: + assert decision == "REVIEW" + else: + assert decision == "FAIL" + + +def test_decision_failure_missing_score() -> None: + result = decision_agent({"application_id": "app-decision-2", "errors": [], "audit_log": []}) + assert result["status"] == "failed" + assert result["errors"] + + +def test_decision_uses_fallback_on_invalid_model_output( + monkeypatch: pytest.MonkeyPatch, decision_state: dict[str, object] +) -> None: + monkeypatch.setattr("app.agents.decision_agent.generate_json_response", lambda **_kwargs: "invalid-json") + result = decision_agent(decision_state) + assert result["status"] == "decided" + assert result["decision"] == "PASS" + assert "threshold" in str(result["decision_reason"]).lower() diff --git a/tests/test_evaluation_agent.py b/tests/test_evaluation_agent.py new file mode 100644 index 0000000..f196889 --- /dev/null +++ b/tests/test_evaluation_agent.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import json + +import pytest + +from app.agents.evaluation_agent import _build_evaluation_prompt, evaluation_agent + + +@pytest.fixture +def evaluation_state() -> dict[str, object]: + return { + "application_id": "app-eval-1", + "extracted_json": { + "name": "Jane Doe", + "email": "jane@example.com", + "skills": ["Python", "SQL"], + "experience": [{"title": "Engineer", "company": "Acme", "duration": "3 years"}], + "education": [{"degree": "BSc", "institution": "State U", "year": "2020"}], + "other_details": [], + }, + "errors": [], + "audit_log": [], + } + + +def test_evaluation_success(monkeypatch: pytest.MonkeyPatch, evaluation_state: dict[str, object]) -> None: + monkeypatch.setattr( + "app.agents.evaluation_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "evaluation_score": 82.0, + "evaluation_reasoning": "Strong technical match with relevant experience.", + } + ), + ) + + result = evaluation_agent(evaluation_state) + assert result["status"] == "evaluated" + assert result["evaluation_score"] == 82.0 + assert "technical" in str(result["evaluation_reasoning"]).lower() + + +def test_evaluation_edge_zero_score( + monkeypatch: pytest.MonkeyPatch, evaluation_state: dict[str, object] +) -> None: + monkeypatch.setattr( + "app.agents.evaluation_agent.generate_json_response", + lambda **_kwargs: json.dumps( + {"evaluation_score": 0.0, "evaluation_reasoning": "No qualifying evidence was provided."} + ), + ) + + result = evaluation_agent(evaluation_state) + assert result["evaluation_score"] == 0.0 + + +def test_evaluation_failure_missing_extracted_json() -> None: + result = evaluation_agent({"application_id": "app-eval-2", "errors": [], "audit_log": []}) + assert result["status"] == "failed" + assert result["errors"] + + +def test_evaluation_prompt_includes_schema() -> None: + prompt = _build_evaluation_prompt({"extracted_json": {"name": "Jane"}}) + assert '"evaluation_score"' in prompt + assert '"evaluation_reasoning"' in prompt + + +def test_evaluation_masks_email_in_audit_log( + monkeypatch: pytest.MonkeyPatch, evaluation_state: dict[str, object] +) -> None: + monkeypatch.setattr( + "app.agents.evaluation_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "evaluation_score": 65.0, + "evaluation_reasoning": "Reach out to jane@example.com for verification.", + } + ), + ) + + result = evaluation_agent(evaluation_state) + entry = result["audit_log"][0] + output_summary = str(entry.get("output_summary", "")) + assert "jane@" not in output_summary + assert "****@" in output_summary diff --git a/tests/test_notification_agent.py b/tests/test_notification_agent.py new file mode 100644 index 0000000..435beec --- /dev/null +++ b/tests/test_notification_agent.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json + +import pytest + +from app.agents.notification_agent import _build_notification_prompt, notification_agent + + +@pytest.fixture +def notification_state() -> dict[str, object]: + return { + "application_id": "app-notify-1", + "decision": "PASS", + "decision_reason": "Strong score.", + "report_applicant": "You passed.", + "extracted_json": {"name": "Jane Doe", "email": "jane@example.com"}, + "errors": [], + "audit_log": [], + } + + +def test_notification_success( + monkeypatch: pytest.MonkeyPatch, notification_state: dict[str, object] +) -> None: + class DummySender: + @staticmethod + def invoke(*_args: object, **_kwargs: object) -> dict[str, str]: + return {"status": "sent"} + + monkeypatch.setattr( + "app.agents.notification_agent.generate_json_response", + lambda **_kwargs: json.dumps( + {"subject": "Application outcome", "body": "You have passed the initial screening."} + ), + ) + monkeypatch.setattr("app.agents.notification_agent.send_notification_tool", DummySender()) + + result = notification_agent(notification_state) + assert result["status"] == "completed" + assert result["notification_status"] == "sent" + + +def test_notification_edge_invalid_email(notification_state: dict[str, object]) -> None: + state = {**notification_state, "extracted_json": {"name": "Jane Doe", "email": "bad-email"}} + result = notification_agent(state) + assert result["status"] == "completed" + assert result["notification_status"] == "failed" + assert result["errors"] + + +def test_notification_failure_missing_decision() -> None: + result = notification_agent({"application_id": "app-notify-2", "errors": [], "audit_log": []}) + assert result["status"] == "failed" + assert result["errors"] + + +def test_notification_prompt_contract() -> None: + prompt = _build_notification_prompt({"decision": "REVIEW"}, "candidate@example.com") + assert '"subject"' in prompt + assert '"body"' in prompt + + +def test_notification_masks_email_in_audit_log( + monkeypatch: pytest.MonkeyPatch, notification_state: dict[str, object] +) -> None: + class DummySender: + @staticmethod + def invoke(*_args: object, **_kwargs: object) -> dict[str, str]: + return {"status": "sent"} + + monkeypatch.setattr( + "app.agents.notification_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "subject": "Contact jane@example.com", + "body": "Reply to jane@example.com for details.", + } + ), + ) + monkeypatch.setattr("app.agents.notification_agent.send_notification_tool", DummySender()) + + result = notification_agent(notification_state) + output_summary = str(result["audit_log"][0].get("output_summary", "")) + assert "jane@" not in output_summary diff --git a/tests/test_report_agent.py b/tests/test_report_agent.py new file mode 100644 index 0000000..f484e10 --- /dev/null +++ b/tests/test_report_agent.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from app.agents.report_agent import _build_report_prompt, report_agent + + +@pytest.fixture +def report_state() -> dict[str, object]: + return { + "application_id": "app-report-1", + "decision": "REVIEW", + "decision_reason": "Needs additional verification.", + "evaluation_score": 68.0, + "evaluation_reasoning": "Good baseline with some gaps.", + "extracted_json": { + "name": "Jane Doe", + "email": "jane@example.com", + "skills": ["Python"], + "other_details": ["AWS Certified"], + }, + "errors": [], + "audit_log": [], + } + + +def test_report_success(monkeypatch: pytest.MonkeyPatch, report_state: dict[str, object]) -> None: + class DummyWriter: + @staticmethod + def invoke(*_args: object, **_kwargs: object) -> dict[str, object]: + return {} + + monkeypatch.setattr( + "app.agents.report_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "report_applicant": "# Applicant Report\n\nYour application is under review.", + "report_internal": "# Internal Report\n\n- Decision: REVIEW", + } + ), + ) + monkeypatch.setattr("app.agents.report_agent.write_reports_tool", DummyWriter()) + + result = report_agent(report_state) + assert result["status"] == "reported" + assert "# Applicant Report" in str(result["report_applicant"]) + assert "# Internal Report" in str(result["report_internal"]) + + +def test_report_writes_markdown_files( + monkeypatch: pytest.MonkeyPatch, report_state: dict[str, object], tmp_path: Path +) -> None: + class DummySettings: + ollama_base_url = "http://localhost:11434" + report_model = "gemma3:1b-it-q4_K_M" + ollama_timeout_seconds = 30.0 + reports_dir = str(tmp_path) + + monkeypatch.setattr("app.agents.report_agent.get_settings", lambda: DummySettings()) + monkeypatch.setattr( + "app.agents.report_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "report_applicant": "# Applicant Report\n\nHello.", + "report_internal": "# Internal Report\n\nDetails.", + } + ), + ) + + result = report_agent({**report_state, "application_id": "app-report-2"}) + assert result["status"] == "reported" + assert (tmp_path / "app-report-2_applicant.md").exists() + assert (tmp_path / "app-report-2_internal.md").exists() + + +def test_report_failure_missing_decision() -> None: + result = report_agent({"application_id": "app-report-3", "errors": [], "audit_log": []}) + assert result["status"] == "failed" + assert result["errors"] + + +def test_report_prompt_includes_markdown_contract() -> None: + prompt = _build_report_prompt({"decision": "PASS"}) + assert "markdown" in prompt.lower() + assert '"report_applicant"' in prompt + assert '"report_internal"' in prompt + + +def test_report_masks_email_in_audit_log( + monkeypatch: pytest.MonkeyPatch, report_state: dict[str, object] +) -> None: + class DummyWriter: + @staticmethod + def invoke(*_args: object, **_kwargs: object) -> dict[str, object]: + return {} + + monkeypatch.setattr( + "app.agents.report_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "report_applicant": "Please contact jane@example.com for follow-up.", + "report_internal": "Internal note with jane@example.com reference.", + } + ), + ) + monkeypatch.setattr("app.agents.report_agent.write_reports_tool", DummyWriter()) + + result = report_agent(report_state) + output_summary = str(result["audit_log"][0].get("output_summary", "")) + assert "jane@" not in output_summary + assert "****@" in output_summary diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 76a0317..cf12004 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -7,6 +7,11 @@ def test_workflow_runs_with_stubbed_agents(monkeypatch, tmp_path: Path) -> None: + class DummyWriter: + @staticmethod + def invoke(*_args: object, **_kwargs: object) -> dict[str, object]: + return {} + candidate_file = tmp_path / "candidate.txt" candidate_file.write_text("Test Candidate", encoding="utf-8") monkeypatch.setattr("app.agents.extraction_agent.update_application", lambda *_args, **_kwargs: None) @@ -25,6 +30,43 @@ def test_workflow_runs_with_stubbed_agents(monkeypatch, tmp_path: Path) -> None: } ), ) + monkeypatch.setattr( + "app.agents.evaluation_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "evaluation_score": 72.0, + "evaluation_reasoning": "Candidate has solid fundamentals and moderate experience.", + } + ), + ) + monkeypatch.setattr( + "app.agents.decision_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "decision_reason": "Score lands in review range based on thresholds.", + "confidence": 0.7, + } + ), + ) + monkeypatch.setattr( + "app.agents.report_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "report_applicant": "# Applicant Report\n\nYour application is under review.", + "report_internal": "# Internal Report\n\n- Decision: REVIEW", + } + ), + ) + monkeypatch.setattr("app.agents.report_agent.write_reports_tool", DummyWriter()) + monkeypatch.setattr( + "app.agents.notification_agent.generate_json_response", + lambda **_kwargs: json.dumps( + { + "subject": "Application update", + "body": "Your application is currently under review.", + } + ), + ) workflow = build_workflow() state = { From e36c798b9c909f467bbaef56405d942851089610 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:20:23 +0000 Subject: [PATCH 3/7] refactor: keep extraction persona changes and restore mocked downstream agents Agent-Logs-Url: https://github.com/nmdra/AgentHire/sessions/09a8ad7f-3dd2-49a1-8c0d-d0de3f8c4748 Co-authored-by: nmdra <73674803+nmdra@users.noreply.github.com> --- .env.example | 4 - README.md | 40 ---------- app/agents/decision_agent.py | 101 ++----------------------- app/agents/evaluation_agent.py | 72 ++---------------- app/agents/notification_agent.py | 104 +------------------------- app/agents/personas.py | 82 +-------------------- app/agents/report_agent.py | 102 +++----------------------- app/config.py | 6 -- app/tools/notification_dispatch.py | 42 ----------- app/tools/report_writer.py | 51 ------------- pyproject.toml | 1 - tests/test_agent_prompts.py | 48 ------------ tests/test_decision_agent.py | 78 -------------------- tests/test_evaluation_agent.py | 87 ---------------------- tests/test_notification_agent.py | 85 --------------------- tests/test_report_agent.py | 114 ----------------------------- tests/test_workflow.py | 42 ----------- 17 files changed, 27 insertions(+), 1032 deletions(-) delete mode 100644 app/tools/notification_dispatch.py delete mode 100644 app/tools/report_writer.py delete mode 100644 tests/test_agent_prompts.py delete mode 100644 tests/test_decision_agent.py delete mode 100644 tests/test_evaluation_agent.py delete mode 100644 tests/test_notification_agent.py delete mode 100644 tests/test_report_agent.py diff --git a/.env.example b/.env.example index 02667aa..19d68a0 100644 --- a/.env.example +++ b/.env.example @@ -4,10 +4,6 @@ REPORTS_DIR=reports MAX_UPLOAD_SIZE_BYTES=10485760 OLLAMA_BASE_URL=http://localhost:11434 EXTRACTION_MODEL=smollm:360m -EVALUATION_MODEL=gemma3:1b-it-q4_K_M -DECISION_MODEL=phi4-mini:3.8b-q4_K_M -REPORT_MODEL=gemma3:1b-it-q4_K_M -NOTIFICATION_MODEL=smollm:360m OLLAMA_TIMEOUT_SECONDS=30 RESEND_API_KEY= RESEND_FROM_EMAIL=noreply@example.com diff --git a/README.md b/README.md index 5a6835d..ed47806 100644 --- a/README.md +++ b/README.md @@ -30,42 +30,6 @@ Decision routing: --- -## Agent Persona & Constraints - -All persona specs are defined in code in `app/agents/personas.py` and are injected into every agent prompt via structured sections (`SYSTEM`, `TASK`, `CONTEXT`, `OUTPUT`). - -### Extraction Agent -- **Persona:** precise document parser for applicant facts -- **Owned state fields:** `extracted_json` -- **Tool usage:** `parse_pdf_tool`, `parse_text_tool`, `parse_json_tool`, `generate_json_response` -- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, JSON-only schema output - -### Evaluation Agent -- **Persona:** rubric-style evaluator of extracted data -- **Owned state fields:** `evaluation_score`, `evaluation_reasoning` -- **Tool usage:** `generate_json_response` -- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, score must be `0..100` - -### Decision Agent -- **Persona:** deterministic threshold decision explainer -- **Owned state fields:** `decision`, `confidence`, `decision_reason` -- **Tool usage:** deterministic threshold classifier + `generate_json_response` for reason metadata -- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, confidence must be `0.0..1.0` - -### Report Agent -- **Persona:** report writer for applicant/internal stakeholders -- **Owned state fields:** `report_applicant`, `report_internal` -- **Tool usage:** `generate_json_response`, `write_reports_tool` -- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, deterministic markdown output shape - -### Notification Agent -- **Persona:** safe notification composer and dispatcher -- **Owned state fields:** `notification_status` -- **Tool usage:** `generate_json_response`, `send_notification_tool` -- **Non-negotiable rules:** no hallucinated fields, no secret/API key leakage, no overwriting other agents' owned state, deterministic plain-text subject/body output - ---- - ## Prerequisites - Python **3.11+** @@ -97,10 +61,6 @@ Settings are loaded from environment variables and `.env` (if present). | `MAX_UPLOAD_SIZE_BYTES` | `10485760` | Max upload size (10 MB) | | `OLLAMA_BASE_URL` | `http://localhost:11434` | Local Ollama base URL | | `EXTRACTION_MODEL` | `smollm:360m` | Extraction model label | -| `EVALUATION_MODEL` | `gemma3:1b-it-q4_K_M` | Evaluation model label | -| `DECISION_MODEL` | `phi4-mini:3.8b-q4_K_M` | Decision model label | -| `REPORT_MODEL` | `gemma3:1b-it-q4_K_M` | Report model label | -| `NOTIFICATION_MODEL` | `smollm:360m` | Notification model label | | `RESEND_API_KEY` | empty | Resend API key (optional) | | `RESEND_FROM_EMAIL` | `noreply@example.com` | Sender email | | `RETRY_ATTEMPTS` | `2` | Retries per workflow node | diff --git a/app/agents/decision_agent.py b/app/agents/decision_agent.py index 369c8ce..0d52611 100644 --- a/app/agents/decision_agent.py +++ b/app/agents/decision_agent.py @@ -1,104 +1,17 @@ -"""Decision agent implementation.""" +"""Mock decision agent for Phase 1 scaffold.""" from __future__ import annotations -import json - -from pydantic import BaseModel, Field, ValidationError - -from app.agents.personas import DECISION_PERSONA, build_structured_prompt -from app.config import get_settings from app.observability import traced -from app.state import ApplicationState, Decision -from app.tools.ollama import generate_json_response - -PASS_THRESHOLD = 75.0 -REVIEW_THRESHOLD = 60.0 - - -class DecisionOutput(BaseModel): - """Structured output expected from the decision reason model.""" - - decision_reason: str = Field(min_length=1) - confidence: float = Field(ge=0.0, le=1.0) - - -def _classify_score(score: float) -> Decision: - """Apply deterministic threshold logic for pass/review/fail.""" - if score >= PASS_THRESHOLD: - return "PASS" - if score >= REVIEW_THRESHOLD: - return "REVIEW" - return "FAIL" - - -def _build_decision_prompt(state: ApplicationState, decision: Decision) -> str: - task = ( - "Given the deterministic decision and evaluation context, provide concise human-readable reason and confidence." - ) - context = json.dumps( - { - "evaluation_score": state.get("evaluation_score"), - "evaluation_reasoning": state.get("evaluation_reasoning"), - "decision": decision, - "pass_threshold": PASS_THRESHOLD, - "review_threshold": REVIEW_THRESHOLD, - }, - ensure_ascii=False, - ) - output = ( - "Return strict JSON only:\n" - '{\n' - ' "decision_reason": string,\n' - ' "confidence": number (0.0 to 1.0)\n' - '}' - ) - return build_structured_prompt( - persona=DECISION_PERSONA, - task=task, - context=context, - output=output, - ) - - -def _run_decision_prompt(prompt: str, fallback_decision: Decision, score: float) -> DecisionOutput: - settings = get_settings() - response_text = generate_json_response( - base_url=settings.ollama_base_url, - model=settings.decision_model, - prompt=prompt, - temperature=0.0, - top_p=0.1, - stop=["```"], - timeout_seconds=settings.ollama_timeout_seconds, - ) - try: - payload = json.loads(response_text) - return DecisionOutput.model_validate(payload) - except (json.JSONDecodeError, ValidationError): - return DecisionOutput( - decision_reason=( - f"Decision {fallback_decision} from score {score:.2f} using threshold rules." - ), - confidence=0.75, - ) +from app.state import ApplicationState @traced("decision_agent") -def decision_agent(state: ApplicationState) -> dict[str, object]: - """Classify applicant decision deterministically and generate reason metadata.""" - raw_score = state.get("evaluation_score") - if raw_score is None: - raise ValueError("evaluation_score is required for decision") - - score = float(raw_score) - decision = _classify_score(score) - prompt = _build_decision_prompt(state, decision) - details = _run_decision_prompt(prompt, decision, score) - +def decision_agent(_state: ApplicationState) -> dict[str, object]: + """Return deterministic mocked decision output.""" return { "status": "decided", - "decision": decision, - "confidence": float(details.confidence), - "decision_reason": details.decision_reason, + "decision": "REVIEW", + "confidence": 0.7, + "decision_reason": "Mocked decision based on static evaluation score.", } diff --git a/app/agents/evaluation_agent.py b/app/agents/evaluation_agent.py index 2075931..0fa7796 100644 --- a/app/agents/evaluation_agent.py +++ b/app/agents/evaluation_agent.py @@ -1,78 +1,16 @@ -"""Evaluation agent implementation.""" +"""Mock evaluation agent for Phase 1 scaffold.""" from __future__ import annotations -import json - -from pydantic import BaseModel, Field, ValidationError - -from app.agents.personas import EVALUATION_PERSONA, build_structured_prompt -from app.config import get_settings from app.observability import traced from app.state import ApplicationState -from app.tools.ollama import generate_json_response - - -class EvaluationOutput(BaseModel): - """Structured output expected from the evaluation model.""" - - evaluation_score: float = Field(ge=0.0, le=100.0) - evaluation_reasoning: str = Field(min_length=1) - - -def _build_evaluation_prompt(state: ApplicationState) -> str: - extracted_json = state.get("extracted_json") - task = ( - "Evaluate candidate quality from extracted data using evidence-based reasoning and assign a score." - ) - context = json.dumps( - {"extracted_json": extracted_json}, - ensure_ascii=False, - ) - output = ( - "Return strict JSON only:\n" - '{\n' - ' "evaluation_score": number (0 to 100),\n' - ' "evaluation_reasoning": string\n' - '}' - ) - return build_structured_prompt( - persona=EVALUATION_PERSONA, - task=task, - context=context, - output=output, - ) - - -def _run_evaluation_prompt(prompt: str) -> EvaluationOutput: - settings = get_settings() - response_text = generate_json_response( - base_url=settings.ollama_base_url, - model=settings.evaluation_model, - prompt=prompt, - temperature=0.1, - top_p=0.1, - stop=["```"], - timeout_seconds=settings.ollama_timeout_seconds, - ) - try: - payload = json.loads(response_text) - return EvaluationOutput.model_validate(payload) - except (json.JSONDecodeError, ValidationError) as exc: - raise ValueError(f"Invalid evaluation output: {exc}") from exc @traced("evaluation_agent") -def evaluation_agent(state: ApplicationState) -> dict[str, object]: - """Evaluate extracted applicant data and produce score with reasoning.""" - extracted_json = state.get("extracted_json") - if not isinstance(extracted_json, dict): - raise ValueError("extracted_json is required for evaluation") - - prompt = _build_evaluation_prompt(state) - evaluation = _run_evaluation_prompt(prompt) +def evaluation_agent(_state: ApplicationState) -> dict[str, object]: + """Return deterministic mocked evaluation output.""" return { "status": "evaluated", - "evaluation_score": float(evaluation.evaluation_score), - "evaluation_reasoning": evaluation.evaluation_reasoning, + "evaluation_score": 72.0, + "evaluation_reasoning": "Mocked rubric evaluation for scaffold run.", } diff --git a/app/agents/notification_agent.py b/app/agents/notification_agent.py index e479e13..f3069a1 100644 --- a/app/agents/notification_agent.py +++ b/app/agents/notification_agent.py @@ -1,111 +1,15 @@ -"""Notification agent implementation.""" +"""Mock notification agent for Phase 1 scaffold.""" from __future__ import annotations -import json -import re -from typing import Any - -from pydantic import BaseModel, Field, ValidationError - -from app.agents.personas import NOTIFICATION_PERSONA, build_structured_prompt -from app.config import get_settings from app.observability import traced from app.state import ApplicationState -from app.tools.notification_dispatch import send_notification_tool -from app.tools.ollama import generate_json_response - -EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") - - -class NotificationContent(BaseModel): - """Structured output expected from notification model.""" - - subject: str = Field(min_length=1) - body: str = Field(min_length=1) - - -def _build_notification_prompt(state: ApplicationState, recipient: str) -> str: - task = "Generate a concise plain-text notification email subject and body." - context = json.dumps( - { - "recipient": recipient, - "name": (state.get("extracted_json") or {}).get("name") - if isinstance(state.get("extracted_json"), dict) - else None, - "decision": state.get("decision"), - "decision_reason": state.get("decision_reason"), - "report_applicant": state.get("report_applicant"), - }, - ensure_ascii=False, - ) - output = ( - "Return strict JSON only:\n" - '{\n' - ' "subject": string,\n' - ' "body": string\n' - '}' - ) - return build_structured_prompt( - persona=NOTIFICATION_PERSONA, - task=task, - context=context, - output=output, - ) - - -def _run_notification_prompt(prompt: str) -> NotificationContent: - settings = get_settings() - response_text = generate_json_response( - base_url=settings.ollama_base_url, - model=settings.notification_model, - prompt=prompt, - temperature=0.2, - top_p=0.1, - stop=["```"], - timeout_seconds=settings.ollama_timeout_seconds, - ) - try: - payload = json.loads(response_text) - return NotificationContent.model_validate(payload) - except (json.JSONDecodeError, ValidationError) as exc: - raise ValueError(f"Invalid notification output: {exc}") from exc - - -def _extract_recipient(state: ApplicationState) -> str | None: - extracted = state.get("extracted_json") - if not isinstance(extracted, dict): - return None - email = extracted.get("email") - if isinstance(email, str) and email: - return email - return None @traced("notification_agent") -def notification_agent(state: ApplicationState) -> dict[str, Any]: - """Generate and dispatch a notification payload.""" - if state.get("decision") is None: - raise ValueError("decision is required for notification") - - recipient = _extract_recipient(state) - if recipient is None or not EMAIL_PATTERN.fullmatch(recipient): - return { - "status": "completed", - "notification_status": "failed", - "errors": ["notification_agent failed: missing or invalid recipient email"], - } - - prompt = _build_notification_prompt(state, recipient) - content = _run_notification_prompt(prompt) - dispatch_result = send_notification_tool.invoke( - { - "to_address": recipient, - "subject": content.subject, - "body": content.body, - } - ) +def notification_agent(_state: ApplicationState) -> dict[str, object]: + """Simulate notification dispatch without external email integration.""" return { "status": "completed", - "notification_status": dispatch_result.get("status", "failed"), + "notification_status": "sent", } diff --git a/app/agents/personas.py b/app/agents/personas.py index 84ca3af..989c03d 100644 --- a/app/agents/personas.py +++ b/app/agents/personas.py @@ -1,4 +1,4 @@ -"""Persona specifications and prompt helpers for agent nodes.""" +"""Persona specifications and prompt helpers for extraction agent.""" from __future__ import annotations @@ -54,86 +54,6 @@ def system_section(self) -> str: ) -EVALUATION_PERSONA = PersonaSpec( - role_identity=( - "You are the Evaluation Agent. You evaluate extracted applicant data against rubric-style criteria." - ), - scope_boundaries=( - "Only produce evaluation score and evaluation reasoning.", - "Do not alter extraction, decision, report, or notification fields.", - "Use only data available in context.", - ), - hard_constraints=( - *GLOBAL_GUARDRAILS, - "Keep score in the range 0 to 100.", - ), - output_contract=( - "Return strict JSON with keys: evaluation_score, evaluation_reasoning.", - "evaluation_reasoning must be concise and evidence-based.", - ), -) - - -DECISION_PERSONA = PersonaSpec( - role_identity=( - "You are the Decision Agent. You explain deterministic decision outcomes from evaluation results." - ), - scope_boundaries=( - "Do not choose the decision label when the threshold decision is already provided.", - "Only provide decision_reason and confidence metadata.", - "Do not mutate other agent-owned fields.", - ), - hard_constraints=( - *GLOBAL_GUARDRAILS, - "Keep confidence in the range 0.0 to 1.0.", - ), - output_contract=( - "Return strict JSON with keys: decision_reason, confidence.", - "decision_reason must align with provided score and threshold-based decision.", - ), -) - - -REPORT_PERSONA = PersonaSpec( - role_identity=( - "You are the Report Agent. You create applicant-facing and internal markdown reports." - ), - scope_boundaries=( - "Use current workflow outputs to draft reports.", - "Do not make hiring decisions or send notifications.", - "Do not include secrets or hidden system information.", - ), - hard_constraints=( - *GLOBAL_GUARDRAILS, - "Produce deterministic markdown shape: applicant report and internal report.", - ), - output_contract=( - "Return strict JSON with keys: report_applicant, report_internal.", - "Both values must be markdown strings.", - ), -) - - -NOTIFICATION_PERSONA = PersonaSpec( - role_identity=( - "You are the Notification Agent. You generate safe decision email content and trigger delivery tooling." - ), - scope_boundaries=( - "Only create notification text and status for the provided decision and recipient.", - "Do not alter evaluation, decision, or report fields.", - "Use templates/structured format required by the output contract.", - ), - hard_constraints=( - *GLOBAL_GUARDRAILS, - "Never include credentials, tokens, or environment values in generated content.", - ), - output_contract=( - "Return strict JSON with keys: subject, body.", - "subject and body must be plain text suitable for an email sender tool.", - ), -) - - def build_structured_prompt( *, persona: PersonaSpec, task: str, context: str, output: str ) -> str: diff --git a/app/agents/report_agent.py b/app/agents/report_agent.py index f7e5ed6..7bbf50b 100644 --- a/app/agents/report_agent.py +++ b/app/agents/report_agent.py @@ -1,104 +1,22 @@ -"""Report agent implementation.""" +"""Mock report agent for Phase 1 scaffold.""" from __future__ import annotations -import json - -from pydantic import BaseModel, Field, ValidationError - -from app.agents.personas import REPORT_PERSONA, build_structured_prompt -from app.config import get_settings from app.observability import traced from app.state import ApplicationState -from app.tools.ollama import generate_json_response -from app.tools.report_writer import write_reports_tool - - -class ReportOutput(BaseModel): - """Structured report output expected from the report model.""" - - report_applicant: str = Field(min_length=1) - report_internal: str = Field(min_length=1) - - -def _build_report_prompt(state: ApplicationState) -> str: - task = ( - "Generate two markdown reports: an applicant-facing summary and an internal structured report." - ) - context = json.dumps( - { - "name": (state.get("extracted_json") or {}).get("name") - if isinstance(state.get("extracted_json"), dict) - else None, - "decision": state.get("decision"), - "decision_reason": state.get("decision_reason"), - "evaluation_score": state.get("evaluation_score"), - "evaluation_reasoning": state.get("evaluation_reasoning"), - "skills": (state.get("extracted_json") or {}).get("skills") - if isinstance(state.get("extracted_json"), dict) - else None, - "other_details": (state.get("extracted_json") or {}).get("other_details") - if isinstance(state.get("extracted_json"), dict) - else None, - }, - ensure_ascii=False, - ) - output = ( - "Return strict JSON only:\n" - '{\n' - ' "report_applicant": string markdown,\n' - ' "report_internal": string markdown\n' - '}' - ) - return build_structured_prompt( - persona=REPORT_PERSONA, - task=task, - context=context, - output=output, - ) - - -def _run_report_prompt(prompt: str) -> ReportOutput: - settings = get_settings() - response_text = generate_json_response( - base_url=settings.ollama_base_url, - model=settings.report_model, - prompt=prompt, - temperature=0.3, - top_p=0.2, - stop=["```"], - timeout_seconds=settings.ollama_timeout_seconds, - ) - try: - payload = json.loads(response_text) - return ReportOutput.model_validate(payload) - except (json.JSONDecodeError, ValidationError) as exc: - raise ValueError(f"Invalid report output: {exc}") from exc @traced("report_agent") def report_agent(state: ApplicationState) -> dict[str, object]: - """Generate applicant and internal reports from pipeline state.""" - if state.get("decision") is None: - raise ValueError("decision is required for reporting") - - settings = get_settings() - prompt = _build_report_prompt(state) - reports = _run_report_prompt(prompt) - - application_id = state.get("application_id") - if isinstance(application_id, str) and application_id: - write_reports_tool.invoke( - { - "reports_dir": settings.reports_dir, - "application_id": application_id, - "report_applicant": reports.report_applicant, - "report_internal": reports.report_internal, - } - ) - + """Generate placeholder applicant and internal reports.""" + decision = state.get("decision", "REVIEW") return { "status": "reported", - "report_applicant": reports.report_applicant, - "report_internal": reports.report_internal, + "report_applicant": f"Your application is currently marked as {decision}. We will follow up soon.", + "report_internal": ( + "# Internal Report\n\n" + f"- Decision: {decision}\n" + f"- Evaluation score: {state.get('evaluation_score', 72.0)}\n" + "- Notes: Placeholder report generated by scaffold.\n" + ), } diff --git a/app/config.py b/app/config.py index 11fba26..a828ced 100644 --- a/app/config.py +++ b/app/config.py @@ -16,13 +16,7 @@ class Settings: max_upload_size_bytes: int = int(getenv("MAX_UPLOAD_SIZE_BYTES", "10485760")) ollama_base_url: str = getenv("OLLAMA_BASE_URL", "http://localhost:11434") extraction_model: str = getenv("EXTRACTION_MODEL", "smollm:360m") - evaluation_model: str = getenv("EVALUATION_MODEL", "gemma3:1b-it-q4_K_M") - decision_model: str = getenv("DECISION_MODEL", "phi4-mini:3.8b-q4_K_M") - report_model: str = getenv("REPORT_MODEL", "gemma3:1b-it-q4_K_M") - notification_model: str = getenv("NOTIFICATION_MODEL", "smollm:360m") ollama_timeout_seconds: float = float(getenv("OLLAMA_TIMEOUT_SECONDS", "30")) - resend_api_key: str = getenv("RESEND_API_KEY", "") - resend_from_email: str = getenv("RESEND_FROM_EMAIL", "noreply@example.com") def get_settings() -> Settings: diff --git a/app/tools/notification_dispatch.py b/app/tools/notification_dispatch.py deleted file mode 100644 index 6193e3c..0000000 --- a/app/tools/notification_dispatch.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Notification dispatch tool.""" - -from __future__ import annotations - -import re - -from langchain.tools import tool - -EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") - - -@tool -def send_notification_tool(to_address: str, subject: str, body: str) -> dict[str, str]: - """Dispatch a notification payload. - - Args: - to_address: Recipient email address. - subject: Subject line for the email message. - body: Plain-text email body. - - Returns: - A status dictionary with dispatch result. - - Raises: - ValueError: If to_address is missing or invalid. - - Example: - send_notification_tool.invoke( - { - "to_address": "candidate@example.com", - "subject": "Application update", - "body": "Thank you for applying.", - } - ) - """ - if not EMAIL_PATTERN.fullmatch(to_address): - raise ValueError("Invalid recipient email address") - return { - "status": "sent", - "message": f"Notification prepared for {to_address} with subject '{subject[:80]}'", - "body_preview": body[:120], - } diff --git a/app/tools/report_writer.py b/app/tools/report_writer.py deleted file mode 100644 index 658b666..0000000 --- a/app/tools/report_writer.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Markdown report writing tool.""" - -from __future__ import annotations - -from pathlib import Path - -from langchain.tools import tool - - -@tool -def write_reports_tool( - reports_dir: str, application_id: str, report_applicant: str, report_internal: str -) -> dict[str, str]: - """Write applicant and internal reports to markdown files. - - Args: - reports_dir: Directory where report files should be written. - application_id: Unique application identifier used for file naming. - report_applicant: Applicant-facing markdown report. - report_internal: Internal markdown report. - - Returns: - Mapping with generated report paths. - - Raises: - ValueError: If application_id is empty. - - Example: - write_reports_tool.invoke( - { - "reports_dir": "reports", - "application_id": "app-1", - "report_applicant": "# Applicant", - "report_internal": "# Internal", - } - ) - """ - if not application_id: - raise ValueError("application_id is required to write reports") - - output_dir = Path(reports_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - applicant_path = output_dir / f"{application_id}_applicant.md" - internal_path = output_dir / f"{application_id}_internal.md" - applicant_path.write_text(report_applicant, encoding="utf-8") - internal_path.write_text(report_internal, encoding="utf-8") - return { - "applicant_path": str(applicant_path), - "internal_path": str(internal_path), - } diff --git a/pyproject.toml b/pyproject.toml index ff70b6d..5349c60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ dependencies = [ dev = [ "pytest>=8.0.0", "pytest-mock>=3.14.0", - "hypothesis>=6.112.1", "ruff>=0.6.0", "mypy>=1.11.0", ] diff --git a/tests/test_agent_prompts.py b/tests/test_agent_prompts.py deleted file mode 100644 index 545a490..0000000 --- a/tests/test_agent_prompts.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -from app.agents.decision_agent import _build_decision_prompt -from app.agents.evaluation_agent import _build_evaluation_prompt -from app.agents.extraction_agent import _build_extraction_prompt -from app.agents.notification_agent import _build_notification_prompt -from app.agents.report_agent import _build_report_prompt - - -def _assert_guardrails(prompt: str) -> None: - lower = prompt.lower() - assert "system section" in lower - assert "task section" in lower - assert "context section" in lower - assert "output section" in lower - assert "no hallucinated fields" in lower - assert "no secret/api key leakage" in lower - assert "no overwriting other agents' owned state" in lower - - -def test_extraction_prompt_contains_persona_and_guardrails() -> None: - prompt = _build_extraction_prompt("Jane Doe\nPython") - _assert_guardrails(prompt) - assert "extraction agent" in prompt.lower() - - -def test_evaluation_prompt_contains_persona_and_guardrails() -> None: - prompt = _build_evaluation_prompt({"extracted_json": {"name": "Jane"}}) - _assert_guardrails(prompt) - assert "evaluation agent" in prompt.lower() - - -def test_decision_prompt_contains_persona_and_guardrails() -> None: - prompt = _build_decision_prompt({"evaluation_score": 77.0}, "PASS") - _assert_guardrails(prompt) - assert "decision agent" in prompt.lower() - - -def test_report_prompt_contains_persona_and_guardrails() -> None: - prompt = _build_report_prompt({"decision": "REVIEW"}) - _assert_guardrails(prompt) - assert "report agent" in prompt.lower() - - -def test_notification_prompt_contains_persona_and_guardrails() -> None: - prompt = _build_notification_prompt({"decision": "PASS"}, "candidate@example.com") - _assert_guardrails(prompt) - assert "notification agent" in prompt.lower() diff --git a/tests/test_decision_agent.py b/tests/test_decision_agent.py deleted file mode 100644 index bcec74e..0000000 --- a/tests/test_decision_agent.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -import json - -import pytest -from hypothesis import given -from hypothesis import strategies as st - -from app.agents.decision_agent import ( - PASS_THRESHOLD, - REVIEW_THRESHOLD, - _classify_score, - decision_agent, -) - - -@pytest.fixture -def decision_state() -> dict[str, object]: - return { - "application_id": "app-decision-1", - "evaluation_score": 82.0, - "evaluation_reasoning": "Strong profile.", - "errors": [], - "audit_log": [], - } - - -def test_decision_success(monkeypatch: pytest.MonkeyPatch, decision_state: dict[str, object]) -> None: - monkeypatch.setattr( - "app.agents.decision_agent.generate_json_response", - lambda **_kwargs: json.dumps( - {"decision_reason": "Score exceeds pass threshold.", "confidence": 0.91} - ), - ) - - result = decision_agent(decision_state) - assert result["status"] == "decided" - assert result["decision"] == "PASS" - assert float(result["confidence"]) == pytest.approx(0.91) - - -@pytest.mark.parametrize( - ("score", "expected"), - [ - (75.0, "PASS"), - (60.0, "REVIEW"), - (59.99, "FAIL"), - ], -) -def test_decision_threshold_boundaries(score: float, expected: str) -> None: - assert _classify_score(score) == expected - - -@given(st.floats(min_value=0.0, max_value=100.0, allow_nan=False, allow_infinity=False)) -def test_decision_threshold_property(score: float) -> None: - decision = _classify_score(score) - if score >= PASS_THRESHOLD: - assert decision == "PASS" - elif score >= REVIEW_THRESHOLD: - assert decision == "REVIEW" - else: - assert decision == "FAIL" - - -def test_decision_failure_missing_score() -> None: - result = decision_agent({"application_id": "app-decision-2", "errors": [], "audit_log": []}) - assert result["status"] == "failed" - assert result["errors"] - - -def test_decision_uses_fallback_on_invalid_model_output( - monkeypatch: pytest.MonkeyPatch, decision_state: dict[str, object] -) -> None: - monkeypatch.setattr("app.agents.decision_agent.generate_json_response", lambda **_kwargs: "invalid-json") - result = decision_agent(decision_state) - assert result["status"] == "decided" - assert result["decision"] == "PASS" - assert "threshold" in str(result["decision_reason"]).lower() diff --git a/tests/test_evaluation_agent.py b/tests/test_evaluation_agent.py deleted file mode 100644 index f196889..0000000 --- a/tests/test_evaluation_agent.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import annotations - -import json - -import pytest - -from app.agents.evaluation_agent import _build_evaluation_prompt, evaluation_agent - - -@pytest.fixture -def evaluation_state() -> dict[str, object]: - return { - "application_id": "app-eval-1", - "extracted_json": { - "name": "Jane Doe", - "email": "jane@example.com", - "skills": ["Python", "SQL"], - "experience": [{"title": "Engineer", "company": "Acme", "duration": "3 years"}], - "education": [{"degree": "BSc", "institution": "State U", "year": "2020"}], - "other_details": [], - }, - "errors": [], - "audit_log": [], - } - - -def test_evaluation_success(monkeypatch: pytest.MonkeyPatch, evaluation_state: dict[str, object]) -> None: - monkeypatch.setattr( - "app.agents.evaluation_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "evaluation_score": 82.0, - "evaluation_reasoning": "Strong technical match with relevant experience.", - } - ), - ) - - result = evaluation_agent(evaluation_state) - assert result["status"] == "evaluated" - assert result["evaluation_score"] == 82.0 - assert "technical" in str(result["evaluation_reasoning"]).lower() - - -def test_evaluation_edge_zero_score( - monkeypatch: pytest.MonkeyPatch, evaluation_state: dict[str, object] -) -> None: - monkeypatch.setattr( - "app.agents.evaluation_agent.generate_json_response", - lambda **_kwargs: json.dumps( - {"evaluation_score": 0.0, "evaluation_reasoning": "No qualifying evidence was provided."} - ), - ) - - result = evaluation_agent(evaluation_state) - assert result["evaluation_score"] == 0.0 - - -def test_evaluation_failure_missing_extracted_json() -> None: - result = evaluation_agent({"application_id": "app-eval-2", "errors": [], "audit_log": []}) - assert result["status"] == "failed" - assert result["errors"] - - -def test_evaluation_prompt_includes_schema() -> None: - prompt = _build_evaluation_prompt({"extracted_json": {"name": "Jane"}}) - assert '"evaluation_score"' in prompt - assert '"evaluation_reasoning"' in prompt - - -def test_evaluation_masks_email_in_audit_log( - monkeypatch: pytest.MonkeyPatch, evaluation_state: dict[str, object] -) -> None: - monkeypatch.setattr( - "app.agents.evaluation_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "evaluation_score": 65.0, - "evaluation_reasoning": "Reach out to jane@example.com for verification.", - } - ), - ) - - result = evaluation_agent(evaluation_state) - entry = result["audit_log"][0] - output_summary = str(entry.get("output_summary", "")) - assert "jane@" not in output_summary - assert "****@" in output_summary diff --git a/tests/test_notification_agent.py b/tests/test_notification_agent.py deleted file mode 100644 index 435beec..0000000 --- a/tests/test_notification_agent.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -import json - -import pytest - -from app.agents.notification_agent import _build_notification_prompt, notification_agent - - -@pytest.fixture -def notification_state() -> dict[str, object]: - return { - "application_id": "app-notify-1", - "decision": "PASS", - "decision_reason": "Strong score.", - "report_applicant": "You passed.", - "extracted_json": {"name": "Jane Doe", "email": "jane@example.com"}, - "errors": [], - "audit_log": [], - } - - -def test_notification_success( - monkeypatch: pytest.MonkeyPatch, notification_state: dict[str, object] -) -> None: - class DummySender: - @staticmethod - def invoke(*_args: object, **_kwargs: object) -> dict[str, str]: - return {"status": "sent"} - - monkeypatch.setattr( - "app.agents.notification_agent.generate_json_response", - lambda **_kwargs: json.dumps( - {"subject": "Application outcome", "body": "You have passed the initial screening."} - ), - ) - monkeypatch.setattr("app.agents.notification_agent.send_notification_tool", DummySender()) - - result = notification_agent(notification_state) - assert result["status"] == "completed" - assert result["notification_status"] == "sent" - - -def test_notification_edge_invalid_email(notification_state: dict[str, object]) -> None: - state = {**notification_state, "extracted_json": {"name": "Jane Doe", "email": "bad-email"}} - result = notification_agent(state) - assert result["status"] == "completed" - assert result["notification_status"] == "failed" - assert result["errors"] - - -def test_notification_failure_missing_decision() -> None: - result = notification_agent({"application_id": "app-notify-2", "errors": [], "audit_log": []}) - assert result["status"] == "failed" - assert result["errors"] - - -def test_notification_prompt_contract() -> None: - prompt = _build_notification_prompt({"decision": "REVIEW"}, "candidate@example.com") - assert '"subject"' in prompt - assert '"body"' in prompt - - -def test_notification_masks_email_in_audit_log( - monkeypatch: pytest.MonkeyPatch, notification_state: dict[str, object] -) -> None: - class DummySender: - @staticmethod - def invoke(*_args: object, **_kwargs: object) -> dict[str, str]: - return {"status": "sent"} - - monkeypatch.setattr( - "app.agents.notification_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "subject": "Contact jane@example.com", - "body": "Reply to jane@example.com for details.", - } - ), - ) - monkeypatch.setattr("app.agents.notification_agent.send_notification_tool", DummySender()) - - result = notification_agent(notification_state) - output_summary = str(result["audit_log"][0].get("output_summary", "")) - assert "jane@" not in output_summary diff --git a/tests/test_report_agent.py b/tests/test_report_agent.py deleted file mode 100644 index f484e10..0000000 --- a/tests/test_report_agent.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from app.agents.report_agent import _build_report_prompt, report_agent - - -@pytest.fixture -def report_state() -> dict[str, object]: - return { - "application_id": "app-report-1", - "decision": "REVIEW", - "decision_reason": "Needs additional verification.", - "evaluation_score": 68.0, - "evaluation_reasoning": "Good baseline with some gaps.", - "extracted_json": { - "name": "Jane Doe", - "email": "jane@example.com", - "skills": ["Python"], - "other_details": ["AWS Certified"], - }, - "errors": [], - "audit_log": [], - } - - -def test_report_success(monkeypatch: pytest.MonkeyPatch, report_state: dict[str, object]) -> None: - class DummyWriter: - @staticmethod - def invoke(*_args: object, **_kwargs: object) -> dict[str, object]: - return {} - - monkeypatch.setattr( - "app.agents.report_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "report_applicant": "# Applicant Report\n\nYour application is under review.", - "report_internal": "# Internal Report\n\n- Decision: REVIEW", - } - ), - ) - monkeypatch.setattr("app.agents.report_agent.write_reports_tool", DummyWriter()) - - result = report_agent(report_state) - assert result["status"] == "reported" - assert "# Applicant Report" in str(result["report_applicant"]) - assert "# Internal Report" in str(result["report_internal"]) - - -def test_report_writes_markdown_files( - monkeypatch: pytest.MonkeyPatch, report_state: dict[str, object], tmp_path: Path -) -> None: - class DummySettings: - ollama_base_url = "http://localhost:11434" - report_model = "gemma3:1b-it-q4_K_M" - ollama_timeout_seconds = 30.0 - reports_dir = str(tmp_path) - - monkeypatch.setattr("app.agents.report_agent.get_settings", lambda: DummySettings()) - monkeypatch.setattr( - "app.agents.report_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "report_applicant": "# Applicant Report\n\nHello.", - "report_internal": "# Internal Report\n\nDetails.", - } - ), - ) - - result = report_agent({**report_state, "application_id": "app-report-2"}) - assert result["status"] == "reported" - assert (tmp_path / "app-report-2_applicant.md").exists() - assert (tmp_path / "app-report-2_internal.md").exists() - - -def test_report_failure_missing_decision() -> None: - result = report_agent({"application_id": "app-report-3", "errors": [], "audit_log": []}) - assert result["status"] == "failed" - assert result["errors"] - - -def test_report_prompt_includes_markdown_contract() -> None: - prompt = _build_report_prompt({"decision": "PASS"}) - assert "markdown" in prompt.lower() - assert '"report_applicant"' in prompt - assert '"report_internal"' in prompt - - -def test_report_masks_email_in_audit_log( - monkeypatch: pytest.MonkeyPatch, report_state: dict[str, object] -) -> None: - class DummyWriter: - @staticmethod - def invoke(*_args: object, **_kwargs: object) -> dict[str, object]: - return {} - - monkeypatch.setattr( - "app.agents.report_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "report_applicant": "Please contact jane@example.com for follow-up.", - "report_internal": "Internal note with jane@example.com reference.", - } - ), - ) - monkeypatch.setattr("app.agents.report_agent.write_reports_tool", DummyWriter()) - - result = report_agent(report_state) - output_summary = str(result["audit_log"][0].get("output_summary", "")) - assert "jane@" not in output_summary - assert "****@" in output_summary diff --git a/tests/test_workflow.py b/tests/test_workflow.py index cf12004..76a0317 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -7,11 +7,6 @@ def test_workflow_runs_with_stubbed_agents(monkeypatch, tmp_path: Path) -> None: - class DummyWriter: - @staticmethod - def invoke(*_args: object, **_kwargs: object) -> dict[str, object]: - return {} - candidate_file = tmp_path / "candidate.txt" candidate_file.write_text("Test Candidate", encoding="utf-8") monkeypatch.setattr("app.agents.extraction_agent.update_application", lambda *_args, **_kwargs: None) @@ -30,43 +25,6 @@ def invoke(*_args: object, **_kwargs: object) -> dict[str, object]: } ), ) - monkeypatch.setattr( - "app.agents.evaluation_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "evaluation_score": 72.0, - "evaluation_reasoning": "Candidate has solid fundamentals and moderate experience.", - } - ), - ) - monkeypatch.setattr( - "app.agents.decision_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "decision_reason": "Score lands in review range based on thresholds.", - "confidence": 0.7, - } - ), - ) - monkeypatch.setattr( - "app.agents.report_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "report_applicant": "# Applicant Report\n\nYour application is under review.", - "report_internal": "# Internal Report\n\n- Decision: REVIEW", - } - ), - ) - monkeypatch.setattr("app.agents.report_agent.write_reports_tool", DummyWriter()) - monkeypatch.setattr( - "app.agents.notification_agent.generate_json_response", - lambda **_kwargs: json.dumps( - { - "subject": "Application update", - "body": "Your application is currently under review.", - } - ), - ) workflow = build_workflow() state = { From 6ba7a00b7f9e81c4db9201436a7c6df7eb476b86 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:09:48 +0000 Subject: [PATCH 4/7] feat(extraction): add pymupdf4llm for PDF parsing Agent-Logs-Url: https://github.com/nmdra/AgentHire/sessions/9aa47908-a01a-4b1e-a3f4-ec74a9327676 Co-authored-by: nmdra <73674803+nmdra@users.noreply.github.com> --- app/tools/parse_pdf.py | 5 ++--- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/tools/parse_pdf.py b/app/tools/parse_pdf.py index e0376aa..ca5aafa 100644 --- a/app/tools/parse_pdf.py +++ b/app/tools/parse_pdf.py @@ -4,8 +4,8 @@ from pathlib import Path -import fitz from langchain.tools import tool +import pymupdf4llm @tool @@ -29,8 +29,7 @@ def parse_pdf_tool(path: str) -> str: if not file_path.exists(): raise FileNotFoundError(f"File does not exist: {path}") - with fitz.open(path) as document: - text = "\n".join(page.get_text("text") for page in document) + text = str(pymupdf4llm.to_markdown(path)) cleaned = text.strip() if not cleaned: diff --git a/pyproject.toml b/pyproject.toml index 5349c60..969df35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "python-dotenv>=1.0.1", "httpx>=0.27.0", "pymupdf>=1.24.0", + "pymupdf4llm>=1.27.2.1", ] [project.optional-dependencies] From c68874585d574453af453dc136f9b9a3724382b6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:17:51 +0000 Subject: [PATCH 5/7] fix(extraction): apply PR review thread feedback Agent-Logs-Url: https://github.com/nmdra/AgentHire/sessions/db48781c-bc90-4b92-85c8-17294dc3126b Co-authored-by: nmdra <73674803+nmdra@users.noreply.github.com> --- app/agents/extraction_agent.py | 24 +++++++-- app/tools/validate_extraction.py | 8 ++- tests/test_extraction_agent.py | 85 ++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 5 deletions(-) diff --git a/app/agents/extraction_agent.py b/app/agents/extraction_agent.py index f73effa..28e6ae9 100644 --- a/app/agents/extraction_agent.py +++ b/app/agents/extraction_agent.py @@ -22,6 +22,23 @@ MAX_INPUT_CHARS = 32000 +def _strip_markdown_json_fences(text: str) -> str: + stripped = text.strip() + if not stripped.startswith("```"): + return stripped + + lines = stripped.splitlines() + if not lines: + return stripped + if lines[-1].strip() != "```": + return stripped + + body = lines[1:-1] + if lines[0].strip().lower() in {"```json", "```"}: + return "\n".join(body).strip() + return stripped + + def _read_input(file_path: str) -> str: extension = Path(file_path).suffix.lower() if extension == ".pdf": @@ -48,8 +65,8 @@ def _build_extraction_prompt(raw_text: str, correction_error: str | None = None) ' "phone": string or null,\n' ' "website": string or null,\n' ' "skills": [string, ...],\n' - ' "experience": [{"title": string, "company": string, "duration": string}, ...],\n' - ' "education": [{"degree": string, "institution": string, "year": string}, ...],\n' + ' "experience": [{"title": string or null, "company": string or null, "duration": string or null}, ...],\n' + ' "education": [{"degree": string or null, "institution": string or null, "year": string or null}, ...],\n' ' "other_details": [string, ...]\n' '}' ) @@ -73,11 +90,10 @@ def _extract_with_retry( prompt=_build_extraction_prompt(raw_text, correction_error=error), temperature=0.0, top_p=0.1, - stop=["```"], timeout_seconds=timeout_seconds, ) try: - payload = json.loads(response_text) + payload = json.loads(_strip_markdown_json_fences(response_text)) validated = CandidateExtraction.model_validate(payload) return validated.model_dump() except (json.JSONDecodeError, ValidationError) as exc: diff --git a/app/tools/validate_extraction.py b/app/tools/validate_extraction.py index e561537..4a203f2 100644 --- a/app/tools/validate_extraction.py +++ b/app/tools/validate_extraction.py @@ -2,12 +2,14 @@ from __future__ import annotations -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class ExperienceEntry(BaseModel): """Structured experience entry extracted from a candidate profile.""" + model_config = ConfigDict(extra="forbid") + title: str | None = Field(default=None) company: str | None = Field(default=None) duration: str | None = Field(default=None) @@ -16,6 +18,8 @@ class ExperienceEntry(BaseModel): class EducationEntry(BaseModel): """Structured education entry extracted from a candidate profile.""" + model_config = ConfigDict(extra="forbid") + degree: str | None = Field(default=None) institution: str | None = Field(default=None) year: str | None = Field(default=None) @@ -24,6 +28,8 @@ class EducationEntry(BaseModel): class CandidateExtraction(BaseModel): """Structured extraction output from extraction agent.""" + model_config = ConfigDict(extra="forbid") + name: str | None = Field(default=None) email: str | None = Field(default=None) phone: str | None = Field(default=None) diff --git a/tests/test_extraction_agent.py b/tests/test_extraction_agent.py index 9f9614b..f8db26f 100644 --- a/tests/test_extraction_agent.py +++ b/tests/test_extraction_agent.py @@ -127,3 +127,88 @@ def test_missing_optional_fields_default_to_null( assert extracted["experience"] == [] assert extracted["education"] == [] assert extracted["other_details"] == [] + + +def test_extraction_accepts_fenced_json( + monkeypatch: pytest.MonkeyPatch, base_state: dict[str, object] +) -> None: + monkeypatch.setattr("app.agents.extraction_agent.update_application", lambda *_args, **_kwargs: None) + call_count = {"count": 0} + valid_payload = { + "name": "Jane Doe", + "email": "jane@example.com", + "phone": None, + "website": None, + "skills": ["Python"], + "experience": [{"title": "Engineer", "company": None, "duration": "3 years"}], + "education": [], + "other_details": [], + } + + def mock_generate_fenced(**_kwargs: object) -> str: + call_count["count"] += 1 + return f"```json\n{json.dumps(valid_payload)}\n```" + + monkeypatch.setattr("app.agents.extraction_agent.generate_json_response", mock_generate_fenced) + + result = extraction_agent(base_state) + + assert result["status"] == "extracted" + assert result["extracted_json"]["email"] == "jane@example.com" + assert call_count["count"] == 1 + + +def test_extraction_retries_when_extra_fields_present( + monkeypatch: pytest.MonkeyPatch, base_state: dict[str, object] +) -> None: + monkeypatch.setattr("app.agents.extraction_agent.update_application", lambda *_args, **_kwargs: None) + call_count = {"count": 0} + responses = iter([ + json.dumps( + { + "name": "Jane Doe", + "email": "jane@example.com", + "phone": None, + "website": None, + "skills": ["Python"], + "experience": [ + { + "title": "Engineer", + "company": "Acme", + "duration": "3 years", + "extra_nested_key": "unexpected", + } + ], + "education": [], + "other_details": [], + "extra_key": "unexpected", + } + ), + json.dumps( + { + "name": "Jane Doe", + "email": "jane@example.com", + "phone": None, + "website": None, + "skills": ["Python"], + "experience": [{"title": "Engineer", "company": "Acme", "duration": "3 years"}], + "education": [], + "other_details": [], + } + ), + ]) + + def mock_generate_with_retry(**_kwargs: object) -> str: + call_count["count"] += 1 + return next(responses) + + monkeypatch.setattr( + "app.agents.extraction_agent.generate_json_response", + mock_generate_with_retry, + ) + + result = extraction_agent(base_state) + + assert result["status"] == "extracted" + assert result["extracted_json"]["name"] == "Jane Doe" + assert call_count["count"] == 2 From 9bef4c9fb892821f80e8c5989481860ee17e6dcc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:19:01 +0000 Subject: [PATCH 6/7] chore(extraction): add helper docstring and test naming cleanup Agent-Logs-Url: https://github.com/nmdra/AgentHire/sessions/db48781c-bc90-4b92-85c8-17294dc3126b Co-authored-by: nmdra <73674803+nmdra@users.noreply.github.com> --- app/agents/extraction_agent.py | 12 ++++++++++++ tests/test_extraction_agent.py | 6 ++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/app/agents/extraction_agent.py b/app/agents/extraction_agent.py index 28e6ae9..9877b1a 100644 --- a/app/agents/extraction_agent.py +++ b/app/agents/extraction_agent.py @@ -23,6 +23,18 @@ def _strip_markdown_json_fences(text: str) -> str: + """Strip wrapping markdown JSON fences from model output. + + Args: + text: Raw model output that may include fenced JSON. + + Returns: + The unfenced JSON string when standard markdown fences are present, + otherwise the original trimmed text. + + Example: + _strip_markdown_json_fences("```json\\n{\\"name\\": \\"A\\"}\\n```") + """ stripped = text.strip() if not stripped.startswith("```"): return stripped diff --git a/tests/test_extraction_agent.py b/tests/test_extraction_agent.py index f8db26f..1d1a689 100644 --- a/tests/test_extraction_agent.py +++ b/tests/test_extraction_agent.py @@ -145,7 +145,8 @@ def test_extraction_accepts_fenced_json( "other_details": [], } - def mock_generate_fenced(**_kwargs: object) -> str: + def mock_generate_fenced(**kwargs: object) -> str: + _ = kwargs call_count["count"] += 1 return f"```json\n{json.dumps(valid_payload)}\n```" @@ -198,7 +199,8 @@ def test_extraction_retries_when_extra_fields_present( ), ]) - def mock_generate_with_retry(**_kwargs: object) -> str: + def mock_generate_with_retry(**kwargs: object) -> str: + _ = kwargs call_count["count"] += 1 return next(responses) From 4b740c199bfc99d2fe89a68a6d3a2d0d8e6dd2d6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:20:34 +0000 Subject: [PATCH 7/7] Changes before error encountered Agent-Logs-Url: https://github.com/nmdra/AgentHire/sessions/db48781c-bc90-4b92-85c8-17294dc3126b Co-authored-by: nmdra <73674803+nmdra@users.noreply.github.com> --- app/agents/extraction_agent.py | 1 + tests/test_extraction_agent.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/agents/extraction_agent.py b/app/agents/extraction_agent.py index 9877b1a..89e64c9 100644 --- a/app/agents/extraction_agent.py +++ b/app/agents/extraction_agent.py @@ -34,6 +34,7 @@ def _strip_markdown_json_fences(text: str) -> str: Example: _strip_markdown_json_fences("```json\\n{\\"name\\": \\"A\\"}\\n```") + '{"name": "A"}' """ stripped = text.strip() if not stripped.startswith("```"): diff --git a/tests/test_extraction_agent.py b/tests/test_extraction_agent.py index 1d1a689..aa1c124 100644 --- a/tests/test_extraction_agent.py +++ b/tests/test_extraction_agent.py @@ -146,7 +146,6 @@ def test_extraction_accepts_fenced_json( } def mock_generate_fenced(**kwargs: object) -> str: - _ = kwargs call_count["count"] += 1 return f"```json\n{json.dumps(valid_payload)}\n```" @@ -200,7 +199,6 @@ def test_extraction_retries_when_extra_fields_present( ]) def mock_generate_with_retry(**kwargs: object) -> str: - _ = kwargs call_count["count"] += 1 return next(responses)