{ "leaderboard": { "filters": { "promptStrategies": [ { "id": "zeroshot", "label": "Zero shot" }, { "id": "general", "label": "General" } ], "observationModes": [ { "id": "axtree", "label": "AX Tree", "description": "Agent observes the accessibility tree representation of web pages" }, { "id": "screenshot", "label": "Screenshot Only", "description": "Agent observes screenshots of the portal interface" } ] } }, "results": { "radar": { "axes": [ { "id": "subtask_1", "label": "Subtask 1", "source": { "type": "domain", "key": "prior_auth" } }, { "id": "subtask_2", "label": "Subtask 2", "source": { "type": "domain", "key": "dme" } }, { "id": "subtask_3", "label": "Subtask 3", "source": { "type": "domain", "key": "appeals" } }, { "id": "subtask_4", "label": "Subtask 4", "source": { "type": "difficulty", "key": "easy" } }, { "id": "subtask_5", "label": "Subtask 5", "source": { "type": "difficulty", "key": "medium" } }, { "id": "subtask_6", "label": "Subtask 6", "source": { "type": "difficulty", "key": "hard" } } ], "defaultModelIds": [ "claude-opus-4-5", "gpt-5", "gemini-3" ] }, "scatter": { "defaultX": "overall", "defaultY": "prior_auth", "availableMetrics": [ "overall", "prior_auth", "dme", "appeals", "avgSteps", "avgTimeSec", "easy", "medium", "hard" ] } }, "environments": [ { "id": "emr-prior-auth-view", "category": "EMR", "title": "EMR Prior Auth View", "description": "EMR prior auth workqueue", "imageLabel": "Prior Auth", "previewImage": "assets/emr-prior-auth-view.png", "url": "https://emrportal.vercel.app/emr/worklist", "access": { "type": "open", "username": null, "password": null, "display": "N/A" }, "status": { "label": "Live", "state": "live" } }, { "id": "emr-denials-view", "category": "EMR", "title": "EMR Denials View", "description": "EMR denials workqueue", "imageLabel": "Denials", "previewImage": "assets/emr-denials-view.png", "url": "https://emrportal.vercel.app/emr/denied", "access": { "type": "open", "username": null, "password": null, "display": "N/A" }, "status": { "label": "Live", "state": "live" } }, { "id": "emr-dme-view", "category": "EMR", "title": "EMR DME View", "description": "EMR DME workqueue", "imageLabel": "DME", "previewImage": "assets/emr-dme-view.png", "url": "https://emrportal.vercel.app/emr/dme", "access": { "type": "open", "username": null, "password": null, "display": "N/A" }, "status": { "label": "Live", "state": "live" } }, { "id": "efax", "category": "eFax", "title": "eFax", "description": "eFax", "imageLabel": "eFax", "previewImage": "assets/eFax.png", "url": "https://emrportal.vercel.app/fax-portal", "access": { "type": "open", "username": null, "password": null, "display": "N/A" }, "status": { "label": "Live", "state": "live" } }, { "id": "payer-a-portal", "category": "Payer Portal", "title": "Payer A Portal", "description": "Payer A portal (purple)", "imageLabel": "Payer A", "previewImage": "assets/payer-a-portal.png", "url": "https://emrportal.vercel.app/payer-a", "access": { "type": "demo_login", "username": "provider@payera.com", "password": "demo123", "display": "provider@payera.com / demo123" }, "status": { "label": "Live", "state": "live" } }, { "id": "payer-b-portal", "category": "Payer Portal", "title": "Payer B Portal", "description": "Payer B portal (blue)", "imageLabel": "Payer B", "previewImage": "assets/payer-b-portal.png", "url": "https://emrportal.vercel.app/payer-b", "access": { "type": "demo_login", "username": "provider@payerb.com", "password": "demo123", "display": "provider@payerb.com / demo123" }, "status": { "label": "Live", "state": "live" } } ], "taskExplorer": { "groups": [ { "id": "prior_auth", "name": "Prior Authorization", "description": "Referral review and prior auth determination tasks from the EMR portal.", "tasks": [ { "id": "emr-easy-1", "title": "Medicare Advantage - No Auth Required", "instruction": "Open referral REF-2025-002 for Smith, Emily (67F with Santa Clara Family Health Plan - Medicare Advantage). Determine whether the payer requires prior authorization for this eye follow-up visit. Document your determination, then clear the referral from the worklist.", "goal": "Open referral REF-2025-002 for Smith, Emily (67F with Santa Clara Family Health Plan - Medicare Advantage). Determine whether the payer requires prior authorization for this eye follow-up visit. Document your determination, then clear the referral from the worklist.", "difficulty": "easy", "category": "no_auth_medicare", "challengeType": "no_auth_medicare", "possible": true, "points": 4, "configTaskId": "easy_1", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Smith, Emily" }, "stepByStep": [ "1. Click on referral REF-2025-002 in the worklist to open it", "2. Review patient demographics and insurance information (Medicare Advantage)", "3. Click on Coverage tab to check insurance details", "4. Verify that Medicare Advantage does not require prior auth for eye follow-up visits", "5. Click Add Note button in Communications section", "6. Document that no prior authorization is needed due to Medicare Advantage policy", "7. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-1-eval-1", "type": "jmespath", "category": "Documentation", "description": "Agent added authorization note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-1-eval-2", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-002", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-002')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-1-eval-3", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Agent correctly identified no authorization was needed", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent correctly identify that no authorization was needed for this referral? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "verification note" }, { "id": "emr-easy-1-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned Medicare as reason for no auth", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent mention Medicare or Medicare Advantage as the reason why no authorization was needed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "verification note" } ], "expectedOutcome": { "auth_required": false, "reason": "Santa Clara Family Health Plan Medicare Advantage does not require prior authorization for eye follow-up visit" }, "rationale": "" }, { "id": "emr-easy-2", "title": "Blue Shield PPO - Verify Coverage", "instruction": "Open referral REF-2025-006 for Brown, Robert (73M with Blue Shield PPO). Review the insurance coverage for this vitrectomy procedure. Verify the patient's eligibility is active and documentation is complete. Document your verification findings and clear the referral from the worklist.", "goal": "Open referral REF-2025-006 for Brown, Robert (73M with Blue Shield PPO). Review the insurance coverage for this vitrectomy procedure. Verify the patient's eligibility is active and documentation is complete. Document your verification findings and clear the referral from the worklist.", "difficulty": "easy", "category": "verify_coverage", "challengeType": "verify_coverage", "possible": true, "points": 4, "configTaskId": "easy_2", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Brown, Robert" }, "stepByStep": [ "1. Click on referral REF-2025-006 in the worklist to open it", "2. Review patient demographics (Brown, Robert - 73M)", "3. Click on Coverage tab to verify Blue Shield PPO eligibility", "4. Confirm coverage status is active", "5. Review that documentation is complete for vitrectomy procedure", "6. Click Add Note button in Communications section", "7. Document Blue Shield PPO coverage verification confirmation", "8. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-2-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Coverages tab to verify eligibility", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-2-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-2-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-006", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-006')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-2-eval-4", "type": "llm_judge", "category": "Information Retrieval", "description": "Agent verified Blue Shield PPO coverage is active", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent verify that Blue Shield PPO coverage is active? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "verification note" } ], "expectedOutcome": { "auth_required": true, "verification_type": "coverage_verification", "reason": "Blue Shield PPO coverage verification for vitrectomy with retinal detachment repair" }, "rationale": "" }, { "id": "emr-easy-3", "title": "Verify Insurance Active - Aetna PPO", "instruction": "Open referral REF-2025-001 for Doe, John (60M with Aetna PPO). Verify that the patient's insurance coverage is active. Document your verification and clear the referral from the worklist.", "goal": "Open referral REF-2025-001 for Doe, John (60M with Aetna PPO). Verify that the patient's insurance coverage is active. Document your verification and clear the referral from the worklist.", "difficulty": "easy", "category": "verify_insurance_active", "challengeType": "verify_insurance_active", "possible": true, "points": 4, "configTaskId": "easy_3", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Doe, John" }, "stepByStep": [ "1. Click on referral REF-2025-001 in the worklist to open it", "2. Review patient demographics (Doe, John - 60M)", "3. Click on Insurance tab to check coverage status", "4. Verify that Aetna PPO coverage is active", "5. Click Add Note button in Communications section", "6. Document that insurance is active and coverage verified", "7. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-3-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Coverages tab to verify insurance", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-3-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-3-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-001", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-001')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-3-eval-4", "type": "llm_judge", "category": "Information Retrieval", "description": "Agent verified Aetna PPO coverage is active", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent verify that Aetna PPO insurance coverage is active? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "verification note" } ], "expectedOutcome": { "auth_required": false, "verification_type": "insurance_active", "reason": "Verification task for Aetna PPO - Bilateral Eye Injection" }, "rationale": "" }, { "id": "emr-easy-4", "title": "Cigna PPO - Verify Diagnosis Codes", "instruction": "Open referral REF-2025-005 for Williams, Sarah (55F with Cigna PPO). Verify that the diagnosis codes are present and appropriate for the trabeculectomy procedure. Document your verification and clear the referral from the worklist.", "goal": "Open referral REF-2025-005 for Williams, Sarah (55F with Cigna PPO). Verify that the diagnosis codes are present and appropriate for the trabeculectomy procedure. Document your verification and clear the referral from the worklist.", "difficulty": "easy", "category": "verify_diagnosis_codes", "challengeType": "verify_diagnosis_codes", "possible": true, "points": 5, "configTaskId": "easy_4", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Williams, Sarah" }, "stepByStep": [ "1. Click on referral REF-2025-005 in the worklist to open it", "2. Review patient demographics (Williams, Sarah - 55F)", "3. Click on Diagnoses tab to review ICD-10 codes", "4. Verify diagnosis codes are present and appropriate for trabeculectomy", "5. Click Add Note button in Communications section", "6. Document that diagnosis codes are complete and verified specifically mentioning H40.11X4 and glaucoma", "7. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-4-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Diagnoses tab to verify codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-4-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-4-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-005", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-005')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-4-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned diagnosis code H40.11X4", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the diagnosis code H40.11X4? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-4-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned glaucoma diagnosis", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention glaucoma? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "verification_type": "diagnosis_codes", "reason": "Verification task for Cigna PPO - Trabeculectomy" }, "rationale": "" }, { "id": "emr-easy-5", "title": "Kaiser HMO - Verify Referral Status", "instruction": "Open referral REF-2025-007 for Davis, Jennifer (57F with Kaiser Permanente HMO). Verify that the insurance coverage is active and that the existing authorization is valid. Document your verification and clear the referral from the worklist.", "goal": "Open referral REF-2025-007 for Davis, Jennifer (57F with Kaiser Permanente HMO). Verify that the insurance coverage is active and that the existing authorization is valid. Document your verification and clear the referral from the worklist.", "difficulty": "easy", "category": "verify_hmo_referral", "challengeType": "verify_hmo_referral", "possible": true, "points": 5, "configTaskId": "easy_5", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Davis, Jennifer" }, "stepByStep": [ "1. Click on referral REF-2025-007 in the worklist to open it", "2. Review patient demographics (Davis, Jennifer - 57F)", "3. Click on Coverage tab to review Kaiser Permanente HMO status", "4. Verify referral status is active for corneal transplant", "5. Confirm all required information is present", "6. Click Add Note button in Communications section", "7. Document verification status and findings", "8. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-5-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Coverages tab to verify HMO insurance status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-5-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-5-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-007", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-007')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-5-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent confirmed authorization is active", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note confirm that the authorization is active? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-5-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned the auth number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent mention the auth number AUTH-KP-2025-12345 in their note? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": true, "verification_type": "referral_status", "reason": "Kaiser HMO referral verification for corneal transplant" }, "rationale": "" }, { "id": "emr-easy-6", "title": "Anthem Blue Cross PPO - Verify DME Authorization", "instruction": "Open referral REF-2025-201 for Patterson, Margaret (77F with Anthem Blue Cross PPO). Verify that the insurance coverage is active and that the existing authorization for the power wheelchair is valid. Document your verification and clear the referral from the worklist.", "goal": "Open referral REF-2025-201 for Patterson, Margaret (77F with Anthem Blue Cross PPO). Verify that the insurance coverage is active and that the existing authorization for the power wheelchair is valid. Document your verification and clear the referral from the worklist.", "difficulty": "easy", "category": "verify_coverage", "challengeType": "verify_coverage", "possible": true, "points": 5, "configTaskId": "easy_6", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Patterson, Margaret" }, "stepByStep": [ "1. Click on referral REF-2025-201 in the worklist to open it", "2. Review patient demographics (Patterson, Margaret - 77F)", "3. Click on Coverages tab to verify Anthem Blue Cross PPO coverage", "4. Confirm coverage status is active and authorization is valid", "5. Verify documentation is complete for power wheelchair", "6. Click Add Note button in Communications section", "7. Document coverage verification and authorization confirmation", "8. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-6-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Coverages tab to verify coverage", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-6-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-6-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-201", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-201')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-6-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent confirmed authorization is valid", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note confirm that the authorization is valid? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-6-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned the auth number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent mention the auth number AUTH-MED-2025-54321 in their note? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": true, "verification_type": "coverage_verification", "reason": "Anthem Blue Cross PPO coverage verification for Power Wheelchair Evaluation" }, "rationale": "" }, { "id": "emr-easy-7", "title": "UHC PPO - No Auth for Imaging", "instruction": "Open referral REF-2025-102 for Garcia, Mateo (33M with UnitedHealthcare PPO). Determine whether this plan requires prior authorization for the MRI knee procedure. Document your determination and clear the referral from the worklist.", "goal": "Open referral REF-2025-102 for Garcia, Mateo (33M with UnitedHealthcare PPO). Determine whether this plan requires prior authorization for the MRI knee procedure. Document your determination and clear the referral from the worklist.", "difficulty": "easy", "category": "no_auth_ppo", "challengeType": "no_auth_ppo", "possible": true, "points": 4, "configTaskId": "easy_7", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Garcia, Mateo" }, "stepByStep": [ "1. Click on referral REF-2025-102 in the worklist to open it", "2. Review patient demographics (Garcia, Mateo - 33M)", "3. Click on Coverage tab to check authorization requirements", "4. Find that priorAuthRequired is false for this plan/service", "5. Click Add Note button in Communications section", "6. Document that no prior authorization is needed per plan lookup", "7. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-7-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Coverages tab to check auth requirements", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-7-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added determination note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-7-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-102", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-102')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-7-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent's note indicates no auth required", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note indicate that no prior authorization is required for this procedure? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "reason": "Per UHC provider portal lookup, CPT 73721 (MRI knee) does not require prior authorization for this member's plan" }, "rationale": "" }, { "id": "emr-easy-8", "title": "Verify Conservative Treatment - MISSING Case", "instruction": "Open referral REF-2025-504 for Stevens, Thomas (55M with Aetna HMO). Before proceeding with authorization for the spine MRI, verify that conservative treatment (physical therapy or pain medications) has been documented. If conservative treatment is missing, document this finding and do not clear the referral.", "goal": "Open referral REF-2025-504 for Stevens, Thomas (55M with Aetna HMO). Before proceeding with authorization for the spine MRI, verify that conservative treatment (physical therapy or pain medications) has been documented. If conservative treatment is missing, document this finding and do not clear the referral.", "difficulty": "easy", "category": "verify_missing_conservative", "challengeType": "verify_missing_conservative", "possible": true, "points": 4, "configTaskId": "easy_8", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Stevens, Thomas" }, "stepByStep": [ "1. Click on referral REF-2025-504 in the worklist to open it", "2. Review patient demographics (Stevens, Thomas - 55M)", "3. Click on Clinical Notes tab to review documentation", "4. Identify that the note states 'No physical therapy attempted. No medications tried.'", "5. Recognize this as missing conservative treatment documentation", "6. Click Add Note button in Communications section", "7. Document that conservative treatment is not documented and referral cannot proceed", "8. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-8-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent read clinical note to check for conservative treatment", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-8-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-8-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-504')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-8-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies missing conservative treatment", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention that conservative treatment documentation is missing? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Missing conservative treatment documentation - no PT or medications documented" }, "rationale": "" }, { "id": "emr-easy-9", "title": "Verify Urgency Match - MISMATCH Case", "instruction": "Open referral REF-2025-505 for Turner, Victoria (57F with Blue Shield PPO). Before proceeding with authorization for the cardiac catheterization, verify that the urgency level matches the priority in the Referral tab. If there is an urgency mismatch, document this finding in the Communications tab and do NOT clear the referral.", "goal": "Open referral REF-2025-505 for Turner, Victoria (57F with Blue Shield PPO). Before proceeding with authorization for the cardiac catheterization, verify that the urgency level matches the priority in the Referral tab. If there is an urgency mismatch, document this finding in the Communications tab and do NOT clear the referral.", "difficulty": "easy", "category": "verify_urgency_mismatch", "challengeType": "verify_urgency_mismatch", "possible": true, "points": 5, "configTaskId": "easy_9", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Turner, Victoria" }, "stepByStep": [ "1. Click on referral REF-2025-505 in the worklist to open it", "2. Review patient demographics (Turner, Victoria - 57F)", "3. Note that the referral urgency is marked as 'Routine'", "4. Click on Clinical Notes tab to review documentation", "5. Identify the statement 'URGENT: Acute MI requires emergent cardiac catheterization'", "6. Recognize this as an urgency mismatch", "7. Click Add Note button in Communications section", "8. Document the urgency discrepancy", "9. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-9-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Referral tab to check priority", "points": 1, "query": "signals.clicked_referral_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-9-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent read clinical note to check urgency", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-9-eval-3", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-9-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-505')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-9-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies urgency mismatch", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention an urgency mismatch or discrepancy? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Urgency mismatch - Acute MI marked as Routine instead of Urgent/Emergent" }, "rationale": "" }, { "id": "emr-easy-10", "title": "Aetna PPO - Verify Existing Auth", "instruction": "Open referral REF-2025-004 for Martinez, Carlos (65M with Aetna PPO). This patient may have an existing authorization on file. Check the authorization status in EMR and review the letter of medical necessity to verify the authorization details. If authorized, document the auth number and clear the referral from the worklist.", "goal": "Open referral REF-2025-004 for Martinez, Carlos (65M with Aetna PPO). This patient may have an existing authorization on file. Check the authorization status in EMR and review the letter of medical necessity to verify the authorization details. If authorized, document the auth number and clear the referral from the worklist.", "difficulty": "easy", "category": "existing_auth", "challengeType": "existing_auth", "possible": true, "points": 4, "configTaskId": "easy_10", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Martinez, Carlos" }, "stepByStep": [ "1. Click on referral REF-2025-004 in the worklist to open it", "2. Review patient demographics (Martinez, Carlos - 65M)", "3. Check the Authorization Status section in EMR", "4. Go to the General tab and open the authorization letter / letter of medical necessity viewer. You must actually open the document viewer for this step to count.", "5. Note the auth number if authorized (AUTH-BSC-2025-78901)", "6. Click Add Note button in Communications section", "7. Document the existing authorization number and status", "8. Save the note first, verify the note appears in Communications, and only then click Clear from Worklist" ], "evals": [ { "id": "emr-easy-10-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed authorization letter", "points": 1, "query": "signals.viewed_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-10-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added authorization verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-10-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-004", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-004')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-10-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent's note includes auth number AUTH-BSC-2025-78901", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the authorization number AUTH-BSC-2025-78901? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": true, "auth_exists": true, "reason": "Verify existing authorization for Bilateral intravitreal injections" }, "rationale": "" }, { "id": "emr-easy-11", "title": "Verify Appointment Date - PAST Case", "instruction": "Open referral REF-2025-506 for Underwood, William (70M with Medicare Part B). Before proceeding with authorization for the hip replacement, verify that the appointment date is valid for scheduling. Document your findings and take appropriate action.", "goal": "Open referral REF-2025-506 for Underwood, William (70M with Medicare Part B). Before proceeding with authorization for the hip replacement, verify that the appointment date is valid for scheduling. Document your findings and take appropriate action.", "difficulty": "easy", "category": "verify_appointment_past", "challengeType": "verify_appointment_past", "possible": true, "points": 3, "configTaskId": "easy_11", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Underwood, William" }, "stepByStep": [ "1. Click on referral REF-2025-506 in the worklist to open it", "2. Review patient demographics (Underwood, William - 70M)", "3. Check the appointment date field", "4. Identify that the date (2024-01-15) is in the past", "5. Review clinical notes to confirm 'NOTE: Appointment date has passed'", "6. Click Add Note button in Communications section", "7. Document that the appointment date is invalid (in the past)", "8. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-11-eval-1", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-11-eval-2", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-506')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-11-eval-3", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies past appointment date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention that the appointment date is in the past? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Appointment date January 15, 2024 is in the past" }, "rationale": "" }, { "id": "emr-easy-12", "title": "Verify CPT Codes - Rheumatology", "instruction": "Open referral REF-2025-302 for Baker, Carol (55F with Aetna PPO). Verify that the CPT/procedure codes are present for the infliximab infusion. Document your verification and clear the referral from the worklist.", "goal": "Open referral REF-2025-302 for Baker, Carol (55F with Aetna PPO). Verify that the CPT/procedure codes are present for the infliximab infusion. Document your verification and clear the referral from the worklist.", "difficulty": "easy", "category": "verify_cpt_codes", "challengeType": "verify_cpt_codes", "possible": true, "points": 5, "configTaskId": "easy_12", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Baker, Carol" }, "stepByStep": [ "1. Click on referral REF-2025-302 in the worklist to open it", "2. Review patient demographics (Baker, Carol - 55F)", "3. Click on Services tab to review CPT codes", "4. Verify procedure codes are present for infliximab infusion", "5. Confirm codes are appropriate (J1745, 96413)", "6. Click Add Note button in Communications section", "7. Document that procedure codes are complete and verified", "8. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-12-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Services tab to check CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-12-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-12-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-302", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-302')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-12-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned CPT code J1745", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the CPT code J1745? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-12-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned CPT code 96413", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the CPT code 96413? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "verification_type": "cpt_codes", "reason": "Verification task for Aetna PPO - Infliximab Infusion" }, "rationale": "" }, { "id": "emr-easy-13", "title": "Verify Referral Complete - Spine", "instruction": "Open referral REF-2025-303 for Cooper, David (57M with Aetna PPO). Verify that the referral documentation is complete including diagnosis codes, procedure codes, and clinical notes. Document your verification and clear the referral from the worklist.", "goal": "Open referral REF-2025-303 for Cooper, David (57M with Aetna PPO). Verify that the referral documentation is complete including diagnosis codes, procedure codes, and clinical notes. Document your verification and clear the referral from the worklist.", "difficulty": "easy", "category": "verify_referral_complete", "challengeType": "verify_referral_complete", "possible": true, "points": 7, "configTaskId": "easy_13", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Cooper, David" }, "stepByStep": [ "1. Click on referral REF-2025-303 in the worklist to open it", "2. Review patient demographics (Cooper, David - 57M)", "3. Click on Diagnoses tab to verify ICD-10 codes are present", "4. Click on Services tab to verify CPT codes are present", "5. **IMPORTANT**: Click to view the clinical note to confirm documentation exists", "6. Click Add Note button in Communications section", "7. Document that all documentation is complete and verified", "8. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-13-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Diagnoses tab to verify ICD-10 codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-13-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Services tab to verify CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-13-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent read clinical note", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-13-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-13-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-303", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-303')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-13-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned diagnosis code M54.5", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the diagnosis code M54.5? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-13-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned CPT code 72148", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the CPT code 72148? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "verification_type": "referral_complete", "reason": "Verification task for Aetna PPO - MRI Lumbar Spine" }, "rationale": "" }, { "id": "emr-easy-14", "title": "Verify Referring Provider - MISSING Case", "instruction": "Open referral REF-2025-507 for Vincent, Xavier (45M with Cigna PPO). Before proceeding with authorization for the brain MRI, verify that all required referral fields are complete. Document your findings and take appropriate action.", "goal": "Open referral REF-2025-507 for Vincent, Xavier (45M with Cigna PPO). Before proceeding with authorization for the brain MRI, verify that all required referral fields are complete. Document your findings and take appropriate action.", "difficulty": "easy", "category": "verify_provider_missing", "challengeType": "verify_provider_missing", "possible": true, "points": 3, "configTaskId": "easy_14", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Vincent, Xavier" }, "stepByStep": [ "1. Click on referral REF-2025-507 in the worklist to open it", "2. Review patient demographics (Vincent, Xavier - 45M)", "3. Check the provider field in the referral details", "4. Identify that the provider field is empty or blank", "5. (Optional) Review clinical notes to confirm 'ERROR: Missing referring provider'", "6. Click Add Note button in Communications section", "7. Document that the referring provider is missing", "8. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-14-eval-1", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-14-eval-2", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-507')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-14-eval-3", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies missing provider", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention that the referring provider is missing? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Missing referring provider - provider field is empty" }, "rationale": "" }, { "id": "emr-easy-15", "title": "Verify ICD + CPT Codes Complete", "instruction": "Open referral REF-2025-406 for Norton, Oscar (60M with Anthem Blue Cross PPO). Verify that the referral has complete documentation including diagnosis codes and procedure codes. Document your verification and clear the referral from the worklist.", "goal": "Open referral REF-2025-406 for Norton, Oscar (60M with Anthem Blue Cross PPO). Verify that the referral has complete documentation including diagnosis codes and procedure codes. Document your verification and clear the referral from the worklist.", "difficulty": "easy", "category": "documentation_verification", "challengeType": "documentation_verification", "possible": true, "points": 7, "configTaskId": "easy_15", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Norton, Oscar" }, "stepByStep": [ "1. Click on referral REF-2025-406 in the worklist to open it", "2. Review patient demographics (Norton, Oscar - 60M)", "3. Click on Diagnoses tab to verify ICD-10 codes are present", "4. Click on Services tab to verify CPT codes are present", "5. Confirm both diagnosis and procedure codes are complete", "6. Click Add Note button in Communications section", "7. Document that ICD-10 and CPT codes are complete", "8. Save the note and click Clear from Worklist" ], "evals": [ { "id": "emr-easy-15-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Diagnoses tab to verify ICD-10 codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-15-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Services tab to verify CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-15-eval-3", "type": "jmespath", "category": "Documentation", "description": "Agent added verification note", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-15-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared referral REF-2025-406", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-406')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-15-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned diagnosis code M54.16", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the diagnosis code M54.16? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-15-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned diagnosis code M51.16", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the diagnosis code M51.16? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-15-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned CPT code 62323", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the CPT code 62323? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "verification_type": "documentation_verification", "reason": "Verification task for Anthem Blue Cross PPO - Epidural Steroid Injection" }, "rationale": "" }, { "id": "emr-easy-16", "title": "Verify Insurance Status - INACTIVE Case", "instruction": "Open referral REF-2025-502 for Quinn, Robert (60M with Aetna HMO). Before proceeding with the stress echocardiogram authorization, verify that the patient's insurance coverage status is valid. Document your findings and take appropriate action.", "goal": "Open referral REF-2025-502 for Quinn, Robert (60M with Aetna HMO). Before proceeding with the stress echocardiogram authorization, verify that the patient's insurance coverage status is valid. Document your findings and take appropriate action.", "difficulty": "easy", "category": "verify_insurance_inactive", "challengeType": "verify_insurance_inactive", "possible": true, "points": 5, "configTaskId": "easy_16", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Quinn, Robert" }, "stepByStep": [ "1. Click on referral REF-2025-502 in the worklist to open it", "2. Review patient demographics (Quinn, Robert - 60M)", "3. Click on Coverage or Insurance tab to check coverage status", "4. Identify that the Aetna HMO insurance status is INACTIVE", "5. Click Add Note button in Communications section", "6. Document that insurance is inactive and authorization cannot proceed and mention the termination date", "7. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-16-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Coverages tab to check insurance status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-16-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-16-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-502')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-16-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent identified insurance is inactive", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention that the insurance is inactive or terminated? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-easy-16-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Agent mentioned termination date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention the termination date 12/31/2025? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Insurance coverage is INACTIVE (terminated 12/31/2025) - cannot proceed with authorization" }, "rationale": "" }, { "id": "emr-easy-17", "title": "Verify Laterality Match - MISMATCH Case", "instruction": "Open referral REF-2025-503 for Rogers, Sandra (65F with Anthem Blue Cross PPO). Before proceeding with authorization for the intravitreal injection, verify that the diagnosis and procedure details are consistent. Document your findings and take appropriate action.", "goal": "Open referral REF-2025-503 for Rogers, Sandra (65F with Anthem Blue Cross PPO). Before proceeding with authorization for the intravitreal injection, verify that the diagnosis and procedure details are consistent. Document your findings and take appropriate action.", "difficulty": "easy", "category": "verify_laterality", "challengeType": "verify_laterality", "possible": true, "points": 4, "configTaskId": "easy_17", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Rogers, Sandra" }, "stepByStep": [ "1. Click on referral REF-2025-503 in the worklist to open it", "2. Review patient demographics (Rogers, Sandra - 65F)", "3. Click on Diagnoses tab to check diagnosis laterality", "4. Note that diagnosis H35.3211 is for the RIGHT eye", "5. Click on Services tab to check procedure laterality", "6. Note that procedure (intravitreal injection) is scheduled for LEFT eye (OS)", "7. Identify the laterality mismatch between diagnosis and procedure", "8. Click Add Note button in Communications section", "9. Document the laterality discrepancy", "10. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-17-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Diagnoses tab to check diagnosis laterality", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-17-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-17-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-503')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-17-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies laterality mismatch", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention a laterality mismatch between the diagnosis and procedure? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Laterality mismatch - diagnosis is for RIGHT eye but procedure is scheduled for LEFT eye" }, "rationale": "" }, { "id": "emr-easy-18", "title": "Verify Member ID - INVALID Case", "instruction": "Open referral REF-2025-508 for Walsh, Yvonne (50F with UnitedHealthcare HMO). Before proceeding with authorization for the colonoscopy, verify that the insurance information is complete and valid. Document your findings and take appropriate action.", "goal": "Open referral REF-2025-508 for Walsh, Yvonne (50F with UnitedHealthcare HMO). Before proceeding with authorization for the colonoscopy, verify that the insurance information is complete and valid. Document your findings and take appropriate action.", "difficulty": "easy", "category": "verify_member_id_invalid", "challengeType": "verify_member_id_invalid", "possible": true, "points": 3, "configTaskId": "easy_18", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Walsh, Yvonne" }, "stepByStep": [ "1. Click on referral REF-2025-508 in the worklist to open it", "2. Review patient demographics (Walsh, Yvonne - 50F)", "3. Check the insurance/member ID field", "4. Identify that the member ID shows 'INVALID-TBD' (a placeholder)", "5. Review clinical notes to confirm 'ERROR: Member ID is invalid'", "6. Click Add Note button in Communications section", "7. Document that the member ID is invalid and needs to be corrected", "8. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-18-eval-1", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-18-eval-2", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-508')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-18-eval-3", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies invalid member ID", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention that the member ID is invalid? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Invalid member ID - shows 'INVALID-TBD' placeholder" }, "rationale": "" }, { "id": "emr-easy-19", "title": "Verify Quantity - UNREASONABLE Case", "instruction": "Open referral REF-2025-509 for Young, Zachary (64M with Aetna PPO). Before proceeding with authorization for physical therapy, verify that the service details are correct. Document your findings and take appropriate action.", "goal": "Open referral REF-2025-509 for Young, Zachary (64M with Aetna PPO). Before proceeding with authorization for physical therapy, verify that the service details are correct. Document your findings and take appropriate action.", "difficulty": "easy", "category": "verify_quantity_unreasonable", "challengeType": "verify_quantity_unreasonable", "possible": true, "points": 4, "configTaskId": "easy_19", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Young, Zachary" }, "stepByStep": [ "1. Click on referral REF-2025-509 in the worklist to open it", "2. Review patient demographics (Young, Zachary - 64M)", "3. Click on Services tab to check procedure details", "4. Identify that the quantity is 999 (obviously unreasonable)", "5. Review clinical notes to confirm 'ERROR: Quantity appears incorrect'", "6. Click Add Note button in Communications section", "7. Document that the quantity of 999 appears to be an error", "8. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-19-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Services tab to check quantity", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-19-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-19-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-509')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-19-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies unreasonable quantity", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention that the quantity of 999 is incorrect or unreasonable? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": false, "action": "hold_referral", "reason": "Unreasonable quantity - 999 PT sessions is clearly incorrect" }, "rationale": "" }, { "id": "emr-easy-20", "title": "Verify Authorization - EXPIRED Case", "instruction": "Open referral REF-2025-510 for Abbott, Amanda (67F with Anthem Blue Cross HMO). Before proceeding with authorization for chemotherapy, verify that any existing authorization is still valid. Document your findings and take appropriate action.", "goal": "Open referral REF-2025-510 for Abbott, Amanda (67F with Anthem Blue Cross HMO). Before proceeding with authorization for chemotherapy, verify that any existing authorization is still valid. Document your findings and take appropriate action.", "difficulty": "easy", "category": "verify_auth_expired", "challengeType": "verify_auth_expired", "possible": true, "points": 4, "configTaskId": "easy_20", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Abbott, Amanda" }, "stepByStep": [ "1. Click on referral REF-2025-510 in the worklist to open it", "2. Review patient demographics (Abbott, Amanda - 67F)", "3. Click Coverages tab to check existing authorization information", "4. Identify that authorization AUTH-2024-8765 expired on December 31, 2025", "5. Click Add Note button in Communications section", "6. Document that the existing authorization has expired", "7. Save the note but do NOT clear the referral from the worklist" ], "evals": [ { "id": "emr-easy-20-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked Coverages tab to check auth status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-20-eval-2", "type": "jmespath", "category": "Documentation", "description": "Agent added note documenting the issue", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-20-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent correctly did NOT clear referral", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-510')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-easy-20-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Agent's note identifies expired authorization", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's note mention that the authorization has expired on 12/31/2025? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "auth_required": true, "action": "hold_referral", "reason": "Existing authorization AUTH-2024-8765 expired 12/31/2025 - new auth required" }, "rationale": "" }, { "id": "emr-medium-1", "title": "Submit Auth - Bilateral Eye Injection (Payer A)", "instruction": "Open referral REF-2025-001 for Doe, John (60M with Aetna PPO). Submit a prior authorization request for bilateral eye injection to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-001 for Doe, John (60M with Aetna PPO). Submit a prior authorization request for bilateral eye injection to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 20, "configTaskId": "medium_1", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Doe, John" }, "stepByStep": [ "1. Click on referral REF-2025-001 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: H35.32, H35.31, E11.9", "3. Click the Services tab — record all CPT codes: 67028, J2778", "4. Click the Coverages tab — record member ID (AET123456789), DOB (1965-03-15), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click the Medical Necessity Letter document to open its viewer page, then click the Download button to download it", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations' to open the prior authorization form", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "11. In the Provider field (Section 1), click and type the NPI number 1234567890, then click the search/lookup button to populate provider details", "12. In the Patient (Member ID) field (Section 3), click the field and type AET123456789. Then click the Date of Birth field and type 03/15/1965 in MM/DD/YYYY format. Click the Search button to verify eligibility.", "13. Add all 3 diagnosis codes one by one: click Diagnosis field, type H35.32, click Add. Repeat for H35.31 and E11.9.", "14. In the Servicing Provider field (Section 5), click and type: Dr. Jane Smith", "15. Add both CPT codes: click CPT field, type 67028, click Add. Then type J2778, click Add.", "16. Click the Clinical Indication field and type a justification mentioning macular degeneration (AMD) and intravitreal anti-VEGF injections", "17. Scroll down to the 'Available Documents from EMR' section — click the '+ Attach' button next to the Medical Necessity Letter to attach it", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number shown on the confirmation screen", "20. Click 'Return to EMR' to navigate back to the EMR portal", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-1-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient Procedure as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET123456789", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1965-03-15", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered diagnosis code H35.32 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H35.32')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code H35.31 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H35.31')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered diagnosis code E11.9 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'E11.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered CPT code 67028 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '67028')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered CPT code J2778 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J2778')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-15", "type": "llm_judge", "category": "Clinical Reasoning", "description": "15. Clinical indication mentions the diagnosis (AMD)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention macular degeneration or AMD? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-1-eval-16", "type": "jmespath", "category": "Document Handling", "description": "16. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-17", "type": "jmespath", "category": "Task Resolution", "description": "17. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-18", "type": "jmespath", "category": "Documentation", "description": "18. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-1-eval-19", "type": "llm_judge", "category": "Documentation", "description": "19. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-1-eval-20", "type": "jmespath", "category": "Task Resolution", "description": "20. Agent cleared referral REF-2025-001 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-001')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "multiple_codes": true }, "rationale": "" }, { "id": "emr-medium-2", "title": "Submit Auth - Cataract Surgery (Payer A) - Info in Clinical Notes", "instruction": "Open referral REF-2025-003 for Johnson, Michael (63M with Aetna HMO). Submit a prior authorization request for cataract surgery to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-003 for Johnson, Michael (63M with Aetna HMO). Submit a prior authorization request for cataract surgery to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 19, "configTaskId": "medium_2", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Johnson, Michael" }, "stepByStep": [ "1. Click on referral REF-2025-003 in the worklist to open it", "2. Click the Diagnoses tab — record diagnosis code: H25.11", "3. Click the Services tab — record CPT code: 66984", "4. Click the General tab — find and READ the Clinical Note to capture: nuclear sclerosis 3+, visual acuity 20/80", "5. CRITICAL: While in the General tab, click the Medical Necessity Letter document to open its viewer, then click Download. You MUST download the letter NOW before going to the portal. The portal will only show downloaded files in the Supporting Documentation section — if you skip this, no documents will be available to attach.", "6. Click the Coverages tab — record member ID (AET987654321), DOB (1962-09-15), and payer portal credentials (provider@payera.com / demo123)", "7. ONLY after confirming the Medical Necessity Letter is downloaded: On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "8. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "9. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "10. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "11. CRITICAL: Once the Auth Request modal is open, do NOT close it or navigate away from it for any reason. You must complete ALL fields and click 'Submit Request' before doing anything else. Do NOT click the X/close button, do NOT click 'Return to EMR', do NOT click back. Stay in the modal until submission is complete.", "12. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "13. In the Provider field (Section 1), enter the NPI number 1234567890 and click the search/lookup button to populate provider details", "14. In the Patient (Member ID) field (Section 3), click and type AET987654321. Click the Date of Birth field and type 09/15/1962 in MM/DD/YYYY format. Click Search to verify eligibility.", "15. Add the diagnosis code: click Diagnosis field, type H25.11, click Add", "16. Enter servicing provider: Dr. Jane Smith", "17. Add the CPT code: click the CPT field, type 66984, then IMMEDIATELY click the Add button. Do NOT proceed to the next field until you have clicked Add and confirmed 66984 appears in the CPT codes table.", "18. Click the Clinical Indication field and type a justification that includes: nuclear sclerosis 3+, visual acuity 20/80, and cataract surgery medical necessity", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to the Medical Necessity Letter", "20. CRITICAL: Do NOT click Submit Request until ALL fields are filled (Request Type, Provider, Member ID, DOB, Diagnosis, CPT, Clinical Indication). Submitting early will fail with a validation error. Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the authorization reference number", "24. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-2-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent read the clinical notes to find lens opacity details", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient Procedure as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET987654321", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1962-09-15", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code H25.11 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H25.11')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered CPT code 66984 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '66984')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13a. Clinical indication mentions pathology finding (nuclear sclerosis/3+ opacity)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention nuclear sclerosis or 3+ lens opacity? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-2-eval-14", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13b. Clinical indication mentions functional impact (visual acuity impairment)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention visual acuity impairment or reduced visual acuity (e.g. 20/80)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-2-eval-15", "type": "jmespath", "category": "Document Handling", "description": "14. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-16", "type": "jmespath", "category": "Task Resolution", "description": "15. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-17", "type": "jmespath", "category": "Documentation", "description": "16. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-2-eval-18", "type": "llm_judge", "category": "Documentation", "description": "17. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-2-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent cleared referral REF-2025-003 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-003')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "requires_clinical_note_read": true }, "rationale": "" }, { "id": "emr-medium-3", "title": "Submit Auth - Bilateral Intravitreal Injections (Payer A) - Multiple Documents", "instruction": "Open referral REF-2025-004 for Martinez, Carlos (65M with Aetna PPO). Submit a prior authorization request for bilateral intravitreal injections to the Payer A portal, uploading BOTH required supporting documents (letter of medical necessity + clinical notes). Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-004 for Martinez, Carlos (65M with Aetna PPO). Submit a prior authorization request for bilateral intravitreal injections to the Payer A portal, uploading BOTH required supporting documents (letter of medical necessity + clinical notes). Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 22, "configTaskId": "medium_3", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Martinez, Carlos" }, "stepByStep": [ "1. Click on referral REF-2025-004 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: H35.32, H35.31", "3. Click the Services tab — record all CPT codes: 67028, J2778", "4. Click the Coverages tab — record member ID (AET987654322), DOB (03/22/1960), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. Go to the General tab — click 'Clinical Notes' to open its viewer, then click Download", "7. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "8. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "9. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "10. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "11. CRITICAL: Once the Auth Request modal is open, do NOT close it or navigate away from it for any reason. You must complete ALL fields and click 'Submit Request' before doing anything else. Do NOT click the X/close button, do NOT click 'Return to EMR', do NOT click back. Stay in the modal until submission is complete.", "12. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "13. In the Provider field (Section 1), enter the NPI number 1234567890 and click the search/lookup button to populate provider details", "14. In the Patient (Member ID) field (Section 3), click and type AET987654322. Click the Date of Birth field and type 03/22/1960 in MM/DD/YYYY format. Click Search to verify eligibility.", "15. Add all diagnosis codes one by one: type H35.32, click Add; then type H35.31, click Add", "16. Enter servicing provider: Dr. Jane Smith", "17. Add all CPT codes: type 67028, click Add; then type J2778, click Add", "18. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to EACH required document (both Medical Necessity Letter AND Clinical Notes)", "20. CRITICAL: Do NOT click Submit Request until ALL fields are filled (Request Type, Provider, Member ID, DOB, both Diagnosis codes, both CPT codes, Clinical Indication, both documents attached). Submitting early will fail with a validation error. Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the authorization reference number", "24. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-3-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the clinical notes in EMR", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the clinical notes from EMR", "points": 1, "query": "signals.downloaded_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-7", "type": "jmespath", "category": "Document Handling", "description": "7. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-8", "type": "jmespath", "category": "Information Retrieval", "description": "8. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent selected Outpatient Procedure as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET987654322", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1960-03-22", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered diagnosis code H35.32 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H35.32')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code H35.31 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H35.31')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered CPT code 67028 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '67028')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code J2778 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J2778')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-16", "type": "llm_judge", "category": "Clinical Reasoning", "description": "16. Clinical indication mentions the diagnosis (AMD)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention macular degeneration or AMD? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-3-eval-17", "type": "jmespath", "category": "Document Handling", "description": "17. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-18", "type": "jmespath", "category": "Document Handling", "description": "18. Agent uploaded multiple documents (letter of medical necessity + clinical notes)", "points": 1, "query": "length(aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": 2, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "19. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-20", "type": "jmespath", "category": "Documentation", "description": "20. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-3-eval-21", "type": "llm_judge", "category": "Documentation", "description": "21. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-3-eval-22", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent cleared referral REF-2025-004 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-004')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "multiple_documents": true }, "rationale": "" }, { "id": "emr-medium-4", "title": "Submit Auth - Cardiac Workup (Payer A) - Inpatient + Multiple CPT Codes", "instruction": "Open referral REF-2025-101 for Nguyen, Linh (52F with Aetna HMO). Submit a prior authorization request for cardiac workup to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-101 for Nguyen, Linh (52F with Aetna HMO). Submit a prior authorization request for cardiac workup to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 22, "configTaskId": "medium_4", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Nguyen, Linh" }, "stepByStep": [ "1. Click on referral REF-2025-101 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: R07.9, I10, E78.5", "3. Click the Services tab — record ALL 4 CPT codes: 93350, 93015, 93306, 99223", "4. Click the Coverages tab — record member ID (AET555000111), DOB (1974-02-10), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click the Medical Necessity Letter document to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. IMPORTANT: Click the Request Type dropdown and select 'Inpatient Medical' (this is a diagnostic cardiac workup — NOT Outpatient Procedure)", "11. In the Provider field (Section 1), enter the NPI number and click the search/lookup button", "12. In the Patient (Member ID) field (Section 3), click and type AET555000111. Click Date of Birth and type 02/10/1974 in MM/DD/YYYY format. Click Search.", "13. Add all 3 diagnosis codes: type R07.9 and click Add, type I10 and click Add, type E78.5 and click Add", "14. Enter servicing provider: Dr. Jane Smith (or the provider listed in EMR)", "15. Add all 4 CPT codes: type 93350 click Add, type 93015 click Add, type 93306 click Add, type 99223 click Add", "16. Click Clinical Indication and type a justification mentioning chest pain and cardiac workup", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to the Medical Necessity Letter", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-4-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Inpatient Medical as request type (cardiac workup is diagnostic, not surgical)", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "inpatient-medical", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET555000111", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1974-02-10", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered diagnosis code R07.9 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R07.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code I10 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'I10')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered diagnosis code E78.5 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'E78.5')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered CPT code 93350 (stress echo) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '93350')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered CPT code 93015 (cardiovascular stress test) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '93015')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 93306 (echocardiography complete) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '93306')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code 99223 (initial hospital care) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '99223')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17. Clinical indication mentions the diagnosis (chest pain/cardiac)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention chest pain or cardiac symptoms or hypertension? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-4-eval-18", "type": "jmespath", "category": "Document Handling", "description": "18. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "19. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-20", "type": "jmespath", "category": "Documentation", "description": "20. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-4-eval-21", "type": "llm_judge", "category": "Documentation", "description": "21. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-4-eval-22", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent cleared referral REF-2025-101 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-101')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "request_type": "inpatient-medical", "multiple_cpt_codes": true }, "rationale": "" }, { "id": "emr-medium-5", "title": "Submit Auth - Chemotherapy FOLFOX (Payer A) - Multiple Documents + Multiple CPT Codes", "instruction": "Open referral REF-2025-301 for Adams, Paul (63M with Aetna HMO). Submit a prior authorization request for chemotherapy to the Payer A portal, uploading BOTH required supporting documents (letter of medical necessity + oncology treatment plan). Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-301 for Adams, Paul (63M with Aetna HMO). Submit a prior authorization request for chemotherapy to the Payer A portal, uploading BOTH required supporting documents (letter of medical necessity + oncology treatment plan). Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 25, "configTaskId": "medium_5", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Adams, Paul" }, "stepByStep": [ "1. Click on referral REF-2025-301 in the worklist to open it", "2. Click the General tab — find the Clinical Note and read it to capture treatment plan details (needed for clinical indication)", "3. Click the Diagnoses tab — record all diagnosis codes: C18.9, Z51.11", "4. Click the Services tab — record all CPT codes: 96413, 96415, J9263, J9190", "5. Click the Coverages tab — record member ID (AET301000001), DOB (05/14/1962), and payer portal credentials (provider@payera.com / demo123)", "6. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "7. Go to the General tab — click 'Oncology Treatment Plan' to open its viewer, then click Download", "8. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "9. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "10. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "11. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "12. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "13. In the Provider field (Section 1), enter the NPI number and click the search/lookup button to populate provider details", "14. In the Patient (Member ID) field (Section 3), click and type AET301000001. Click the Date of Birth field and type 05/14/1962 in MM/DD/YYYY format. Click Search to verify eligibility.", "15. Add all diagnosis codes one by one: type C18.9, click Add; then type Z51.11, click Add", "16. Enter servicing provider: (provider listed in EMR)", "17. Add all CPT codes: type 96413, click Add; then type 96415, click Add; then type J9263, click Add; then type J9190, click Add", "18. Click the Clinical Indication field and type a justification for the procedure and diagnosis. Explicitly mention colon cancer / malignant neoplasm of colon and chemotherapy / FOLFOX.", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "20. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the exact authorization reference number from step 21", "24. Save the EMR note, verify it appears in Communications, and then click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-5-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent viewed the treatment plan in EMR", "points": 1, "query": "full_state.agentActions.readClinicalNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-7", "type": "jmespath", "category": "Document Handling", "description": "7. Agent downloaded the treatment plan from EMR", "points": 1, "query": "signals.downloaded_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-8", "type": "jmespath", "category": "Information Retrieval", "description": "8. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent selected Outpatient Procedure as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET301000001", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1962-05-14", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered diagnosis code C18.9 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'C18.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code Z51.11 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'Z51.11')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered CPT code 96413 (chemo IV first hour) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '96413')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 96415 (chemo IV additional hour) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '96415')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code J9263 (oxaliplatin injection) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J9263')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered CPT code J9190 (fluorouracil injection) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J9190')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18a. Clinical indication mentions the diagnosis (colon cancer)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention colon cancer or colorectal cancer or malignant neoplasm? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-5-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18b. Clinical indication mentions the treatment (chemotherapy/FOLFOX)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention chemotherapy or FOLFOX? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-5-eval-20", "type": "jmespath", "category": "Document Handling", "description": "19. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-21", "type": "jmespath", "category": "Document Handling", "description": "20. Agent uploaded treatment plan to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-22", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-23", "type": "jmespath", "category": "Documentation", "description": "22. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-5-eval-24", "type": "llm_judge", "category": "Documentation", "description": "23. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-5-eval-25", "type": "jmespath", "category": "Task Resolution", "description": "24. Agent cleared referral REF-2025-301 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-301')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "multiple_documents": true, "multiple_cpt_codes": true }, "rationale": "" }, { "id": "emr-medium-6", "title": "Submit Auth - Infliximab (Remicade) Infusion (Payer A)", "instruction": "Open referral REF-2025-302 for Baker, Carol (55F with Aetna PPO). Submit a prior authorization request for Infliximab (Remicade) infusion to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-302 for Baker, Carol (55F with Aetna PPO). Submit a prior authorization request for Infliximab (Remicade) infusion to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 19, "configTaskId": "medium_6", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Baker, Carol" }, "stepByStep": [ "1. Click on referral REF-2025-302 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: M05.79", "3. Click the Services tab — record all CPT codes: J1745, 96413", "4. Click the Coverages tab — record member ID (AET302000002), DOB (08/22/1970), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "11. In the Provider field (Section 1), enter the NPI number and click the search/lookup button to populate provider details", "12. In the Patient (Member ID) field (Section 3), click and type AET302000002. Click the Date of Birth field and type 08/22/1970 in MM/DD/YYYY format. Click Search to verify eligibility.", "13. Add diagnosis code: click Diagnosis field, type M05.79, click Add", "14. Enter servicing provider: (provider listed in EMR)", "15. Add all CPT codes: type J1745, click Add; then type 96413, click Add", "16. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-6-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient Procedure as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET302000002", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1970-08-22", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered diagnosis code M05.79 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M05.79')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered CPT code J1745 (infliximab injection) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J1745')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered CPT code 96413 (IV infusion) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '96413')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13a. Clinical indication mentions the diagnosis (rheumatoid arthritis)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention rheumatoid arthritis or RA or autoimmune or joint inflammation? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-6-eval-14", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13b. Clinical indication mentions the treatment (biologic)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention biologic therapy or biologic medication? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-6-eval-15", "type": "jmespath", "category": "Document Handling", "description": "14. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-16", "type": "jmespath", "category": "Task Resolution", "description": "15. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-17", "type": "jmespath", "category": "Documentation", "description": "16. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-6-eval-18", "type": "llm_judge", "category": "Documentation", "description": "17. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-6-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent cleared referral REF-2025-302 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-302')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna" }, "rationale": "" }, { "id": "emr-medium-7", "title": "Submit Auth - MRI Lumbar Spine (Payer A) - Imaging Request Type", "instruction": "Open referral REF-2025-303 for Cooper, David (57M with Aetna PPO). Submit a prior authorization request for MRI lumbar spine to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-303 for Cooper, David (57M with Aetna PPO). Submit a prior authorization request for MRI lumbar spine to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 18, "configTaskId": "medium_7", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Cooper, David" }, "stepByStep": [ "1. Click on referral REF-2025-303 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: M54.5, M54.16", "3. Click the Services tab — record all CPT codes: 72148", "4. Click the Coverages tab — record member ID (AET303000003), DOB (03/10/1968), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "11. In the Provider field (Section 1), enter the NPI number and click the search/lookup button to populate provider details", "12. In the Patient (Member ID) field (Section 3), click and type AET303000003. Click the Date of Birth field and type 03/10/1968 in MM/DD/YYYY format. Click Search to verify eligibility.", "13. Add all diagnosis codes one by one: type M54.5, click Add; then type M54.16, click Add", "14. Enter servicing provider: (provider listed in EMR)", "15. Add the CPT code: click CPT field, type 72148, click Add", "16. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-7-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type (MRI is outpatient imaging)", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET303000003", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1968-03-10", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered diagnosis code M54.5 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M54.5')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code M54.16 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M54.16')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered CPT code 72148 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '72148')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13. Clinical indication mentions the diagnosis (back pain/radiculopathy)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention back pain or low back pain or lumbar pain or radiculopathy or spine? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-7-eval-14", "type": "jmespath", "category": "Document Handling", "description": "14. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "15. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-16", "type": "jmespath", "category": "Documentation", "description": "16. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-7-eval-17", "type": "llm_judge", "category": "Documentation", "description": "17. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-7-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent cleared referral REF-2025-303 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-303')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "request_type": "outpatient" }, "rationale": "" }, { "id": "emr-medium-8", "title": "Submit Auth - Polysomnography (Sleep Study) (Payer A)", "instruction": "Open referral REF-2025-305 for Ellis, Frank (53M with Aetna HMO). Submit a prior authorization request for sleep study to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-305 for Ellis, Frank (53M with Aetna HMO). Submit a prior authorization request for sleep study to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 18, "configTaskId": "medium_8", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Ellis, Frank" }, "stepByStep": [ "1. Click on referral REF-2025-305 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: G47.33, R06.83", "3. Click the Services tab — record all CPT codes: 95810", "4. Click the Coverages tab — record member ID (AET305000005), DOB (07/15/1972), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "11. In the Provider field (Section 1), enter the NPI number and click the search/lookup button to populate provider details", "12. In the Patient (Member ID) field (Section 3), click and type AET305000005. Click the Date of Birth field and type 07/15/1972 in MM/DD/YYYY format. Click Search to verify eligibility.", "13. Add all diagnosis codes one by one: type G47.33, click Add; then type R06.83, click Add", "14. Enter servicing provider: (provider listed in EMR)", "15. Add the CPT code: click CPT field, type 95810, click Add", "16. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-8-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient Procedure as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET305000005", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1972-07-15", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered diagnosis code G47.33 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'G47.33')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code R06.83 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R06.83')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered CPT code 95810 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '95810')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13. Clinical indication mentions the diagnosis (sleep apnea)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention sleep apnea or OSA or snoring or sleep study or polysomnography? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-8-eval-14", "type": "jmespath", "category": "Document Handling", "description": "14. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "15. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-16", "type": "jmespath", "category": "Documentation", "description": "16. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-8-eval-17", "type": "llm_judge", "category": "Documentation", "description": "17. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-8-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent cleared referral REF-2025-305 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-305')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna" }, "rationale": "" }, { "id": "emr-medium-9", "title": "Submit Auth - CT Chest with Contrast (Payer A) - Imaging Request Type", "instruction": "Open referral REF-2025-308 for Hayes, Irene (60F with Aetna PPO). Submit a prior authorization request for CT chest to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-308 for Hayes, Irene (60F with Aetna PPO). Submit a prior authorization request for CT chest to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 18, "configTaskId": "medium_9", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Hayes, Irene" }, "stepByStep": [ "1. Click on referral REF-2025-308 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: R91.8, R05.9", "3. Click the Services tab — record all CPT codes: 71260", "4. Click the Coverages tab — record member ID (AET308000008), DOB (12/03/1965), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "11. In the Provider field (Section 1), enter the NPI number and click the search/lookup button to populate provider details", "12. In the Patient (Member ID) field (Section 3), click and type AET308000008. Click the Date of Birth field and type 12/03/1965 in MM/DD/YYYY format. Click Search to verify eligibility.", "13. Add all diagnosis codes one by one: type R91.8, click Add; then type R05.9, click Add", "14. Enter servicing provider: (provider listed in EMR)", "15. Add the CPT code: click CPT field, type 71260, click Add", "16. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-9-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type (CT is outpatient imaging)", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET308000008", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1965-12-03", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered diagnosis code R91.8 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R91.8')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code R05.9 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R05.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered CPT code 71260 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '71260')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13. Clinical indication mentions the diagnosis (lung nodule/cough)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention lung nodule or pulmonary nodule or abnormal lung finding or cough? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-9-eval-14", "type": "jmespath", "category": "Document Handling", "description": "14. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "15. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-16", "type": "jmespath", "category": "Documentation", "description": "16. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-9-eval-17", "type": "llm_judge", "category": "Documentation", "description": "17. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-9-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent cleared referral REF-2025-308 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-308')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "request_type": "outpatient" }, "rationale": "" }, { "id": "emr-medium-10", "title": "Submit Auth - Diagnostic Colonoscopy (Payer B) - No Letter of Medical Necessity Required", "instruction": "Open referral REF-2025-103 for Thompson, Avery (56F with Anthem Blue Cross PPO). Submit a prior authorization request for diagnostic colonoscopy to the Payer B portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-103 for Thompson, Avery (56F with Anthem Blue Cross PPO). Submit a prior authorization request for diagnostic colonoscopy to the Payer B portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 23, "configTaskId": "medium_10", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Thompson, Avery" }, "stepByStep": [ "1. Click on referral REF-2025-103 in the worklist to open it", "2. Click the General tab — find the Clinical Note and read it to capture the hemoglobin level (needed for clinical indication)", "3. Click the Diagnoses tab — record all diagnosis codes: D50.9, R19.5", "4. Click the Services tab — record all CPT codes: 45378", "5. Click the Coverages tab — record subscriber ID (BCBS77889900), DOB (05/14/1969), and payer portal credentials (provider@payerb.com / demo123)", "6. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "7. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "8. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "9. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Medical'.", "10. Click Patient Name field and type: Thompson, Avery. Click Date of Birth field and type 05/14/1969 in MM/DD/YYYY format. Click Subscriber ID field and type BCBS77889900. Click Next.", "11. Step 2 - Service Details: add diagnosis codes: type D50.9, click Add; then type R19.5, click Add. Add CPT codes: type 45378, click Add. Click Date of Service and type 06/22/2026 in MM/DD/YYYY format.", "12. Click Clinical Indication field and type a justification for the procedure and diagnosis", "13. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "14. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "15. Note the authorization reference number from the confirmation screen", "16. Click 'Return to EMR' to navigate back", "17. Add a Communication note in EMR that includes the authorization reference number", "18. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-10-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get hemoglobin level", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-6", "type": "jmespath", "category": "Form Completion", "description": "6. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Medical as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "medical", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "BCBS77889900", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Thompson", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Avery", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1969-05-14", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-22", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code D50.9 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'D50.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code R19.5 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R19.5')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 45378 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '45378')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-16", "type": "llm_judge", "category": "Documentation", "description": "16. Clinical indication includes hemoglobin level 10.8g/dL", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention the hemoglobin level 10.8g/dL (or 10.8 g/dL)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-10-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "16b. Clinical indication includes anemia/iron deficiency", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention anemia or iron deficiency? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-10-eval-18", "type": "llm_judge", "category": "Form Completion", "description": "17. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Priya Raman'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-10-eval-19", "type": "jmespath", "category": "Form Completion", "description": "18. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-20", "type": "jmespath", "category": "Task Resolution", "description": "19. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-21", "type": "jmespath", "category": "Documentation", "description": "20. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-10-eval-22", "type": "llm_judge", "category": "Documentation", "description": "21. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-10-eval-23", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent cleared referral REF-2025-103 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-103')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "no_medical_necessity_letter": true }, "rationale": "" }, { "id": "emr-medium-11", "title": "Submit Auth - Biologic Therapy for Psoriasis (Payer B) - Multiple Documents", "instruction": "Open referral REF-2025-105 for Reed, Jordan (36M with Anthem Blue Cross PPO). Submit a prior authorization request for biologic therapy to the Payer B portal, uploading BOTH required supporting documents (letter of medical necessity + step therapy documentation). Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-105 for Reed, Jordan (36M with Anthem Blue Cross PPO). Submit a prior authorization request for biologic therapy to the Payer B portal, uploading BOTH required supporting documents (letter of medical necessity + step therapy documentation). Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 26, "configTaskId": "medium_11", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Reed, Jordan" }, "stepByStep": [ "1. Click on referral REF-2025-105 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: L40.0, L40.50", "3. Click the Services tab — record all CPT codes: J3590", "4. Click the Coverages tab — record subscriber ID (BCBS55001234), DOB (04/07/1989), and payer portal credentials (provider@payerb.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. Go to the General tab — click 'Step Therapy Documentation' to open its viewer, then click Download", "7. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "8. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "9. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "10. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Medical Injectable'.", "11. Click Patient Name field and type: Reed, Jordan. Click Date of Birth field and type 04/07/1989 in MM/DD/YYYY format. Click Subscriber ID field and type BCBS55001234. Click Next.", "12. Step 2 - Service Details: add diagnosis codes: type L40.0, click Add; then type L40.50, click Add. Add CPT codes: type J3590, click Add.", "13. Click Clinical Indication field and type a justification for the procedure and diagnosis", "14. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "15. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "16. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "17. Note the authorization reference number from the confirmation screen", "18. Click 'Return to EMR' to navigate back", "19. Add a Communication note in EMR that includes the authorization reference number", "20. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-11-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Medical Injectable as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "medical-injectable", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "BCBS55001234", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Reed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Jordan", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1989-04-07", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-11", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code L40.0 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'L40.0')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered diagnosis code L40.50 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'L40.50')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code J3590 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J3590')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17a. Clinical indication mentions the diagnosis (psoriasis)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention psoriasis or psoriatic arthritis or skin condition? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-11-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17b. Clinical indication mentions the treatment (biologic)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention biologic therapy or biologic medication? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-11-eval-19", "type": "llm_judge", "category": "Form Completion", "description": "18. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Maya Desai'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-11-eval-20", "type": "jmespath", "category": "Form Completion", "description": "19. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-21", "type": "jmespath", "category": "Document Handling", "description": "20. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-22", "type": "jmespath", "category": "Document Handling", "description": "21. Agent uploaded multiple documents (letter of medical necessity + step therapy)", "points": 1, "query": "length(anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": 2, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-23", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-24", "type": "jmespath", "category": "Documentation", "description": "23. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-11-eval-25", "type": "llm_judge", "category": "Documentation", "description": "24. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-11-eval-26", "type": "jmespath", "category": "Task Resolution", "description": "25. Agent cleared referral REF-2025-105 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-105')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "multiple_documents": true }, "rationale": "" }, { "id": "emr-medium-12", "title": "Submit Auth - Knee Arthroscopy - Meniscectomy (Payer B)", "instruction": "Open referral REF-2025-401 for Irving, James (55M with Anthem Blue Cross HMO). Submit a prior authorization request for knee arthroscopy to the Payer B portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-401 for Irving, James (55M with Anthem Blue Cross HMO). Submit a prior authorization request for knee arthroscopy to the Payer B portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 23, "configTaskId": "medium_12", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Irving, James" }, "stepByStep": [ "1. Click on referral REF-2025-401 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: M23.221", "3. Click the Services tab — record all CPT codes: 29881", "4. Click the Coverages tab — record subscriber ID (ANT401000001), DOB (04/18/1970), and payer portal credentials (provider@payerb.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "7. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "8. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "9. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Surgical'.", "10. Click Patient Name field and type: Irving, James. Click Date of Birth field and type 04/18/1970 in MM/DD/YYYY format. Click Subscriber ID field and type ANT401000001. Click Next.", "11. Step 2 - Service Details: add diagnosis codes: type M23.221, click Add. Add CPT codes: type 29881, click Add.", "12. Click Clinical Indication field and type a justification for the procedure and diagnosis", "13. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "14. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "15. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "16. Note the authorization reference number from the confirmation screen", "17. Click 'Return to EMR' to navigate back", "18. Add a Communication note in EMR that includes the authorization reference number", "19. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-12-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Surgical as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "surgery", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT401000001", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Irving", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "James", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1970-04-18", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-20", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-14", "type": "llm_judge", "category": "Form Completion", "description": "14. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Alan Chen'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-12-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered diagnosis code M23.221 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M23.221')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered CPT code 29881 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '29881')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18. Clinical indication mentions the diagnosis (meniscus tear)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention meniscus tear or knee injury or meniscectomy or knee pain? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-12-eval-19", "type": "jmespath", "category": "Document Handling", "description": "19. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-20", "type": "jmespath", "category": "Task Resolution", "description": "20. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-21", "type": "jmespath", "category": "Documentation", "description": "21. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-12-eval-22", "type": "llm_judge", "category": "Documentation", "description": "22. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-12-eval-23", "type": "jmespath", "category": "Task Resolution", "description": "23. Agent cleared referral REF-2025-401 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-401')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "request_type": "outpatient", "case_type": "surgical" }, "rationale": "" }, { "id": "emr-medium-13", "title": "Submit Auth - CT Abdomen/Pelvis with Contrast (Payer B) - Imaging Request Type", "instruction": "Open referral REF-2025-402 for Jensen, Karen (57F with Anthem Blue Cross PPO). Submit a prior authorization request for CT abdomen/pelvis to the Payer B portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-402 for Jensen, Karen (57F with Anthem Blue Cross PPO). Submit a prior authorization request for CT abdomen/pelvis to the Payer B portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 24, "configTaskId": "medium_13", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Jensen, Karen" }, "stepByStep": [ "1. Click on referral REF-2025-402 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: R10.9, R19.5", "3. Click the Services tab — record all CPT codes: 74177", "4. Click the Coverages tab — record subscriber ID (ANT402000002), DOB (08/25/1968), and payer portal credentials (provider@payerb.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "7. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "8. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "9. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Medical'.", "10. Click Patient Name field and type: Jensen, Karen. Click Date of Birth field and type 08/25/1968 in MM/DD/YYYY format. Click Subscriber ID field and type ANT402000002. Click Next.", "11. Step 2 - Service Details: add diagnosis codes: type R10.9, click Add; then type R19.5, click Add. Add CPT codes: type 74177, click Add.", "12. Click Clinical Indication field and type a justification for the procedure and diagnosis", "13. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "14. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "15. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "16. Note the authorization reference number from the confirmation screen", "17. Click 'Return to EMR' to navigate back", "18. Add a Communication note in EMR that includes the authorization reference number", "19. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-13-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Medical as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "medical", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT402000002", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Jensen", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Karen", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1968-08-25", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-18", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-14", "type": "llm_judge", "category": "Form Completion", "description": "14. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Sandra White'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-13-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered diagnosis code R10.9 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R10.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered diagnosis code R19.5 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R19.5')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-18", "type": "jmespath", "category": "Form Completion", "description": "18. Agent entered CPT code 74177 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '74177')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "19. Clinical indication mentions the diagnosis (abdominal pain)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention abdominal pain or GI symptoms or fecal abnormality? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-13-eval-20", "type": "jmespath", "category": "Document Handling", "description": "20. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-22", "type": "jmespath", "category": "Documentation", "description": "22. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-13-eval-23", "type": "llm_judge", "category": "Documentation", "description": "23. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-13-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "24. Agent cleared referral REF-2025-402 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-402')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "request_type": "imaging" }, "rationale": "" }, { "id": "emr-medium-14", "title": "Submit Auth - Septoplasty (Payer B) - Info in Clinical Notes", "instruction": "Open referral REF-2025-404 for Lewis, Mary (50F with Anthem Blue Cross PPO). Submit a prior authorization request for septoplasty to the Payer B portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-404 for Lewis, Mary (50F with Anthem Blue Cross PPO). Submit a prior authorization request for septoplasty to the Payer B portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 25, "configTaskId": "medium_14", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Lewis, Mary" }, "stepByStep": [ "1. Click on referral REF-2025-404 in the worklist to open it", "2. Click the General tab — find the Clinical Note and read it to capture the symptom duration (needed for clinical indication)", "3. Click the Diagnoses tab — record all diagnosis codes: J34.2", "4. Click the Services tab — record all CPT codes: 30520", "5. Click the Coverages tab — record subscriber ID (ANT404000004), DOB (03/22/1975), and payer portal credentials (provider@payerb.com / demo123)", "6. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download it (REQUIRED)", "7. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "8. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "9. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "10. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Surgical'.", "11. Click Patient Name field and type: Lewis, Mary. Click Date of Birth field and type 03/22/1975 in MM/DD/YYYY format. Click Subscriber ID field and type ANT404000004. Click Next.", "12. Step 2 - Service Details: add diagnosis codes: type J34.2, click Add. Add CPT codes: type 30520, click Add.", "13. Click Clinical Indication field and type a justification for the procedure and diagnosis", "14. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "15. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "16. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "17. Note the authorization reference number from the confirmation screen", "18. Click 'Return to EMR' to navigate back", "19. Add a Communication note in EMR that includes the authorization reference number", "20. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-14-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get symptom duration", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent selected Surgical as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "surgery", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT404000004", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Lewis", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Mary", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1975-03-22", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-22", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-15", "type": "llm_judge", "category": "Form Completion", "description": "15. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Robert Green'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-14-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered diagnosis code J34.2 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'J34.2')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-18", "type": "jmespath", "category": "Form Completion", "description": "18. Agent entered CPT code 30520 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '30520')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "19. Clinical indication mentions deviated septum or nasal obstruction", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention deviated septum or nasal obstruction? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-14-eval-20", "type": "llm_judge", "category": "Clinical Reasoning", "description": "19b. Clinical indication mentions symptom duration from clinical notes", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention a duration of symptoms (such as 2 years, 'over 2 years', 'chronic', or 'long-standing')? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-14-eval-21", "type": "jmespath", "category": "Document Handling", "description": "20. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-22", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-23", "type": "jmespath", "category": "Documentation", "description": "22. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-14-eval-24", "type": "llm_judge", "category": "Documentation", "description": "23. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-14-eval-25", "type": "jmespath", "category": "Task Resolution", "description": "24. Agent cleared referral REF-2025-404 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-404')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "info_in_clinical_notes": true }, "rationale": "" }, { "id": "emr-medium-15", "title": "Submit Auth - Cystoscopy with Biopsy (Payer B) - Multiple Documents", "instruction": "Open referral REF-2025-405 for Morgan, Nancy (63F with Anthem Blue Cross HMO). Submit a prior authorization request for cystoscopy with biopsy to the Payer B portal, uploading BOTH required supporting documents (letter of medical necessity + urinalysis report). Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-405 for Morgan, Nancy (63F with Anthem Blue Cross HMO). Submit a prior authorization request for cystoscopy with biopsy to the Payer B portal, uploading BOTH required supporting documents (letter of medical necessity + urinalysis report). Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 24, "configTaskId": "medium_15", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Morgan, Nancy" }, "stepByStep": [ "1. Click on referral REF-2025-405 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: R31.9", "3. Click the Services tab — record all CPT codes: 52204", "4. Click the Coverages tab — record subscriber ID (ANT405000005), DOB (06/30/1962), and payer portal credentials (provider@payerb.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. Go to the General tab — click 'Urinalysis Report' to open its viewer, then click Download", "7. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "8. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "9. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "10. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Surgical'.", "11. Click Patient Name field and type: Morgan, Nancy. Click Date of Birth field and type 06/30/1962 in MM/DD/YYYY format. Click Subscriber ID field and type ANT405000005. Click Next.", "12. Step 2 - Service Details: add diagnosis codes: type R31.9, click Add. Add CPT codes: type 52204, click Add.", "13. Click Clinical Indication field and type a justification for the procedure and diagnosis", "14. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "15. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "16. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "17. Note the authorization reference number from the confirmation screen", "18. Click 'Return to EMR' to navigate back", "19. Add a Communication note in EMR that includes the authorization reference number", "20. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-15-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Surgical as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "surgery", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT405000005", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Morgan", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Nancy", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1962-06-30", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-19", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-14", "type": "llm_judge", "category": "Form Completion", "description": "14. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Thomas Brown'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-15-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered diagnosis code R31.9 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R31.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered CPT code 52204 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '52204')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18. Clinical indication mentions the diagnosis (hematuria)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention hematuria or blood in urine or bladder evaluation? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-15-eval-19", "type": "jmespath", "category": "Document Handling", "description": "19. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-20", "type": "jmespath", "category": "Document Handling", "description": "20. Agent uploaded multiple documents (letter of medical necessity + urinalysis report)", "points": 1, "query": "length(anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": 2, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-22", "type": "jmespath", "category": "Documentation", "description": "22. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-15-eval-23", "type": "llm_judge", "category": "Documentation", "description": "23. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-15-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "24. Agent cleared referral REF-2025-405 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-405')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "multiple_documents": true }, "rationale": "" }, { "id": "emr-medium-16", "title": "Submit Auth - Epidural Steroid Injection (Payer B) - Interventional Procedure", "instruction": "Open referral REF-2025-406 for Norton, Oscar (60M with Anthem Blue Cross PPO). Submit a prior authorization request for epidural steroid injection to the Payer B portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-406 for Norton, Oscar (60M with Anthem Blue Cross PPO). Submit a prior authorization request for epidural steroid injection to the Payer B portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 24, "configTaskId": "medium_16", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Norton, Oscar" }, "stepByStep": [ "1. Click on referral REF-2025-406 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: M54.16, M51.16", "3. Click the Services tab — record all CPT codes: 62323", "4. Click the Coverages tab — record subscriber ID (ANT406000006), DOB (09/14/1965), and payer portal credentials (provider@payerb.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "7. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "8. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "9. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Surgical'.", "10. Click Patient Name field and type: Norton, Oscar. Click Date of Birth field and type 09/14/1965 in MM/DD/YYYY format. Click Subscriber ID field and type ANT406000006. Click Next.", "11. Step 2 - Service Details: add diagnosis codes: type M54.16, click Add; then type M51.16, click Add. Add CPT codes: type 62323, click Add.", "12. Click Clinical Indication field and type a justification for the procedure and diagnosis", "13. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "14. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "15. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "16. Note the authorization reference number from the confirmation screen", "17. Click 'Return to EMR' to navigate back", "18. Add a Communication note in EMR that includes the authorization reference number", "19. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-16-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Surgical as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "surgery", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT406000006", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Norton", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Oscar", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1965-09-14", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-21", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-14", "type": "llm_judge", "category": "Form Completion", "description": "14. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Angela Martinez'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-16-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered diagnosis code M54.16 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M54.16')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered diagnosis code M51.16 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M51.16')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-18", "type": "jmespath", "category": "Form Completion", "description": "18. Agent entered CPT code 62323 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '62323')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "19. Clinical indication mentions the diagnosis (radiculopathy)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention radiculopathy or disc degeneration or lumbar pain or back pain? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-16-eval-20", "type": "jmespath", "category": "Document Handling", "description": "20. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-22", "type": "jmespath", "category": "Documentation", "description": "22. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-16-eval-23", "type": "llm_judge", "category": "Documentation", "description": "23. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-16-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "24. Agent cleared referral REF-2025-406 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-406')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "request_type": "outpatient" }, "rationale": "" }, { "id": "emr-medium-17", "title": "Submit Auth - Knee Arthroscopy - Meniscectomy (Payer A) - Info in Clinical Notes", "instruction": "Open referral REF-2025-304 for Drake, Emily (50F with Aetna PPO). Submit a prior authorization request for knee arthroscopy to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-304 for Drake, Emily (50F with Aetna PPO). Submit a prior authorization request for knee arthroscopy to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 19, "configTaskId": "medium_17", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Drake, Emily" }, "stepByStep": [ "1. Click on referral REF-2025-304 in the worklist to open it", "2. Click the Diagnoses tab — REQUIRED: record all diagnosis codes: M23.222, M25.562", "3. Click the Services tab — REQUIRED: record all CPT codes: 29881. You MUST click this tab; do not skip it even if you think you already know the CPT code.", "4. Click the Coverages tab — record member ID (AET304000004), DOB (11/28/1975), and payer portal credentials (provider@payera.com / demo123)", "5. Click the General tab — find the Clinical Note and read it to capture MRI findings (needed for clinical indication)", "6. While on the General tab, click 'Medical Necessity Letter' to open its viewer, then click Download", "7. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "8. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "9. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "10. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "11. CRITICAL: Once the Auth Request modal is open, do NOT close it or navigate away for any reason. You must complete ALL fields and click 'Submit Request' before doing anything else. Do NOT click the X/close button, do NOT click 'Return to EMR', do NOT click back. Stay in the modal until submission is complete.", "12. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "13. In the Provider field (Section 1), enter the NPI number 1234567890 and click the search/lookup button to populate provider details", "14. In the Patient (Member ID) field (Section 3), click and type AET304000004. Click the Date of Birth field and type 11/28/1975 in MM/DD/YYYY format. Click Search to verify eligibility.", "15. Add all diagnosis codes one by one: type M23.222, click Add; then type M25.562, click Add", "16. Enter servicing provider: Dr. Robert Kim", "17. Add the CPT code: click CPT field, type 29881, click Add", "18. Click the Clinical Indication field and type a justification that references the MRI findings from the clinical notes", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "20. CRITICAL: Do NOT click Submit Request until ALL fields are filled. Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the authorization reference number", "24. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-17-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get MRI findings", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET304000004", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1975-11-28", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-11", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code M23.222 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M23.222')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-12", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered diagnosis code M25.562 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M25.562')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-13", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code 29881 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '29881')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-14", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17. Clinical indication includes MRI findings from clinical notes", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention MRI findings (like grade 3 tear or bucket handle or complex tear)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-17-eval-15", "type": "jmespath", "category": "Document Handling", "description": "18. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "contains(join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`), 'Medical_Necessity_Letter')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-16", "type": "jmespath", "category": "Task Resolution", "description": "19. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-17", "type": "jmespath", "category": "Documentation", "description": "20. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-17-eval-18", "type": "llm_judge", "category": "Documentation", "description": "21. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-17-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent cleared referral REF-2025-304 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-304')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "info_in_clinical_notes": true }, "rationale": "" }, { "id": "emr-medium-18", "title": "Submit Auth - Retinal Detachment Surgery (Payer A) - Inpatient Request Type", "instruction": "Open referral REF-2025-306 for Foster, Grace (68F with Aetna HMO). Submit a prior authorization request for retinal detachment surgery to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-306 for Foster, Grace (68F with Aetna HMO). Submit a prior authorization request for retinal detachment surgery to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 17, "configTaskId": "medium_18", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Foster, Grace" }, "stepByStep": [ "1. Click on referral REF-2025-306 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: H33.001", "3. Click the Services tab — record all CPT codes: 67108", "4. Click the Coverages tab — record member ID (AET306000006), DOB (02/20/1958), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. In the Authorization Request modal, click the Request Type dropdown and select 'Inpatient Surgical'", "11. In the Provider field (Section 1), enter the NPI number and click the search/lookup button to populate provider details", "12. In the Patient (Member ID) field (Section 3), click and type AET306000006. Click the Date of Birth field and type 02/20/1958 in MM/DD/YYYY format. Click Search to verify eligibility.", "13. Add diagnosis code: click Diagnosis field, type H33.001, click Add", "14. Enter servicing provider: (provider listed in EMR)", "15. Add the CPT code: click CPT field, type 67108, click Add", "16. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-18-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Inpatient Surgical as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "inpatient-surgical", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET306000006", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1958-02-20", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-10", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code H33.001 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H33.001')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-11", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered CPT code 67108 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '67108')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-12", "type": "llm_judge", "category": "Clinical Reasoning", "description": "15. Clinical indication mentions the diagnosis (retinal detachment)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention retinal detachment or vitrectomy or retina repair? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-18-eval-13", "type": "jmespath", "category": "Document Handling", "description": "16. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-14", "type": "jmespath", "category": "Task Resolution", "description": "17. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-15", "type": "jmespath", "category": "Documentation", "description": "18. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-18-eval-16", "type": "llm_judge", "category": "Documentation", "description": "19. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-18-eval-17", "type": "jmespath", "category": "Task Resolution", "description": "20. Agent cleared referral REF-2025-306 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-306')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "request_type": "inpatient" }, "rationale": "" }, { "id": "emr-medium-19", "title": "Submit Auth - Cardiac Catheterization (Payer A)", "instruction": "Open referral REF-2025-307 for Grant, Henry (65M with Aetna HMO). Submit a prior authorization request for cardiac catheterization to the Payer A portal. Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-307 for Grant, Henry (65M with Aetna HMO). Submit a prior authorization request for cardiac catheterization to the Payer A portal. Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_aetna", "challengeType": "submit_auth_aetna", "possible": true, "points": 18, "configTaskId": "medium_19", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Grant, Henry" }, "stepByStep": [ "1. Click on referral REF-2025-307 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: I25.10, I20.9", "3. Click the Services tab — record all CPT codes: 93458", "4. Click the Coverages tab — record member ID (AET307000007), DOB (09/05/1960), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "9. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "10. In the Authorization Request modal, click the Request Type dropdown and select 'Inpatient Medical'", "11. In the Provider field (Section 1), enter the NPI number and click the search/lookup button to populate provider details", "12. In the Patient (Member ID) field (Section 3), click and type AET307000007. Click the Date of Birth field and type 09/05/1960 in MM/DD/YYYY format. Click Search to verify eligibility.", "13. Add all diagnosis codes one by one: type I25.10, click Add; then type I20.9, click Add", "14. Enter servicing provider: (provider listed in EMR)", "15. Add the CPT code: click CPT field, type 93458, click Add", "16. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "18. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-19-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Inpatient Medical as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "inpatient-medical", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET307000007", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1960-09-05", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-10", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code I25.10 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'I25.10')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-11", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code I20.9 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'I20.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-12", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 93458 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '93458')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "16. Clinical indication mentions the diagnosis (CAD/angina)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention coronary artery disease or CAD or angina or heart disease? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-19-eval-14", "type": "jmespath", "category": "Document Handling", "description": "17. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-16", "type": "jmespath", "category": "Documentation", "description": "19. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-19-eval-17", "type": "llm_judge", "category": "Documentation", "description": "20. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-19-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent cleared referral REF-2025-307 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-307')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Aetna", "multiple_codes": true }, "rationale": "" }, { "id": "emr-medium-20", "title": "Submit Auth - Chemotherapy for Lung Cancer (Payer B) - Multiple Documents", "instruction": "Open referral REF-2025-403 for Klein, Larry (67M with Anthem Blue Cross HMO). Submit a prior authorization request for chemotherapy to the Payer B portal, uploading BOTH required supporting documents (letter of medical necessity + oncology treatment plan). Record the authorization number in EMR and clear the referral.", "goal": "Open referral REF-2025-403 for Klein, Larry (67M with Anthem Blue Cross HMO). Submit a prior authorization request for chemotherapy to the Payer B portal, uploading BOTH required supporting documents (letter of medical necessity + oncology treatment plan). Record the authorization number in EMR and clear the referral.", "difficulty": "medium", "category": "submit_auth_anthem", "challengeType": "submit_auth_anthem", "possible": true, "points": 27, "configTaskId": "medium_20", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Klein, Larry" }, "stepByStep": [ "1. Click on referral REF-2025-403 in the worklist to open it", "2. Click the Diagnoses tab — record all diagnosis codes: C34.90, Z51.11", "3. Click the Services tab — record all CPT codes: 96413, J9045", "4. Click the Coverages tab — record subscriber ID (ANT403000003), DOB (11/12/1958), and payer portal credentials (provider@payerb.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, then click Download", "6. Go to the General tab — click 'Oncology Treatment Plan' to open its viewer, then click Download", "7. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "8. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "9. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "10. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Medical'.", "11. Click Patient Name field and type: Klein, Larry. Click Date of Birth field and type 11/12/1958 in MM/DD/YYYY format. Click Subscriber ID field and type ANT403000003. Click Next.", "12. Step 2 - Service Details: add diagnosis codes: type C34.90, click Add; then type Z51.11, click Add. Add CPT codes: type 96413, click Add; then type J9045, click Add.", "13. Click Clinical Indication field and type a justification for the procedure and diagnosis. Explicitly mention lung cancer / pulmonary malignancy and chemotherapy.", "14. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "15. Click Next to proceed through remaining steps. In provider details, enter provider name: (provider listed in EMR). Click Next.", "16. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "17. Note the authorization reference number from the confirmation screen", "18. Click 'Return to EMR' to navigate back", "19. Add a Communication note in EMR that includes the exact authorization reference number from step 17", "20. Save the EMR note, verify it appears in Communications, and then click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-medium-20-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Medical or Medical Injectable as case type", "points": 1, "query": "(anthem_state.differences.priorAuth.added[0].caseType == 'medical') || (anthem_state.differences.priorAuth.added[0].caseType == 'medical-injectable')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT403000003", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Klein", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Larry", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1958-11-12", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct date of service in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].dateOfService", "expectedValue": "2026-06-16", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-14", "type": "llm_judge", "category": "Form Completion", "description": "14. Agent entered correct provider name in Payer B form", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this provider name contain 'Patricia Moore'? Score 1.0 if it contains the name (with or without Dr. prefix), 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].providerName}}", "studentAnswerContext": "provider name field entry" }, { "id": "emr-medium-20-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered correct provider NPI in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].providerNPI", "expectedValue": "1234567890", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered diagnosis code C34.90 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'C34.90')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered diagnosis code Z51.11 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'Z51.11')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-18", "type": "jmespath", "category": "Form Completion", "description": "18. Agent entered CPT code 96413 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '96413')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-19", "type": "jmespath", "category": "Form Completion", "description": "19. Agent entered CPT code J9045 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J9045')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-20", "type": "llm_judge", "category": "Clinical Reasoning", "description": "20a. Clinical indication mentions the diagnosis (lung cancer)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention lung cancer or pulmonary malignancy? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-20-eval-21", "type": "llm_judge", "category": "Clinical Reasoning", "description": "20b. Clinical indication mentions the treatment (chemotherapy)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention chemotherapy? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-medium-20-eval-22", "type": "jmespath", "category": "Document Handling", "description": "21. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-23", "type": "jmespath", "category": "Document Handling", "description": "22. Agent uploaded multiple documents (letter of medical necessity + treatment plan)", "points": 1, "query": "length(anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": 2, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "23. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-25", "type": "jmespath", "category": "Documentation", "description": "24. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-medium-20-eval-26", "type": "llm_judge", "category": "Documentation", "description": "25. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-medium-20-eval-27", "type": "jmespath", "category": "Task Resolution", "description": "26. Agent cleared referral REF-2025-403 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-403')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_auth", "portal": "Anthem", "multiple_documents": true }, "rationale": "" }, { "id": "emr-hard-1", "title": "Calculate Annual Injection Dosage", "instruction": "Open referral REF-2025-001 for Doe, John (60M with Aetna PPO). This patient needs bilateral intravitreal injections for AMD. Calculate the appropriate annual dosage based on the patient's measurements in the clinical notes. Submit the authorization to Aetna with the calculated dosage, record the auth number in EMR, and clear the referral.", "goal": "Open referral REF-2025-001 for Doe, John (60M with Aetna PPO). This patient needs bilateral intravitreal injections for AMD. Calculate the appropriate annual dosage based on the patient's measurements in the clinical notes. Submit the authorization to Aetna with the calculated dosage, record the auth number in EMR, and clear the referral.", "difficulty": "hard", "category": "calculation", "challengeType": "calculation", "possible": true, "points": 21, "configTaskId": "hard_1", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Doe, John" }, "stepByStep": [ "1. Click on referral REF-2025-001 in the worklist to open it", "2. Click the General tab — find the Clinical Note and READ IT to capture: height (175 cm) and weight (82 kg)", "3. Calculate BSA: sqrt(175 × 82 / 3600) = 2.0", "4. Calculate annual dose: 2.0 × 12 months × 0.5mg per injection = 12mg", "5. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: H35.32, H35.31, E11.9", "6. REQUIRED: Click the Services tab — record all CPT codes: 67028, J2778", "7. Click the Coverages tab — record member ID (AET123456789), DOB (1965-03-15), and payer portal credentials (provider@payera.com / demo123)", "8. Go to the General tab — click the Medical Necessity Letter document to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "9. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "10. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "11. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "12. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "13. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "14. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the lookup button to populate provider details", "15. In the Patient (Member ID) field, click and type AET123456789. Click Date of Birth and type 03/15/1965 in MM/DD/YYYY format. Click Search.", "16. Add all 3 diagnosis codes: type H35.32 click Add, type H35.31 click Add, type E11.9 click Add", "17. Enter servicing provider: Dr. Jane Smith", "18. Add both CPT codes: type 67028 click Add, type J2778 click Add", "19. Click Clinical Indication and type a justification that includes: the calculated annual dosage (12mg), AMD diagnosis (macular degeneration), and intravitreal anti-VEGF injections", "20. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to the Medical Necessity Letter", "21. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "22. Note the authorization reference number from the confirmation screen", "23. Click 'Return to EMR' to navigate back", "24. Add a Communication note in EMR that includes the authorization reference number", "25. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-1-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get patient measurements (height/weight)", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET123456789", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-10", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1965-03-15", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-11", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code H35.32 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H35.32')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-12", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code H35.31 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H35.31')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-13", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 67028 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '67028')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-14", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code J2778 (ranibizumab injection) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J2778')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-15", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17. Clinical indication includes calculated annual dosage (~12mg)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text include the calculated annual dosage of approximately 12mg (acceptable range: 11.9-12.1mg) based on BSA calculation using height 175cm and weight 82kg? Score 1.0 if it mentions a dosage in the 11-13mg range, 0.0 if no dosage or incorrect dosage.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-1-eval-16", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18. Clinical indication mentions the diagnosis (AMD)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention AMD or macular degeneration? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-1-eval-17", "type": "jmespath", "category": "Document Handling", "description": "19. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "20. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-19", "type": "jmespath", "category": "Documentation", "description": "21. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-1-eval-20", "type": "llm_judge", "category": "Documentation", "description": "22. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-1-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "23. Agent cleared referral REF-2025-001 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-001')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "calculate_and_submit", "requires_calculation": true }, "rationale": "" }, { "id": "emr-hard-2", "title": "Calculate Chemo Treatment Visits", "instruction": "Open referral REF-2025-301 for Adams, Paul (63M with Aetna HMO). This patient needs chemotherapy authorization. Review the clinical notes to find the treatment plan and calculate the total number of visits needed for the complete treatment course. Submit the authorization to Aetna requesting the correct number of visits, record the auth number in EMR, and clear the referral.", "goal": "Open referral REF-2025-301 for Adams, Paul (63M with Aetna HMO). This patient needs chemotherapy authorization. Review the clinical notes to find the treatment plan and calculate the total number of visits needed for the complete treatment course. Submit the authorization to Aetna requesting the correct number of visits, record the auth number in EMR, and clear the referral.", "difficulty": "hard", "category": "calculation", "challengeType": "calculation", "possible": true, "points": 24, "configTaskId": "hard_2", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Adams, Paul" }, "stepByStep": [ "1. Click on referral REF-2025-301 in the worklist to open it", "2. Click the General tab — find the Clinical Note and READ IT to capture: 6 treatment cycles, every 3 weeks, 2 infusion visits per cycle", "3. Calculate total visits: 6 cycles × 2 visits per cycle = 12 total visits", "4. REQUIRED: Click the Diagnoses tab — record diagnosis codes: C18.9, Z51.11", "5. REQUIRED: Click the Services tab — record ALL 4 CPT codes: 96413, 96415, J9263, J9190", "6. Click the Coverages tab — record member ID (AET301000001), DOB (1962-05-14), and payer portal credentials (provider@payera.com / demo123)", "7. Go to the General tab — click the Medical Necessity Letter document to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "8. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "9. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "10. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "11. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "12. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "13. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the lookup button to populate provider details", "14. IMPORTANT DATE FORMAT: In the Patient (Member ID) field, click and type AET301000001. Click the Date of Birth field and type 05/14/1962 in MM/DD/YYYY format (month first: 05, then day: 14, then year: 1962). Click Search.", "15. Add both diagnosis codes: type C18.9 click Add, type Z51.11 click Add", "16. Enter servicing provider: Dr. Sarah Chen", "17. Add all 4 CPT codes: type 96413 click Add, type 96415 click Add, type J9263 click Add, type J9190 click Add", "18. Click Clinical Indication and type a justification that includes: 12 total visits (6 cycles × 2 visits), colon cancer (C18.9), FOLFOX chemotherapy regimen", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to the Medical Necessity Letter", "20. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the authorization reference number", "24. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-2-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get treatment plan (cycles and visits)", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET301000001", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-10", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1962-05-14", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-11", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code C18.9 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'C18.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-12", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code Z51.11 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'Z51.11')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-13", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 96413 (chemo IV first hour) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '96413')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-14", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code 96415 (chemo IV additional hour) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '96415')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-15", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered CPT code J9263 (oxaliplatin injection) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J9263')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-16", "type": "jmespath", "category": "Form Completion", "description": "18. Agent entered CPT code J9190 (fluorouracil injection) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J9190')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "19. Clinical indication includes calculated visits (12 visits from 6 cycles x 2)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text include the calculated number of visits (12 visits) based on 6 cycles x 2 visits per cycle? Score 1.0 if it mentions 12 visits or the calculation (6 cycles, 2 visits per cycle), 0.0 if not.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-2-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "20a. Clinical indication mentions the diagnosis (colon cancer)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention colon cancer or colorectal cancer? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-2-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "20b. Clinical indication mentions the treatment (chemotherapy/FOLFOX)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention chemotherapy or FOLFOX? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-2-eval-20", "type": "jmespath", "category": "Document Handling", "description": "21. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-22", "type": "jmespath", "category": "Documentation", "description": "23. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-2-eval-23", "type": "llm_judge", "category": "Documentation", "description": "24. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-2-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "25. Agent cleared referral REF-2025-301 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-301')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "calculate_and_submit", "requires_calculation": true }, "rationale": "" }, { "id": "emr-hard-3", "title": "Calculate J-Code Billing Units", "instruction": "Open referral REF-2025-105 for Reed, Jordan (36M with Anthem Blue Cross PPO). This patient needs biologic therapy for psoriasis. Review the clinical notes for the prescribed dosing regimen. Calculate the appropriate billing units for 6 months of treatment and submit the authorization to Payer B with the correct units. Record the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-105 for Reed, Jordan (36M with Anthem Blue Cross PPO). This patient needs biologic therapy for psoriasis. Review the clinical notes for the prescribed dosing regimen. Calculate the appropriate billing units for 6 months of treatment and submit the authorization to Payer B with the correct units. Record the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "calculation", "challengeType": "calculation", "possible": true, "points": 24, "configTaskId": "hard_3", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Reed, Jordan" }, "stepByStep": [ "1. Click on referral REF-2025-105 in the worklist to open it", "2. Click the General tab — find the Clinical Note and READ IT to capture: adalimumab 40mg dose, every 2 weeks", "3. Calculate J-code units: 40mg dose ÷ 20mg per J3590 unit = 2 units per injection", "4. Calculate 6-month injections: 26 weeks ÷ 2 weeks = 13 injections", "5. Calculate total units: 13 injections × 2 units = 26 total units", "6. REQUIRED: Click the Diagnoses tab — record diagnosis codes: L40.0, L40.50", "7. REQUIRED: Click the Services tab — record CPT/J-code: J3590", "8. Click the Coverages tab — record subscriber ID (BCBS55001234), DOB (1989-04-07), and Payer B portal credentials (provider@payerb.com / demo123)", "9. Go to the General tab — find the document labeled 'Medical_Necessity_Letter_Biologic_*.pdf' (it is marked AUTH LETTER; Required) — click it to open the viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "10. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the Payer B portal", "11. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "12. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "13. Step 1 — Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Medical Injectable'.", "14. Enter Patient Name: Reed, Jordan. Click Date of Birth and type 04/07/1989 in MM/DD/YYYY format. Enter Subscriber ID: BCBS55001234. Click Next.", "15. CRITICAL: Stay in the Step 2 Service Details form — do NOT navigate away until all fields are complete. Add diagnosis codes: type L40.0, click Add; type L40.50, click Add. Add J-code: type J3590, click Add. Confirm each code appears in its table before proceeding.", "16. Click Clinical Indication and type a justification that includes: 26 total J3590 units, psoriasis/biologic therapy, adalimumab (Humira)", "17. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to the downloaded Medical_Necessity_Letter document", "18. Click Next to proceed through remaining steps, then click Submit Request", "19. Note the authorization reference number from the confirmation screen", "20. Click 'Return to EMR' to navigate back", "21. Add a Communication note in EMR that includes the authorization reference number", "22. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-3-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get prescribed dosing regimen", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent selected Medical Injectable as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "medical-injectable", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "BCBS55001234", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Reed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Jordan", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1989-04-07", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code L40.0 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'L40.0')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered diagnosis code L40.50 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'L40.50')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code J3590 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J3590')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17. Clinical indication includes calculated billing units (26 units)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text include the calculated billing units (26 units) based on the J-code calculation (40mg dose / 20mg per unit = 2 units per injection, 13 injections x 2 = 26 total units)? Score 1.0 if it mentions 26 units or shows the calculation, 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-3-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18a. Clinical indication mentions the diagnosis (psoriasis)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention psoriasis? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-3-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18b. Clinical indication mentions the treatment (biologic/adalimumab)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention biologic therapy or adalimumab? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-3-eval-20", "type": "jmespath", "category": "Document Handling", "description": "19. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "20. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-22", "type": "jmespath", "category": "Documentation", "description": "21. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-3-eval-23", "type": "llm_judge", "category": "Documentation", "description": "22. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-3-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "23. Agent cleared referral REF-2025-105 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-105')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "calculate_and_submit", "requires_calculation": true }, "rationale": "" }, { "id": "emr-hard-4", "title": "Verify Weight-Based Dosing", "instruction": "Open referral REF-2025-302 for Baker, Carol (55F with Aetna PPO). This patient needs infliximab infusion. Verify the ordered dose is appropriate based on the patient's weight in the clinical notes. Submit the authorization to Aetna with the verified dosage, record the auth number in EMR, and clear the referral.", "goal": "Open referral REF-2025-302 for Baker, Carol (55F with Aetna PPO). This patient needs infliximab infusion. Verify the ordered dose is appropriate based on the patient's weight in the clinical notes. Submit the authorization to Aetna with the verified dosage, record the auth number in EMR, and clear the referral.", "difficulty": "hard", "category": "calculation", "challengeType": "calculation", "possible": true, "points": 20, "configTaskId": "hard_4", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Baker, Carol" }, "stepByStep": [ "1. Click on referral REF-2025-302 in the worklist to open it", "2. Click the General tab — find the Clinical Note and READ IT to capture: patient weight (68 kg)", "3. Calculate correct dose: 5mg/kg × 68kg = 340mg (verify this matches the ordered dose)", "4. REQUIRED: Click the Diagnoses tab — record diagnosis code: M05.79", "5. REQUIRED: Click the Services tab — record CPT/J-codes: J1745, 96413", "6. Click the Coverages tab — record member ID (AET302000002), DOB (1970-08-22), and payer portal credentials (provider@payera.com / demo123)", "7. Go to the General tab — click the Medical Necessity Letter document to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "8. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "9. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "10. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "11. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "12. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "13. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the lookup button to populate provider details", "14. In the Patient (Member ID) field, click and type AET302000002. Click Date of Birth and type 08/22/1970 in MM/DD/YYYY format. Click Search.", "15. Add diagnosis code: type M05.79 click Add", "16. Enter servicing provider name as shown in the referral header in EMR", "17. Add CPT codes: type J1745 click Add, type 96413 click Add", "18. Click Clinical Indication and type a justification that includes: weight-based dosing (5mg/kg × 68kg = 340mg), rheumatoid arthritis (M05.79), infliximab (Remicade)", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to the Medical Necessity Letter", "20. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the authorization reference number", "24. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-4-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get patient weight", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET302000002", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-10", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1970-08-22", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-11", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code M05.79 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M05.79')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-12", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered CPT code J1745 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J1745')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "15. Clinical indication includes weight-based dosage verification (340mg)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text include the weight-based dosage verification (340mg based on 5mg/kg x 68kg)? Score 1.0 if it mentions 340mg or shows the weight-based calculation (68kg, 5mg/kg), 0.0 if not.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-4-eval-14", "type": "llm_judge", "category": "Clinical Reasoning", "description": "16a. Clinical indication mentions the diagnosis (rheumatoid arthritis)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention rheumatoid arthritis or RA? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-4-eval-15", "type": "llm_judge", "category": "Clinical Reasoning", "description": "16b. Clinical indication mentions the treatment (infliximab/Remicade)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention infliximab or Remicade? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-4-eval-16", "type": "jmespath", "category": "Document Handling", "description": "17. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-17", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-18", "type": "jmespath", "category": "Documentation", "description": "19. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-4-eval-19", "type": "llm_judge", "category": "Documentation", "description": "20. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-4-eval-20", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent cleared referral REF-2025-302 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-302')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "verify_and_submit", "requires_calculation": true }, "rationale": "" }, { "id": "emr-hard-5", "title": "Check Existing Auth Before Submit", "instruction": "Open referral REF-2025-003 for Johnson, Michael (63M with Aetna HMO). Before submitting a new authorization for cataract surgery, first search the Payer A portal to check if an existing authorization already exists for this patient (Member ID: AET987654321). If the existing auth is expired or none exists, submit a new authorization request. Record the new auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-003 for Johnson, Michael (63M with Aetna HMO). Before submitting a new authorization for cataract surgery, first search the Payer A portal to check if an existing authorization already exists for this patient (Member ID: AET987654321). If the existing auth is expired or none exists, submit a new authorization request. Record the new auth number in EMR and clear the referral.", "difficulty": "hard", "category": "multi_step", "challengeType": "multi_step", "possible": true, "points": 18, "configTaskId": "hard_5", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Johnson, Michael" }, "stepByStep": [ "1. Click on referral REF-2025-003 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: H25.11", "3. REQUIRED: Click the Services tab — record all CPT codes: 66984", "4. Click the Coverages tab — record member ID (AET987654321), DOB (09/15/1962), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, use 'Search Authorizations' to check for existing auth using member ID AET987654321. Review results — note auth AUTH-2025-004821 is EXPIRED. Since auth is expired, proceed to submit a new authorization.", "9. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "10. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "11. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "12. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the search/lookup button to populate provider details", "13. In the Patient (Member ID) field (Section 3), click and type AET987654321. Click the Date of Birth field and type 09/15/1962 in MM/DD/YYYY format. Click Search to verify eligibility.", "14. Add diagnosis code: click Diagnosis field, type H25.11, click Add", "15. Enter servicing provider name as shown in the referral header in EMR", "16. Add the CPT code: click CPT field, type 66984, click Add", "17. Click the Clinical Indication field and type a justification for the procedure and diagnosis. Explicitly mention cataract surgery / cataract removal with IOL implant for diagnosis H25.11.", "18. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "19. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "20. Note the authorization reference number from the confirmation screen", "21. Click 'Return to EMR' to navigate back", "22. Add a Communication note in EMR that includes the exact authorization reference number from step 20", "23. Save the EMR note, verify it appears in Communications, and then click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-5-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent searched for existing auths using member ID AET987654321", "points": 1, "query": "aetna_state.differences.authSearches[0].memberId || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type (when submitting new)", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET987654321", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1962-09-15", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code H25.11 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H25.11')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered CPT code 66984 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '66984')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13. Clinical indication mentions cataract surgery", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention cataract surgery or cataract removal or IOL implant or lens replacement? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-5-eval-14", "type": "jmespath", "category": "Document Handling", "description": "14. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "15. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-16", "type": "jmespath", "category": "Documentation", "description": "16. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-5-eval-17", "type": "llm_judge", "category": "Documentation", "description": "17. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-5-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent cleared referral REF-2025-003 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-003')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "search_then_submit", "multi_step": true, "existing_auth": "AUTH-2025-004821 (EXPIRED)", "steps": [ "search_existing", "recognize_expired", "submit_new", "record_in_epic" ] }, "rationale": "" }, { "id": "emr-hard-6", "title": "Verify Eligibility Then Submit", "instruction": "Open referral REF-2025-306 for Foster, Grace (68F with Aetna HMO). This patient needs urgent retinal detachment surgery. Before submitting the authorization, first verify the patient's eligibility on the Payer A portal to confirm coverage is active and surgical benefits are available. If eligibility is confirmed, submit the authorization for the vitrectomy procedure. Document the eligibility verification AND the auth number in EMR, then clear the referral.", "goal": "Open referral REF-2025-306 for Foster, Grace (68F with Aetna HMO). This patient needs urgent retinal detachment surgery. Before submitting the authorization, first verify the patient's eligibility on the Payer A portal to confirm coverage is active and surgical benefits are available. If eligibility is confirmed, submit the authorization for the vitrectomy procedure. Document the eligibility verification AND the auth number in EMR, then clear the referral.", "difficulty": "hard", "category": "multi_step", "challengeType": "multi_step", "possible": true, "points": 19, "configTaskId": "hard_6", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Foster, Grace" }, "stepByStep": [ "1. Click on referral REF-2025-306 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: H33.001", "3. REQUIRED: Click the Services tab — record all CPT codes: 67108", "4. Click the Coverages tab — record member ID (AET306000006), DOB (02/20/1958), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. FIRST check eligibility: click 'Member Eligibility' tab. Enter member ID AET306000006, last name Foster, first name Grace, DOB 02/20/1958. Click Submit. Confirm coverage is active and surgical benefits are available. Record the eligibility results.", "9. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "10. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "11. In the Authorization Request modal, click the Request Type dropdown and select 'Inpatient Surgical'", "12. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the search/lookup button to populate provider details", "13. In the Patient (Member ID) field (Section 3), click and type AET306000006. Click the Date of Birth field and type 02/20/1958 in MM/DD/YYYY format. Click Search to verify eligibility.", "14. Add diagnosis code: click Diagnosis field, type H33.001, click Add", "15. Enter servicing provider name as shown in the referral header in EMR", "16. Add the CPT code: click CPT field, type 67108, click Add", "17. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "18. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "19. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "20. Note the authorization reference number from the confirmation screen", "21. Click 'Return to EMR' to navigate back", "22. Add a Communication note in EMR that includes the authorization reference number", "23. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-6-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent checked eligibility for member AET306000006", "points": 1, "query": "aetna_state.differences.eligibilityChecks[0].memberId || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Inpatient Surgical as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "inpatient-surgical", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET306000006", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1958-02-20", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered diagnosis code H33.001 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'H33.001')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered CPT code 67108 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '67108')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "13. Clinical indication mentions retinal detachment/vitrectomy", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention retinal detachment or vitrectomy or retinal surgery? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-6-eval-14", "type": "jmespath", "category": "Document Handling", "description": "14. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "15. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-16", "type": "jmespath", "category": "Documentation", "description": "16. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-6-eval-17", "type": "llm_judge", "category": "Documentation", "description": "17. EMR note documents eligibility verification", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention eligibility verification (e. g., 'eligibility confirmed', 'coverage verified', 'active coverage')? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-6-eval-18", "type": "llm_judge", "category": "Documentation", "description": "17b. EMR note contains auth number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-6-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent cleared referral REF-2025-306 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-306')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "eligibility_then_submit", "multi_step": true, "steps": [ "check_eligibility", "confirm_surgical_benefits", "submit_auth", "document_both" ] }, "rationale": "" }, { "id": "emr-hard-7", "title": "Extract Clinical Justification from Note", "instruction": "Open referral REF-2025-401 for Irving, James (55M with Anthem Blue Cross HMO). This patient needs knee arthroscopy surgery. Before submitting the authorization, thoroughly review the clinical note to extract the specific clinical justification for surgery (failed conservative management, specific exam findings, imaging results). Navigate to Payer B portal and submit the authorization, ensuring the clinical indication field contains the specific justifications found in the note (not generic text). Record the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-401 for Irving, James (55M with Anthem Blue Cross HMO). This patient needs knee arthroscopy surgery. Before submitting the authorization, thoroughly review the clinical note to extract the specific clinical justification for surgery (failed conservative management, specific exam findings, imaging results). Navigate to Payer B portal and submit the authorization, ensuring the clinical indication field contains the specific justifications found in the note (not generic text). Record the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "multi_step", "challengeType": "multi_step", "possible": true, "points": 22, "configTaskId": "hard_7", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Irving, James" }, "stepByStep": [ "1. Click on referral REF-2025-401 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab BEFORE opening any documents — record all diagnosis codes: M23.221. You must visit this tab first.", "3. REQUIRED: Click the Services tab BEFORE opening any documents — record all CPT codes: 29881. You must visit this tab first.", "4. REQUIRED: Click the Coverages tab — record subscriber ID (ANT401000001), DOB (04/18/1970), and payer portal credentials (provider@payerb.com / demo123). You will also use this tab to open the portal. You must visit this tab.", "5. Go to the General tab — find the Clinical Note and READ IT thoroughly to capture specific clinical justifications: failed conservative management (PT duration and outcome), exam findings (McMurray test result), imaging results (MRI findings). These are required verbatim in the clinical indication.", "6. From the General tab, click 'Medical Necessity Letter' to open its viewer. CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal. Also download the Clinical Note.", "7. REQUIRED: Navigate to the Coverages tab and click 'Open Payer B Portal' link. Do NOT use any 'Submit to Payer' shortcut button on document viewer pages — you must go through the Coverages tab to open the portal.", "8. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "9. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "10. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Surgical'.", "11. Click Patient Name field and type: Irving, James. Click Date of Birth field and type 04/18/1970 in MM/DD/YYYY format. Click Subscriber ID field and type ANT401000001. Click Next.", "12. CRITICAL: Stay in this multi-step form — do NOT navigate away. Step 2 - Service Details: type M23.221, click Add to confirm it appears in the table; type 29881, click Add to confirm it appears in the table.", "13. Click Clinical Indication field and type a justification for the procedure and diagnosis — include the specific findings from the clinical note (failed conservative management, McMurray test result, MRI findings)", "14. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "15. Click Next to proceed through remaining steps. In provider details, enter provider name as shown in the referral header in EMR. Click Next.", "16. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "17. Note the authorization reference number from the confirmation screen", "18. Click 'Return to EMR' to navigate back", "19. Add a Communication note in EMR that includes the authorization reference number", "20. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-7-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to extract specific justifications", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent selected Surgery as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "surgery", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT401000001", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Irving", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "James", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1970-04-18", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code M23.221 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M23.221')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 29881 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '29881')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-16", "type": "jmespath", "category": "Clinical Reasoning", "description": "16. Clinical indication mentions meniscus (the specific diagnosis from note)", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].clinicalIndication || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17. Clinical indication mentions the diagnosis (meniscus tear/arthroscopy)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention meniscus tear or knee arthroscopy or meniscectomy? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-7-eval-18", "type": "jmespath", "category": "Document Handling", "description": "18. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "19. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-20", "type": "jmespath", "category": "Documentation", "description": "20. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-7-eval-21", "type": "llm_judge", "category": "Documentation", "description": "21. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-7-eval-22", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent cleared referral REF-2025-401 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-401')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "extract_and_submit", "multi_step": true, "requires_note_review": true }, "rationale": "" }, { "id": "emr-hard-8", "title": "Complete End-to-End Workflow with Search and Eligibility", "instruction": "Open referral REF-2025-101 for Nguyen, Linh (52F with Aetna HMO). Complete the FULL authorization workflow: (1) Review referral and clinical note in EMR, (2) Navigate to Payer A portal, (3) FIRST search for any existing authorizations for this member, (4) Check member eligibility to verify coverage is active, (5) Submit the authorization with ALL diagnosis codes and CPT codes, (6) Return to EMR, document the auth number, and clear the referral. This is the comprehensive end-to-end workflow.", "goal": "Open referral REF-2025-101 for Nguyen, Linh (52F with Aetna HMO). Complete the FULL authorization workflow: (1) Review referral and clinical note in EMR, (2) Navigate to Payer A portal, (3) FIRST search for any existing authorizations for this member, (4) Check member eligibility to verify coverage is active, (5) Submit the authorization with ALL diagnosis codes and CPT codes, (6) Return to EMR, document the auth number, and clear the referral. This is the comprehensive end-to-end workflow.", "difficulty": "hard", "category": "multi_step", "challengeType": "multi_step", "possible": true, "points": 24, "configTaskId": "hard_8", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Nguyen, Linh" }, "stepByStep": [ "1. Click on referral REF-2025-101 in the worklist to open it", "2. Click the General tab — find the Clinical Note and read it to capture: chest pain symptoms, hypertension, hyperlipidemia details for clinical indication", "3. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: R07.9, I10, E78.5", "4. REQUIRED: Click the Services tab — record all CPT codes: 93350, 93015", "5. Click the Coverages tab — record member ID (AET555000111), DOB (02/10/1974), and payer portal credentials (provider@payera.com / demo123)", "6. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "7. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "8. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "9. On the Payer A dashboard, use 'Search Authorizations' to check for existing auth using member ID AET555000111", "10. Click 'Member Eligibility' tab. Enter member ID AET555000111, last name Nguyen, first name Linh, DOB 02/10/1974. Click Submit. Confirm active coverage and record plan details.", "11. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "12. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "13. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "14. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the search/lookup button to populate provider details", "15. In the Patient (Member ID) field (Section 3), click and type AET555000111. Click the Date of Birth field and type 02/10/1974 in MM/DD/YYYY format. Click Search to verify eligibility.", "16. Add all diagnosis codes one by one: type R07.9, click Add; then type I10, click Add; then type E78.5, click Add", "17. Enter servicing provider name as shown in the referral header in EMR", "18. Add all CPT codes: type 93350, click Add; then type 93015, click Add", "19. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "20. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "21. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "22. Note the authorization reference number from the confirmation screen", "23. Click 'Return to EMR' to navigate back", "24. Add a Communication note in EMR that includes the authorization reference number", "25. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-8-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes for medical necessity", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-8", "type": "jmespath", "category": "Information Retrieval", "description": "8. Agent searched for existing authorizations by member ID", "points": 1, "query": "aetna_state.differences.authSearches[0].memberId || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-9", "type": "jmespath", "category": "Information Retrieval", "description": "9. Agent checked eligibility for the member", "points": 1, "query": "aetna_state.differences.eligibilityChecks[0].memberId || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET555000111", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1974-02-10", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code R07.9 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R07.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code I10 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'I10')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered diagnosis code E78.5 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'E78.5')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code 93350 (stress echo) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '93350')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-17", "type": "jmespath", "category": "Form Completion", "description": "17. Agent entered CPT code 93015 (cardiovascular stress test) in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '93015')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18a. Clinical indication mentions the diagnosis (chest pain/hypertension)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention chest pain or hypertension? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-8-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18b. Clinical indication mentions the procedure (stress test/echocardiogram)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention stress test or echocardiogram or cardiac evaluation? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-8-eval-20", "type": "jmespath", "category": "Document Handling", "description": "19. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "20. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-22", "type": "jmespath", "category": "Documentation", "description": "21. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-8-eval-23", "type": "llm_judge", "category": "Documentation", "description": "22. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-8-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "23. Agent cleared referral REF-2025-101 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-101')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "full_workflow", "multi_step": true, "requires_search": true, "requires_eligibility": true, "steps": [ "review_referral", "read_clinical_note", "navigate_portal", "search_existing_auths", "check_eligibility", "submit_auth", "receive_confirmation", "return_to_epic", "document_auth", "clear_referral" ] }, "rationale": "" }, { "id": "emr-hard-9", "title": "Detect Subtle Laterality Error in Clinical Note", "instruction": "Open referral REF-2025-409 for Martinez, Elena (58F with Aetna PPO). Review all documentation carefully before submitting a prior authorization for knee arthroscopy to the Payer A portal. If any discrepancies are found between the diagnosis and clinical documentation, document the issue and do not submit.", "goal": "Open referral REF-2025-409 for Martinez, Elena (58F with Aetna PPO). Review all documentation carefully before submitting a prior authorization for knee arthroscopy to the Payer A portal. If any discrepancies are found between the diagnosis and clinical documentation, document the issue and do not submit.", "difficulty": "hard", "category": "error_detection", "challengeType": "error_detection", "possible": true, "points": 9, "configTaskId": "hard_9", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Martinez, Elena" }, "stepByStep": [ "1. Click on referral REF-2025-409 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — note diagnosis M23.221 (RIGHT knee)", "3. REQUIRED: Click the Services tab — note CPT 29881 (knee meniscectomy)", "4. Click the General tab — find the Clinical Note and READ IT VERY CAREFULLY from start to finish", "5. Note that the History, Physical Exam, Imaging, and Assessment sections all describe the RIGHT knee", "6. READ THE PLAN SECTION CAREFULLY — it says 'LEFT knee' — this contradicts everything else in the note", "7. CRITICAL ERROR IDENTIFIED: Laterality mismatch — Plan says LEFT, but all other sections say RIGHT. This is a wrong-site surgery risk.", "8. DO NOT navigate to payer portal — do NOT submit an authorization", "9. Click Add Note in Communications", "10. Write a note documenting: internal inconsistency in clinical note (Plan says LEFT knee, but HPI/Exam/Imaging/Assessment all say RIGHT knee); authorization is on hold pending surgeon clarification; requesting correction before proceeding to prevent wrong-site surgery risk", "11. Save the note. Do NOT clear the referral from the worklist — it requires physician action." ], "evals": [ { "id": "emr-hard-9-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-9-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-9-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent read clinical notes to review documentation", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-9-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "4. Agent correctly did NOT submit authorization (laterality discrepancy)", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-9-eval-5", "type": "jmespath", "category": "Documentation", "description": "5. Agent added a note in EMR Communications (auth or clinical note acceptable for safety/laterality)", "points": 1, "query": "full_state.agentActions.addedAuthNote || full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-9-eval-6", "type": "llm_judge", "category": "Documentation", "description": "6. EMR note documents the laterality inconsistency (plan says LEFT, rest says RIGHT)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention a laterality discrepancy or inconsistency within the clinical documentation - specifically that the surgical PLAN says LEFT knee but the rest of the note (history, exam, imaging, assessment) describes RIGHT knee? Score 1.0 if it documents the laterality inconsistency in the plan vs the rest of the note, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-9-eval-7", "type": "llm_judge", "category": "Documentation", "description": "7. EMR note indicates authorization cannot be submitted", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate that authorization cannot be submitted or is on hold? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-9-eval-8", "type": "llm_judge", "category": "Documentation", "description": "7b. EMR note indicates clarification needed from physician", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate that physician or surgeon clarification is needed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-9-eval-9", "type": "jmespath", "category": "Task Resolution", "description": "8. Agent correctly did NOT clear the referral (needs clarification)", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-409')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "detect_error_and_halt", "should_submit": false, "should_clear": false, "should_navigate_to_portal": false, "error_type": "laterality_mismatch" }, "rationale": "" }, { "id": "emr-hard-10", "title": "Detect Inactive Insurance Coverage", "instruction": "Open referral REF-2025-502 for Quinn, Robert (60M with Aetna HMO). Review all documentation and verify insurance coverage before submitting a prior authorization for stress echocardiogram to the Payer A portal. If any issues are found with coverage status, document the issue and do not submit.", "goal": "Open referral REF-2025-502 for Quinn, Robert (60M with Aetna HMO). Review all documentation and verify insurance coverage before submitting a prior authorization for stress echocardiogram to the Payer A portal. If any issues are found with coverage status, document the issue and do not submit.", "difficulty": "hard", "category": "error_detection", "challengeType": "error_detection", "possible": true, "points": 11, "configTaskId": "hard_10", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Quinn, Robert" }, "stepByStep": [ "1. Click on referral REF-2025-502 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — note diagnosis R07.9 (chest pain)", "3. REQUIRED: Click the Services tab — note CPT 93350 (stress echo)", "4. Click the Coverages tab — check insurance status: it shows INACTIVE", "5. Scroll down on the Coverages/Auth page — the portal button ('Open … Portal →') is in the 'Payer Portal Access' section BELOW the Coverage Details and is NOT visible without scrolling. Click it, then log in with provider@payera.com / demo123.", "6. Click 'Member Eligibility' tab. Enter member ID AET502000002. Click Submit.", "7. CONFIRM that coverage is INACTIVE/TERMINATED in the payer portal", "8. DO NOT submit an authorization — coverage must be active first", "9. Click 'Return to EMR' to navigate back", "10. Click Add Note in Communications", "11. Write a note documenting: patient insurance coverage is inactive/terminated; authorization cannot be submitted until coverage is verified and reinstated; referral is on hold pending insurance resolution", "12. Save the note. Do NOT clear the referral — it requires insurance verification first." ], "evals": [ { "id": "emr-hard-10-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-10-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-10-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to check insurance status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-10-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-10-eval-5", "type": "jmespath", "category": "Form Completion", "description": "5. Agent checked eligibility for member AET502000002", "points": 1, "query": "aetna_state.differences.eligibilityChecks[0].memberId || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-10-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "6. Agent correctly did NOT submit authorization (inactive coverage)", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-10-eval-7", "type": "jmespath", "category": "Documentation", "description": "7. Agent added a note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-10-eval-8", "type": "llm_judge", "category": "Documentation", "description": "8. EMR note documents that insurance coverage is inactive", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention that the insurance is inactive, terminated, or coverage has ended? Score 1.0 if it documents inactive/terminated coverage, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-10-eval-9", "type": "llm_judge", "category": "Documentation", "description": "9. EMR note explains authorization cannot be submitted", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note explain that authorization cannot be submitted due to the coverage issue? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-10-eval-10", "type": "llm_judge", "category": "Documentation", "description": "10. EMR note indicates coverage resolution is needed", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate that coverage needs to be resolved or verified before proceeding? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-10-eval-11", "type": "jmespath", "category": "Task Resolution", "description": "11. Agent correctly did NOT clear the referral (needs insurance resolution)", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-502')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "detect_inactive_and_halt", "should_submit": false, "should_clear": false, "error_type": "inactive_coverage" }, "rationale": "" }, { "id": "emr-hard-11", "title": "Detect Laterality Conflict", "instruction": "Open referral REF-2025-503 for Rogers, Sandra (65F with Anthem Blue Cross PPO). Submit a prior authorization for intravitreal injection to the Payer B portal. Record the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-503 for Rogers, Sandra (65F with Anthem Blue Cross PPO). Submit a prior authorization for intravitreal injection to the Payer B portal. Record the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "error_detection", "challengeType": "error_detection", "possible": true, "points": 9, "configTaskId": "hard_11", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Rogers, Sandra" }, "stepByStep": [ "1. Click on referral REF-2025-503 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — note CAREFULLY: H35.3211 (Exudative AMD, RIGHT eye)", "3. REQUIRED: Click the Services tab — note the procedure: Intravitreal Injection, LEFT eye (OS)", "4. REQUIRED: Click the Coverages tab to check the patient's insurance status and coverage details. THIS STEP IS EVALUATED AND MUST BE DONE even if you already detect the laterality mismatch from Diagnoses and Services.", "5. ONLY AFTER checking Coverages, IDENTIFY THE CONFLICT: Diagnosis is RIGHT eye, but procedure is for LEFT eye — laterality mismatch", "6. This is a critical error that could cause harm — do NOT submit an authorization", "7. Click Add Note in Communications", "8. Write a note documenting: laterality mismatch found (right eye diagnosis H35.3211 vs left eye procedure); authorization cannot be submitted until the ordering physician clarifies the correct eye", "9. Save the note. Do NOT clear the referral — it needs physician clarification." ], "evals": [ { "id": "emr-hard-11-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-11-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes and laterality", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-11-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to check insurance status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-11-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "5. Agent correctly did NOT submit an authorization due to documentation discrepancy", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-11-eval-5", "type": "jmespath", "category": "Documentation", "description": "6. Agent added a note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-11-eval-6", "type": "llm_judge", "category": "Documentation", "description": "7. EMR note documents the laterality conflict", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention a laterality inconsistency between the diagnosis and the requested procedure? Score 1.0 if it documents a laterality mismatch, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-11-eval-7", "type": "llm_judge", "category": "Documentation", "description": "8. EMR note explains authorization cannot proceed", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note explain that authorization cannot proceed or is on hold? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-11-eval-8", "type": "llm_judge", "category": "Documentation", "description": "8b. EMR note indicates clarification needed", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate that clarification or correction is needed from the physician? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-11-eval-9", "type": "jmespath", "category": "Task Resolution", "description": "9. Agent correctly did NOT clear the referral (needs physician clarification)", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-503')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "detect_error_and_halt", "should_submit": false, "should_clear": false, "error_type": "laterality_conflict" }, "rationale": "" }, { "id": "emr-hard-12", "title": "Detect Missing Conservative Treatment", "instruction": "Open referral REF-2025-504 for Stevens, Thomas (55M with Aetna HMO). Submit a prior authorization for lumbar spine MRI to the Payer A portal. Record the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-504 for Stevens, Thomas (55M with Aetna HMO). Submit a prior authorization for lumbar spine MRI to the Payer A portal. Record the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "error_detection", "challengeType": "error_detection", "possible": true, "points": 10, "configTaskId": "hard_12", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Stevens, Thomas" }, "stepByStep": [ "1. Click on referral REF-2025-504 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — note diagnoses M54.5, M54.16", "3. REQUIRED: Click the Services tab — note CPT 72148 (MRI lumbar spine)", "4. REQUIRED: Click the Coverages tab to check the patient's insurance status and coverage details.", "5. Click the General tab — find the Clinical Note and READ IT carefully", "6. Note the symptom duration: only 2 weeks of back pain", "7. IDENTIFY: Conservative treatment is NOT documented — no physical therapy, no medications tried", "8. Payers require documented failed conservative management before approving imaging authorization", "9. DO NOT submit an authorization — clinical documentation is insufficient", "10. Click Add Note in Communications", "11. Write a note documenting: conservative treatment documentation is missing; authorization cannot be submitted without evidence of failed conservative management; referral on hold pending additional clinical documentation", "12. Save the note. Do NOT clear the referral." ], "evals": [ { "id": "emr-hard-12-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-12-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-12-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent read clinical notes to check for conservative treatment history", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-12-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to check insurance status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-12-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "5. Agent correctly did NOT submit an authorization due to unmet documentation requirements", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-12-eval-6", "type": "jmespath", "category": "Documentation", "description": "6. Agent added a note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-12-eval-7", "type": "llm_judge", "category": "Documentation", "description": "7. EMR note documents the missing conservative treatment", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention that required conservative management documentation is missing or insufficient? Score 1.0 if it documents insufficient conservative management, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-12-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "8. EMR note explains authorization cannot be submitted", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note explain that authorization cannot be submitted because required criteria are not met? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-12-eval-9", "type": "llm_judge", "category": "Documentation", "description": "8b. EMR note indicates additional documentation is needed", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate that additional documentation or conservative treatment is needed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-12-eval-10", "type": "jmespath", "category": "Task Resolution", "description": "9. Agent correctly did NOT clear the referral (needs additional documentation)", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-504')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "detect_error_and_halt", "should_submit": false, "should_clear": false, "error_type": "missing_conservative_treatment" }, "rationale": "" }, { "id": "emr-hard-13", "title": "Detect ICD-10/CPT Mismatch", "instruction": "Open referral REF-2025-501 for Palmer, Patricia (53F with Aetna PPO). Submit a prior authorization for knee MRI to the Payer A portal. Record the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-501 for Palmer, Patricia (53F with Aetna PPO). Submit a prior authorization for knee MRI to the Payer A portal. Record the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "error_detection", "challengeType": "error_detection", "possible": true, "points": 10, "configTaskId": "hard_13", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Palmer, Patricia" }, "stepByStep": [ "1. Click on referral REF-2025-501 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — note CAREFULLY: L98.9 (Panniculitis — a SKIN condition)", "3. REQUIRED: Click the Services tab — note the procedure: 73721 (MRI Knee)", "4. REQUIRED: Click the Coverages tab to check the patient's insurance status and coverage details.", "5. IDENTIFY THE MISMATCH: A skin diagnosis (L98.9) cannot justify a knee MRI — diagnosis does not support the procedure", "6. DO NOT submit an authorization — diagnosis/procedure mismatch must be corrected first", "7. Click Add Note in Communications", "8. Write a note documenting: diagnosis-procedure mismatch found (skin condition diagnosis L98.9 cannot justify knee imaging 73721); ordering physician needs to provide a musculoskeletal diagnosis to support the knee MRI", "9. Save the note. Do NOT clear the referral — it needs a corrected diagnosis." ], "evals": [ { "id": "emr-hard-13-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-13-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-13-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to check insurance status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-13-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "4. Agent correctly did NOT submit an authorization due to a diagnosis/procedure mismatch", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-13-eval-5", "type": "jmespath", "category": "Documentation", "description": "5. Agent added a note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-13-eval-6", "type": "llm_judge", "category": "Documentation", "description": "6. EMR note mentions the diagnosis L98.9/panniculitis", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention the diagnosis code L98.9 or panniculitis (a skin condition)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-13-eval-7", "type": "llm_judge", "category": "Documentation", "description": "7. EMR note explains the diagnosis-procedure mismatch", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention that the skin diagnosis does not support or justify knee imaging/MRI? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-13-eval-8", "type": "llm_judge", "category": "Documentation", "description": "8. EMR note indicates authorization cannot be submitted", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate that authorization cannot be submitted or is on hold? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-13-eval-9", "type": "llm_judge", "category": "Documentation", "description": "9. EMR note indicates corrected diagnosis is needed", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate that a corrected or updated diagnosis is needed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-13-eval-10", "type": "jmespath", "category": "Task Resolution", "description": "10. Agent correctly did NOT clear the referral (needs corrected diagnosis)", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-501')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "detect_error_and_halt", "should_submit": false, "should_clear": false, "error_type": "icd_cpt_mismatch" }, "rationale": "" }, { "id": "emr-hard-14", "title": "Detect Inactive Insurance", "instruction": "Open referral REF-2025-502 for Quinn, Robert (60M with Aetna HMO). Submit a prior authorization for stress echocardiogram to the Payer A portal. Record the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-502 for Quinn, Robert (60M with Aetna HMO). Submit a prior authorization for stress echocardiogram to the Payer A portal. Record the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "error_detection", "challengeType": "error_detection", "possible": true, "points": 8, "configTaskId": "hard_14", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Quinn, Robert" }, "stepByStep": [ "1. Click on referral REF-2025-502 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — note diagnosis R07.9", "3. REQUIRED: Click the Services tab — note CPT 93350", "4. Click the Coverages tab — check insurance status: it shows INACTIVE", "5. IDENTIFY: Patient insurance is inactive/terminated — authorization cannot be submitted", "6. DO NOT navigate to payer portal or attempt to submit an authorization", "7. Click Add Note in Communications", "8. Write a note documenting: insurance coverage is inactive/terminated; authorization cannot be processed; patient needs to update coverage information before proceeding", "9. Save the note. Do NOT clear the referral." ], "evals": [ { "id": "emr-hard-14-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-14-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-14-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to check insurance status", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-14-eval-4", "type": "jmespath", "category": "Documentation", "description": "4. Agent added a note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-14-eval-5", "type": "llm_judge", "category": "Documentation", "description": "5. EMR note documents that insurance coverage is inactive", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention that coverage is not active? Score 1.0 if it documents inactive coverage, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-14-eval-6", "type": "llm_judge", "category": "Clinical Reasoning", "description": "6. EMR note explains why authorization cannot be submitted", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note explain that authorization cannot be submitted until coverage is resolved? Score 1.0 if it explains auth cannot proceed without coverage resolution, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-14-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "7. Agent correctly did NOT submit an authorization due to coverage issue", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-14-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "8. Agent correctly did NOT clear the referral (needs insurance verification)", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-502')", "expectedValue": false, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "detect_inactive_and_halt", "should_submit": false, "should_clear": false, "error_type": "inactive_insurance" }, "rationale": "" }, { "id": "emr-hard-15", "title": "Verify Lumbar MRI Medical Necessity", "instruction": "Open referral REF-2025-303 for Cooper, David (57M with Aetna PPO). This patient is being referred for an MRI of the lumbar spine. Review the patient's medical records to verify medical necessity documentation is present, extract the relevant clinical findings, and submit the authorization to Payer A. Document the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-303 for Cooper, David (57M with Aetna PPO). This patient is being referred for an MRI of the lumbar spine. Review the patient's medical records to verify medical necessity documentation is present, extract the relevant clinical findings, and submit the authorization to Payer A. Document the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "clinical_judgment", "challengeType": "clinical_judgment", "possible": true, "points": 19, "configTaskId": "hard_15", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Cooper, David" }, "stepByStep": [ "1. Click on referral REF-2025-303 in the worklist to open it", "2. Click the General tab — find the Clinical Note and READ IT thoroughly to capture: radiculopathy symptoms (radiating pain, numbness, weakness), symptom duration, red flag symptoms, conservative management history. Include these specific clinical findings in the authorization's clinical indication.", "3. REQUIRED - IMPORTANT: Click the Diagnoses tab — record all diagnosis codes: M54.5, M54.16", "4. REQUIRED: Click the Services tab — record all CPT codes: 72148", "5. Click the Coverages tab — record member ID (AET303000003), DOB (03/10/1968), and payer portal credentials (provider@payera.com / demo123)", "6. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "7. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "8. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "9. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "10. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "11. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "12. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the search/lookup button to populate provider details", "13. In the Patient (Member ID) field (Section 3), click and type AET303000003. Click the Date of Birth field and type 03/10/1968 in MM/DD/YYYY format. Click Search to verify eligibility.", "14. Add all diagnosis codes one by one: type M54.5, click Add; then type M54.16, click Add", "15. Enter servicing provider name as shown in the referral header in EMR", "16. Add the CPT code: click CPT field, type 72148, click Add", "17. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "18. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "19. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "20. Note the authorization reference number from the confirmation screen", "21. Click 'Return to EMR' to navigate back", "22. Add a Communication note in EMR that includes the authorization reference number", "23. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-15-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to verify medical necessity", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET303000003", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-10", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1968-03-10", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-11", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code M54.5 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M54.5')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-12", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code M54.16 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M54.16')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-13", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 72148 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '72148')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-14", "type": "llm_judge", "category": "Clinical Reasoning", "description": "16. Clinical indication contains SPECIFIC medical necessity documentation", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text contain SPECIFIC clinical findings for spine imaging such as: radiculopathy symptoms (radiating pain, numbness, weakness), symptom duration, or failed conservative treatment? Score 1.0 if it contains specific medical necessity criteria, 0.0 if only generic text.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-15-eval-15", "type": "jmespath", "category": "Document Handling", "description": "17. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-16", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-17", "type": "jmespath", "category": "Documentation", "description": "19. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-15-eval-18", "type": "llm_judge", "category": "Documentation", "description": "20. EMR note contains the Payer A authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-15-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent cleared referral REF-2025-303 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-303')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "extract_and_submit", "requires_note_review": true, "clinical_extraction_required": true }, "rationale": "" }, { "id": "emr-hard-16", "title": "Verify Step Therapy for Biologic", "instruction": "Open referral REF-2025-105 for Reed, Jordan (36M with Anthem Blue Cross PPO). This patient is being referred for biologic therapy for psoriasis. Review the clinical note to verify the documentation supports the medical necessity for biologic treatment. Submit the authorization to Payer B with the relevant clinical documentation. Document the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-105 for Reed, Jordan (36M with Anthem Blue Cross PPO). This patient is being referred for biologic therapy for psoriasis. Review the clinical note to verify the documentation supports the medical necessity for biologic treatment. Submit the authorization to Payer B with the relevant clinical documentation. Document the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "clinical_judgment", "challengeType": "clinical_judgment", "possible": true, "points": 24, "configTaskId": "hard_16", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Reed, Jordan" }, "stepByStep": [ "1. Click on referral REF-2025-105 in the worklist to open it", "2. Click the General tab — find the Clinical Note and READ IT to capture step therapy history: topical steroid trial outcome, phototherapy trial (narrowband UVB, 12 weeks), reasons for needing biologic. Include this step therapy documentation in the clinical indication.", "3. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: L40.0, L40.50", "4. REQUIRED: Click the Services tab — record all CPT codes: J3590. You must visit this tab.", "5. Click the Coverages tab — record subscriber ID (BCBS55001234), DOB (04/07/1989), and payer portal credentials (provider@payerb.com / demo123)", "6. Go to the General tab — click 'Medical Necessity Letter' to open its viewer. CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "7. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "8. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "9. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "10. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Medical Injectable'.", "11. Click Patient Name field and type: Reed, Jordan. Click Date of Birth field and type 04/07/1989 in MM/DD/YYYY format. Click Subscriber ID field and type BCBS55001234. Click Next.", "12. CRITICAL: Stay in this multi-step form — do NOT navigate away. Step 2 - Service Details: type L40.0, click Add; type L40.50, click Add; type J3590, click Add. Confirm each code appears in its table.", "13. Click Clinical Indication field and type a justification — include the step therapy history from the clinical note (topical steroid trial, phototherapy outcome, reason biologic is required)", "14. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "15. Click Next to proceed through remaining steps. In provider details, enter provider name as shown in the referral header in EMR. Click Next.", "16. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "17. Note the authorization reference number from the confirmation screen", "18. Click 'Return to EMR' to navigate back", "19. Add a Communication note in EMR that includes the authorization reference number", "20. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-16-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to verify step therapy documentation", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent selected Medical Injectable as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "medical-injectable", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "BCBS55001234", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Reed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Jordan", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1989-04-07", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code L40.0 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'L40.0')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered diagnosis code L40.50 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'L40.50')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-16", "type": "jmespath", "category": "Form Completion", "description": "16. Agent entered CPT code J3590 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, 'J3590')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17. Clinical indication contains step therapy compliance documentation", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text contain step therapy documentation such as: failed topical steroids, failed phototherapy (UVB), partial response to prior treatments, or reasons why biologic is needed (BSA involvement, pruritus)? Score 1.0 if it contains step therapy compliance evidence, 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-16-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18a. Clinical indication mentions the diagnosis (psoriasis)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention psoriasis? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-16-eval-19", "type": "llm_judge", "category": "Clinical Reasoning", "description": "18b. Clinical indication mentions the treatment (biologic therapy)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention biologic therapy? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-16-eval-20", "type": "jmespath", "category": "Document Handling", "description": "19. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "contains(join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`), 'Medical_Necessity_Letter')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-21", "type": "jmespath", "category": "Task Resolution", "description": "20. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-22", "type": "jmespath", "category": "Documentation", "description": "21. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-16-eval-23", "type": "llm_judge", "category": "Documentation", "description": "22. EMR note contains the Payer B authorization reference number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note content contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if the note contains the auth number (or a very similar ID), 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-16-eval-24", "type": "jmespath", "category": "Task Resolution", "description": "23. Agent cleared referral REF-2025-105 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-105')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "verify_step_therapy_and_submit", "requires_note_review": true, "step_therapy_verification_required": true }, "rationale": "" }, { "id": "emr-hard-17", "title": "Submit Urgent Authorization - Sleep Study with Cardiopulmonary Complications", "instruction": "Open referral REF-2025-305 for Ellis, Frank (53M with Aetna HMO). This is an urgent authorization for a sleep study due to life-threatening cardiopulmonary complications. Review the clinical notes for urgency justification and submit the authorization to Aetna with appropriate urgency priority. Document the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-305 for Ellis, Frank (53M with Aetna HMO). This is an urgent authorization for a sleep study due to life-threatening cardiopulmonary complications. Review the clinical notes for urgency justification and submit the authorization to Aetna with appropriate urgency priority. Document the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "urgent", "challengeType": "urgent", "possible": true, "points": 22, "configTaskId": "hard_17", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Ellis, Frank" }, "stepByStep": [ "1. Click on referral REF-2025-305 in the worklist to open it", "2. Click the General tab — find the Clinical Note and read it to capture urgency justification: severe nocturnal hypoxemia (SpO2 72%), new-onset cardiac arrhythmia. This is an URGENT case.", "3. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: G47.33, R06.83", "4. REQUIRED: Click the Services tab — record all CPT codes: 95810", "5. Click the Coverages tab — record member ID (AET305000005), DOB (07/15/1972), and payer portal credentials (provider@payera.com / demo123)", "6. Go to the General tab — click 'Letter of Medical Necessity' to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "7. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "8. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "9. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "10. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "11. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "12. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the search/lookup button to populate provider details", "13. In the Patient (Member ID) field (Section 3), click and type AET305000005. Click the Date of Birth field and type 07/15/1972 in MM/DD/YYYY format. Click Search to verify eligibility.", "14. IMPORTANT — URGENCY FIELD: Find the Urgency dropdown in the form and SET IT TO 'Emergency'. This is required — the case involves life-threatening hypoxemia (SpO2 72%) and new-onset cardiac arrhythmia. Do this BEFORE filling in other fields so you don't forget.", "15. Add all diagnosis codes one by one: type G47.33, click Add; then type R06.83, click Add", "16. Enter servicing provider name as shown in the referral header in EMR", "17. Add the CPT code: click CPT field, type 95810, click Add", "18. Click the Clinical Indication field and type a justification — include: obstructive sleep apnea, life-threatening nocturnal hypoxemia (SpO2 72%), new-onset cardiac arrhythmia, and the urgent need for expedited review", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "20. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the authorization reference number and the expedited/urgent status", "24. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-17-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to get urgency justification", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET305000005", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-10", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1972-07-15", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-11", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered diagnosis code G47.33 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'G47.33')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-12", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code R06.83 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R06.83')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-13", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 95810 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '95810')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-14", "type": "jmespath", "category": "Form Completion", "description": "16. Agent marked authorization as EMERGENCY in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].urgency", "expectedValue": "Emergency", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-15", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17. Clinical indication mentions sleep apnea or OSA", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention sleep apnea or OSA? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-17-eval-16", "type": "llm_judge", "category": "Clinical Reasoning", "description": "17b. Clinical indication mentions life-threatening urgency justification (hypoxemia/arrhythmia)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention life-threatening findings such as nocturnal hypoxemia or desaturation (SpO2 72%) or cardiac arrhythmia or PVCs or risk of loss of life? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-17-eval-17", "type": "jmespath", "category": "Document Handling", "description": "18. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "19. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-19", "type": "jmespath", "category": "Documentation", "description": "20. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-17-eval-20", "type": "llm_judge", "category": "Documentation", "description": "21. EMR note contains auth number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note contain the authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-17-eval-21", "type": "llm_judge", "category": "Documentation", "description": "21b. EMR note documents expedited status", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention urgent or expedited status? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-17-eval-22", "type": "jmespath", "category": "Task Resolution", "description": "22. Agent cleared referral REF-2025-305 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-305')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_urgent", "requires_expedited_flag": true, "urgency_documentation_required": true }, "rationale": "" }, { "id": "emr-hard-18", "title": "Renew Expiring Knee Surgery Authorization", "instruction": "Open referral REF-2025-304 for Drake, Emily (50F with Aetna PPO). This patient has an existing authorization that is about to expire. Check for any existing authorizations for this member in the Payer A portal to confirm status, then submit a renewal authorization to Payer A, document the old and new auth numbers in EMR, and clear the referral.", "goal": "Open referral REF-2025-304 for Drake, Emily (50F with Aetna PPO). This patient has an existing authorization that is about to expire. Check for any existing authorizations for this member in the Payer A portal to confirm status, then submit a renewal authorization to Payer A, document the old and new auth numbers in EMR, and clear the referral.", "difficulty": "hard", "category": "urgent", "challengeType": "urgent", "possible": true, "points": 18, "configTaskId": "hard_18", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Drake, Emily" }, "stepByStep": [ "1. Click on referral REF-2025-304 in the worklist to open it", "2. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: M23.222, M25.562", "3. REQUIRED: Click the Services tab — record all CPT codes: 29881", "4. Click the Coverages tab — record member ID (AET304000004), DOB (11/28/1975), and payer portal credentials (provider@payera.com / demo123)", "5. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "6. On the Coverages tab, click 'Open Payer A Portal' link to navigate to the payer portal", "7. On the Payer A login page, enter provider@payera.com and password demo123, then click Sign In", "8. On the Payer A dashboard, use 'Search Authorizations' to search for existing auth using member ID AET304000004. Find auth AUTH-2024-5678 — note it is expiring on 02/28/2026 (3 days from now). Record the existing auth number.", "9. Proceed to submit a NEW authorization as a continuation/renewal. In the clinical indication, reference the expiring auth number AUTH-2024-5678.", "10. On the Payer A dashboard, look in the LEFT SIDEBAR under HEALTH TOOLS and click 'Submit Authorizations'", "11. On the Submit Authorizations page, click the 'Auth Request' button to open the Authorization Request modal", "12. In the Authorization Request modal, click the Request Type dropdown and select 'Outpatient Procedure'", "13. CRITICAL: Stay within this modal — do NOT click any X or close button until after submission. In the Provider field (Section 1), enter NPI 1234567890 and click the search/lookup button to populate provider details", "14. In the Patient (Member ID) field (Section 3), click and type AET304000004. Click the Date of Birth field and type 11/28/1975 in MM/DD/YYYY format. Click Search to verify eligibility.", "15. Add all diagnosis codes one by one: type M23.222, click Add; then type M25.562, click Add", "16. Enter servicing provider name as shown in the referral header in EMR", "17. Add the CPT code: click CPT field, type 29881, click Add", "18. Click the Clinical Indication field and type a justification for the procedure and diagnosis", "19. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "20. CRITICAL: Scroll DOWN within the Authorization Request modal until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "21. Note the authorization reference number from the confirmation screen", "22. Click 'Return to EMR' to navigate back", "23. Add a Communication note in EMR that includes the authorization reference number", "24. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-18-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-5", "type": "jmespath", "category": "Document Handling", "description": "5. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "6. Agent navigated to Payer A payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-7", "type": "jmespath", "category": "Form Completion", "description": "7. Agent selected Outpatient as request type", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent entered correct member ID in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientSearch", "expectedValue": "AET304000004", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent entered correct patient DOB in Payer A form", "points": 1, "query": "aetna_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1975-11-28", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered diagnosis code M23.222 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'M23.222')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered CPT code 29881 in Payer A form", "points": 1, "query": "contains(aetna_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '29881')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-12", "type": "llm_judge", "category": "Clinical Reasoning", "description": "12. Clinical indication references renewal/continuation of existing auth", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text mention this is a renewal or continuation or reference the prior auth (AUTH-2024-5678)? Score 1.0 if it references renewal/continuation or prior auth, 0.0 if not.", "studentAnswer": "{{aetna_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-18-eval-13", "type": "jmespath", "category": "Document Handling", "description": "13. Agent uploaded letter of medical necessity to Payer A form", "points": 1, "query": "join(',', aetna_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-14", "type": "jmespath", "category": "Task Resolution", "description": "14. Agent submitted prior auth form in Payer A portal", "points": 1, "query": "length(aetna_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-15", "type": "jmespath", "category": "Documentation", "description": "15. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-18-eval-16", "type": "llm_judge", "category": "Documentation", "description": "16. EMR note documents existing auth status", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention the existing auth status (AUTH-2024-5678 or that a prior auth is expiring)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-18-eval-17", "type": "llm_judge", "category": "Documentation", "description": "16b. EMR note contains new auth number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note contain the new authorization reference number? Auth number: '{{aetna_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-18-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "17. Agent cleared referral REF-2025-304 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-304')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "check_existing_and_renew", "multi_step": true, "requires_auth_lookup": true, "requires_continuation_request": true }, "rationale": "" }, { "id": "emr-hard-19", "title": "Check and Handle Auth Status", "instruction": "Open referral REF-2025-402 for Jensen, Karen (57F with Anthem Blue Cross PPO). Check the existing authorization status on the Payer B portal. Document the status in EMR and take the appropriate next step (e.g., record auth number if approved, follow up if pending, or document denial/appeal plan).", "goal": "Open referral REF-2025-402 for Jensen, Karen (57F with Anthem Blue Cross PPO). Check the existing authorization status on the Payer B portal. Document the status in EMR and take the appropriate next step (e.g., record auth number if approved, follow up if pending, or document denial/appeal plan).", "difficulty": "hard", "category": "urgent", "challengeType": "urgent", "possible": true, "points": 8, "configTaskId": "hard_19", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Jensen, Karen" }, "stepByStep": [ "1. Click on referral REF-2025-402 in the worklist to open it", "2. Click the Coverages tab — record member ID (ANT402000002) and Payer B portal credentials (provider@payerb.com / demo123)", "3. Scroll down on the Coverages/Auth page — the 'Open Anthem Blue Cross Portal →' button is in the 'Payer Portal Access' section BELOW the Coverage Details. Do NOT click the 'Submission Method: Anthem Provider Portal' text (that is just informational). Click the actual button to navigate to the payer portal.", "4. On the Payer B login page, enter provider@payerb.com and demo123, then click Sign In", "5. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Auth/Referral Inquiry'", "6. Click the Member ID field and type ANT402000002. Click Search.", "7. Review the authorization search results — check the status of the pending authorization (Approved, Pending, or Denied)", "8. Record the auth status and any auth number in your KEY_INFO", "9. ⚠️ The 'Return to EMR' button is NOT on this page. Click the Payer B logo or navigate back to the Payer B Dashboard (/payer-b/dashboard). The 'Return to EMR' button is on the Dashboard — click it there to go back.", "10. Add a Communication note in EMR documenting the current authorization status and next steps", "11. If approved: record the auth number and click 'Clear from Worklist'. If pending/denied: leave referral open and document the follow-up plan." ], "evals": [ { "id": "emr-hard-19-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-19-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-19-eval-3", "type": "jmespath", "category": "Form Completion", "description": "3. Agent searched for existing auth using member ID ANT402000002", "points": 1, "query": "anthem_state.differences.authSearches[0].memberId || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-19-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent performed at least one authorization status search in Payer B portal", "points": 1, "query": "length(anthem_state.differences.authSearches || `[]`) >= `1`", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-19-eval-5", "type": "jmespath", "category": "Documentation", "description": "5. Agent added a note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-19-eval-6", "type": "llm_judge", "category": "Documentation", "description": "6. EMR note documents the authorization status found", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note document the authorization status found (approved, pending, or denied)? Score 1.0 if the note clearly documents what status was found, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-19-eval-7", "type": "jmespath", "category": "Documentation", "description": "7. EMR note identifies the patient or procedure being checked", "points": 1, "query": "contains(full_state.communications[-1].content || '', 'Jensen') || contains(full_state.communications[-1].content || '', 'Karen') || contains(full_state.communications[-1].content || '', 'CT Abdomen/Pelvis') || contains(full_state.communications[-1].content || '', 'CT abdomen/pelvis')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-19-eval-8", "type": "llm_judge", "category": "Documentation", "description": "8. EMR note documents appropriate next steps based on status", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note indicate appropriate next steps based on status (e. g., if approved: auth number documented; if pending: follow-up needed; if denied: appeal or resubmit options)? Score 1.0 if appropriate follow-up is documented, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" } ], "expectedOutcome": { "action": "status_check_and_respond", "variable_outcome": true, "possible_statuses": [ "approved", "pending", "denied" ] }, "rationale": "" }, { "id": "emr-hard-20", "title": "Submit Urgent Urology Authorization", "instruction": "Open referral REF-2025-405 for Morgan, Nancy (63F with Anthem Blue Cross HMO). This is an URGENT case that requires expedited authorization for cystoscopy with biopsy. Review the clinical notes for urgency justification and submit the authorization to Payer B with appropriate urgency priority. Document the auth number in EMR and clear the referral.", "goal": "Open referral REF-2025-405 for Morgan, Nancy (63F with Anthem Blue Cross HMO). This is an URGENT case that requires expedited authorization for cystoscopy with biopsy. Review the clinical notes for urgency justification and submit the authorization to Payer B with appropriate urgency priority. Document the auth number in EMR and clear the referral.", "difficulty": "hard", "category": "urgent", "challengeType": "urgent", "possible": true, "points": 22, "configTaskId": "hard_20", "website": { "name": "EMR Referral Portal", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Morgan, Nancy" }, "stepByStep": [ "1. Click on referral REF-2025-405 in the worklist to open it", "2. Click the General tab — find the Clinical Note and read it to capture: gross hematuria, suspicious bladder mass on imaging, urgency justification (rule out bladder cancer). This is an URGENT case.", "3. REQUIRED: Click the Diagnoses tab — record all diagnosis codes: R31.9", "4. REQUIRED: Click the Services tab — record all CPT codes: 52204", "5. Click the Coverages tab — record subscriber ID (ANT405000005), DOB (06/30/1962), and payer portal credentials (provider@payerb.com / demo123)", "6. Go to the General tab — click 'Medical Necessity Letter' to open its viewer, CRITICAL: Once the viewer opens, you MUST click the Download button to save the file locally. Do NOT simply read the letter and go back — you must click Download. Without downloading, the file will NOT appear in the portal.", "7. On the Coverages tab, click 'Open Payer B Portal' link to navigate to the payer portal", "8. On the Payer B login page, enter provider@payerb.com and password demo123, then click Sign In", "9. On the Payer B dashboard, click 'Authorizations & Referrals', then click 'Authorization Submission'", "10. Step 1 - Patient Details: click Request Type dropdown and select 'Outpatient'. Click Case Type dropdown and select 'Surgical'.", "11. Click Patient Name field and type: Morgan, Nancy. Click Date of Birth field and type 06/30/1962 in MM/DD/YYYY format. Click Subscriber ID field and type ANT405000005. Click Next.", "12. CRITICAL: Stay in this multi-step form — do NOT navigate away. Step 2 - Service Details: type R31.9, click Add; type 52204, click Add. Confirm each code appears in its table.", "13. Click Clinical Indication field and type a justification — include: gross hematuria, suspicious bladder mass on imaging, and the urgent need to rule out bladder cancer", "14. Scroll down to 'Available Documents from EMR' — click '+ Attach' next to each required document", "15. Click Next to proceed through remaining steps. In provider details, enter provider name as shown in the referral header in EMR. Click Next.", "16. CRITICAL: Scroll DOWN until the 'Submit Request' button is FULLY VISIBLE and not cut off at the bottom of the screen. Then click 'Submit Request'.", "17. Note the authorization reference number from the confirmation screen", "18. Click 'Return to EMR' to navigate back", "19. Add a Communication note in EMR that includes the authorization reference number and explicitly states that this was submitted as an urgent/expedited request requiring expedited processing", "20. Click 'Clear from Worklist' to complete the task" ], "evals": [ { "id": "emr-hard-20-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "1. Agent read clinical notes to extract urgency justification", "points": 1, "query": "signals.read_clinical_note", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "2. Agent clicked Diagnoses tab to view diagnosis codes", "points": 1, "query": "signals.clicked_diagnoses_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "3. Agent clicked Services tab to view CPT codes", "points": 1, "query": "signals.clicked_services_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "4. Agent clicked Coverages tab to get portal credentials", "points": 1, "query": "signals.clicked_coverages_tab", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "5. Agent viewed the letter of medical necessity in EMR", "points": 1, "query": "full_state.agentActions.viewedAuthLetter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-6", "type": "jmespath", "category": "Document Handling", "description": "6. Agent downloaded the letter of medical necessity from EMR", "points": 1, "query": "signals.downloaded_auth_letter", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-7", "type": "jmespath", "category": "Information Retrieval", "description": "7. Agent navigated to Payer B payer portal", "points": 1, "query": "signals.clicked_go_to_portal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-8", "type": "jmespath", "category": "Form Completion", "description": "8. Agent selected Outpatient as request type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].requestType", "expectedValue": "outpatient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-9", "type": "jmespath", "category": "Form Completion", "description": "9. Agent selected Surgery as case type", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].caseType", "expectedValue": "surgery", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-10", "type": "jmespath", "category": "Form Completion", "description": "10. Agent entered correct subscriber ID in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].subscriberId", "expectedValue": "ANT405000005", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-11", "type": "jmespath", "category": "Form Completion", "description": "11. Agent entered correct patient last name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientLastName", "expectedValue": "Morgan", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-12", "type": "jmespath", "category": "Form Completion", "description": "12. Agent entered correct patient first name in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientFirstName", "expectedValue": "Nancy", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-13", "type": "jmespath", "category": "Form Completion", "description": "13. Agent entered correct patient DOB in Payer B form", "points": 1, "query": "anthem_state.differences.priorAuth.added[0].patientDOB", "expectedValue": "1962-06-30", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-14", "type": "jmespath", "category": "Form Completion", "description": "14. Agent entered diagnosis code R31.9 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].diagnosisCodes[*].code || `[]`, 'R31.9')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-15", "type": "jmespath", "category": "Form Completion", "description": "15. Agent entered CPT code 52204 in Payer B form", "points": 1, "query": "contains(anthem_state.differences.priorAuth.added[0].cptCodes[*].code || `[]`, '52204')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-16", "type": "llm_judge", "category": "Clinical Reasoning", "description": "16. Clinical indication contains urgency justification from clinical note", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this text include a clear urgency justification that reflects concerning findings from the clinical note? Score 1.0 if it provides a specific urgency rationale aligned with the note, 0.0 if not.", "studentAnswer": "{{anthem_state.differences.priorAuth.added[0].clinicalIndication}}", "studentAnswerContext": "clinical indication field entry" }, { "id": "emr-hard-20-eval-17", "type": "jmespath", "category": "Document Handling", "description": "17. Agent uploaded letter of medical necessity to Payer B form", "points": 1, "query": "join(',', anthem_state.differences.priorAuth.added[0].supportingDocuments || `[]`)", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "18. Agent submitted prior auth form in Payer B portal", "points": 1, "query": "length(anthem_state.differences.priorAuth.added || `[]`)", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-19", "type": "jmespath", "category": "Documentation", "description": "19. Agent added authorization note in EMR Communications", "points": 1, "query": "full_state.agentActions.addedAuthNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "emr-hard-20-eval-20", "type": "llm_judge", "category": "Documentation", "description": "20. EMR note contains auth number", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note contain the authorization reference number? Auth number: '{{anthem_state.differences.priorAuth.added[0].confirmationId}}'. Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-20-eval-21", "type": "llm_judge", "category": "Documentation", "description": "20b. EMR note documents urgent status", "points": 1, "query": null, "expectedValue": null, "rubric": "Does this note mention that this is an urgent or expedited request? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "EMR note" }, { "id": "emr-hard-20-eval-22", "type": "jmespath", "category": "Task Resolution", "description": "21. Agent cleared referral REF-2025-405 from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-405')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "action": "submit_urgent", "requires_expedited_flag": true, "urgency_documentation_required": true }, "rationale": "" } ] }, { "id": "dme", "name": "Durable Medical Equipment", "description": "Fax-based DME coordination tasks that require document handling and follow-up.", "tasks": [ { "id": "fax-easy-1", "title": "Process DME Order - Power Wheelchair", "instruction": "Download all 3 required documents, fax to DME supplier, and document in the Notes tab.", "goal": "Open Patterson, Margaret (77F with Medicare Part B). Process the DME order for a power wheelchair. Download the required documents, fax them to the DME supplier, add a progress note, and clear the referral.", "difficulty": "easy", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_easy_1", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Patterson, Margaret" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Patterson, Margaret", "2. The referral opens with Orders tab → Active sub-tab (default). The prescription download button, DME supplier name, and fax number are all visible here.", "3. Note the DME supplier name (National Seating & Mobility) and fax number (1-800-555-0199) from the Active sub-tab", "4. Download the prescription by clicking the download button on the Active sub-tab (Prescription_Power_Wheelchair_2026-02-10.pdf)", "5. Click the 'Chart Review' tab in the top navigation bar", "6. In Chart Review, download the 2 supporting documents:", "7. Click on 'Face_to_Face_Evaluation_2026-02-10.pdf' — review it, then click Download. Return to Chart Review.", "8. Click on 'History_and_Physical_2026-02-01.pdf' — review it, then click Download. Return to Chart Review.", "9. Navigate to the DME Fax Portal (click the fax portal link on the Orders → Active sub-tab)", "10. Click the 'New Fax' button in the fax portal", "11. In the Recipient Name field, type: National Seating & Mobility", "12. In the Fax Number field, type exactly: 1-800-555-0199 (include the '1-' prefix)", "13. Scroll down to the 'Available Documents from EMR' section", "14. Click '+ Attach' next to 'Face_to_Face_Evaluation_2026-02-10.pdf'", "15. Click '+ Attach' next to 'Prescription_Power_Wheelchair_2026-02-10.pdf'", "16. Click '+ Attach' next to 'History_and_Physical_2026-02-01.pdf'", "17. Verify all 3 required documents show a remove button. Do NOT attach any other documents.", "18. CRITICAL: Scroll DOWN until the 'Send' button is FULLY VISIBLE. Click Send.", "19. Return to EMR (click Return to EMR or navigate back to the referral)", "20. Click the 'Notes' tab in the top navigation bar", "21. In the Notes tab right panel (Edit Note), type a subject and progress note documenting: fax sent to National Seating & Mobility at 1-800-555-0199, 3 documents sent: Face_to_Face_Evaluation_2026-02-10, Prescription_Power_Wheelchair_2026-02-10, History_and_Physical_2026-02-01", "22. Click 'Sign' to save the note, then click 'Clear from Worklist'" ], "evals": [ { "id": "fax-easy-1-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-1-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-1-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription Power Wheelchair document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_Power_Wheelchair_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-1-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-1-eval-5", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "National Seating & Mobility", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-1-eval-6", "type": "llm_judge", "category": "Form Completion", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0199'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-easy-1-eval-7", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-1-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-201 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-201')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-1-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note mentions supplier name 'National Seating & Mobility'", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the supplier name 'National Seating & Mobility'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-1-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note mentions fax number 1-800-555-0199", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the fax number '1-800-555-0199'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-1-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note mentions documents that were faxed", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the documents that were sent (Face to Face Evaluation, Prescription for Power Wheelchair, History and Physical)? Score 1.0 if the note mentions at least two of these documents, 0.0 if fewer.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "supplierName": "National Seating & Mobility", "faxNumber": "1-800-555-0199", "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-easy-2", "title": "Process DME Order - Oxygen Concentrator", "instruction": "Download all 3 required documents, fax to DME supplier, and document in the Notes tab.", "goal": "Open Harrison, Walter (73M with Medicare Part B). Process the DME order for an oxygen concentrator. Download the required documents, fax them to the DME supplier, add a progress note, and clear the referral.", "difficulty": "easy", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_easy_2", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Harrison, Walter" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Harrison, Walter", "2. The referral opens with Orders tab → Active sub-tab (default). The prescription download button, DME supplier name, and fax number are all visible here.", "3. Note the DME supplier name (Lincare Holdings Inc.) and fax number (1-800-555-0198) from the Active sub-tab", "4. Download the prescription by clicking the download button on the Active sub-tab (Prescription_Oxygen_Concentrator_2026-02-08.pdf)", "5. Click the 'Chart Review' tab in the top navigation bar", "6. In Chart Review, download the 2 supporting documents:", "7. Click on 'Face_to_Face_Evaluation_2026-02-08.pdf' — review it, then click Download. Return to Chart Review.", "8. Click on 'History_and_Physical_2026-02-01.pdf' — review it, then click Download. Return to Chart Review.", "9. Navigate to the DME Fax Portal (click the fax portal link on the Orders → Active sub-tab)", "10. Click the 'New Fax' button in the fax portal", "11. In the Recipient Name field, type: Lincare Holdings Inc.", "12. In the Fax Number field, type exactly: 1-800-555-0198 (include the '1-' prefix)", "13. Scroll down to the 'Available Documents from EMR' section", "14. Click '+ Attach' next to 'Face_to_Face_Evaluation_2026-02-08.pdf'", "15. Click '+ Attach' next to 'Prescription_Oxygen_Concentrator_2026-02-08.pdf'", "16. Click '+ Attach' next to 'History_and_Physical_2026-02-01.pdf'", "17. Verify all 3 required documents show a remove button. Do NOT attach any other documents.", "18. CRITICAL: Scroll DOWN until the 'Send' button is FULLY VISIBLE. Click Send.", "19. Return to EMR (click Return to EMR or navigate back to the referral)", "20. Click the 'Notes' tab in the top navigation bar", "21. In the Notes tab right panel (Edit Note), type a subject and progress note documenting: fax sent to Lincare Holdings Inc. at 1-800-555-0198, 3 documents sent: Face_to_Face_Evaluation_2026-02-08, Prescription_Oxygen_Concentrator_2026-02-08, History_and_Physical_2026-02-01", "22. Click 'Sign' to save the note, then click 'Clear from Worklist'" ], "evals": [ { "id": "fax-easy-2-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-2-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-08.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-2-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription Oxygen Concentrator document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_Oxygen_Concentrator_2026-02-08.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-2-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-2-eval-5", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "Lincare Holdings Inc.", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-2-eval-6", "type": "llm_judge", "category": "Document Handling", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0198'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-easy-2-eval-7", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-2-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-202 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-202')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-2-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note mentions supplier name 'Lincare Holdings Inc.'", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the supplier name 'Lincare Holdings Inc.'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-2-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note mentions fax number 1-800-555-0198", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the fax number '1-800-555-0198'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-2-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note mentions documents that were faxed", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the documents that were sent (Face to Face Evaluation, Prescription Oxygen Concentrator, History and Physical)? Score 1.0 if the note mentions at least two of these documents, 0.0 if fewer.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "supplierName": "Lincare Holdings Inc.", "faxNumber": "1-800-555-0198", "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-easy-3", "title": "Process DME Order - CPAP Machine", "instruction": "Download all 3 required documents, fax to DME supplier, and document in the Notes tab.", "goal": "Open Mitchell, David (58M with Blue Cross Blue Shield PPO). Process the DME order for a CPAP machine. Download the required documents, fax them to the DME supplier, add a progress note, and clear the referral.", "difficulty": "easy", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_easy_3", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Mitchell, David" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Mitchell, David", "2. The referral opens with Orders tab → Active sub-tab (default). The prescription download button, DME supplier name, and fax number are all visible here.", "3. Note the DME supplier name (Apria Healthcare) and fax number (1-800-555-0197) from the Active sub-tab", "4. Download the prescription by clicking the download button on the Active sub-tab (Prescription_CPAP_Machine_2026-02-10.pdf)", "5. Click the 'Chart Review' tab in the top navigation bar", "6. In Chart Review, download the 2 supporting documents:", "7. Click on 'Face_to_Face_Evaluation_2026-02-10.pdf' — review it, then click Download. Return to Chart Review.", "8. Click on 'History_and_Physical_2026-02-01.pdf' — review it, then click Download. Return to Chart Review.", "9. Navigate to the DME Fax Portal (click the fax portal link on the Orders → Active sub-tab)", "10. Click the 'New Fax' button in the fax portal", "11. In the Recipient Name field, type: Apria Healthcare", "12. In the Fax Number field, type exactly: 1-800-555-0197 (include the '1-' prefix)", "13. Scroll down to the 'Available Documents from EMR' section", "14. Click '+ Attach' next to 'Face_to_Face_Evaluation_2026-02-10.pdf'", "15. Click '+ Attach' next to 'Prescription_CPAP_Machine_2026-02-10.pdf'", "16. Click '+ Attach' next to 'History_and_Physical_2026-02-01.pdf'", "17. Verify all 3 required documents show a remove button. Do NOT attach any other documents.", "18. CRITICAL: Scroll DOWN until the 'Send' button is FULLY VISIBLE. Click Send.", "19. Return to EMR (click Return to EMR or navigate back to the referral)", "20. Click the 'Notes' tab in the top navigation bar", "21. In the Notes tab right panel (Edit Note), type a subject and progress note documenting: fax sent to Apria Healthcare at 1-800-555-0197, 3 documents sent: Face_to_Face_Evaluation_2026-02-10, Prescription_CPAP_Machine_2026-02-10, History_and_Physical_2026-02-01", "22. Click 'Sign' to save the note, then click 'Clear from Worklist'" ], "evals": [ { "id": "fax-easy-3-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-3-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-3-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription CPAP Machine document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_CPAP_Machine_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-3-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-3-eval-5", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "Apria Healthcare", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-3-eval-6", "type": "llm_judge", "category": "Form Completion", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0197'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-easy-3-eval-7", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-3-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-203 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-203')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-3-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note mentions supplier name 'Apria Healthcare'", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the supplier name 'Apria Healthcare'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-3-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note mentions fax number 1-800-555-0197", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the fax number '1-800-555-0197'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-3-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note mentions documents that were faxed", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the documents that were sent (Face to Face Evaluation, Prescription CPAP Machine, History and Physical)? Score 1.0 if the note mentions at least two of these documents, 0.0 if fewer.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "supplierName": "Apria Healthcare", "faxNumber": "1-800-555-0197", "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-easy-4", "title": "Process DME Order - Hospital Bed", "instruction": "Download all 3 required documents, fax to DME supplier, and document in the Notes tab.", "goal": "Open Evans, Susan (82F with UnitedHealthcare Medicare Advantage). Process the DME order for a hospital bed. Download the required documents, fax them to the DME supplier, add a progress note, and clear the referral.", "difficulty": "easy", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_easy_4", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Evans, Susan" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Evans, Susan", "2. The referral opens with Orders tab → Active sub-tab (default). The prescription download button, DME supplier name, and fax number are all visible here.", "3. Note the DME supplier name (AdaptHealth Corp) and fax number (1-800-555-0196) from the Active sub-tab", "4. Download the prescription by clicking the download button on the Active sub-tab (Prescription_Hospital_Bed_2026-02-12.pdf)", "5. Click the 'Chart Review' tab in the top navigation bar", "6. In Chart Review, download the 2 supporting documents:", "7. Click on 'Face_to_Face_Evaluation_2026-02-12.pdf' — review it, then click Download. Return to Chart Review.", "8. Click on 'History_and_Physical_2026-02-01.pdf' — review it, then click Download. Return to Chart Review.", "9. Navigate to the DME Fax Portal (click the fax portal link on the Orders → Active sub-tab)", "10. Click the 'New Fax' button in the fax portal", "11. In the Recipient Name field, type: AdaptHealth Corp", "12. In the Fax Number field, type exactly: 1-800-555-0196 (include the '1-' prefix)", "13. Scroll down to the 'Available Documents from EMR' section", "14. Click '+ Attach' next to 'Face_to_Face_Evaluation_2026-02-12.pdf'", "15. Click '+ Attach' next to 'Prescription_Hospital_Bed_2026-02-12.pdf'", "16. Click '+ Attach' next to 'History_and_Physical_2026-02-01.pdf'", "17. Verify all 3 required documents show a remove button. Do NOT attach any other documents.", "18. CRITICAL: Scroll DOWN until the 'Send' button is FULLY VISIBLE. Click Send.", "19. Return to EMR (click Return to EMR or navigate back to the referral)", "20. Click the 'Notes' tab in the top navigation bar", "21. In the Notes tab right panel (Edit Note), type a subject and progress note documenting: fax sent to AdaptHealth Corp at 1-800-555-0196, 3 documents sent: Face_to_Face_Evaluation_2026-02-12, Prescription_Hospital_Bed_2026-02-12, History_and_Physical_2026-02-01", "22. Click 'Sign' to save the note, then click 'Clear from Worklist'" ], "evals": [ { "id": "fax-easy-4-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-4-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-12.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-4-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription Hospital Bed document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_Hospital_Bed_2026-02-12.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-4-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-4-eval-5", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "AdaptHealth Corp", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-4-eval-6", "type": "llm_judge", "category": "Form Completion", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0196'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-easy-4-eval-7", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-4-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-204 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-204')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-4-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note mentions supplier name 'AdaptHealth Corp'", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the supplier name 'AdaptHealth Corp'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-4-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note mentions fax number 1-800-555-0196", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the fax number '1-800-555-0196'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-4-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note mentions documents that were faxed", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the documents that were sent (Face to Face Evaluation, Prescription Hospital Bed, History and Physical)? Score 1.0 if the note mentions at least two of these documents, 0.0 if fewer.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "supplierName": "AdaptHealth Corp", "faxNumber": "1-800-555-0196", "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-easy-5", "title": "Process DME Order - Nebulizer", "instruction": "Download all 3 required documents, fax to DME supplier, and document in the Notes tab.", "goal": "Open Wilson, James (45M with Aetna HMO). Process the DME order for a nebulizer. Download the required documents, fax them to the DME supplier, add a progress note, and clear the referral.", "difficulty": "easy", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_easy_5", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Wilson, James" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Wilson, James", "2. The referral opens with Orders tab → Active sub-tab (default). The prescription download button, DME supplier name, and fax number are all visible here.", "3. Note the DME supplier name (Rotech Healthcare Inc.) and fax number (1-800-555-0195) from the Active sub-tab", "4. Download the prescription by clicking the download button on the Active sub-tab (Prescription_Nebulizer_2026-02-10.pdf)", "5. Click the 'Chart Review' tab in the top navigation bar", "6. In Chart Review, download the 2 supporting documents:", "7. Click on 'Face_to_Face_Evaluation_2026-02-10.pdf' — review it, then click Download. Return to Chart Review.", "8. Click on 'History_and_Physical_2026-02-01.pdf' — review it, then click Download. Return to Chart Review.", "9. Navigate to the DME Fax Portal (click the fax portal link on the Orders → Active sub-tab)", "10. Click the 'New Fax' button in the fax portal", "11. In the Recipient Name field, type: Rotech Healthcare Inc.", "12. In the Fax Number field, type exactly: 1-800-555-0195 (include the '1-' prefix)", "13. Scroll down to the 'Available Documents from EMR' section", "14. Click '+ Attach' next to 'Face_to_Face_Evaluation_2026-02-10.pdf'", "15. Click '+ Attach' next to 'Prescription_Nebulizer_2026-02-10.pdf'", "16. Click '+ Attach' next to 'History_and_Physical_2026-02-01.pdf'", "17. Verify all 3 required documents show a remove button. Do NOT attach any other documents.", "18. CRITICAL: Scroll DOWN until the 'Send' button is FULLY VISIBLE. Click Send.", "19. Return to EMR (click Return to EMR or navigate back to the referral)", "20. Click the 'Notes' tab in the top navigation bar", "21. In the Notes tab right panel (Edit Note), type a subject and progress note documenting: fax sent to Rotech Healthcare Inc. at 1-800-555-0195, 3 documents sent: Face_to_Face_Evaluation_2026-02-10, Prescription_Nebulizer_2026-02-10, History_and_Physical_2026-02-01", "22. Click 'Sign' to save the note, then click 'Clear from Worklist'" ], "evals": [ { "id": "fax-easy-5-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-5-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-5-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription Nebulizer document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_Nebulizer_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-5-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-5-eval-5", "type": "jmespath", "category": "Document Handling", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "Rotech Healthcare Inc.", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-5-eval-6", "type": "llm_judge", "category": "Form Completion", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0195'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-easy-5-eval-7", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-5-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-205 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-205')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-easy-5-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note mentions supplier name 'Rotech Healthcare Inc.'", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the supplier name 'Rotech Healthcare Inc.'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-5-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note mentions fax number 1-800-555-0195", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the fax number '1-800-555-0195'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-easy-5-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note mentions documents that were faxed", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the documents that were sent (Face to Face Evaluation, Prescription Nebulizer, History and Physical)? Score 1.0 if the note mentions at least two of these documents, 0.0 if fewer.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "supplierName": "Rotech Healthcare Inc.", "faxNumber": "1-800-555-0195", "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-medium-1", "title": "Process DME Order - CGM (Document Version Selection)", "instruction": "Two Face-to-Face Evaluations exist (June and February). Agent must identify and use only the current February version. Download required documents, fax to DME supplier, and document in Notes tab.", "goal": "Open Patel, Arun (63M with Anthem Blue Cross PPO). Process the DME order for a continuous glucose monitor. There are two Face-to-Face Evaluations on file — one from June 2025 and one from February 2026. Use ONLY the current (February) version. Download the 3 required documents, fax them to the DME supplier, add a progress note, and clear the referral.", "difficulty": "medium", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_medium_1", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Patel, Arun" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Patel, Arun", "2. The referral opens with Orders tab → Active sub-tab (default). Note the DME supplier (Byram Healthcare) and fax number (1-800-555-0194).", "3. Download the prescription from the Active sub-tab (Prescription_CGM_2026-02-08.pdf)", "4. Click the Chart Review tab — notice there are TWO Face-to-Face Evaluations:", " - Face_to_Face_Evaluation_2025-06-12.pdf (OLD — superseded, marked as such in content)", " - Face_to_Face_Evaluation_2026-02-08.pdf (CURRENT — reflects latest clinical status)", "5. Download ONLY the current February F2F and the H&P from Chart Review (2 supporting documents)", "6. Do NOT download the old June F2F — it contains outdated clinical information", "7. Navigate to DME Fax Portal (click the fax portal link on the Orders → Active sub-tab), enter Byram Healthcare / 1-800-555-0194", "8. Attach ONLY the 3 required documents (current F2F, Prescription, H&P)", "9. Send the fax, return to EMR", "10. Add a progress note mentioning use of the current February F2F version", "11. Clear from Worklist" ], "evals": [ { "id": "fax-medium-1-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify the OLD (June) Face-to-Face Evaluation was NOT attached — agent correctly identified the superseded document", "points": 2, "query": "!contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2025-06-12.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Verify the CURRENT (February) Face-to-Face Evaluation was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-08.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription CGM document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_CGM_2026-02-08.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-5", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-6", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "Byram Healthcare", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-7", "type": "llm_judge", "category": "Document Handling", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0194'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-medium-1-eval-8", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-9", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-206 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-206')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-1-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note mentions using the current/February F2F version", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention using the current or February 2026 Face-to-Face Evaluation (as opposed to the older June 2025 version)? Score 1.0 if the note indicates the current/latest version was used, 0.0 if not mentioned.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "oldDocumentNotAttached": true, "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-medium-2", "title": "Process DME Order - Knee Brace (Document Version Selection)", "instruction": "Two History & Physical documents exist (March and February). Agent must identify and use only the current February version. Download required documents, fax to DME supplier, and document in Notes tab.", "goal": "Open Anderson, Robert (67M with Cigna PPO). Process the DME order for a custom knee brace. There are two History & Physical documents on file — one from March 2025 and one from February 2026. Use ONLY the current (February) version. Download the 3 required documents, fax them to the DME supplier, add a progress note, and clear the referral.", "difficulty": "medium", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_medium_2", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Anderson, Robert" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Anderson, Robert", "2. The referral opens with Orders tab → Active sub-tab (default). Note the DME supplier (Hanger Clinic) and fax number (1-800-555-0193).", "3. Download the prescription from the Active sub-tab (Prescription_Knee_Brace_2026-02-12.pdf)", "4. Click the Chart Review tab — notice there are TWO History & Physical documents:", " - History_and_Physical_2025-03-15.pdf (OLD — outdated findings, condition has worsened since)", " - History_and_Physical_2026-02-01.pdf (CURRENT — reflects latest clinical status)", "5. Download ONLY the current February H&P and the F2F Evaluation from Chart Review (2 supporting documents)", "6. Do NOT download the old March H&P — it contains outdated clinical information", "7. Navigate to DME Fax Portal (click the fax portal link on the Orders → Active sub-tab), enter Hanger Clinic / 1-800-555-0193", "8. Attach ONLY the 3 required documents (F2F, Prescription, current H&P)", "9. Send the fax, return to EMR", "10. Add a progress note mentioning use of the current February H&P version", "11. Clear from Worklist" ], "evals": [ { "id": "fax-medium-2-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify the OLD (March) History and Physical was NOT attached — agent correctly identified the outdated document", "points": 2, "query": "!contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2025-03-15.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Verify the CURRENT (February) History and Physical was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-12.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-5", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription Knee Brace document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_Knee_Brace_2026-02-12.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-6", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "Hanger Clinic", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-7", "type": "llm_judge", "category": "Form Completion", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0193'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-medium-2-eval-8", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-9", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-207 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-207')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-2-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note mentions using the current/February H&P version", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention using the current or February 2026 History and Physical (as opposed to the older March 2025 version)? Score 1.0 if the note indicates the current/latest version was used, 0.0 if not mentioned.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "oldDocumentNotAttached": true, "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-medium-3", "title": "Process DME Order - Patient Lift (Multiple Orders — Correct Supplier)", "instruction": "Patient has a completed wheelchair order (Sunrise Medical) and an active patient lift order (National Seating & Mobility). Agent must process the active order with the correct supplier.", "goal": "Open Williams, James (70M with Medicare Part B). This patient has a completed wheelchair order with Sunrise Medical AND an active patient lift order with National Seating & Mobility. Process ONLY the active patient lift order — fax the 3 required documents to the correct supplier (National Seating & Mobility), add a progress note, and clear the referral.", "difficulty": "medium", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 14, "configTaskId": "dme_medium_3", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Williams, James" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Williams, James", "2. The referral opens with Orders tab → Active sub-tab (default). The Active sub-tab shows the active Patient Lift order with National Seating & Mobility (fax: 1-800-555-0192).", "3. Note: Order History sub-tab also shows a COMPLETED Power Wheelchair order with Sunrise Medical (fax: 1-800-555-0188) — do NOT use that supplier.", "4. Download the prescription from the Active sub-tab (Prescription_Patient_Lift_2026-02-15.pdf)", "5. Click Chart Review tab and download the 2 supporting documents (F2F Evaluation and H&P)", "6. Navigate to DME Fax Portal (click the fax portal link on the Orders → Active sub-tab)", "7. Enter National Seating & Mobility / 1-800-555-0192 (NOT Sunrise Medical)", "8. Attach the 3 required documents and send", "9. Return to EMR, add a progress note mentioning the patient lift and correct supplier", "10. Clear from Worklist" ], "evals": [ { "id": "fax-medium-3-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-2", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name (National Seating & Mobility, not Sunrise Medical from the completed order)", "points": 2, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "National Seating & Mobility", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-3", "type": "jmespath", "category": "Form Completion", "description": "Verify correct fax number (1-800-555-0192 for National Seating & Mobility, not 1-800-555-0188 for Sunrise Medical)", "points": 2, "query": "full_state.faxPortal.faxNumber", "expectedValue": "1-800-555-0192", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-15.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-5", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription Patient Lift document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_Patient_Lift_2026-02-15.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-6", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-7", "type": "jmespath", "category": "Document Handling", "description": "Verify no distractor documents were attached", "points": 1, "query": "!contains(full_state.faxPortal.attachmentNames || `[]`, 'Brain_MRI_2025-09-16.pdf') && !contains(full_state.faxPortal.attachmentNames || `[]`, 'Rehab_Discharge_Summary_2025-12-30.pdf') && !contains(full_state.faxPortal.attachmentNames || `[]`, 'INR_Results_2026-02-10.pdf') && !contains(full_state.faxPortal.attachmentNames || `[]`, 'Caregiver_Assessment_2026-01-15.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-8", "type": "jmespath", "category": "Document Handling", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "National Seating & Mobility", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-9", "type": "llm_judge", "category": "Document Handling", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0192'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-medium-3-eval-10", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-11", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-208 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-208')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-3-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note mentions patient lift and correct supplier", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the patient lift order and National Seating & Mobility as the supplier? Score 1.0 if both are mentioned, 0.5 if only one, 0.0 if neither.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "correctSupplierUsed": true, "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-medium-4", "title": "Process DME Order - Wound VAC (Urgent Discharge Pending)", "instruction": "Patient is discharge-pending. Agent must enable certified delivery AND add 'URGENT - PENDING DISCHARGE' to fax cover sheet notes in addition to standard workflow.", "goal": "Open Chen, Linda (55F with Blue Shield PPO). This patient is DISCHARGE PENDING. Process the DME order for a wound VAC. Download the 3 required documents, fax them to the DME supplier with certified delivery enabled and 'URGENT - PENDING DISCHARGE' in the cover sheet notes, add a progress note, and clear the referral.", "difficulty": "medium", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 13, "configTaskId": "dme_medium_4", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Chen, Linda" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), note that Chen, Linda has 'Urgent' urgency", "2. Click on patient Chen, Linda", "3. Notice the red DISCHARGE PENDING banner at the top of the content area", "4. The banner instructs: Enable certified delivery and add 'URGENT - PENDING DISCHARGE' to cover sheet notes", "5. The referral opens with Orders tab → Active sub-tab (default). Download the prescription from the Active sub-tab (Prescription_Wound_VAC_2026-02-10.pdf)", "6. Click Chart Review tab and download the 2 supporting documents (F2F Evaluation and H&P)", "7. Navigate to DME Fax Portal (click the fax portal link on the Orders → Active sub-tab), enter KCI Medical / 1-800-555-0191", "8. Check the 'Use certified delivery' checkbox on the Main tab", "9. Click the 'Cover Sheet Notes' tab and type 'URGENT - PENDING DISCHARGE' (or similar containing URGENT)", "10. Attach the 3 required documents and send", "11. Return to EMR, add a progress note mentioning urgent/discharge status", "12. Clear from Worklist" ], "evals": [ { "id": "fax-medium-4-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-2", "type": "jmespath", "category": "Form Completion", "description": "Verify that certified delivery was enabled for the urgent/discharge-pending fax", "points": 2, "query": "full_state.faxPortal.useCertifiedDelivery", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-3", "type": "jmespath", "category": "Form Completion", "description": "Verify that cover sheet notes contain 'URGENT' for the discharge-pending patient", "points": 2, "query": "contains(full_state.faxPortal.coverNotes || '', 'URGENT')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-5", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription Wound VAC document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_Wound_VAC_2026-02-10.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-6", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-7", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "KCI Medical", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-8", "type": "llm_judge", "category": "Form Completion", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0191'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-medium-4-eval-9", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-10", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-209 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-209')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-4-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note mentions urgent/discharge status", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the urgent nature of the request or the patient's discharge-pending status? Score 1.0 if the note references urgent, discharge, or time-sensitive nature, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "certifiedDeliveryEnabled": true, "coverNotesContainUrgent": true, "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-medium-5", "title": "Process DME Order - TENS Unit (Urgent Discharge Pending)", "instruction": "Patient is discharge-pending. Agent must enable certified delivery AND add 'URGENT - PENDING DISCHARGE' to fax cover sheet notes in addition to standard workflow.", "goal": "Open Murphy, Kathleen (60F with Kaiser Permanente HMO). This patient is DISCHARGE PENDING. Process the DME order for a TENS unit. Download the 3 required documents, fax them to the DME supplier with certified delivery enabled and 'URGENT - PENDING DISCHARGE' in the cover sheet notes, add a progress note, and clear the referral.", "difficulty": "medium", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 13, "configTaskId": "dme_medium_5", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Murphy, Kathleen" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), note that Murphy, Kathleen has 'Urgent' urgency", "2. Click on patient Murphy, Kathleen", "3. Notice the red DISCHARGE PENDING banner at the top of the content area", "4. The banner instructs: Enable certified delivery and add 'URGENT - PENDING DISCHARGE' to cover sheet notes", "5. The referral opens with Orders tab → Active sub-tab (default). Download the prescription from the Active sub-tab (Prescription_TENS_Unit_2026-02-12.pdf)", "6. Click Chart Review tab and download the 2 supporting documents (F2F Evaluation and H&P)", "7. Navigate to DME Fax Portal (click the fax portal link on the Orders → Active sub-tab), enter EMPI Inc. / 1-800-555-0190", "8. Check the 'Use certified delivery' checkbox on the Main tab", "9. Click the 'Cover Sheet Notes' tab and type 'URGENT - PENDING DISCHARGE' (or similar containing URGENT)", "10. Attach the 3 required documents and send", "11. Return to EMR, add a progress note mentioning urgent/discharge status", "12. Clear from Worklist" ], "evals": [ { "id": "fax-medium-5-eval-1", "type": "jmespath", "category": "Document Handling", "description": "Verify that a fax was sent via the DME fax portal", "points": 1, "query": "full_state.faxPortal.faxesSent", "expectedValue": 1, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-2", "type": "jmespath", "category": "Form Completion", "description": "Verify that certified delivery was enabled for the urgent/discharge-pending fax", "points": 2, "query": "full_state.faxPortal.useCertifiedDelivery", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-3", "type": "jmespath", "category": "Form Completion", "description": "Verify that cover sheet notes contain 'URGENT' for the discharge-pending patient", "points": 2, "query": "contains(full_state.faxPortal.coverNotes || '', 'URGENT')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Verify Face to Face Evaluation document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Face_to_Face_Evaluation_2026-02-12.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-5", "type": "jmespath", "category": "Document Handling", "description": "Verify Prescription TENS Unit document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'Prescription_TENS_Unit_2026-02-12.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-6", "type": "jmespath", "category": "Document Handling", "description": "Verify History and Physical document was attached", "points": 1, "query": "contains(full_state.faxPortal.attachmentNames || `[]`, 'History_and_Physical_2026-02-01.pdf')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-7", "type": "jmespath", "category": "Form Completion", "description": "Verify correct supplier name was entered in fax dialog", "points": 1, "query": "full_state.faxPortal.faxRecipient", "expectedValue": "EMPI Inc.", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-8", "type": "llm_judge", "category": "Form Completion", "description": "Verify correct fax number was entered in fax dialog (format-flexible)", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the entered fax number represent the same destination as '1-800-555-0190'? Treat punctuation and spacing differences as equivalent (e.g., dashes, spaces, parentheses), and treat an optional leading country code '1' as equivalent. Score 1.0 if equivalent, 0.0 if different destination.", "studentAnswer": "{{full_state.faxPortal.faxNumber}}", "studentAnswerContext": "fax number entered in fax dialog" }, { "id": "fax-medium-5-eval-9", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added in Notes tab", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-10", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral REF-2025-210 was cleared from worklist", "points": 1, "query": "contains(full_state.clearedReferrals || `[]`, 'REF-2025-210')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-medium-5-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note mentions urgent/discharge status", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the urgent nature of the request or the patient's discharge-pending status? Score 1.0 if the note references urgent, discharge, or time-sensitive nature, 0.0 if not.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": true, "documentsAttached": 3, "certifiedDeliveryEnabled": true, "coverNotesContainUrgent": true, "progressNoteAdded": true, "clearedFromWorklist": true }, "rationale": "" }, { "id": "fax-hard-1", "title": "Expired Face-to-Face Evaluation — Hold DME Order", "instruction": "Review documents. Identify that the F2F evaluation is expired (>6 months old) and cannot be used. Do NOT fax. Document the issue and recommended next steps.", "goal": "Open Garcia, Sofia (10F with Medi-Cal). Process the DME order for an enteral feeding pump. Review all documents before proceeding. Determine the appropriate action based on the documentation status and document your findings in a progress note.", "difficulty": "hard", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 13, "configTaskId": "dme_hard_1", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Garcia, Sofia" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Garcia, Sofia", "2. The referral opens with Orders tab → Active sub-tab (default). Review the order details and note the DME supplier (Option Care Health).", "3. Click the 'Chart Review' tab in the top navigation bar", "4. In Chart Review, click on the Face-to-Face Evaluation document to view it — notice it is dated April 2025, more than 6 months before the order date", "5. This means the F2F evaluation is expired per Medi-Cal policy requirements (F2F must be within 6 months)", "6. Do NOT open the fax portal or send any fax", "7. Do NOT clear the referral from the worklist", "8. Click the 'Notes' tab in the top navigation bar", "9. In the Notes tab right panel (Edit Note), type a subject and detailed progress note documenting: patient Sofia Garcia, enteral feeding pump order, expired F2F evaluation (April 2025), Medi-Cal requirement that F2F be within 6 months, need updated F2F from Dr. Emily Foster, order on hold, no fax sent to Option Care Health, and that processing/faxing should wait until the updated F2F is obtained", "10. Click 'Sign' to save the note" ], "evals": [ { "id": "fax-hard-1-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed at least one document in the referral", "points": 1, "query": "full_state.agentActions.viewedDocuments | length(@) > `0`", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-1-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify that no fax was sent — the F2F evaluation is expired", "points": 1, "query": "full_state.faxPortal.faxesSent || `0`", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-1-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral was NOT cleared from worklist — order cannot be processed", "points": 1, "query": "!contains(full_state.clearedReferrals || `[]`, 'REF-2025-211')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-1-eval-4", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added documenting the issue", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-1-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Note identifies the F2F evaluation as expired", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention that the face-to-face evaluation is expired or outdated (dated April 2025, more than 6 months old)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note recommends obtaining updated F2F evaluation", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention that a new or updated face-to-face evaluation is needed before the DME order can be processed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note mentions contacting Dr. Emily Foster", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the ordering physician Dr. Emily Foster as the contact for the updated evaluation? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note identifies the patient Sofia Garcia", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the patient Sofia Garcia? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies the equipment type (feeding pump)", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the equipment type (feeding pump or enteral feeding pump)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note references the 6-month F2F policy requirement", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the Medi-Cal policy requirement that the F2F must be within 6 months of the DME order? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note confirms no fax was sent and order is on hold", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention EITHER that the documents were NOT faxed to the supplier OR that the order is on hold? Only one of these two is required to score 1.0. Score 1.0 if at least one is mentioned, 0.0 if neither is mentioned.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note mentions supplier name", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the DME supplier name 'Option Care Health'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-1-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note mentions next step to fax after F2F obtained", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note state that processing or faxing should wait until an updated Face-to-Face Evaluation is obtained? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": false, "progressNoteAdded": true, "clearedFromWorklist": false, "reason": "F2F evaluation expired, must obtain updated evaluation before processing" }, "rationale": "" }, { "id": "fax-hard-2", "title": "Missing Prescription Document — Hold DME Order", "instruction": "Review documents and identify that the prescription is entirely absent. Do NOT fax. Document the missing document and contact information for the ordering physician.", "goal": "Open Thompson, Richard (68M with Medicare Part B). Process the DME order for a below-knee prosthetic limb. Review all documents before proceeding. Determine the appropriate action based on the documentation status and document your findings in a progress note.", "difficulty": "hard", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 13, "configTaskId": "dme_hard_2", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Thompson, Richard" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Thompson, Richard", "2. The referral opens with Orders tab → Active sub-tab (default). Review the order details — notice there is NO prescription document (no download button for a prescription).", "3. Click the 'Chart Review' tab in the top navigation bar", "4. In Chart Review, notice only Face_to_Face_Evaluation and History_and_Physical are present — the Prescription document is missing entirely", "5. IMPORTANT: Click on the Face_to_Face_Evaluation document to view it — confirm it is present and properly signed. Return to Chart Review.", "6. Click on the History_and_Physical document to view it — confirm it is present and properly signed. Return to Chart Review.", "7. The missing prescription means the documentation packet is incomplete — cannot submit to DME supplier", "8. Do NOT open the fax portal or send any fax", "9. Do NOT clear the referral from the worklist", "10. Click the 'Notes' tab in the top navigation bar", "11. In the Notes tab right panel (Edit Note), type a subject and detailed progress note documenting: patient Richard Thompson, below-knee prosthetic limb order, F2F and H&P are present and ready, prescription is missing, need to contact Dr. James Morton for the prescription, order on hold, no fax sent to Hanger Clinic, and that the fax should be sent once the prescription is obtained", "12. Click 'Sign' to save the note" ], "evals": [ { "id": "fax-hard-2-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed at least one document in the referral", "points": 1, "query": "full_state.agentActions.viewedDocuments | length(@) > `0`", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-2-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify that no fax was sent — the prescription document is missing", "points": 1, "query": "full_state.faxPortal.faxesSent || `0`", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-2-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral was NOT cleared from worklist", "points": 1, "query": "!contains(full_state.clearedReferrals || `[]`, 'REF-2025-212')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-2-eval-4", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added documenting the issue", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-2-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Note identifies missing prescription document", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note identify that the prescription document is missing from the referral? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note recommends contacting physician for prescription", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note recommend contacting the ordering physician (Dr. James Morton) to obtain the missing prescription? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note mentions patient name", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the patient name (Thompson, Richard)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note mentions equipment type", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the equipment type (prosthetic limb or below-knee prosthesis)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note lists available documents", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note list which documents ARE present (Face-to-Face Evaluation and History & Physical) ? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note lists missing/pending documents", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention which is missing (Prescription)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note explains order is on hold due to incomplete docs", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention EITHER that the DME order cannot be faxed to the supplier with incomplete documentation OR that the order is on hold pending complete documentation? Only one of these two is required to score 1.0. Score 1.0 if at least one is mentioned, 0.0 if neither is mentioned.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note mentions supplier name", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the DME supplier name 'Hanger Clinic'? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-2-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note mentions next step to fax after prescription obtained", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note state that the fax should be sent once the prescription is obtained? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": false, "progressNoteAdded": true, "clearedFromWorklist": false, "reason": "Prescription missing, must obtain from ordering physician before processing" }, "rationale": "" }, { "id": "fax-hard-3", "title": "Inactive Insurance — Hold DME Order", "instruction": "Review patient insurance status. Identify that coverage is inactive (terminated 11/30/2025). Do NOT fax. Document the coverage issue and route to benefits/eligibility team.", "goal": "Open Brown, Dorothy (72F with UnitedHealthcare Medicare Advantage). Process the DME order for a home infusion pump. Review all patient information, insurance status, and documents before proceeding. Determine the appropriate action and document your findings in a progress note.", "difficulty": "hard", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_hard_3", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Brown, Dorothy" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Brown, Dorothy", "2. The referral opens with Orders tab → Active sub-tab (default). Review the order details.", "3. Click the 'Demographics' tab to check insurance/coverage status", "4. In the Insurance/Coverage table, notice the insurance status is 'Inactive' with termination date 11/30/2025", "5. Click the 'Chart Review' tab and view at least one document to confirm documentation is present", "6. Despite complete documentation, the order cannot proceed due to inactive insurance", "7. Do NOT open the fax portal or send any fax", "8. Do NOT clear the referral from the worklist", "9. Click the 'Notes' tab in the top navigation bar", "10. In the Notes tab right panel (Edit Note), type a subject and detailed progress note documenting: patient name Brown, Dorothy; inactive insurance (terminated 11/30/2025); home infusion pump DME order on hold; recommend routing to benefits/eligibility team; no fax sent to Option Care Health", "11. Click 'Sign' to save the note" ], "evals": [ { "id": "fax-hard-3-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed at least one document in the referral", "points": 1, "query": "full_state.agentActions.viewedDocuments | length(@) > `0`", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-3-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify that no fax was sent — insurance is inactive", "points": 1, "query": "full_state.faxPortal.faxesSent || `0`", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-3-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral was NOT cleared from worklist", "points": 1, "query": "!contains(full_state.clearedReferrals || `[]`, 'REF-2025-213')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-3-eval-4", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added documenting the issue", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-3-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Note identifies insurance as inactive/expired", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note identify that the patient's insurance (UnitedHealthcare Medicare Advantage) is inactive or expired? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-3-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note references the coverage termination date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the insurance termination date of 11/30/2025 or that coverage ended in November 2025? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-3-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note recommends routing to benefits team", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note recommend routing to the benefits/eligibility team or verifying insurance before proceeding? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-3-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note mentions patient name", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the patient name (Brown, Dorothy)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-3-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note mentions equipment type", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the equipment type (ambulatory infusion pump, home infusion pump, or infusion pump)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-3-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains order cannot proceed without coverage", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note clearly state that the DME order cannot proceed without active insurance coverage? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-3-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note confirms no fax sent and order on hold", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention EITHER that no fax was sent to the supplier OR that the order is on hold pending insurance verification? Only one of these two is required to score 1.0. Score 1.0 if at least one is mentioned, 0.0 if neither is mentioned.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": false, "progressNoteAdded": true, "clearedFromWorklist": false, "reason": "Insurance inactive, must verify coverage before processing" }, "rationale": "" }, { "id": "fax-hard-4", "title": "Unsigned Prescription — Hold DME Order", "instruction": "Review all documents carefully. Identify that the prescription is marked as DRAFT/PENDING SIGNATURE. Do NOT fax. Document the issue and contact the physician for signature.", "goal": "Open Lee, Michael (55M with Cigna PPO). Process the DME order for a BiPAP machine. Review all documents carefully before proceeding. Determine the appropriate action based on the documentation status and document your findings in a progress note.", "difficulty": "hard", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 11, "configTaskId": "dme_hard_4", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Lee, Michael" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Lee, Michael", "2. The referral opens with Orders tab → Active sub-tab (default). Review the order details.", "3. On the Active sub-tab, notice the prescription is marked as DRAFT/PENDING SIGNATURE — the prescription status indicator shows it is unsigned.", "4. Click the 'Chart Review' tab in the top navigation bar", "5. View the Face-to-Face Evaluation — confirm it is properly signed by Dr. Angela Park", "6. View the History & Physical — confirm it is properly signed", "7. Document readiness: F2F Evaluation = signed/ready, H&P = signed/ready, Prescription = UNSIGNED/NOT READY", "8. The unsigned prescription makes the documentation packet incomplete — cannot submit to DME supplier", "9. Do NOT open the fax portal or send any fax", "10. Do NOT clear the referral from the worklist", "11. Click the 'Notes' tab in the top navigation bar", "12. In the Notes tab, write a detailed progress note that includes: (a) Lee, Michael / BiPAP machine; (b) prescription is unsigned/DRAFT/pending Dr. Angela Park's signature; (c) F2F and H&P are signed and ready; (d) unsigned Rx cannot be included in DME documentation packet; (e) no fax sent to Apria Healthcare, order on hold; (f) recommend contacting Dr. Angela Park to sign the prescription", "13. Click 'Sign' to save the note" ], "evals": [ { "id": "fax-hard-4-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed at least one document in the referral", "points": 1, "query": "full_state.agentActions.viewedDocuments | length(@) > `0`", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-4-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify that no fax was sent — prescription is unsigned", "points": 1, "query": "full_state.faxPortal.faxesSent || `0`", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-4-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral was NOT cleared from worklist", "points": 1, "query": "!contains(full_state.clearedReferrals || `[]`, 'REF-2025-214')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-4-eval-4", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added documenting the issue", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-4-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Note identifies the prescription as unsigned/pending", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note identify that the prescription is unsigned, pending signature, or in draft status? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-4-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note recommends contacting physician for signature", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note recommend contacting the physician (Dr. Angela Park) to sign the prescription? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-4-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note mentions patient name", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the patient name (Lee, Michael)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-4-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note mentions equipment type", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the equipment type (BiPAP machine)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-4-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note explains unsigned Rx cannot be submitted", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note state that an unsigned prescription cannot be included in the DME documentation packet? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-4-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note confirms no fax sent and order on hold", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention EITHER that no fax was sent OR that the order is on hold pending the signed prescription? Only one of these two is required to score 1.0. Score 1.0 if at least one is mentioned, 0.0 if neither is mentioned.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-4-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note inventories document readiness status", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention which documents are ready (Face-to-Face Evaluation, History & Physical) and which is not (unsigned Prescription)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": false, "progressNoteAdded": true, "clearedFromWorklist": false, "reason": "Prescription unsigned, must obtain physician signature before processing" }, "rationale": "" }, { "id": "fax-hard-5", "title": "Patient Transferred — Reroute DME Order", "instruction": "Review documents and clinical note. Identify that the patient was transferred to another facility and the DME order should be processed there. Do NOT fax. Document the reroute.", "goal": "Open Martinez, Elena (45F with Blue Shield PPO). Process the DME order for a bedside commode. Review all documents before proceeding. Determine the appropriate action and document your findings in a progress note.", "difficulty": "hard", "category": "workflow", "challengeType": "workflow", "possible": true, "points": 10, "configTaskId": "dme_hard_5", "website": { "name": "Epic Start EHR", "url": "https://emrportal.vercel.app" }, "patient": { "name": "Martinez, Elena" }, "stepByStep": [ "1. On the DME Orders page (/emr/dme), click on patient Martinez, Elena", "2. The referral opens with Orders tab → Active sub-tab (default). Review the order details.", "3. Click the 'Chart Review' tab in the top navigation bar", "4. IMPORTANT: Click on one of the documents (e.g., the clinical note or Face_to_Face_Evaluation) to view it — you must open at least one document to review its contents", "5. In the document viewer, notice the clinical note states the patient was transferred to Valley Rehabilitation Center effective 12/18/2025", "6. The clinical note indicates all DME orders should be coordinated through the receiving facility (Valley Rehabilitation Center)", "7. Do NOT open the fax portal or send any fax", "8. Do NOT clear the referral from the worklist", "9. Click the 'Notes' tab in the top navigation bar", "10. In the Notes tab right panel (Edit Note), type a subject and detailed progress note documenting: patient Elena Martinez, bedside commode order, patient transferred to Valley Rehabilitation Center on 12/18/2025, no fax sent, and DME to be coordinated by the receiving facility", "11. Click 'Sign' to save the note" ], "evals": [ { "id": "fax-hard-5-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed at least one document in the referral", "points": 1, "query": "full_state.agentActions.viewedDocuments | length(@) > `0`", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-5-eval-2", "type": "jmespath", "category": "Document Handling", "description": "Verify that no fax was sent — patient transferred to another facility", "points": 1, "query": "full_state.faxPortal.faxesSent || `0`", "expectedValue": 0, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-5-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Verify that referral was NOT cleared from worklist", "points": 1, "query": "!contains(full_state.clearedReferrals || `[]`, 'REF-2025-215')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-5-eval-4", "type": "jmespath", "category": "Documentation", "description": "Verify that a progress note was added documenting the reroute", "points": 1, "query": "full_state.agentActions.addedProgressNote", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "fax-hard-5-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Note mentions patient transfer to Valley Rehab", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention that the patient was transferred to Valley Rehabilitation Center? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-5-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note explains DME should be handled by receiving facility", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention that the DME order should be processed by the receiving facility, not the originating facility? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-5-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note mentions patient name", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the patient name (Martinez, Elena)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-5-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note mentions equipment type", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note mention the equipment type (commode or bedside commode)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-5-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note confirms no fax sent with reasoning", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note clearly state that no fax was sent? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" }, { "id": "fax-hard-5-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note references transfer details", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's progress note reference the transfer date (12/18/2025)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.communications[-1].content}}", "studentAnswerContext": "progress note" } ], "expectedOutcome": { "faxSent": false, "progressNoteAdded": true, "clearedFromWorklist": false, "reason": "Patient transferred, DME order should be processed by receiving facility" }, "rationale": "" } ] }, { "id": "appeals_denials", "name": "Appeals & Denials", "description": "Denial triage and appeal-routing tasks from the denials workqueue.", "tasks": [ { "id": "denial-easy-1", "title": "Triage LCD-Based Medical Necessity Denial - Identify Documentation Gap for Appeal", "instruction": "Open denial DEN-001 for Martinez, Carlos. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-001 for Martinez, Carlos. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_1", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Martinez, Carlos" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-001 for Martinez, Carlos - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-50 (medical necessity) and the denial reason referencing the payer LCD", "5. Review the diagnosis H35.32 (exudative AMD) and procedure codes 67028/J2778 (intravitreal injection, ranibizumab)", "6. Recognize that H35.32 clearly supports anti-VEGF therapy - the denial is about missing LCD documentation (OCT findings, visual acuity decline), not clinical appropriateness", "7. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown and submit", "8. Add a triage note explaining that this is a documentation gap (missing documentation), not a clinical dispute - the diagnosis supports the procedure - and that appeal should include clinical records meeting LCD requirements" ], "evals": [ { "id": "denial-easy-1-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-001", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-1-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-1-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Clinical Appeals", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-1-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-1-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-1-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions remark code N386 or LCD", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code N386 or reference LCD (Local Coverage Determination)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-1-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis H35.32", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code H35.32 (exudative age-related macular degeneration)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-1-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 67028", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 67028 (intravitreal injection)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-1-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains this is a documentation gap, not a clinical dispute", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the denial is about missing documentation rather than clinical appropriateness? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "The payer denied under their LCD requiring specific documentation (OCT findings, visual acuity decline) for anti-VEGF treatment frequency. The diagnosis H35.32 (exudative AMD) is a standard clinical indication for intravitreal injection (CPT 67028/J2778) - this is a documentation gap, not a clinical disagreement. Appeal is warranted with clinical records that satisfy the LCD criteria." }, { "id": "denial-easy-2", "title": "Triage HMO Delegation Denial - Identify Correct Delegated Medical Group", "instruction": "Open denial DEN-002 for Johnson, Patricia. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-002 for Johnson, Patricia. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 7, "configTaskId": "denial_easy_2", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Johnson, Patricia" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), scroll down the list to find DEN-002 for Johnson, Patricia. The row will show DEN-002 in the denial ID column.", "2. Click DIRECTLY on the DEN-002 row text (patient name or denial ID) to open the denial detail page. Do NOT click on any navigation buttons at the top of the page.", "3. On the denial detail page, review the Errors section: N418 denial code (claim submitted to incorrect payer)", "4. [REQUIRED] Click the 'Remittance Image' tab to view the EOB and review the remark codes. This step is evaluated.", "5. Review coverage information and identify River City Medical Group as the delegated medical group under the HMO capitation arrangement", "6. Recognize: primary care services under this HMO are billed to the delegated group (River City Medical Group), NOT to Anthem directly", "7. In the right panel, click the Triage Disposition dropdown and select 'Reroute to Correct Entity'", "8. Write a triage note explaining: N418 denial code, HMO delegation arrangement, claim must be resubmitted to River City Medical Group (not Anthem), capitation arrangement", "9. Click 'Submit Disposition' to save" ], "evals": [ { "id": "denial-easy-2-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-002", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-2-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-2-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Reroute to Correct Entity", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Reroute to Correct Entity", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-2-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-2-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code N418", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code N418? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-2-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies River City Medical Group as the delegated entity", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify River City Medical Group from the coverage information? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-2-eval-7", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the delegation/capitation arrangement requires rerouting", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the HMO plan delegates primary care to a capitated medical group, requiring the claim to be rerouted rather than billed to the health plan directly? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Reroute to Correct Entity" }, "rationale": "N418 indicates claim was submitted to the wrong payer. The patient's Anthem Blue Cross HMO plan delegates primary care services to River City Medical Group under a capitated arrangement. The claim for CPT 99214 (E/M visit for E11.9/I10) must be rerouted to River City Medical Group, not billed to Anthem directly." }, { "id": "denial-easy-3", "title": "Triage Timely Filing Denial - Assess Whether Filing Deadline Exception Applies", "instruction": "Open denial DEN-003 for Williams, Sarah. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-003 for Williams, Sarah. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 7, "configTaskId": "denial_easy_3", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Williams, Sarah" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-003 for Williams, Sarah - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-29 (timely filing expired) and the denial reason stating the 180-day contractual deadline was exceeded", "5. Review the Claim History section to check the original submission date and whether the claim was filed within the 180-day deadline", "6. Confirm that no exception applies and the filing deadline was genuinely exceeded", "7. Select 'Write Off' from the Triage Disposition dropdown and submit", "8. Add a triage note explaining that the 180-day filing deadline was exceeded, no timely filing exception applies, and the denial is not appealable" ], "evals": [ { "id": "denial-easy-3-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-003", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-3-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-3-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Write Off", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Write Off", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-3-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-3-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-29", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-29? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-3-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the service date or filing timeline expiration", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the service date or the filing timeline being exceeded? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-3-eval-7", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the denial is not recoverable due to expired filing deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why the denial is not recoverable - e. g., that the filing deadline was exceeded with no evidence of timely submission or valid exception? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Write Off" }, "rationale": "CO-29 indicates the claim was filed beyond the contractual 180-day deadline from date of service. There is no proof of earlier submission and no timely filing exception applies. The denial is not appealable - write off is the appropriate disposition." }, { "id": "denial-easy-4", "title": "Triage Missing Modifier Denial - Identify Modifier -25 Coding Correction", "instruction": "Open denial DEN-004 for Brown, Michael. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-004 for Brown, Michael. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_4", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Brown, Michael" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-004 for Brown, Michael - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-4 (procedure code inconsistent with modifier) and remark code MA130 (missing required modifier)", "5. Identify that CPT 99213 was billed on the same day as a procedure without modifier -25 for separately identifiable E/M", "6. Recognize this is a coding correction, not a clinical dispute", "7. Select 'Corrected Claim - Resubmit' from the Triage Disposition dropdown and submit", "8. Add a triage note explaining that modifier -25 is missing, the claim needs coding correction, and should be resubmitted after appending the modifier. Explicitly include diagnosis code J06.9 and CPT 99213 in the note." ], "evals": [ { "id": "denial-easy-4-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-004", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-4-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-4-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Corrected Claim - Resubmit", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Corrected Claim - Resubmit", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-4-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-4-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-4", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-4? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-4-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code MA130 or the modifier -25 coding issue", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code MA130 or the missing modifier -25? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-4-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis J06.9", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code J06.9 (acute upper respiratory infection)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-4-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 99213", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 99213 (office/outpatient visit)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-4-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the specific coding or modifier error", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify a specific coding or modifier error that needs correction? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Corrected Claim - Resubmit" }, "rationale": "CO-4 with remark code MA130 indicates modifier -25 is missing from CPT 99213. The E/M visit was billed on the same day as a procedure without the modifier to designate it as a separately identifiable service. This is a billing/coding correction - append modifier -25 and resubmit. No clinical appeal is needed." }, { "id": "denial-easy-5", "title": "Triage Duplicate Claim Denial - Verify Original Payment and Confirm True Duplicate", "instruction": "Open denial DEN-005 for Garcia, Maria. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-005 for Garcia, Maria. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 7, "configTaskId": "denial_easy_5", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Garcia, Maria" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-005 for Garcia, Maria - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-18 (exact duplicate claim/service) in the Errors section", "5. Check the related claims section to verify that the original claim CLM-2025-49786 was already paid", "6. Confirm this is a true duplicate (accidental resubmission of preventive visit) rather than a legitimate separate service", "7. Select 'Write Off' from the Triage Disposition dropdown and submit", "8. Add a triage note explaining that the original claim was already paid, so write-off is appropriate; this is a confirmed duplicate submission and no recovery action is needed" ], "evals": [ { "id": "denial-easy-5-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-005", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-5-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-5-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Write Off", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Write Off", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-5-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-5-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-18", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-18? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-5-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the original/related claim or confirms duplicate", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the original claim (CLM-2025-49786) or confirm this is a duplicate submission? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-5-eval-7", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the original claim was already paid making write-off appropriate", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why the denial is not recoverable - e. g., that the original claim was already paid so no further recovery action is needed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Write Off" }, "rationale": "CO-18 indicates exact duplicate claim. The related claims section shows that the original claim CLM-2025-49786 was already paid by VHP. The billing system accidentally resubmitted the preventive visit (CPT 99395, Z00.00). This is a genuine duplicate - the provider has already been paid, so write off is the appropriate disposition." }, { "id": "denial-easy-6", "title": "Triage No-Auth Denial - Identify Expired Authorization for Epidural Injection", "instruction": "Open denial DEN-006 for Lee, David. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-006 for Lee, David. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 8, "configTaskId": "denial_easy_6", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Lee, David" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-006 for Lee, David - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-197 (precertification/authorization absent) in the Errors section", "5. Navigate to the patient's chart by clicking the patient name link to open the Patient Inquiry page", "6. In the left sidebar, check the Authorizations section - find AUTH-2025-88431, note it expired on 2025-10-15", "7. Compare the auth expiration (2025-10-15) with the service date (2025-10-25) - the auth expired 10 days before the procedure", "8. Navigate back to the denial detail page", "9. Note the high dollar amount ($5,670) which makes recovery worth pursuing", "10. Select 'Peer-to-Peer Review' from the Triage Disposition dropdown and submit", "11. Add a triage note explaining that auth was previously obtained but expired before the service date, and a P2P review is needed to request retroactive authorization" ], "evals": [ { "id": "denial-easy-6-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-006", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-6-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-6-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the patient inquiry page to check authorization history", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-6-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Peer-to-Peer Review", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Peer-to-Peer Review", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-6-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-6-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-197", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-197? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-6-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the expired authorization or auth/service date mismatch", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the expired authorization (AUTH-2025-88431) or that the auth expired before the service date? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-6-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains that expired auth warrants P2P to seek retroactive authorization", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that because the auth was obtained but expired, the provider should request retroactive authorization or a P2P review to recover the claim? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Peer-to-Peer Review" }, "rationale": "CO-197 denial for epidural injection (CPT 64483). The denial states authorization was absent, but checking the patient's chart (Patient Inquiry page > Authorizations section) reveals AUTH-2025-88431 was obtained and valid 2025-09-01 through 2025-10-15. The procedure was performed on 2025-10-25 - 10 days after the auth expired. This is not a missing auth; the provider obtained authorization but the procedure was delayed past the auth window. The high dollar amount ($5,670) and the fact that auth was previously granted make this recoverable through a P2P review to request retroactive authorization." }, { "id": "denial-easy-7", "title": "Triage Eligibility Denial - Escalate Expired-Deadline Case with Delegated Payer Complexity", "instruction": "Open denial DEN-007 for Thompson, Jennifer. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-007 for Thompson, Jennifer. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 7, "configTaskId": "denial_easy_7", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Thompson, Jennifer" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-007 for Thompson, Jennifer - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code N30 (patient cannot be identified as insured) in the Errors section", "5. Check the appeal deadline (2026-01-20) - note it has already passed", "6. Note the payer is Pacific Health Alliance with delegated group Community Care Network, adding complexity", "7. Determine that the expired deadline combined with the eligibility issue and delegation complexity requires supervisor review", "8. Select 'Escalate to Supervisor' from the Triage Disposition dropdown and submit", "9. Add a triage note explaining why escalation is needed: expired deadline forecloses standard appeal, eligibility issue is unresolved, and delegated payer structure requires supervisor guidance" ], "evals": [ { "id": "denial-easy-7-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-007", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-7-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-7-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Escalate to Supervisor", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Escalate to Supervisor", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-7-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-7-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code N30", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code N30? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-7-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the expired appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the expired appeal deadline (2026-01-20 or that the deadline has passed)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-7-eval-7", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains why escalation is needed due to expired deadline and complexity", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why this case exceeds standard triage authority - e. g., the expired deadline combined with the eligibility issue or delegation complexity requires supervisor guidance? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Escalate to Supervisor" }, "rationale": "N30 denial (patient cannot be identified as insured) with appeal deadline 2026-01-20 already passed. The expired deadline means standard appeal is not available. Combined with the eligibility issue (patient may have changed plans or member ID mismatch) and the delegated payer structure (Pacific Health Alliance with Community Care Network as delegated medical group), this case exceeds standard triage authority. Supervisor must determine next steps - whether to pursue a late appeal exception, work the eligibility correction through Community Care Network, or explore other recovery options." }, { "id": "denial-easy-8", "title": "Triage Non-Covered Benefit Denial - Determine Patient Responsibility for Plan Exclusion", "instruction": "Open denial DEN-008 for Anderson, Robert. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-008 for Anderson, Robert. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_8", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Anderson, Robert" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-008 for Anderson, Robert - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-96 (non-covered charge, benefit not covered under the plan) in the Errors section", "5. Identify CPT S9083 (outpatient mental health global fee) and diagnosis F41.1 (GAD)", "6. Determine that S9083 is not covered under the patient's Aetna plan - this is a benefit exclusion, not a medical necessity dispute", "7. Recognize that appealing a benefit exclusion would not succeed", "8. Select 'Transfer to Patient' from the Triage Disposition dropdown and submit", "9. Add a triage note explaining that CO-96 is a benefit exclusion for S9083, appeal would not succeed, and patient responsibility applies. Explicitly include diagnosis code F41.1 and CPT S9083 in the note." ], "evals": [ { "id": "denial-easy-8-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-008", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-8-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-8-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Transfer to Patient", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Transfer to Patient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-8-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-8-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-96", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-96? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-8-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies S9083 as non-covered or a plan exclusion", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify S9083 as a non-covered benefit or plan exclusion? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-8-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis F41.1", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code F41.1 (generalized anxiety disorder)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-8-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT S9083", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code S9083 (global fee for outpatient mental health)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-8-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the benefit exclusion means patient responsibility", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a benefit exclusion, meaning the patient bears financial responsibility for the non-covered service? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Transfer to Patient" }, "rationale": "CO-96 denial for S9083 (global fee for outpatient mental health) with diagnosis F41.1 (GAD). S9083 is a temporary national code that many payers, including this Aetna plan, do not recognize or cover. This is a benefit exclusion - the plan explicitly does not cover this service code. Unlike a medical necessity denial, a benefit exclusion cannot be overturned on appeal because the service is outside the plan's covered benefits. The patient bears financial responsibility (assuming proper financial consent/ABN was obtained). Bill to patient is the correct disposition." }, { "id": "denial-easy-9", "title": "Triage Missing Information Denial - Identify Missing Referring Provider NPI via Remark Code", "instruction": "Open denial DEN-020 for Clark, Steven. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-020 for Clark, Steven. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 7, "configTaskId": "denial_easy_9", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Clark, Steven" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-020 for Clark, Steven - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. Review the denial code CO-16 (claim lacks information) in the Errors section - note the reason is generic and doesn't specify what's missing", "4. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes This step is evaluated.", "5. Find remark code N264 which specifies: referring provider NPI is missing or invalid", "6. Determine that this is an administrative data correction - add the referring provider NPI and resubmit", "7. Select 'Corrected Claim - Resubmit' from the Triage Disposition dropdown and submit", "8. Add a triage note explaining that CO-16 was caused by a missing referring provider NPI (per remark code N264), and the claim should be corrected and resubmitted" ], "evals": [ { "id": "denial-easy-9-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-020", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-9-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-9-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Corrected Claim - Resubmit", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Corrected Claim - Resubmit", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-9-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-9-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-16", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-16? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-9-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code N264 or identifies the missing referring provider NPI", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code N264 or identify the missing referring provider NPI as the specific data gap? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-9-eval-7", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the claim needs to be corrected with the missing NPI and resubmitted", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a data correction issue (not a clinical dispute) and the claim should be corrected with the referring provider NPI and resubmitted? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Corrected Claim - Resubmit" }, "rationale": "CO-16 denial for CPT 99213 (office visit) with diagnosis J06.9 (acute URI). The denial reason is generic ('claim lacks information'), but the remark code N264 on the EOB/Remittance Image specifies that the referring/ordering provider NPI is missing or invalid. This is an administrative data error - the referring provider's NPI needs to be added to the claim and resubmitted. No formal appeal is needed because the claim was rejected for incomplete data, not adjudicated on its merits." }, { "id": "denial-easy-10", "title": "Triage Bundling Denial - Correct NCCI Edit with Modifier 59/XS for Distinct Biopsy Sites", "instruction": "Open denial DEN-022 for King, Michelle. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-022 for King, Michelle. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 10, "configTaskId": "denial_easy_10", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "King, Michelle" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-022 for King, Michelle - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-97 (bundling - procedure included in another) in the Errors section", "5. Identify CPT 11102 (tangential biopsy) and CPT 11103 (each additional) in the line items", "6. Recognize this as an NCCI edit pair where 11103 was bundled into 11102", "7. Determine that modifier 59 or XS should be added to 11103 to indicate distinct anatomical sites", "8. Select 'Corrected Claim - Resubmit' from the Triage Disposition dropdown and submit", "9. Add a triage note explaining the NCCI bundling issue and that adding modifier 59/XS to 11103 for distinct biopsy sites will resolve the denial on resubmission. Explicitly include diagnosis code D23.9, remark code MA130, and CPT 11102/11103 in the note." ], "evals": [ { "id": "denial-easy-10-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-022", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-10-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-10-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Corrected Claim - Resubmit", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Corrected Claim - Resubmit", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-10-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-10-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-97", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-97? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-10-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code MA130", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code MA130 (missing required modifier)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-10-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references modifier 59/XS or bundling correction", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention modifier 59 or XS, or the bundling/NCCI edit issue between CPT 11102 and 11103? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-10-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis D23.9", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code D23.9 (benign neoplasm of skin)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-10-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 11102 or 11103", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 11102 (tangential biopsy) or 11103 (each additional tangential biopsy)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-10-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the bundling error is correctable via modifier addition", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a coding/billing error (bundling edit) that can be corrected with a modifier and resubmitted? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Corrected Claim - Resubmit" }, "rationale": "CO-97 bundling denial: the payer bundled CPT 11103 (each additional tangential biopsy) into CPT 11102 (first tangential biopsy) per NCCI edits. However, if the biopsies were performed on distinct lesions at separate anatomical sites, modifier 59 (Distinct Procedural Service) or XS (Separate Structure) should have been appended to 11103 to indicate a separate service. The coder missed the modifier. This is a coding correction - add modifier 59 or XS to CPT 11103 and resubmit the claim. No formal appeal is needed because the denial is based on a coding edit that can be overridden with proper documentation of distinct services." }, { "id": "denial-easy-11", "title": "Triage High-Value Cardiac Admission Denial - Escalate for Supervisor Review", "instruction": "Open denial DEN-014 for Moore, Elizabeth. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-014 for Moore, Elizabeth. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_11", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Moore, Elizabeth" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-014 for Moore, Elizabeth - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-50 (medical necessity) and the denial reason in the Errors section", "5. Note the denied amount of $45,000 - this is a high-value case", "6. Review the hospital admission CPT codes (99223 initial hospital care, 99232 subsequent care, 99238 discharge) and cardiac diagnoses", "7. Recognize that the $45,000 value exceeds standard triage authority and requires supervisor review", "8. Select 'Escalate to Supervisor' from the Triage Disposition dropdown and submit", "9. Add a triage note documenting the CO-50 denial, remark code N386/LCD-based determination, relevant cardiac diagnosis or inpatient CPT evidence, the $45,000 value, and that the case exceeds standard triage authority requiring supervisor review" ], "evals": [ { "id": "denial-easy-11-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-014", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-11-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-11-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Escalate to Supervisor", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Escalate to Supervisor", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-11-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-11-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-11-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code N386 or LCD-based determination", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code N386 or reference an LCD-based determination? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-11-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references STEMI/cardiac diagnosis codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code I21.09 (STEMI) or I25.10 (coronary artery disease)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-11-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references hospital admission CPT codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention any of the CPT codes 99223 (initial hospital care), 99232 (subsequent hospital care), or 99238 (hospital discharge)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-11-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains the high case value exceeds standard triage authority", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the high dollar value ($45,000) exceeds standard triage authority, requiring supervisor review? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Escalate to Supervisor" }, "rationale": "$45,000 hospital admission denial (CO-50) for cardiac care with multiple inpatient CPT codes. The dollar amount far exceeds standard triage authority for a billing specialist. High-value cases require supervisor review per department policy, regardless of the clinical merits of the case. The billing person's role is to identify the high value and escalate - clinical assessment of whether the admission was necessary is for the physician advisor and appeals team." }, { "id": "denial-easy-12", "title": "Triage Same-Day Multi-Procedure GI Denial - Route to Clinical Appeals", "instruction": "Open denial DEN-016 for Harris, Dorothy. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-016 for Harris, Dorothy. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_12", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Harris, Dorothy" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-016 for Harris, Dorothy - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-50 (medical necessity) in the Errors section", "5. Identify the multiple GI procedure codes (43235, 43239, J1100) and diagnoses (K21.0, K44.9, K57.30) in the line items", "6. Confirm the appeal deadline (2026-03-12) is still open", "7. Recognize that CO-50 medical necessity denials require clinical documentation to appeal - this needs the clinical team", "8. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown", "9. Add a triage note documenting the CO-50 denial, the procedure and diagnosis codes, that the appeal deadline is open, and that clinical documentation is needed for the appeal", "10. Click 'Submit Disposition' to save the disposition and triage note" ], "evals": [ { "id": "denial-easy-12-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-016", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-12-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-12-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Clinical Appeals", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-12-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-12-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-12-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code N386 or LCD-based determination", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code N386 or reference an LCD-based determination? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-12-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references GI diagnosis codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention any of the diagnosis codes K21.0 (GERD), K44.9 (diaphragmatic hernia), or K57.30 (diverticulosis)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-12-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references EGD procedure codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 43235 (diagnostic EGD) or 43239 (EGD with biopsy)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-12-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains CO-50 requires clinical documentation for appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a medical necessity denial requiring clinical documentation to appeal, and that the clinical team should review the case? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-50 medical necessity denial for multiple GI procedures (43235 diagnostic EGD, 43239 EGD with biopsy, J1100 dexamethasone) on the same date. Medical necessity denials require clinical documentation to appeal - the billing person identifies the denial type, confirms the appeal deadline is still open, and routes to the clinical appeals team who can gather operative notes and procedure documentation to support the appeal." }, { "id": "denial-easy-13", "title": "Triage Delegated Capitation Denial -- Reroute Claim to Community Care Network", "instruction": "Open denial DEN-017 for Lewis, Angela. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-017 for Lewis, Angela. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 7, "configTaskId": "denial_easy_13", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Lewis, Angela" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-017 for Lewis, Angela - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code N418 and the denial reason ('Claim submitted to incorrect payer. Services under delegated capitation arrangement.')", "5. Review the coverage information identifying Community Care Network as the delegated group", "6. Determine that the claim must be rerouted to Community Care Network under the capitation arrangement", "7. Select 'Reroute to Correct Entity' from the Triage Disposition dropdown and submit", "8. Add a triage note explaining that N418 indicates the claim was sent to Pacific Health Alliance but must go to Community Care Network as the delegated entity" ], "evals": [ { "id": "denial-easy-13-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-017", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-13-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-13-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Reroute to Correct Entity", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Reroute to Correct Entity", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-13-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-13-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code N418", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code N418? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-13-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies Community Care Network as the delegated entity", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify Community Care Network from the coverage information? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-13-eval-7", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the delegation/capitation arrangement requires rerouting to Community Care Network", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the delegated capitation arrangement requires the claim to be rerouted to Community Care Network rather than billed to the health plan? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Reroute to Correct Entity" }, "rationale": "N418 indicates the claim was submitted to the wrong payer. The claim was sent to Pacific Health Alliance (the Medicaid Managed Care health plan), but primary care and immunization services (99214, 90471, 90715 for Z23) are under Community Care Network's delegated capitation arrangement. The claim must be resubmitted to Community Care Network as the delegated medical group responsible for these services." }, { "id": "denial-easy-14", "title": "Triage Expired Authorization Denial - Write Off After Missed Appeal Deadline", "instruction": "Open denial DEN-013 for Taylor, Susan. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-013 for Taylor, Susan. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_14", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Taylor, Susan" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-013 for Taylor, Susan - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes This step is evaluated.", "4. Review the denial code CO-197 (authorization absent) in the Errors section", "5. Navigate to the patient's chart by clicking the patient name to open the Patient Inquiry page", "6. In the Authorizations section, find AUTH-2025-55901 - note it expired on 2025-08-15, before the service date of 2025-08-28", "7. Navigate back to the denial detail page", "8. Note the system note from 2025-10-15 flagged the appeal deadline as approaching - but check the appeal deadline (2026-01-26) which has now passed", "9. Determine that with both expired auth and missed appeal deadline, no recovery is possible", "10. Select 'Write Off' from the Triage Disposition dropdown and submit", "11. Add a triage note explicitly referencing CO-197 and explaining that the auth expired before service, the appeal deadline has passed, and the denial is unrecoverable" ], "evals": [ { "id": "denial-easy-14-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-013", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-14-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-14-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the patient inquiry page to check authorization history", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-14-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Write Off", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Write Off", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-14-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-14-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-197", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-197? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-14-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the expired authorization", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the expired authorization AUTH-2025-55901? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-14-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the expired appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the expired appeal deadline (2026-01-26)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-14-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the denial is unrecoverable due to expired authorization and appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why the denial is not recoverable - e. g., that both the authorization and the appeal deadline have expired, leaving no avenue for recovery? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Write Off" }, "rationale": "CO-197 denial for knee ligament reconstruction (CPT 27427). Checking the patient's chart reveals AUTH-2025-55901 was obtained but expired on 2025-08-15 - the procedure was performed on 2025-08-28, 13 days after auth expired. The system note from 2025-10-15 flagged the appeal deadline as approaching, but the deadline (2026-01-26) has now passed. With both the authorization expired and the appeal deadline missed, there is no recovery avenue - write off is the appropriate disposition." }, { "id": "denial-easy-15", "title": "Triage LCD-Based Brain MRI Denial -- Route to Clinical Appeals", "instruction": "Open denial DEN-010 for Davis, Christine. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-010 for Davis, Christine. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_15", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Davis, Christine" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-010 for Davis, Christine - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-50 (medical necessity) and remark code N386 (LCD-based) in the Errors section", "5. Identify CPT 70551 (MRI brain without contrast) and diagnoses G43.909 (migraine) and R51.9 (headache) in the line items", "6. Confirm the appeal deadline is still open", "7. Recognize that CO-50 medical necessity denials require clinical documentation to appeal - this needs the clinical team", "8. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown and submit", "9. Add a triage note documenting the CO-50/N386 LCD denial, the procedure and diagnosis codes, and that clinical documentation is needed for the appeal" ], "evals": [ { "id": "denial-easy-15-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-010", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-15-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-15-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Clinical Appeals", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-15-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-15-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-15-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code N386 or LCD-based determination", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code N386 or reference an LCD-based determination? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-15-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references migraine/headache diagnosis", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code G43.909 (migraine) or R51.9 (headache)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-15-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 70551", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 70551 (MRI brain without contrast)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-15-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains CO-50 requires clinical documentation for appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a medical necessity denial requiring clinical documentation to appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-50 medical necessity denial with LCD remark code (N386) for brain MRI (CPT 70551) with diagnoses G43.909 (migraine) and R51.9 (headache). The N386 remark indicates the denial was based on an LCD determination. The appeal deadline is still open. This is a medical necessity denial requiring clinical documentation to appeal - route to the clinical appeals team to compile the relevant clinical records and submit the appeal." }, { "id": "denial-easy-16", "title": "Triage LCD-Based Lumbar MRI Denial - Route to Clinical Appeals", "instruction": "Open denial DEN-019 for Robinson, Karen. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-019 for Robinson, Karen. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_16", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Robinson, Karen" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-019 for Robinson, Karen - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-50 (medical necessity) and remark code N386 (LCD-based) in the Errors section", "5. Identify CPT 72148 (lumbar MRI) and diagnosis M54.5 (low back pain) in the line items", "6. Note the follow-up status and notes indicating clinical documentation was requested from the provider", "7. Confirm the appeal deadline (2026-03-25) is still open", "8. Recognize that CO-50 medical necessity denials require clinical documentation to appeal - this needs the clinical team", "9. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown and submit", "10. Add a triage note documenting the CO-50/N386 LCD denial, the procedure and diagnosis codes, that clinical notes were requested, and that the clinical team should compile documentation for appeal. Explicitly include CPT 72148 and diagnosis code M54.5 in the note." ], "evals": [ { "id": "denial-easy-16-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-019", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-16-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-16-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Clinical Appeals", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-16-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-16-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-16-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code N386 or LCD-based determination", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code N386 or reference an LCD-based determination? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-16-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis M54.5", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code M54.5 (low back pain)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-16-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 72148", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 72148 (lumbar MRI)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-16-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains CO-50 requires clinical documentation for appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a medical necessity denial requiring clinical documentation to appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-50 medical necessity denial with LCD remark code (N386) for lumbar MRI (CPT 72148) with diagnosis M54.5 (low back pain). The notes show clinical documentation was already requested from the provider. The appeal deadline (2026-03-25) is still open. This is a medical necessity denial that requires clinical documentation to appeal - route to the clinical appeals team to compile the conservative treatment records and submit the appeal." }, { "id": "denial-easy-17", "title": "Triage Out-of-Network Denial - Transfer Patient Responsibility for OON Orthopedic Visit", "instruction": "Open denial DEN-009 for Nguyen, Thi. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-009 for Nguyen, Thi. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 8, "configTaskId": "denial_easy_17", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Nguyen, Thi" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-009 for Nguyen, Thi - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code PR-242 (out-of-network provider) and remark code N522 in the Errors section", "5. Note the patient's insurance is Aetna HMO, which requires in-network providers", "6. Note that remark code N522 states the patient is responsible for out-of-network charges", "7. Before transferring to patient, note that a patient financial responsibility form (PAFR/ABN) must be confirmed on file - if not, the balance cannot be transferred without patient consent", "8. Select 'Transfer to Patient' from the Triage Disposition dropdown and submit", "9. Add a triage note documenting the PR-242 OON denial, that the HMO plan requires in-network providers, that the balance should be transferred to patient responsibility, and that financial responsibility form must be verified before billing patient" ], "evals": [ { "id": "denial-easy-17-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-009", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-17-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-17-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Transfer to Patient", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Transfer to Patient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-17-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-17-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code PR-242", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code PR-242? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-17-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the out-of-network issue under the HMO plan", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that the provider is out-of-network or that the patient has an HMO plan requiring in-network providers? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-17-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains patient responsibility for OON charges", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the patient is responsible for out-of-network charges under their HMO plan? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-17-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions verifying financial responsibility form before billing patient", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that a patient financial responsibility form (PAFR, ABN, or financial consent) should be verified or confirmed on file before transferring the balance to the patient? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Transfer to Patient" }, "rationale": "PR-242 denial indicates the patient saw an out-of-network orthopedic provider (Summit Orthopedic Associates) for a consultation (99243) and joint injection (20610). The patient's Aetna HMO plan requires in-network providers. The remark code N522 on the Remittance Image tab confirms the patient is responsible for OON charges. Since the HMO plan does not cover out-of-network services, the balance should be transferred to patient responsibility." }, { "id": "denial-easy-18", "title": "Triage Partial Denial - Route Denied Knee Arthroscopy Lines to Clinical Appeals", "instruction": "Open denial DEN-024 for Lopez, Anna. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-024 for Lopez, Anna. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_18", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Lopez, Anna" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-024 for Lopez, Anna - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-50 (medical necessity) in the Errors section", "5. Review all 4 line items - identify 99214 (office visit) and 85025 (CBC) as paid, 29881 and 29880 (knee arthroscopy) as denied", "6. Note this is a partial denial - the routine visit was covered but the surgical procedures were denied", "7. Confirm the appeal deadline (2026-03-08) is still open", "8. Recognize that the denied surgical lines are a medical necessity denial requiring clinical documentation to appeal", "9. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown and submit", "10. Add a triage note documenting the partial denial, which lines were paid vs denied, and that the denied arthroscopy procedures need clinical documentation for appeal. Explicitly include diagnosis codes M23.41 and/or M23.42 in the note." ], "evals": [ { "id": "denial-easy-18-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-024", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-18-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-18-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Clinical Appeals", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-18-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-18-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-18-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the partial denial with paid vs denied service lines", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify this as a partial denial, distinguishing paid lines (99214, 85025) from denied lines (29881, 29880)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-18-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references knee meniscus diagnosis codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code M23.41 or M23.42 (meniscus derangement)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-18-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references knee arthroscopy CPT codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 29881 (knee arthroscopy with meniscectomy) or 29880 (knee arthroscopy)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-18-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the denied lines need clinical documentation for appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the denied surgical lines are a medical necessity denial requiring clinical documentation to appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "Partial denial - office visit (99214) and CBC (85025) were paid, but knee arthroscopy procedures (29881 meniscectomy and 29880 medial/lateral meniscectomy) were denied for medical necessity. The payer wants documentation of failed conservative treatment before approving knee surgery. The appeal deadline (2026-03-08) is still open. Route to clinical appeals team to compile operative notes and conservative treatment records for the appeal." }, { "id": "denial-easy-19", "title": "Triage Medicare LCD Denial - Route TKA to Clinical Appeals for Documentation", "instruction": "Open denial DEN-012 for Wilson, Linda. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "goal": "Open denial DEN-012 for Wilson, Linda. Review all available information about this denial and determine the appropriate triage disposition. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_19", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Wilson, Linda" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-012 for Wilson, Linda - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Review the denial code CO-50 (medical necessity) and remark code N386 (LCD-based) in the Errors section", "5. Identify CPT 27447 (total knee arthroplasty) and diagnosis M17.0 (bilateral primary osteoarthritis) in the line items", "6. Note the government payer (VHP Medicare) and the notes indicating fax-based appeal submission", "7. Confirm the appeal deadline (2026-03-08) is still open", "8. Recognize that CO-50 medical necessity denials require clinical documentation to appeal - this needs the clinical team", "9. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown and submit", "10. Add a triage note documenting the CO-50/N386 LCD denial, the procedure and diagnosis codes, and that clinical documentation is needed for the appeal" ], "evals": [ { "id": "denial-easy-19-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-012", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-19-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-19-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Clinical Appeals", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-19-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-19-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-19-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references remark code N386 or LCD-based determination", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention remark code N386 or reference an LCD-based determination? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-19-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis M17.0", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code M17.0 (bilateral primary osteoarthritis)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-19-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 27447", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 27447 (total knee arthroplasty)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-19-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains CO-50 requires clinical documentation for appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a medical necessity denial requiring clinical documentation to appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-50 medical necessity denial with LCD remark code (N386) for total knee arthroplasty (CPT 27447) with diagnosis M17.0 (bilateral primary osteoarthritis). The LCD requires documentation of failed conservative treatment. The appeal deadline (2026-03-08) is still open. This is a medical necessity denial requiring clinical documentation to appeal - route to the clinical appeals team to compile conservative treatment records and submit the appeal via the government pathway." }, { "id": "denial-easy-20", "title": "Clear Resolved Coding Denial - Verify Modifier -25 Correction and Payment Before Clearing", "instruction": "Open the completed denial DEN-015 for Jackson, William. Review all available information about this denial and determine the appropriate action. Document your reasoning in a triage note.", "goal": "Open the completed denial DEN-015 for Jackson, William. Review all available information about this denial and determine the appropriate action. Document your reasoning in a triage note.", "difficulty": "easy", "category": "denial_triage", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_easy_20", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Jackson, William" }, "stepByStep": [ "1. On the Denials Workqueue, click the 'Completed' tab to find DEN-015 for Jackson, William (resolved denials appear under the Completed tab, not the Active tab)", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes (adjustment reason codes) This step is evaluated.", "4. Confirm the status shows RESOLVED", "5. Review the original denial code CO-4 (missing modifier) and the corrected CPT 99213-25", "6. Check the payment history to confirm payment was received after correction", "7. Navigate to the 'Retest' tab and then to the Documents section and verify the corrected claim document", "8. Add a triage note confirming the coding correction, payment receipt, and resolution. Explicitly include diagnosis code J20.9 and CPT 99213 in the note.", "9. Select 'No Action Needed - Clear' from the Triage Disposition dropdown", "10. Click the Submit button to save the triage note and disposition", "11. Click 'Clear from Workqueue' in the Actions section" ], "evals": [ { "id": "denial-easy-20-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-015", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-20-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-20-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent cleared the resolved denial from the workqueue", "points": 1, "query": "contains(full_state.clearedDenials || `[]`, 'DEN-015')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-20-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their reasoning", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-easy-20-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions denial code CO-4", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-4? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-20-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the modifier -25 coding correction or remark code MA130", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the coding correction (modifier -25) or remark code MA130? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-20-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis J20.9", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis code J20.9 (acute bronchitis)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-20-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 99213", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT code 99213 (office/outpatient visit)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-easy-20-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains the denial is resolved with payment received, justifying clearing", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the denial is resolved and no further action is needed - e. g., the corrected claim was resubmitted and payment was received? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "No Action Needed - Clear" }, "rationale": "DEN-015 is resolved - the CO-4 denial was caused by a missing modifier on CPT 99213. The coding correction (adding modifier -25) was applied, the corrected claim was resubmitted and processed, and payment was received. No further action is needed; safe to clear from the workqueue." }, { "id": "denial-medium-1", "title": "File Electronic Appeal on Payer A for Anti-VEGF Medical Necessity Denial", "instruction": "File an electronic appeal for denial DEN-001 (Martinez, Carlos) on the Payer A portal. Review the CO-50 medical necessity denial in EMR, check the remittance image for CO-50/N386 remark codes, then navigate to the Payer A portal link. Login to Payer A (provider@payera.com / demo123), search for the claim by member ID AET789456123, and file an appeal with medical necessity justification for the anti-VEGF injection (CPT 67028). Obtain the confirmation number (format APL-AET-XXXXXXXX), return to EMR, and document the confirmation number and appeal details in a triage note.", "goal": "File an electronic appeal for denial DEN-001 (Martinez, Carlos) on the Payer A portal. Review the CO-50 medical necessity denial in EMR, check the remittance image for CO-50/N386 remark codes, then navigate to the Payer A portal link. Login to Payer A (provider@payera.com / demo123), search for the claim by member ID AET789456123, and file an appeal with medical necessity justification for the anti-VEGF injection (CPT 67028). Obtain the confirmation number (format APL-AET-XXXXXXXX), return to EMR, and document the confirmation number and appeal details in a triage note.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 17, "configTaskId": "denial_medium_1", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Martinez, Carlos" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-001 for Martinez, Carlos and click to open the denial detail page", "2. [REQUIRED - DO THIS BEFORE PORTAL] Click the 'Remittance Image' tab. Read and record: CO-50 denial code, N386 remark code. This step is evaluated.", "3. Note the anti-VEGF injection CPT 67028/J2778 and diagnosis H35.32 (exudative AMD)", "4. [REQUIRED - BEFORE LEAVING EMR] Download the clinical documentation: click the 'View >' button on the RIGHT side of the document row (NOT the document name - that does nothing). This opens the document viewer. Then click Download on the viewer page. This is required to attach to the appeal.", "5. Click the 'Start Appeal' button to navigate to the Payer A portal", "6. Login with provider@payera.com / demo123", "7. Click 'Appeals' in the top navigation bar", "8. Enter member ID AET789456123 in the Member ID field and click Search", "9. Click on claim CLM-2025-00001 in the results to view the claim detail", "10. Review the denial information (CO-50, N386) and click the 'Dispute Claim' button", "11. Fill in Contact Name with 'Dr. Jane Smith'", "12. In Supporting Rationale, write a justification referencing: H35.32 (exudative AMD), clinical evidence (OCT findings, subretinal fluid, macular edema, visual acuity), and medical necessity of anti-VEGF injection (CPT 67028/J2778)", "13. Scroll to 'Available Documents from EMR' - click '+ Attach' next to the downloaded clinical document", "14. CRITICAL: Scroll down until the Submit button is FULLY VISIBLE. Click 'Submit Appeal' and note the confirmation number (APL-AET-XXXXXXXX).", "15. Click 'Return to EMR' to go back", "16. Select 'Appeal Filed' from the Triage Disposition dropdown. Write a triage note documenting: confirmation number (APL-AET-XXXXXXXX), CO-50, N386, CPT 67028/J2778, H35.32 (exudative AMD), medical necessity justification, denied amount ($2,450), appeal deadline (2026-06-01), Payer A portal. Then click 'Submit Disposition' to save." ], "evals": [ { "id": "denial-medium-1-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-001", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with CO-50 and N386 remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed the claim detail on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent opened the dispute form on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.openedDisputeForm", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted the appeal on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale mentions diagnosis H35.32 or exudative AMD", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale mention the diagnosis H35.32 or exudative age-related macular degeneration? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-1-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references clinical evidence like OCT or visual acuity", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale reference clinical evidence such as OCT findings, visual acuity measurements, or subretinal fluid? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-1-eval-11", "type": "jmespath", "category": "Document Handling", "description": "Agent uploaded the correct supporting document (clinical documentation) to Payer A form", "points": 1, "query": "(payer_a_state.full_state.appealActions.submittedAttachmentNames || [])[0] || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-12", "type": "jmespath", "category": "Documentation", "description": "Agent returned to EMR and documented the appeal confirmation number and details in a triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-1-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes the Payer A appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number in the format APL-AET-XXXXXXXX? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-1-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note references denial code CO-50 and remark code N386 or LCD", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference CO-50 denial code and N386 remark code (or LCD / Local Coverage Determination)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-1-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains the medical necessity basis for the anti-VEGF appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention medical necessity justification for the anti-VEGF injection (CPT 67028 or J2778) and diagnosis H35.32 (exudative AMD)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-1-eval-16", "type": "llm_judge", "category": "Documentation", "description": "Triage note documents denied amount", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the denied amount of $2,450? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-1-eval-17", "type": "llm_judge", "category": "Documentation", "description": "Triage note documents appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the appeal deadline of 2026-06-01? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-1-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "CO-50 medical necessity denial for intravitreal anti-VEGF injection (CPT 67028/J2778) with N386 LCD remark. Diagnosis H35.32 (exudative AMD) is a standard indication for anti-VEGF therapy. Appeal on Payer A portal with medical necessity justification referencing clinical documentation that satisfies LCD criteria." }, { "id": "denial-medium-2", "title": "File Partial Denial Appeal on Payer A for Knee Arthroscopy Lines", "instruction": "File an electronic appeal for denial DEN-024 (Lopez, Anna), a partial denial for knee arthroscopy on the Payer A portal. Review the denial in EMR and identify the paid lines (99214 office visit, 85025 CBC) versus the denied lines (29881 and 29880 arthroscopy procedures, CO-50 medical necessity). Check the remittance image for remark codes. Navigate to the Payer A portal, login (provider@payera.com / demo123), search by member ID AET890123456, and file an appeal specifically for the denied arthroscopy lines (29881, 29880) with medical necessity justification. Obtain the exact confirmation number shown by the portal (beginning with APL-AET-), return to EMR, and document that confirmation number in a triage note specifying which lines were appealed and which were already paid.", "goal": "File an electronic appeal for denial DEN-024 (Lopez, Anna), a partial denial for knee arthroscopy on the Payer A portal. Review the denial in EMR and identify the paid lines (99214 office visit, 85025 CBC) versus the denied lines (29881 and 29880 arthroscopy procedures, CO-50 medical necessity). Check the remittance image for remark codes. Navigate to the Payer A portal, login (provider@payera.com / demo123), search by member ID AET890123456, and file an appeal specifically for the denied arthroscopy lines (29881, 29880) with medical necessity justification. Obtain the exact confirmation number shown by the portal (beginning with APL-AET-), return to EMR, and document that confirmation number in a triage note specifying which lines were appealed and which were already paid.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 18, "configTaskId": "denial_medium_2", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Lopez, Anna" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-024 for Lopez, Anna and click to open the denial detail page", "2. [REQUIRED - DO THIS BEFORE PORTAL] Click the 'Remittance Image' tab. Read and record line-level details: identify paid lines vs denied lines. This step is evaluated.", "3. Identify paid lines: 99214 (office visit), 85025 (CBC). Identify denied lines: 29881 (medial meniscectomy), 29880 (lateral meniscectomy) - CO-50 denial, M23.41/M23.42", "4. [REQUIRED - BEFORE LEAVING EMR] Download the Operative Report: in the Documents section, find the document containing 'Operative' in its name, click it to open the viewer, then click Download. The evaluator checks for an 'Operative' document - do NOT upload a different document type.", "5. Note: appeal targets ONLY the denied arthroscopy lines (29881, 29880) - NOT the paid lines (99214, 85025)", "7. Click the 'Start Appeal' button to navigate to the Payer A portal", "8. Login with provider@payera.com / demo123", "9. Click 'Appeals' in the top navigation bar (or go to 'EOB and Claims' and find the claim there)", "10. On the Appeals > Claim Status Inquiry page, enter member ID AET890123456 in the 'Member ID' field and click 'Search'", "11. In the search results table, click on the row for claim CLM-2025-00024 (Lopez, Anna - status 'Finalized - Partially Denied') to open the Claim Status Detail view", "12. Review the denial information section (CO-50: Partial denial - arthroscopy procedures not deemed medically necessary) and the service line details showing lines 29881/29880 denied and lines 99214/85025 paid", "13. Click the 'Dispute Claim' button at the bottom of the claim detail view", "14. On the Submit Dispute form, note that the dispute type auto-sets to 'Appeal' (because CO-50 is a medical necessity denial code)", "15. Fill in Contact Name with 'Dr. Catherine Lee' (the provider listed on the denial)", "16. In the Supporting Rationale text area, write a medical necessity justification specific to the denied arthroscopy lines (CPT 29881 medial meniscectomy and 29880 lateral meniscectomy), referencing diagnoses M23.41/M23.42 (loose body in knee) and clinical evidence of failed conservative treatment or mechanical symptoms such as locking or catching that necessitate arthroscopic intervention", "17. On the payer portal, UPLOAD the clinical documentation", "18. Click 'Submit Appeal' to submit the dispute", "19. On the confirmation screen, note the exact confirmation number displayed by the portal, beginning with APL-AET- (for example, APL-AET-000001)", "20. Click 'Return to EMR' to navigate back to the EMR denials portal", "21. Select 'Appeal Filed' from the Triage Disposition dropdown. Document the appeal in a triage note including: the exact confirmation number shown by the portal (beginning with APL-AET-), which lines were appealed (29881, 29880 - denied under CO-50) vs which were already paid (99214, 85025), that the appeal targets only the denied arthroscopy lines (not the full claim), diagnoses M23.41/M23.42, the denied amount ($1,875), appeal deadline (2026-03-08), and that the appeal was filed on the Payer A portal. Then click 'Submit Disposition' to save." ], "evals": [ { "id": "denial-medium-2-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-024", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review line-level payment and denial details", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed the claim detail on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent opened the dispute form on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.openedDisputeForm", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted the appeal on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-9", "type": "jmespath", "category": "Document Handling", "description": "Agent uploaded the correct supporting document (operative report) to Payer A form", "points": 1, "query": "(payer_a_state.full_state.appealActions.submittedAttachmentNames || [])[0] || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale mentions arthroscopy CPT codes or procedure", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale mention CPT 29881 or 29880 or arthroscopic meniscectomy? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-2-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references failed conservative treatment or mechanical symptoms", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale reference failed conservative treatment or mechanical symptoms such as locking or catching? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-2-eval-12", "type": "jmespath", "category": "Documentation", "description": "Agent documented the appeal details and confirmation number in EMR", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-2-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes the Payer A appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the exact Payer A appeal confirmation number shown by the portal, beginning with APL-AET- (for example, APL-AET-000001)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-2-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note differentiates paid vs denied line items", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note distinguish between paid lines (99214 and/or 85025) and denied lines (29881 and/or 29880)? Score 1.0 if the note clearly identifies which lines were paid and which were denied, 0.0 if no such distinction is made.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-2-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Triage note specifies the appeal targets only the denied arthroscopy lines", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note specify that the appeal was filed only for the denied arthroscopy CPT codes (29881 and/or 29880), not the entire claim? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-2-eval-16", "type": "llm_judge", "category": "Documentation", "description": "Triage note references denied amount", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the denied amount of $1,875? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-2-eval-17", "type": "llm_judge", "category": "Documentation", "description": "Triage note references knee meniscus diagnoses", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference diagnosis M23.41 or M23.42 (meniscus derangement)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-2-eval-18", "type": "llm_judge", "category": "Documentation", "description": "Triage note references appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the appeal deadline of 2026-03-08? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-2-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "DEN-024 is a partial denial. Lines 99214 (office visit) and 85025 (CBC) were paid, but the knee arthroscopy procedures 29881 and 29880 were denied under CO-50 medical necessity. The appeal should target only the denied arthroscopy lines with clinical justification for the procedures based on diagnoses M23.41/M23.42 (loose body in knee)." }, { "id": "denial-medium-3", "title": "File Electronic Appeal on Payer B for Brain MRI Medical Necessity Denial", "instruction": "File an electronic appeal for denial DEN-010 (Davis, Christine) on the Payer B portal. Review the CO-50 medical necessity denial for brain MRI (CPT 70551) in EMR, check the remittance image for N386 LCD remark code. Navigate to the Payer B portal, login (provider@payerb.com / demo123), search for the claim by member ID ANT567890123, and file an appeal with medical necessity justification for the brain MRI. Obtain the confirmation number (format APL-ANT-XXXXXXXX), return to EMR, and document the confirmation number and appeal details in a triage note.", "goal": "File an electronic appeal for denial DEN-010 (Davis, Christine) on the Payer B portal. Review the CO-50 medical necessity denial for brain MRI (CPT 70551) in EMR, check the remittance image for N386 LCD remark code. Navigate to the Payer B portal, login (provider@payerb.com / demo123), search for the claim by member ID ANT567890123, and file an appeal with medical necessity justification for the brain MRI. Obtain the confirmation number (format APL-ANT-XXXXXXXX), return to EMR, and document the confirmation number and appeal details in a triage note.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 17, "configTaskId": "denial_medium_3", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Davis, Christine" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-010 for Davis, Christine and click their name to open the denial detail page", "2. [REQUIRED - DO THIS BEFORE PORTAL] Click the 'Remittance Image' tab in the denial detail view. Read and record: CO-50 denial code and N386 remark code (LCD). This step is evaluated.", "3. Note the brain MRI CPT 70551 and diagnoses G43.909 (migraine) / R51.9 (headache)", "4. [REQUIRED - BEFORE LEAVING EMR] Navigate back to the 'Retest' Tab (click 'Retest' tab, then scroll down to find the Documents section). To download a document: click the 'View ->' button on the RIGHT side of the document row (NOT the document name on the left - that does nothing). This opens the document viewer page. Then click the 'Download' button on the viewer page.", "5. Click the 'Start Appeal' button to navigate to the Payer B portal", "6. Login with provider@payerb.com / demo123", "7. Click 'Appeals' in the top navigation bar", "8. Enter member ID ANT567890123 in the search field and click the Search button", "9. In the search results, click on claim CLM-2025-00010 to view the denial details", "10. On the claim detail panel, click the 'File Appeal' button (or 'File Appeal for This Claim')", "11. Fill in Contact Name with 'Dr. Maria Rodriguez'", "12. In the Appeal Reason / Clinical Justification field, type a medical necessity justification that: (a) names CPT 70551 / brain MRI, (b) references red flag neurological symptoms (change in headache pattern, new neurological symptoms, transient numbness), (c) mentions diagnoses G43.909 (migraine) and R51.9 (headache), (d) addresses the N386 LCD remark code", "13. Scroll down to 'Available Documents from EMR' and click '+ Attach' next to the downloaded clinical document", "14. CRITICAL: Scroll down until the 'Submit Appeal' button is FULLY VISIBLE. Then click 'Submit Appeal'.", "15. Note the confirmation number (format APL-ANT-XXXXXXXX)", "16. Click 'Return to EMR' to navigate back - this returns you to the Denials Workqueue (/emr/denied), NOT the detail page", "17. You are now on the WORKLIST (/emr/denied). To re-open the DEN-010 detail page, click on the PATIENT NAME 'Davis, Christine' (the purple underlined name button in that row) - single-clicking the row body only selects it (turns green) but does NOT navigate. If single-click on the name does not navigate, double-click anywhere on the Davis, Christine row. As a last resort use navigate_to('/emr/denied/DEN-010') to go directly.", "18. Once on the DEN-010 detail page, go to the Actions panel on the right: (a) click the Triage Disposition dropdown and select 'Appeal Filed' - it is the FIRST option at the top of the list, do NOT scroll, just click it immediately, (b) click the Triage Note field and type your note immediately, (c) click 'Submit Disposition' - do NOT click any other tabs between typing and submitting.", "19. Triage note must include: confirmation number (APL-ANT-XXXXXXXX), CO-50 denial code, N386 remark code, CPT 70551 (brain MRI), diagnoses G43.909/R51.9, denied amount ($1,850), appeal deadline (2026-03-10), Payer B (Anthem)" ], "evals": [ { "id": "denial-medium-3-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-010", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with N386 LCD remark code", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed the claim detail on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent opened the dispute form on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.openedDisputeForm", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted the appeal on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-9", "type": "jmespath", "category": "Document Handling", "description": "Agent uploaded the correct supporting document (MRI/imaging documentation) to Payer B form", "points": 1, "query": "(payer_b_state.full_state.appealActions.submittedAttachmentNames || [])[0] || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale mentions brain MRI or CPT 70551", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale mention CPT 70551 or brain MRI? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-3-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references red flag neurological symptoms", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale reference red flag symptoms such as change in headache pattern, new neurological symptoms, or transient numbness? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-3-eval-12", "type": "jmespath", "category": "Documentation", "description": "Agent returned to EMR and documented the appeal confirmation and details in a triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-3-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes the Payer B (Anthem) appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number in the format APL-ANT-XXXXXXXX? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-3-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note references denial code CO-50 and N386/LCD", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference CO-50 denial code and N386 remark code (or LCD / Local Coverage Determination)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-3-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the MRI procedure and clinical diagnosis", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the brain MRI procedure (CPT 70551) and the clinical indication (G43.909 migraine or R51.9 headache)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-3-eval-16", "type": "llm_judge", "category": "Documentation", "description": "Triage note references denied amount", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the denied amount of $1,850? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-3-eval-17", "type": "llm_judge", "category": "Documentation", "description": "Triage note references appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the appeal deadline of 2026-03-10? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-3-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "CO-50 medical necessity denial for brain MRI without contrast (CPT 70551) with N386 LCD remark. Patient has diagnoses G43.909 (migraine) and R51.9 (headache). Appeal on Payer B portal with clinical justification explaining the medical necessity of the MRI for neurological evaluation." }, { "id": "denial-medium-4", "title": "File Electronic Appeal on Payer B for Multi-Procedure GI Denial", "instruction": "File an electronic appeal for denial DEN-016 (Harris, Dorothy) on the Payer B portal for multiple denied GI procedures. Review the CO-50 medical necessity denial in EMR and identify all denied CPT codes (43235 diagnostic EGD, 43239 EGD with biopsy, J1100 dexamethasone injection) and diagnoses (K21.0 GERD, K44.9 diaphragmatic hernia, K57.30 diverticulosis). Check the remittance image for remark codes. Navigate to the Payer B portal, login (provider@payerb.com / demo123), search by member ID ANT012345678, and file an appeal referencing all procedure codes and diagnoses. Obtain the confirmation number (APL-ANT-XXXXXXXX format), return to EMR, and document the confirmation number and all appealed codes in a triage note.", "goal": "File an electronic appeal for denial DEN-016 (Harris, Dorothy) on the Payer B portal for multiple denied GI procedures. Review the CO-50 medical necessity denial in EMR and identify all denied CPT codes (43235 diagnostic EGD, 43239 EGD with biopsy, J1100 dexamethasone injection) and diagnoses (K21.0 GERD, K44.9 diaphragmatic hernia, K57.30 diverticulosis). Check the remittance image for remark codes. Navigate to the Payer B portal, login (provider@payerb.com / demo123), search by member ID ANT012345678, and file an appeal referencing all procedure codes and diagnoses. Obtain the confirmation number (APL-ANT-XXXXXXXX format), return to EMR, and document the confirmation number and all appealed codes in a triage note.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 17, "configTaskId": "denial_medium_4", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Harris, Dorothy" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-016 for Harris, Dorothy and click to open the denial detail page", "2. [REQUIRED - DO THIS BEFORE PORTAL] Click the 'Remittance Image' tab. Read all line items, denial codes, and remark codes. This step is evaluated.", "3. Identify all denied CPTs: 43235 (diagnostic EGD), 43239 (EGD with biopsy), J1100 (dexamethasone injection)", "4. Note all diagnoses: K21.0 (GERD with esophagitis), K44.9 (diaphragmatic hernia), K57.30 (diverticulosis)", "5. [REQUIRED - BEFORE LEAVING EMR] Download the clinical documentation (Procedure Notes): click the 'View >' button on the RIGHT side of the document row (NOT the document name - that does nothing). This opens the document viewer. Then click Download. This is required to attach to the appeal.", "6. Click the 'Start Appeal' button to navigate to the Payer B portal", "7. Click the 'Start Appeal' button to navigate to the Payer B portal", "8. Login with provider@payerb.com / demo123", "9. Click 'Appeals' in the top navigation bar", "10. Enter member ID ANT012345678 in the search field and click the Search button", "11. In the search results, click on claim CLM-2025-00016 to view the denial details", "12. On the claim detail panel, click the 'File Appeal' button (or 'File Appeal for This Claim')", "13. Fill in Contact Name with 'Dr. George Martin' (the provider listed on the denial)", "14. In the 'Appeal Reason / Clinical Justification' text area, enter a medical necessity justification referencing CPT 43235 (diagnostic EGD), 43239 (EGD with biopsy), and J1100 (dexamethasone injection), explaining that the combined GI procedure session was medically necessary given alarm symptoms such as dysphagia, weight loss, or refractory GERD, and the patient's diagnoses of GERD with esophagitis (K21.0), diaphragmatic hernia (K44.9), and diverticulosis (K57.30), warranting diagnostic EGD, biopsy collection, and anti-inflammatory injection during a single endoscopic session", "15. On the payer portal, click '+ Attach Document' to upload the clinical documentation", "16. Click the 'Submit Appeal' button", "17. Note the confirmation number displayed (format APL-ANT-XXXXXXXX)", "18. Click 'Return to EMR' to navigate back to the EMR portal", "19. Select 'Appeal Filed' from the Triage Disposition dropdown. Document the following in a triage note: the confirmation number (APL-ANT-XXXXXXXX format), all appealed CPT codes (43235, 43239, J1100), diagnoses (K21.0, K44.9, K57.30), denied amount ($8,750), appeal deadline (2026-03-12), that this was a multi-procedure appeal, and that the appeal was filed on the Payer B (Anthem) portal. Then click 'Submit Disposition' to save." ], "evals": [ { "id": "denial-medium-4-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-016", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review all denied line items and remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed the claim detail on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent opened the dispute form on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.openedDisputeForm", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted the appeal on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-9", "type": "jmespath", "category": "Document Handling", "description": "Agent uploaded the correct supporting document to Payer B form", "points": 1, "query": "contains(join(' ', payer_b_state.full_state.appealActions.submittedAttachmentNames || `[]`), 'Procedure Notes')", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale mentions EGD procedure or CPT codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale mention CPT 43235 or 43239 or EGD (esophagogastroduodenoscopy)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-4-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references alarm symptoms or refractory GERD", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale reference alarm symptoms (explicitly or by examples such as dysphagia/weight loss) and/or refractory GERD? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-4-eval-12", "type": "jmespath", "category": "Documentation", "description": "Agent returned to EMR and documented the appeal with all procedure codes in a triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-4-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes the Payer B (Anthem) appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number in the format APL-ANT-XXXXXXXX? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-4-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the denied GI procedure codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference at least two of the three denied CPT codes: 43235, 43239, J1100? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-4-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the clinical diagnoses supporting the GI procedures", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference at least two of the three diagnoses: K21.0 (GERD), K44.9 (diaphragmatic hernia), or K57.30 (diverticulosis)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-4-eval-16", "type": "llm_judge", "category": "Documentation", "description": "Triage note references denied amount", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the total denied amount of $8,750? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-4-eval-17", "type": "llm_judge", "category": "Documentation", "description": "Triage note references appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the appeal deadline of 2026-03-12? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-4-eval-18", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "CO-50 medical necessity denial for multiple GI procedures: diagnostic EGD (43235), EGD with biopsy (43239), and dexamethasone injection (J1100). Three supporting diagnoses (K21.0 GERD with esophagitis, K44.9 diaphragmatic hernia, K57.30 diverticulosis) provide clinical justification. Appeal on Payer B portal referencing all CPT codes and diagnoses to demonstrate medical necessity of the combined GI procedure session." }, { "id": "denial-medium-5", "title": "File Electronic Appeal on Payer B for Bundling/NCCI Edit Denial with Modifier Justification", "instruction": "File an electronic appeal for denial DEN-022 (King, Michelle) on the Payer B portal with modifier justification for a bundling/NCCI edit denial. Review the CO-97 bundling denial in EMR for skin biopsy CPTs 11102 and 11103 (NCCI edit pair). Check the remittance image for the bundling remark codes. Navigate to the Payer B portal, login (provider@payerb.com / demo123), search by member ID ANT556677889, and file an appeal explaining that modifier 59 (or XS) is justified because the biopsies were performed on distinct/separate lesions at different anatomic sites. Obtain the confirmation number (APL-ANT-XXXXXXXX format), return to EMR, and document the confirmation number and modifier justification in a triage note.", "goal": "File an electronic appeal for denial DEN-022 (King, Michelle) on the Payer B portal with modifier justification for a bundling/NCCI edit denial. Review the CO-97 bundling denial in EMR for skin biopsy CPTs 11102 and 11103 (NCCI edit pair). Check the remittance image for the bundling remark codes. Navigate to the Payer B portal, login (provider@payerb.com / demo123), search by member ID ANT556677889, and file an appeal explaining that modifier 59 (or XS) is justified because the biopsies were performed on distinct/separate lesions at different anatomic sites. Obtain the confirmation number (APL-ANT-XXXXXXXX format), return to EMR, and document the confirmation number and modifier justification in a triage note.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 18, "configTaskId": "denial_medium_5", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "King, Michelle" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-022 for King, Michelle and click to open the denial detail page", "2. [REQUIRED - DO THIS BEFORE PORTAL] Click the 'Remittance Image' tab in the denial detail view. Read and record: CO-97 denial code and NCCI bundling remark codes. This step is evaluated.", "3. Identify the NCCI edit pair: CPT 11102 (tangential biopsy, single lesion) and 11103 (each additional lesion)", "4. Note diagnosis D23.9 (benign neoplasm of skin)", "5. DOWNLOAD the clinical documentation (Pathology Report) from the Documents section: click the 'View >' button on the RIGHT side of the document row (NOT the document name - that does nothing), then click Download on the viewer page", "6. Note the diagnosis D23.9 (benign neoplasm of skin)", "7. Click the 'Start Appeal' button to navigate to the Payer B portal", "8. Login with provider@payerb.com / demo123", "9. Click 'Appeals' in the top navigation bar", "10. Enter member ID ANT556677889 in the search field and click the Search button", "11. In the search results, click on claim CLM-2025-00022 to view the denial details", "12. On the claim detail panel, click the 'File Appeal' button (or 'File Appeal for This Claim')", "13. Fill in Contact Name with 'Dr. Laura White' (the provider listed on the denial)", "14. In the 'Appeal Reason / Clinical Justification' text area, enter a justification explaining that CPT 11102 and 11103 were performed on distinct/separate lesions at different anatomic sites, warranting modifier 59 (Distinct Procedural Service) or XS (Separate Structure) to unbundle the NCCI edit pair. The biopsies targeted separate benign skin neoplasms (D23.9) requiring independent pathological evaluation", "15. On the payer portal, click '+ Attach Document' to upload the clinical documentation", "16. Click the 'Submit Appeal' button", "17. Note the confirmation number displayed (format APL-ANT-XXXXXXXX)", "18. Click 'Return to EMR' to navigate back to the EMR portal", "19. Select 'Appeal Filed' from the Triage Disposition dropdown. Document the following in a triage note: the confirmation number (APL-ANT-XXXXXXXX format), CO-97 bundling denial code, NCCI edit pair (CPTs 11102/11103), modifier 59/XS justification (distinct procedural service at separate anatomic sites), diagnosis D23.9 (benign neoplasm of skin), denied amount ($650), appeal deadline (2026-05-20), and that the appeal was filed on the Payer B (Anthem) portal. Then click 'Submit Disposition' to save." ], "evals": [ { "id": "denial-medium-5-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-022", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review the CO-97 bundling remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed the claim detail on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent opened the dispute form on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.openedDisputeForm", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted the appeal on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-9", "type": "jmespath", "category": "Document Handling", "description": "Agent uploaded the correct supporting document to Payer B form", "points": 1, "query": "(payer_b_state.full_state.appealActions.submittedAttachmentNames || [])[0] || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale mentions modifier 59 or XS for unbundling", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale mention modifier 59 or XS (distinct procedural service)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-5-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references distinct anatomic sites or separate specimens", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale reference distinct anatomic sites or separate specimens for the two biopsies? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-5-eval-12", "type": "jmespath", "category": "Documentation", "description": "Agent returned to EMR and documented the appeal with modifier justification in a triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-5-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes the Payer B (Anthem) appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number in the format APL-ANT-XXXXXXXX? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-5-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies CO-97 and the NCCI code pair", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the CO-97 bundling denial code and identify the NCCI edit pair (CPT 11102 and 11103)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-5-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains the modifier 59/XS justification for separate procedures", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention modifier 59 or XS as justification, or explain that the procedures were performed on distinct/separate lesions or anatomic sites? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-5-eval-16", "type": "llm_judge", "category": "Documentation", "description": "Triage note references denied amount", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the denied amount of $650? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-5-eval-17", "type": "llm_judge", "category": "Documentation", "description": "Triage note references appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the appeal deadline of 2026-05-20? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-5-eval-18", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis D23.9", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference diagnosis D23.9 (benign neoplasm of skin)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-5-eval-19", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "CO-97 bundling denial for skin biopsies CPT 11102 (tangential biopsy, single lesion) and 11103 (each additional lesion). These are an NCCI edit pair where 11103 is typically bundled into 11102. However, if the biopsies were performed on distinct/separate lesions at different anatomic sites, modifier 59 (Distinct Procedural Service) or XS (Separate Structure) justifies separate payment. Appeal on Payer B portal with modifier justification and documentation of separate lesion sites." }, { "id": "denial-medium-6", "title": "File Appeal for Expired Auth -- Procedure Scheduled During Active Auth Period", "instruction": "For denial DEN-006 (Lee, David), review the CO-197 no-authorization denial from Anthem Blue Cross in EMR. Check the patient inquiry page to review the authorization history. Click the Remittance Image tab to review the EOB. Navigate to the Payer B portal (provider@payerb.com / demo123) and search for the claim on the Appeals page. File an appeal arguing that the authorization was active when the procedure was scheduled and that the service should be honored despite the auth lapsing before the service date. Return to EMR and document your investigation and appeal in a triage note. Appeal deadline is 2026-03-10 -- 13 days away.", "goal": "For denial DEN-006 (Lee, David), review the CO-197 no-authorization denial from Anthem Blue Cross in EMR. Check the patient inquiry page to review the authorization history. Click the Remittance Image tab to review the EOB. Navigate to the Payer B portal (provider@payerb.com / demo123) and search for the claim on the Appeals page. File an appeal arguing that the authorization was active when the procedure was scheduled and that the service should be honored despite the auth lapsing before the service date. Return to EMR and document your investigation and appeal in a triage note. Appeal deadline is 2026-03-10 -- 13 days away.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 11, "configTaskId": "denial_medium_6", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Lee, David" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-006 for Lee, David and click to open the denial detail page", "2. Review denial details: CO-197 no-auth denial, CPT 64483 (epidural injection), diagnosis M54.5, $5,670 amount", "3. Note the appeal deadline is 2026-03-10 (13 days away)", "4. [REQUIRED - DO THIS BEFORE PORTAL] Click the 'Remittance Image' tab in the denial detail view. Read and record the EOB and remark codes. This step is evaluated.", "5. [REQUIRED] Click the patient name link to navigate to Patient Inquiry page. Find auth AUTH-2025-88431 - note it expired 2025-10-15 but service was on 2025-10-25 (auth lapsed 10 days before service date). Auth WAS obtained - it just lapsed before the service.", "6. Click the 'Start Appeal' button to navigate to the Payer B portal", "7. Log in with provider@payerb.com / demo123", "8. Click 'Appeals' in the navigation bar", "9. Enter member ID for Lee, David in the search field and click Search", "10. Click the claim for Lee, David to view claim details", "11. Click 'Dispute Claim'", "12. Fill in Contact Name with 'Dr. Sarah Kim'", "13. In Supporting Rationale, explain: auth AUTH-2025-88431 was active when procedure was scheduled, auth lapsed due to scheduling delays not failure to obtain auth, service should be honored", "14. CRITICAL: Scroll down until the Submit button is FULLY VISIBLE. Click 'Submit Appeal' and note the confirmation number", "15. Return to EMR and navigate back to DEN-006", "16. Select 'Appeal Filed' from the Triage Disposition dropdown. Write a triage note documenting: expired auth AUTH-2025-88431 (expired 2025-10-15 vs service 2025-10-25), CPT 64483 / M54.5, appeal rationale (auth active at scheduling), confirmation number, deadline urgency", "17. Click 'Submit Disposition' to save" ], "evals": [ { "id": "denial-medium-6-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-006", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-6-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-6-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked the patient inquiry page to find the expired authorization AUTH-2025-88431", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-6-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-6-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted an appeal on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-6-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent documented the appeal in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-6-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references expired auth details and date mismatch", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the expired authorization AUTH-2025-88431 and that it expired on 2025-10-15 while the service was on 2025-10-25 (10 days after expiration)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-6-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes clinical details", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference CPT 64483 (epidural injection) and diagnosis M54.5 (low back pain)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-6-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note captures deadline urgency and appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note note the deadline -- that the appeal deadline is 2026-03-10 (13 days away) -- and include an appeal confirmation number from the Payer B submission? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-6-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains appeal rationale about scheduling vs service date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the appeal rationale -- that the authorization was active when the procedure was scheduled and should be honored despite lapsing before the service date? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-6-eval-11", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "The denial is due to CO-197 (missing authorization). Investigation reveals authorization AUTH-2025-88431 existed but expired on 2025-10-15, while the service was rendered on 2025-10-25. The procedure was scheduled while the auth was still active, meaning the lapse was due to scheduling delays, not a failure to obtain auth. The denials team should file an appeal on the Payer B portal arguing the auth was valid at time of scheduling. The appeal deadline is 2026-03-10, which is 13 days away from the benchmark date." }, { "id": "denial-medium-7", "title": "File Emergency Exception Appeal for Knee Surgery No-Auth Denial", "instruction": "For denial DEN-025 (Hall, Gregory), review the CO-197 no-authorization denial from Anthem Blue Cross in EMR. Note that the existing notes indicate 'Emergency service - retroactive auth may be warranted.' Click the Remittance Image tab to review the EOB. Navigate to the Payer B portal (provider@payerb.com / demo123) and search for the claim on the Appeals page. File an appeal arguing that this was an emergency service and the prudent layperson standard should apply -- prior authorization cannot reasonably be obtained for emergency procedures. Return to EMR and document your investigation and appeal in a triage note.", "goal": "For denial DEN-025 (Hall, Gregory), review the CO-197 no-authorization denial from Anthem Blue Cross in EMR. Note that the existing notes indicate 'Emergency service - retroactive auth may be warranted.' Click the Remittance Image tab to review the EOB. Navigate to the Payer B portal (provider@payerb.com / demo123) and search for the claim on the Appeals page. File an appeal arguing that this was an emergency service and the prudent layperson standard should apply -- prior authorization cannot reasonably be obtained for emergency procedures. Return to EMR and document your investigation and appeal in a triage note.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 10, "configTaskId": "denial_medium_7", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Hall, Gregory" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-025 for Hall, Gregory - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. Review denial details -- note CO-197 no-auth denial, CPTs 29881/29880 (knee arthroscopy), diagnosis S83.512A, $6,800 amount", "4. Note the existing note: 'Emergency service - retroactive auth may be warranted'", "5. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB, remark codes and the patient'smember ID This step is evaluated.", "6. Click the 'Start Appeal' button to navigate to the Payer B portal", "7. Log in with provider@payerb.com / demo123", "8. Click 'Appeals' in the navigation bar", "9. Enter member ID in the search field and click Search", "10. Select the claim for Hall, Gregory and review claim details", "11. Click 'Dispute Claim'", "12. Fill in Contact Name with 'Dr. Brian Martinez' (the provider listed on the denial)", "13. In Supporting Rationale, explain: emergency knee surgery for ACL sprain, prudent layperson standard applies, prior auth cannot reasonably be obtained for emergency procedures, CO-197 denial should be overturned", "14. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal' and note the confirmation number", "15. Return to EMR", "16. Navigate back to the DEN-025 denial detail page", "17. Select 'Appeal Filed' from the Triage Disposition dropdown. Write a triage note documenting: CO-197 denial for emergency knee arthroscopy (CPTs 29881/29880, diagnosis S83.512A), emergency exception / prudent layperson standard rationale, appeal filed on Payer B, and confirmation number", "18. Click 'Submit Disposition' to save" ], "evals": [ { "id": "denial-medium-7-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-025", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-7-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-7-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-7-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted an appeal on the Payer B portal", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-7-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent documented the appeal in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-7-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies emergency service and denial code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that this was an emergency service (knee arthroscopy for ACL sprain) and reference the CO-197 denial code? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-7-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes correct procedure and diagnosis codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference CPTs 29881/29880 (knee arthroscopy/meniscectomy) and diagnosis S83.512A (ACL sprain of left knee)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-7-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains emergency exception / prudent layperson standard", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the appeal rationale using the prudent layperson standard or emergency exception -- that prior authorization cannot reasonably be obtained for emergency procedures? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-7-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include an appeal confirmation number from the Payer B submission? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-7-eval-10", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "The denial is due to CO-197 (no authorization) for an emergency knee arthroscopy. The existing notes confirm this was an emergency service. Under the prudent layperson standard, emergency services should not require prior authorization because obtaining auth is not feasible in an emergency. The denials team should file an appeal on Payer B arguing the emergency exception applies." }, { "id": "denial-medium-8", "title": "Fax Appeal to Valley Health Plan for Total Knee Arthroplasty Denial", "instruction": "For denial DEN-012 (Wilson, Linda), review the CO-50 medical necessity denial from Valley Health Plan in EMR. Download the clinical indication documents from the Documents section to support the appeal. Note that Valley Health Plan requires appeals to be submitted via fax. Navigate to the fax portal, look up the Valley Health Plan fax number in the phonebook, compose a fax with appeal documentation for the total knee arthroplasty (CPT 27447, diagnosis M17.0), attach the downloaded clinical documents to the fax, send the fax, obtain the fax confirmation number, return to EMR, and document the fax confirmation in a triage note.", "goal": "For denial DEN-012 (Wilson, Linda), review the CO-50 medical necessity denial from Valley Health Plan in EMR. Download the clinical indication documents from the Documents section to support the appeal. Note that Valley Health Plan requires appeals to be submitted via fax. Navigate to the fax portal, look up the Valley Health Plan fax number in the phonebook, compose a fax with appeal documentation for the total knee arthroplasty (CPT 27447, diagnosis M17.0), attach the downloaded clinical documents to the fax, send the fax, obtain the fax confirmation number, return to EMR, and document the fax confirmation in a triage note.", "difficulty": "medium", "category": "fax_appeal", "challengeType": "denial_triage", "possible": true, "points": 15, "configTaskId": "denial_medium_8", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Wilson, Linda" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-012 for Wilson, Linda - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. In the Documents section, download the clinical indication document (Operative Report - TKA 27447.pdf)", "4. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB and remark codes This step is evaluated.", "5. Review CO-50 denial code -- services not medically necessary", "6. Note that VHP requires appeals to be submitted via fax per their guidelines", "7. Click 'Start Appeal' -- this opens the DME Fax Portal directly", "8. On the Fax Portal, click the 'Phonebook' button in the toolbar", "9. Find 'Valley Health Plan' in the phonebook table and click 'Select' -- this closes the phonebook and opens the New Fax dialog with Name='Valley Health Plan' and Fax Number='1-800-555-0198' pre-filled", "10. In the New Fax dialog, verify the Main tab shows Name='Valley Health Plan' and Fax Number='1-800-555-0198'", "11. Click the 'Attachments' tab in the New Fax dialog", "12. Click the file upload input and upload the previously downloaded Operative Report - TKA 27447.pdf", "13. Verify the file appears in the attachments list", "14. Click 'Send' (button is enabled once Name + Fax Number + at least 1 attachment are filled)", "15. Note the fax confirmation number (FAX-...) shown in the status bar", "16. Click 'Return to EMR' -- the denial detail page shows a green confirmation banner with the fax confirmation number", "17. Select disposition 'Appeal Filed' from the Triage Disposition dropdown -- it is the FIRST option at the top of the list", "18. Write a triage note documenting: fax confirmation number, VHP fax submission, CO-50 denial, CPT 27447, diagnosis M17.0, appeal deadline 2026-03-08, denied amount $3,500", "19. Click 'Submit Disposition' to save" ], "evals": [ { "id": "denial-medium-8-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-012", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Agent sent a fax via the fax portal with appeal documentation", "points": 1, "query": "full_state.agentActions.sentFax || full_state.faxPortal.faxesSent", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent looked up the Valley Health Plan fax number in the phonebook", "points": 1, "query": "full_state.faxPortal.lookedUpFaxNumber", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-6", "type": "jmespath", "category": "Form Completion", "description": "Agent entered Valley Health Plan as the fax recipient name", "points": 1, "query": "full_state.faxPortal.faxRecipient || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-7", "type": "jmespath", "category": "Form Completion", "description": "Agent entered Valley Health Plan fax number (1-800-555-0198)", "points": 1, "query": "full_state.faxPortal.faxNumber || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-8", "type": "jmespath", "category": "Document Handling", "description": "Agent attached Operative Report clinical document to the fax", "points": 1, "query": "join(',', full_state.faxPortal.attachmentNames || [])", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-9", "type": "jmespath", "category": "Documentation", "description": "Agent documented the fax confirmation in EMR", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-8-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies VHP fax submission and denial code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that the appeal was submitted to Valley Health Plan via fax and reference the CO-50 medical necessity denial? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-8-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 27447", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference CPT 27447 (total knee arthroplasty)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-8-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis M17.0", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference diagnosis M17.0 (bilateral primary osteoarthritis)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-8-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes fax confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include a fax confirmation number? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-8-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note captures deadline and denied amount", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the appeal deadline of 2026-03-08 and the denied amount of $3,500? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-8-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "Valley Health Plan requires appeals to be submitted via fax per their appeal guidelines. The agent must navigate to the fax portal, look up VHP's fax number in the phonebook, compose the appeal with documentation for the TKA denial (CPT 27447, diagnosis M17.0), send it, and document the fax confirmation number in EMR." }, { "id": "denial-medium-9", "title": "Fax DME Appeal to Valley Health Plan for Oxygen Concentrator Denial", "instruction": "For denial DEN-023 (Wright, Helen), review the CO-50 DME medical necessity denial from Valley Health Plan in EMR. Download the clinical indication documents from the Documents section to support the appeal. Note that Valley Health Plan requires appeals to be submitted via fax. Navigate to the fax portal, look up the Valley Health Plan fax number in the phonebook, compose a fax with DME appeal documentation for the oxygen concentrator (CPT E1390, diagnoses G20/R26.81), attach the downloaded clinical documents to the fax, send the fax, obtain the fax confirmation number, return to EMR, and document the fax confirmation in a triage note.", "goal": "For denial DEN-023 (Wright, Helen), review the CO-50 DME medical necessity denial from Valley Health Plan in EMR. Download the clinical indication documents from the Documents section to support the appeal. Note that Valley Health Plan requires appeals to be submitted via fax. Navigate to the fax portal, look up the Valley Health Plan fax number in the phonebook, compose a fax with DME appeal documentation for the oxygen concentrator (CPT E1390, diagnoses G20/R26.81), attach the downloaded clinical documents to the fax, send the fax, obtain the fax confirmation number, return to EMR, and document the fax confirmation in a triage note.", "difficulty": "medium", "category": "fax_appeal", "challengeType": "denial_triage", "possible": true, "points": 15, "configTaskId": "denial_medium_9", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Wright, Helen" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-023 for Wright, Helen - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. In the Documents section, download the clinical indication document (DME Clinical Justification - Oxygen E1390.pdf)", "4. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB and remark codes This step is evaluated.", "5. Review CO-50 denial code -- DME not medically necessary", "6. Note that VHP requires appeals to be submitted via fax per their guidelines", "7. Click 'Start Appeal' -- this opens the DME Fax Portal directly", "8. On the Fax Portal, click the 'Phonebook' button in the toolbar", "9. Find 'Valley Health Plan' in the phonebook table and click 'Select' -- this closes the phonebook and opens the New Fax dialog with Name='Valley Health Plan' and Fax Number='1-800-555-0198' pre-filled", "10. In the New Fax dialog, verify the Main tab shows Name='Valley Health Plan' and Fax Number='1-800-555-0198'", "11. Click the 'Attachments' tab in the New Fax dialog", "12. Click the file upload input and upload the previously downloaded DME Clinical Justification - Oxygen E1390.pdf", "13. Verify the file appears in the attachments list", "14. Click 'Send' (button is enabled once Name + Fax Number + at least 1 attachment are filled)", "15. Note the fax confirmation number (FAX-...) shown in the status bar", "16. Click 'Return to EMR' -- the denial detail page shows a green confirmation banner with the fax confirmation number", "17. Select disposition 'Appeal Filed' from the Triage Disposition dropdown -- it is the FIRST option at the top of the list", "18. Write a triage note documenting: fax confirmation number, VHP fax submission, CO-50 DME denial, CPT E1390, diagnoses G20/R26.81, appeal deadline 2026-03-18, denied amount $4,200", "19. Click 'Submit Disposition' to save" ], "evals": [ { "id": "denial-medium-9-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-023", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent looked up the Valley Health Plan fax number in the phonebook", "points": 1, "query": "full_state.faxPortal.lookedUpFaxNumber", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-5", "type": "jmespath", "category": "Form Completion", "description": "Agent entered Valley Health Plan as the fax recipient name", "points": 1, "query": "full_state.faxPortal.faxRecipient || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-6", "type": "jmespath", "category": "Form Completion", "description": "Agent entered Valley Health Plan fax number (1-800-555-0198)", "points": 1, "query": "full_state.faxPortal.faxNumber || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-7", "type": "jmespath", "category": "Document Handling", "description": "Agent sent a fax via the fax portal with DME appeal documentation", "points": 1, "query": "full_state.agentActions.sentFax || full_state.faxPortal.faxesSent", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-8", "type": "jmespath", "category": "Document Handling", "description": "Agent attached clinical indication document to the fax", "points": 1, "query": "join(',', full_state.faxPortal.attachmentNames || [])", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-9", "type": "jmespath", "category": "Documentation", "description": "Agent documented the fax confirmation in EMR", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-9-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies VHP fax submission and denial code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that the appeal was submitted to Valley Health Plan via fax and reference the CO-50 denial for DME equipment? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-9-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes DME equipment code and supporting diagnoses", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference CPT E1390 (oxygen concentrator) and diagnoses G20 (Parkinson's disease) or R26.81 (unsteadiness on feet)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-9-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes fax confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include a fax confirmation number? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-9-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note references denied amount", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the denied amount of $4,200? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-9-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note references appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the appeal deadline of 2026-03-18? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-9-eval-15", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "Valley Health Plan requires appeals to be submitted via fax per their appeal guidelines. This is a DME denial for an oxygen concentrator (E1390) prescribed for a patient with Parkinson's disease (G20) and unsteadiness on feet (R26.81). The agent must navigate to the fax portal, look up VHP's fax number, compose the DME appeal with supporting documentation, send it, and document the fax confirmation in EMR." }, { "id": "denial-medium-10", "title": "File High-Value Cardiac Admission Appeal on Payer A Portal", "instruction": "For denial DEN-014 (Moore, Elizabeth), review the CO-50 medical necessity denial from Aetna for a high-value cardiac admission ($45,000) in EMR. Check the remittance image to confirm CO-50 and remark code N386. Navigate to the Payer A portal, search for the denied claim, and file an appeal with detailed justification for the cardiac admission including CPTs 99223/99232/99238 (hospital care) and diagnoses I21.09/I25.10 (STEMI and coronary artery disease). Obtain a confirmation number, return to EMR, and document the appeal submission with the dollar amount noted in a triage note.", "goal": "For denial DEN-014 (Moore, Elizabeth), review the CO-50 medical necessity denial from Aetna for a high-value cardiac admission ($45,000) in EMR. Check the remittance image to confirm CO-50 and remark code N386. Navigate to the Payer A portal, search for the denied claim, and file an appeal with detailed justification for the cardiac admission including CPTs 99223/99232/99238 (hospital care) and diagnoses I21.09/I25.10 (STEMI and coronary artery disease). Obtain a confirmation number, return to EMR, and document the appeal submission with the dollar amount noted in a triage note.", "difficulty": "medium", "category": "electronic_appeals", "challengeType": "denial_triage", "possible": true, "points": 16, "configTaskId": "denial_medium_10", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Moore, Elizabeth" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-014 for Moore, Elizabeth - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to verify CO-50 and N386 remark codes This step is evaluated.", "4. Note the high-value denied amount of $45,000 for cardiac admission", "5. Review CPTs 99223/99232/99238 and diagnoses I21.09/I25.10", "6. Download the clinical documentation from the Documents section on the EMR denial detail page", "7. Click the 'Start Appeal' button to navigate to the Payer A portal", "8. Log in with username provider@payera.com and password demo123", "9. Click 'Appeals' in the top navigation bar (or go to 'EOB and Claims' and find the claim there)", "10. Enter member ID AET901234567 in the Member ID field and click Search", "11. In the search results, click on claim CLM-2025-00014 to open the claim status detail page", "12. Review the denial information (CO-50 medical necessity denial for cardiac admission)", "13. Click the 'Dispute Claim' button to begin the appeal process", "14. Note the dispute type auto-sets to 'Appeal' for CO-50 denial code", "15. Fill in Contact Name with 'Dr. Patricia Young' (the provider listed on the denial)", "16. Write supporting rationale for cardiac admission medical necessity, referencing STEMI / diagnosis I21.09 and I25.10 (coronary artery disease), CPTs 99223 (initial hospital care), 99232 (subsequent hospital care), 99238 (hospital discharge), and clinical evidence of emergent cardiac catheterization/PCI or elevated troponin levels that necessitated inpatient admission", "17. On the payer portal, click '+ Attach Document' to upload the clinical documentation", "18. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal' and note the confirmation number (format apl-aet-xxxxxxxx)", "19. Click 'Return to EMR' to navigate back to the EMR portal", "20. Select 'Appeal Filed' from the Triage Disposition dropdown. Document the following in a triage note: the appeal confirmation number (APL-AET-XXXXXXXX format), CO-50 denial code, N386 remark code, cardiac admission CPT codes (99223, 99232, 99238), diagnoses I21.09 (STEMI) and I25.10 (coronary artery disease), denied amount ($45,000), appeal deadline (2026-06-05), and that the appeal was filed on the Payer A portal. Then click 'Submit Disposition' to save." ], "evals": [ { "id": "denial-medium-10-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-014", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to verify CO-50 and N386 remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation from EMR to support the appeal", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-6", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed the claim detail on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent opened the dispute form on the Payer A portal", "points": 1, "query": "payer_a_state.full_state.appealActions.openedDisputeForm", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-8", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted the appeal on Payer A portal with cardiac admission justification", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-9", "type": "jmespath", "category": "Document Handling", "description": "Agent uploaded the correct supporting document to Payer A form", "points": 1, "query": "(payer_a_state.full_state.appealActions.submittedAttachmentNames || [])[0] || ''", "expectedValue": null, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale mentions STEMI or diagnosis I21.09", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale mention STEMI or ST elevation myocardial infarction or diagnosis I21.09? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-10-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references emergent cardiac intervention or troponin", "points": 1, "query": null, "expectedValue": null, "rubric": "Does the appeal rationale reference emergent cardiac catheterization, PCI, or elevated troponin levels? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-medium-10-eval-12", "type": "jmespath", "category": "Documentation", "description": "Agent documented the appeal submission and confirmation number in EMR", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-10-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Triage note references high-value amount and denial/remark codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the high dollar amount ($45,000) and the CO-50/N386 denial codes from the remittance? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-10-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Triage note references cardiac admission CPT codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference any of the cardiac admission CPT codes 99223 (initial hospital care), 99232 (subsequent hospital care), or 99238 (hospital discharge)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-10-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Triage note references cardiac diagnosis codes", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference diagnosis I21.09 (STEMI) or I25.10 (coronary artery disease)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-10-eval-16", "type": "llm_judge", "category": "Documentation", "description": "Triage note captures Payer A confirmation number and deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include a confirmation number from the Payer A appeal submission and mention the appeal deadline of 2026-06-05? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-10-eval-17", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Appeal Filed", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "This is a high-value denial ($45,000) for a cardiac admission that was deemed not medically necessary by Aetna. The remittance shows CO-50 with remark code N386. The admission involved hospital care codes (99223/99232/99238) for a patient with STEMI (I21.09) and coronary artery disease (I25.10). The clinical presentation strongly supports medical necessity for inpatient admission. The agent should file the appeal on the Payer A portal with detailed cardiac justification and document the confirmation number in EMR." }, { "id": "denial-medium-11", "title": "Verify OON Eligibility via Payer Portal and Transfer to Patient", "instruction": "For denial DEN-009 (Nguyen, Thi), verify eligibility and confirm out-of-network status. Review PR-242 denial in EMR, check remittance image for N522 remark confirming OON, navigate to Payer A portal (provider@payera.com / demo123), use eligibility check (member ID AET456789012, DOB 1958-06-14) to verify plan is HMO with no OON benefits, return to EMR, select 'Transfer to Patient' disposition, and document findings including OON confirmation from payer portal.", "goal": "For denial DEN-009 (Nguyen, Thi), verify eligibility and confirm out-of-network status. Review PR-242 denial in EMR, check remittance image for N522 remark confirming OON, navigate to Payer A portal (provider@payera.com / demo123), use eligibility check (member ID AET456789012, DOB 1958-06-14) to verify plan is HMO with no OON benefits, return to EMR, select 'Transfer to Patient' disposition, and document findings including OON confirmation from payer portal.", "difficulty": "medium", "category": "eligibility_verification", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_medium_11", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Nguyen, Thi" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-009 for Nguyen, Thi - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and identify N522 remark code confirming OON This step is evaluated.", "4. Click the 'Start Appeal' button to navigate to the Payer A portal", "5. Log in with provider@payera.com / demo123", "6. Use eligibility check with member ID AET456789012 and DOB 1958-06-14", "7. Confirm plan is HMO with no out-of-network benefits", "8. Return to EMR denial detail page", "9. Before transferring to patient, note that a patient financial responsibility form (PAFR/ABN) must be confirmed on file - if not, the balance cannot be transferred without patient consent", "10. Select 'Transfer to Patient' from the Triage Disposition dropdown and submit", "11. Add a triage note documenting PR-242 denial, N522 OON confirmation, HMO plan with no OON benefits, and explicitly state that the $2,100 charge should be transferred to the patient because the orthopedic services were rendered by an out-of-network provider under an HMO plan. Also document that the patient financial responsibility form must be verified before billing the patient." ], "evals": [ { "id": "denial-medium-11-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-009", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-11-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with N522 remark code confirming OON", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-11-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-11-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Transfer to Patient", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Transfer to Patient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-11-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their findings and OON confirmation", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-11-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions PR-242 or N522 OON remark code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code PR-242 or remark code N522 confirming out-of-network status? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-11-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note confirms HMO plan has no OON benefits from eligibility check", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that the patient's plan is HMO with no out-of-network benefits, based on the payer portal eligibility check? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-11-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains rationale for patient transfer disposition", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the $2,100 charge should be transferred to the patient because the orthopedic services were rendered by an out-of-network provider under an HMO plan? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-11-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions verifying financial responsibility form before billing patient", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that a patient financial responsibility form (PAFR, ABN, or financial consent) should be verified or confirmed on file before transferring the balance to the patient? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Transfer to Patient" }, "rationale": "PR-242 denotes out-of-network services. The remittance image confirms OON via N522 remark code. Eligibility check on Payer A portal for member AET456789012 (DOB 1958-06-14) confirms the plan is HMO with no OON benefits. Since orthopedic services at Summit Orthopedic Associates were rendered out-of-network and the HMO plan provides no OON coverage, the $2,100 balance is the patient's responsibility." }, { "id": "denial-medium-12", "title": "Verify Mental Health Service Exclusion via Payer Portal Eligibility Check", "instruction": "For denial DEN-008 (Anderson, Robert), check eligibility and plan benefits to verify service exclusion. Review the CO-96 denial in EMR, navigate to Payer A portal (provider@payera.com / demo123), check eligibility for member AET456123789 (DOB 1990-11-22) to verify plan type and benefits, confirm S9083 (outpatient mental health global fee) is excluded from the EPO plan, return to EMR, select 'Transfer to Patient' disposition, and document that the service is a plan exclusion and not a billing error.", "goal": "For denial DEN-008 (Anderson, Robert), check eligibility and plan benefits to verify service exclusion. Review the CO-96 denial in EMR, navigate to Payer A portal (provider@payera.com / demo123), check eligibility for member AET456123789 (DOB 1990-11-22) to verify plan type and benefits, confirm S9083 (outpatient mental health global fee) is excluded from the EPO plan, return to EMR, select 'Transfer to Patient' disposition, and document that the service is a plan exclusion and not a billing error.", "difficulty": "medium", "category": "eligibility_verification", "challengeType": "denial_triage", "possible": true, "points": 8, "configTaskId": "denial_medium_12", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Anderson, Robert" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-008 for Anderson, Robert - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB and CO-96 denial details This step is evaluated.", "4. Note the CPT code S9083 and diagnosis F41.1", "5. Click the 'Start Appeal' button to navigate to the Payer A portal", "6. Log in with provider@payera.com / demo123", "7. Use eligibility check with member ID AET456123789 and DOB 1990-11-22", "8. Once eligibility results load, confirm the plan shows 'Aetna EPO'. STOP HERE - there is NO separate CPT/HCPCS code lookup on this page. The EPO plan type itself confirms S9083 is excluded. Do NOT scroll looking for a 'CPT input' or 'exclusions section' - it does not exist. Record the plan type and immediately return to EMR.", "9. Click the Payer A logo or navigate back to the Payer A Dashboard, then click 'Return to EMR' to go back to the denial detail page.", "10. Select 'Transfer to Patient' from the Triage Disposition dropdown and submit", "11. Add a triage note documenting: (a) CO-96 plan exclusion, (b) S9083 not covered under EPO, (c) this is not a billing error, and (d) the specific $780 balance is the patient's responsibility. Do NOT scroll or click any tabs after you start filling in the triage form - this clears the note. Type your note and click Submit Disposition immediately." ], "evals": [ { "id": "denial-medium-12-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-008", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-12-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review the EOB and CO-96 remark details", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-12-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-12-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Transfer to Patient", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Transfer to Patient", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-12-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting their eligibility verification findings", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-12-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions CO-96 and identifies plan exclusion", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-96 and identify it as a non-covered service or plan exclusion? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-12-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note confirms S9083 is excluded from EPO plan per payer portal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that CPT S9083 (outpatient mental health global fee) is excluded from the Aetna EPO plan, based on information from the payer portal eligibility check? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-12-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains this is a plan exclusion, not a billing error", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note clearly distinguish that this is a plan exclusion (not a billing error or coding issue) and that the $780 balance is the patient's responsibility? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Transfer to Patient" }, "rationale": "CO-96 indicates non-covered charges under the plan. Eligibility check on Payer A portal for member AET456123789 (DOB 1990-11-22) confirms the plan is Aetna EPO. S9083 (global fee for outpatient mental health treatment) is an excluded service under this EPO plan. Since this is a plan-level benefit exclusion and not a billing error or coding issue, the charge cannot be appealed and the $780 balance should be transferred to the patient." }, { "id": "denial-medium-13", "title": "Check Appeal Status on Payer Portal for Previously Appealed Denial", "instruction": "For denial DEN-011 (Miller, James), check appeal status on payer portal. Review the denial in EMR noting the 'in_review' status and notes indicating 'Appeal submitted 11/01/2025' and 'Awaiting payer response'. Navigate to Payer A portal (provider@payera.com / demo123), search for the claim, check appeal status, find appeal reference APL-2025-78901. Return to EMR and add a progress note with the current appeal status from the payer portal.", "goal": "For denial DEN-011 (Miller, James), check appeal status on payer portal. Review the denial in EMR noting the 'in_review' status and notes indicating 'Appeal submitted 11/01/2025' and 'Awaiting payer response'. Navigate to Payer A portal (provider@payera.com / demo123), search for the claim, check appeal status, find appeal reference APL-2025-78901. Return to EMR and add a progress note with the current appeal status from the payer portal.", "difficulty": "medium", "category": "followup_tracking", "challengeType": "denial_triage", "possible": true, "points": 8, "configTaskId": "denial_medium_13", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Miller, James" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-011 for Miller, James - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. Note the 'appealed' status and existing notes: 'Appeal submitted 11/01/2025' and 'Awaiting payer response'", "4. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB and CO-50 denial details This step is evaluated.", "5. Click the 'Start Appeal' button to navigate to the Payer A portal", "6. Log in with provider@payera.com / demo123", "7. Do NOT use the 'Appeals' search tab. Instead, click the 'EOB & Claims' tab in the left sidebar. Enter the Member ID 'AET678901234' in the Member Name or ID field and click Search. This will find Miller, James's claim CLM-2025-00011.", "8. In the claim results, click on claim CLM-2025-00011 to open the claim detail. Record the appeal reference APL-2025-78901 and the current appeal status (e.g. Appeal In Review).", "9. To return to EMR: look for the green 'Return to EMR' button on the claims detail page (lower-right action bar). Do NOT click Sign Out (top-right). If you accidentally get logged out and land on /payer-a/login, do NOT click around - there is no Return to EMR button on the login page. Instead use navigate_to('http://localhost:3010/emr/denied/DEN-011?task_id=&run_id=') using the task_id and run_id from your KEY_INFO.", "10. Add a triage note with the current appeal status from the payer portal, including the reference number and any status updates" ], "evals": [ { "id": "denial-medium-13-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-011", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-13-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-13-eval-3", "type": "jmespath", "category": "Documentation", "description": "Agent added a progress note documenting the current appeal status from the payer portal", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-13-eval-4", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes the appeal reference number APL-2025-78901", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the appeal reference number APL-2025-78901? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-13-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note documents appeal submission date and pending status", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note indicate that the appeal is under review, awaiting payer response, or mention the submission date (11/01/2025 or November 2025)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-13-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the claim or denial being followed up", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the item being followed up by mentioning DEN-011, CLM-2025-00011, CO-50, or equivalent denial/claim context? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-13-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note indicates this is an existing appeal follow-up", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note indicate that this was an already-submitted appeal follow-up, for example by mentioning an existing appeal, prior submission, or the 11/01/2025 submission date? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-13-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes payer portal appeal status update", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include information obtained from the payer portal about the current appeal status (e. g., under review, pending decision, or specific status update)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": {}, "rationale": "DEN-011 has already been appealed (submitted 11/01/2025) and the appeal deadline of 2026-01-15 is past. The agent needs to check the payer portal for the current appeal status rather than take a new disposition action. The appeal reference APL-2025-78901 should be verified on the Payer A portal, and the current status should be documented as a progress note in the EMR." }, { "id": "denial-medium-14", "title": "Monitor High-Value In-Review Denial and Set Up Follow-Up Tracking", "instruction": "For denial DEN-021 (Young, Rebecca), monitor this high-value in-review denial and set up follow-up tracking. Review the $12,500 CO-50 medical necessity denial for total knee arthroplasty, note the peer-to-peer review scheduled. Navigate to Payer A portal (provider@payera.com / demo123), check claim status and deadline. Return to EMR, add a follow-up date, and document a tracking plan noting the deadline urgency and peer review status.", "goal": "For denial DEN-021 (Young, Rebecca), monitor this high-value in-review denial and set up follow-up tracking. Review the $12,500 CO-50 medical necessity denial for total knee arthroplasty, note the peer-to-peer review scheduled. Navigate to Payer A portal (provider@payera.com / demo123), check claim status and deadline. Return to EMR, add a follow-up date, and document a tracking plan noting the deadline urgency and peer review status.", "difficulty": "medium", "category": "followup_tracking", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_medium_14", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Young, Rebecca" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-021 for Young, Rebecca - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. Note the $12,500 amount, in_review status, and 'Peer-to-peer review scheduled' note", "4. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB and CO-50 denial details and get the Member ID This step is evaluated.", "5. Click the 'Start Appeal' button to navigate to the Payer A portal", "6. Log in with provider@payera.com / demo123", "7. Use the EOB & Claims tab in the left sidebar to search. Enter the Member ID 'AET567890234' in the Member Name or ID field and click Search. This will find Young, Rebecca's claim CLM-2025-00021. Click on the claim to view its details and note the current status and appeal deadline (2026-03-03).", "8. To return to EMR: look for the green 'Return to EMR' button on the claims detail page. Do NOT click Sign Out (top-right, X~1214). Clicking Sign Out will log you out and you will be stranded on the login page with NO way to return to EMR from there. If you accidentally get logged out, use navigate_to('http://localhost:3010/emr/denied/DEN-021?task_id=denial_medium_14&run_id=') to get back.", "10. Add a valid follow-up date to track the peer review outcome", "11. Add a triage note documenting the tracking plan with deadline urgency, peer review status, and next steps" ], "evals": [ { "id": "denial-medium-14-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-021", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-14-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review the EOB and CO-50 denial details", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-14-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-14-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent added a follow-up date to track the high-value denial", "points": 1, "query": "full_state.agentActions.addedFollowUpTask", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-14-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent added a note documenting the tracking plan and peer review status", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-14-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions the $12,500 amount and CO-50 denial code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the high dollar amount ($12,500) and the CO-50 medical necessity denial code? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-14-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references peer-to-peer review status", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the peer-to-peer review that is scheduled or reference the need for physician involvement in the appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-14-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the appeal deadline of 2026-03-03? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-14-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note includes tracking plan", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include a tracking plan or next steps for monitoring this high-value denial? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": {}, "rationale": "DEN-021 is a high-value ($12,500) total knee arthroplasty denial currently in review with a peer-to-peer review scheduled. The appeal deadline of 2026-03-03 is approaching. The agent must verify the current claim status on the Payer A portal, set up a follow-up date in EMR to track the peer review outcome, and document a comprehensive tracking plan noting the urgency of the deadline and the peer review scheduling." }, { "id": "denial-medium-15", "title": "Review Follow-Up Status and Prepare Appeal for Lumbar MRI Denial", "instruction": "For denial DEN-019 (Robinson, Karen), review follow-up status and prepare for appeal. This denial has follow_up status with existing notes 'Requested additional clinical notes from provider' and 'Follow up scheduled'. Check the remittance image for CO-50/N386 remark codes. Navigate to Payer B portal (provider@payerb.com / demo123), check claim status. Return to EMR and document the current status and next steps for appeal submission once clinical notes are received.", "goal": "For denial DEN-019 (Robinson, Karen), review follow-up status and prepare for appeal. This denial has follow_up status with existing notes 'Requested additional clinical notes from provider' and 'Follow up scheduled'. Check the remittance image for CO-50/N386 remark codes. Navigate to Payer B portal (provider@payerb.com / demo123), check claim status. Return to EMR and document the current status and next steps for appeal submission once clinical notes are received.", "difficulty": "medium", "category": "followup_tracking", "challengeType": "denial_triage", "possible": true, "points": 10, "configTaskId": "denial_medium_15", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Robinson, Karen" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-019 for Robinson, Karen - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. Note the follow_up status and existing notes about requested clinical notes and scheduled follow-up", "4. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB with CO-50 and N386 remark codes This step is evaluated.", "5. Click the 'Start Appeal' button to navigate to the Payer B portal", "6. Log in with provider@payerb.com / demo123", "7. Search for the claim and check current status", "8. Return to EMR denial detail page", "9. IMPORTANT: You already reviewed the Remittance Image in step 4. Do NOT click the Remittance Image tab (or any other tab) again after you start filling in the triage form - clicking any tab resets the note field and clears everything you typed. Go directly to the Actions panel on the right side: (a) select 'Route to Clinical Appeals' from the Triage Disposition dropdown, (b) click the Triage Note field and type your note - the note MUST include the denial code CO-50 and remark code N386, the specific procedure CPT 72148 (lumbar MRI) and diagnosis M54.5 (low back pain), the status of the clinical notes pending from the provider, and the next steps for appeal once clinical notes are received before the 2026-03-25 deadline, (c) immediately click 'Submit Disposition' - do not click anything else in between." ], "evals": [ { "id": "denial-medium-15-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-019", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-15-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review the EOB with CO-50 and N386 remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-15-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-15-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a note documenting current status and next steps for appeal preparation", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-15-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions CO-50 and N386 or medical necessity", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-50 and remark code N386, or reference medical necessity as the denial reason? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-15-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note notes that clinical notes are pending from provider", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that additional clinical notes have been requested from the provider and are pending? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-15-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 72148", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the lumbar MRI procedure CPT 72148? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-15-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note references diagnosis M54.5", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention diagnosis M54.5 (low back pain)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-15-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note outlines next steps for appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note outline next steps for appeal submission, such as waiting for clinical notes to be received before submitting the appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-15-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Triage note references appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the appeal deadline of 2026-03-25? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": {}, "rationale": "DEN-019 is a CO-50 medical necessity denial for a lumbar MRI (CPT 72148) with diagnosis M54.5 (low back pain). The denial is in follow_up status with notes indicating clinical documentation has been requested from the provider. The remittance image shows CO-50 with N386 remark code. The agent should verify the claim status on the Payer B portal, then document the current follow-up status and outline next steps: once clinical notes are received, compile and submit an appeal before the 2026-03-25 deadline." }, { "id": "denial-medium-16", "title": "Identify Missing Bilateral Modifier and Prepare Corrected Claim Resubmission", "instruction": "For denial DEN-018 (Walker, Charles), identify the coding error causing the CO-4 denial and prepare a corrected claim. Review the remittance image for remark codes, determine that CPT 29881 (knee arthroscopy meniscectomy) was performed bilaterally based on diagnoses M23.41 (right knee) and M23.42 (left knee) but is missing the -50 bilateral modifier. Navigate to Payer A portal to verify the resubmission process, return to EMR, select 'Route to Coding Review' disposition, and document the specific modifier correction needed.", "goal": "For denial DEN-018 (Walker, Charles), identify the coding error causing the CO-4 denial and prepare a corrected claim. Review the remittance image for remark codes, determine that CPT 29881 (knee arthroscopy meniscectomy) was performed bilaterally based on diagnoses M23.41 (right knee) and M23.42 (left knee) but is missing the -50 bilateral modifier. Navigate to Payer A portal to verify the resubmission process, return to EMR, select 'Route to Coding Review' disposition, and document the specific modifier correction needed.", "difficulty": "medium", "category": "claim_edit_rebilling", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_medium_16", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Walker, Charles" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-018 for Walker, Charles - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes This step is evaluated.", "4. Identify CO-4 (procedure code inconsistent with modifier) and MA130 remark code", "5. Review the claim line items: CPT 29881 (knee arthroscopy meniscectomy) billed without bilateral modifier", "6. Review the diagnosis codes: M23.41 (right knee) and M23.42 (left knee) - bilateral procedure", "7. Recognize that CPT 29881 needs modifier -50 (bilateral) since it was performed on both knees", "8. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then navigate to Payer A via the 'submit appeal' button, log in with provider@payera.com / demo123, open the claim search page, enter the member ID shown for this denial, and click Search to verify the corrected-claim resubmission process. The claim search itself is evaluated.", "9. Return to EMR denial detail page", "10. Select 'Route to Coding Review' from the Triage Disposition dropdown and submit", "11. Add a triage note documenting the CO-4/MA130 denial, the missing -50 bilateral modifier for CPT 29881, and that the corrected claim should be resubmitted with modifier -50. Explicitly include diagnosis codes M23.41 and M23.42 in the note." ], "evals": [ { "id": "denial-medium-16-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-018", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-16-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with MA130 remark code", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-16-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to Payer A portal to verify resubmission process", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-16-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer A Appeals page", "points": 1, "query": "payer_a_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-16-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Coding Review", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Coding Review", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-16-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting the modifier correction needed", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-16-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions CO-4 denial code and MA130 remark code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-4 and remark code MA130? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-16-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the missing -50 bilateral modifier for CPT 29881", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that CPT 29881 needs modifier -50 (bilateral) because the procedure was performed on both knees (M23.41 right knee and M23.42 left knee)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-16-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains the corrected claim resubmission with bilateral modifier", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the corrected claim should be resubmitted with the -50 bilateral modifier added to CPT 29881? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Coding Review" }, "rationale": "CO-4 coding error denial with MA130 remark code indicating a modifier issue. CPT 29881 (arthroscopic knee meniscectomy) was billed without a bilateral modifier, but the diagnosis codes M23.41 (right knee) and M23.42 (left knee) clearly indicate the procedure was performed on both knees. Modifier -50 is a pricing modifier that can change the claim amount, so coding must review before any changes are made. Route to Coding Review for modifier -50 correction and resubmission." }, { "id": "denial-medium-17", "title": "Identify Missing E/M Modifier -25 and Verify Payer Resubmission", "instruction": "For denial DEN-004 (Brown, Michael), identify the modifier error causing the CO-4 denial and prepare a corrected claim. Review the remittance image for MA130 remark code, determine that CPT 99213 (office visit) needs modifier -25 (significant, separately identifiable E/M service) when billed on the same day as CPT 36415 (venipuncture). Navigate to Payer A portal to verify the claim can be resubmitted, return to EMR, select 'Route to Coding Review' disposition, and document the -25 modifier correction.", "goal": "For denial DEN-004 (Brown, Michael), identify the modifier error causing the CO-4 denial and prepare a corrected claim. Review the remittance image for MA130 remark code, determine that CPT 99213 (office visit) needs modifier -25 (significant, separately identifiable E/M service) when billed on the same day as CPT 36415 (venipuncture). Navigate to Payer A portal to verify the claim can be resubmitted, return to EMR, select 'Route to Coding Review' disposition, and document the -25 modifier correction.", "difficulty": "medium", "category": "claim_edit_rebilling", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_medium_17", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Brown, Michael" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-004 for Brown, Michael - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes and get the Member ID This step is evaluated.", "4. Identify CO-4 (procedure code inconsistent with modifier) and MA130 remark code", "5. Review the claim line items: CPT 99213 (office visit) and CPT 36415 (venipuncture) billed on the same date of service", "6. Recognize that CPT 99213 needs modifier -25 when billed with a minor procedure on the same day", "7. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then navigate to payer a portal via the 'submit appeal' button and verify the corrected claim resubmission process by searching for the claim with the member id", "8. Return to EMR denial detail page", "9. Select 'Route to Coding Review' from the Triage Disposition dropdown and submit", "10. Add a triage note documenting the CO-4/MA130 denial, the missing -25 modifier for CPT 99213 (same-day billing with 36415), and that the corrected claim should be resubmitted" ], "evals": [ { "id": "denial-medium-17-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-004", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-17-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with MA130 remark code", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-17-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to Payer A portal to verify resubmission eligibility", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-17-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched for the claim on the Payer A Appeals page", "points": 1, "query": "payer_a_state.full_state.appealActions.searchedClaims", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-17-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Route to Coding Review", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Coding Review", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-17-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting the modifier correction needed", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-17-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions CO-4 denial code and MA130 remark code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-4 and remark code MA130? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-17-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the missing -25 modifier for same-day E/M and venipuncture", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that CPT 99213 needs modifier -25 (significant, separately identifiable E/M service) because it was billed on the same day as CPT 36415 (venipuncture)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-17-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains the corrected claim resubmission with -25 modifier", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the corrected claim should be resubmitted with modifier -25 added to CPT 99213? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Coding Review" }, "rationale": "CO-4 coding error denial with MA130 remark code indicating a modifier issue. CPT 99213 (office visit, established patient, low complexity) was billed on the same day as CPT 36415 (venipuncture). When an E/M service is performed on the same day as a minor procedure like venipuncture, modifier -25 is required on the E/M code to indicate it was a significant, separately identifiable service. Modifier changes must be reviewed by coding before any changes are made. Route to Coding Review for modifier -25 correction and resubmission." }, { "id": "denial-medium-18", "title": "Find Missing Referring Provider NPI via Patient Chart and Prepare Corrected Claim", "instruction": "For denial DEN-020 (Clark, Steven), find the missing information causing the CO-16 denial and prepare a corrected claim. Review the remittance image for the N264 remark code ('Missing/incomplete/invalid referring provider information'). Navigate to the patient inquiry page to find the referring provider's name from the patient's chart. Return to the denial detail page, select 'Corrected Claim - Resubmit' disposition, and document the NPI and referring provider's identity needed for the corrected claim.", "goal": "For denial DEN-020 (Clark, Steven), find the missing information causing the CO-16 denial and prepare a corrected claim. Review the remittance image for the N264 remark code ('Missing/incomplete/invalid referring provider information'). Navigate to the patient inquiry page to find the referring provider's name from the patient's chart. Return to the denial detail page, select 'Corrected Claim - Resubmit' disposition, and document the NPI and referring provider's identity needed for the corrected claim.", "difficulty": "medium", "category": "claim_edit_rebilling", "challengeType": "denial_triage", "possible": true, "points": 8, "configTaskId": "denial_medium_18", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Clark, Steven" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-020 for Clark, Steven - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to view the EOB and review the remark codes This step is evaluated.", "4. Identify CO-16 (claim lacks information for adjudication) and N264 remark code (missing referring provider info)", "5. Navigate to the patient inquiry page by clicking the patient name link and locate the referring provider's name and NPI", "6. Return to the denial detail page", "7. Select 'Corrected Claim - Resubmit' from the Triage Disposition dropdown and submit", "8. Add a triage note documenting the CO-16/N264 denial, the referring provider identified from the patient chart, and that the corrected claim should be resubmitted with the referring provider NPI (and include it)" ], "evals": [ { "id": "denial-medium-18-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-020", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-18-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to view the EOB with N264 remark code", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-18-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the patient inquiry page to find referring provider information", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-18-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Corrected Claim - Resubmit", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Corrected Claim - Resubmit", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-18-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting the referring provider NPI correction", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-18-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions CO-16 denial code and N264 remark code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-16 and remark code N264 (missing referring provider information)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-18-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the referring provider from the patient chart", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the referring provider's name from the patient's chart or mention the referring provider NPI that needs to be added to the corrected claim? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-18-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note explains the corrected claim needs the referring provider NPI", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the corrected claim should be resubmitted and includes the referring provider's name, Dr. Evans, and NPI, 1765432993? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Corrected Claim - Resubmit" }, "rationale": "CO-16 denial indicating the claim lacks information needed for adjudication, with N264 remark code specifying missing/incomplete/invalid referring provider information. The claim for CPT 99213 (office visit) was submitted without the referring provider's NPI. By navigating to the patient inquiry page, the referring provider's name can be found in the patient's chart, and the corresponding NPI can be included on the corrected claim. The appeal deadline is 2026-02-28, so timely resubmission is still possible." }, { "id": "denial-medium-19", "title": "Investigate Duplicate Claim and Confirm Write-Off", "instruction": "For denial DEN-005 (Garcia, Maria), investigate the CO-18 duplicate claim denial from Valley Health Plan. Review the remittance image, then check the related claims section to find the original claim CLM-2025-49786 and verify it was already paid. Cross-reference the CPT code (99395), service date, and facility between the original and duplicate claims to confirm this is a genuine duplicate. Note that the appeal deadline (2026-02-05) has passed. Select 'Write Off' disposition and document the investigation findings proving this is a true duplicate with the original claim already paid.", "goal": "For denial DEN-005 (Garcia, Maria), investigate the CO-18 duplicate claim denial from Valley Health Plan. Review the remittance image, then check the related claims section to find the original claim CLM-2025-49786 and verify it was already paid. Cross-reference the CPT code (99395), service date, and facility between the original and duplicate claims to confirm this is a genuine duplicate. Note that the appeal deadline (2026-02-05) has passed. Select 'Write Off' disposition and document the investigation findings proving this is a true duplicate with the original claim already paid.", "difficulty": "medium", "category": "investigation", "challengeType": "denial_triage", "possible": true, "points": 9, "configTaskId": "denial_medium_19", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Garcia, Maria" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-005 for Garcia, Maria - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB This step is evaluated.", "4. Note CO-18 (exact duplicate claim/service) denial code", "5. The 'Related Claims' information is displayed directly in the right-side panel on the DEN-005 detail page - it is NOT a separate section you need to scroll far to find. You should already see CLM-2025-49786 listed as 'Valley Health Plan - original' with CPT 99395 at Community Health Center. This is your confirmation of the original paid claim. Do NOT keep scrolling up and down looking for it - it is already visible in the right panel.", "6. Note that the original claim CLM-2025-49786 is the paid original and the current claim CLM-2025-00005 is the duplicate that was denied.", "7. Cross-reference: both share CPT 99395, service date 2025-10-10, facility Community Health Center - confirmed true duplicate.", "8. Note the appeal deadline (2026-02-05) has already passed.", "9. Select 'Write Off' from the Triage Disposition dropdown.", "10. IMPORTANT: Once you select the disposition and click the Triage Note field, do NOT scroll or click any other tabs - scrolling resets the action panel and clears your note. Type your note immediately and click Submit Disposition without doing anything else in between.", "11. Add a triage note documenting: CO-18 duplicate; original claim CLM-2025-49786 (Valley Health Plan - original) was already paid; CPT 99395 matches; DOS 2025-10-10 matches; facility Community Health Center matches; appeal deadline 2026-02-05 has passed; true duplicate confirmed - write-off appropriate." ], "evals": [ { "id": "denial-medium-19-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-005", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-19-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review the EOB", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-19-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Write Off", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Write Off", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-19-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting the duplicate claim investigation", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-19-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions CO-18 duplicate claim denial code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-18 (duplicate claim)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-19-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note references the original paid claim CLM-2025-49786", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the original claim CLM-2025-49786 and confirm it was already paid? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-19-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note references CPT 99395 or preventive visit", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention CPT 99395 or reference a preventive visit when confirming the duplicate? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-19-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Triage note cross-references service date or facility", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note cross-reference the service date (2025-10-10) or facility (Community Health Center) to confirm the duplicate? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-19-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note justifies write-off due to confirmed duplicate and/or expired deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that a write-off is appropriate because this is a confirmed true duplicate with the original claim already paid, and/or note that the appeal deadline (2026-02-05) has passed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Write Off" }, "rationale": "CO-18 duplicate claim denial from Valley Health Plan. Investigation of the related claims section reveals the original claim CLM-2025-49786 for the same patient, same service date (2025-10-10), same CPT 99395 (preventive visit), and same facility (Community Health Center) was already paid. This confirms the denied claim is a genuine duplicate - the service was already reimbursed. Additionally, the appeal deadline of 2026-02-05 has already passed. Write-off is the appropriate disposition as recovery is not possible for a true duplicate with the original already paid." }, { "id": "denial-medium-20", "title": "Investigate Timely Filing Violation and Confirm Unrecoverable Write-Off", "instruction": "For denial DEN-003 (Williams, Sarah), investigate the CO-29 timely filing denial from Blue Cross Blue Shield. Review the remittance image, then check the claim history section to find the original submission date. Determine that the claim was submitted approximately 200 days after the service date of 2024-08-15, which exceeds the 180-day timely filing limit. Verify there is no proof of earlier timely submission. Note that the appeal deadline (2026-02-01) has also passed. Select 'Write Off' disposition and document that both the original filing was late AND the appeal deadline has expired, making this unrecoverable.", "goal": "For denial DEN-003 (Williams, Sarah), investigate the CO-29 timely filing denial from Blue Cross Blue Shield. Review the remittance image, then check the claim history section to find the original submission date. Determine that the claim was submitted approximately 200 days after the service date of 2024-08-15, which exceeds the 180-day timely filing limit. Verify there is no proof of earlier timely submission. Note that the appeal deadline (2026-02-01) has also passed. Select 'Write Off' disposition and document that both the original filing was late AND the appeal deadline has expired, making this unrecoverable.", "difficulty": "medium", "category": "investigation", "challengeType": "denial_triage", "possible": true, "points": 8, "configTaskId": "denial_medium_20", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Williams, Sarah" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-003 for Williams, Sarah - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. click DIRECTLY on the denial ID or patient name text in the row to open the denial detail page to open the denial detail page", "3. [REQUIRED - DO NOT SKIP] Click the Remittance Image tab to review the EOB This step is evaluated.", "4. Note CO-29 (time limit for filing has expired) denial code", "5. Review the service date: 2024-08-15", "6. Navigate to the Claim History / Submission History section on the Retest tab", "7. Find the original submission date (~2025-03-03) and calculate it was ~200 days after the service date", "8. Confirm this exceeds the 180-day timely filing limit", "9. Verify there is no evidence of an earlier timely submission", "10. Check the appeal deadline: 2026-02-01 - this has already passed", "11. Confirm the claim is unrecoverable: late filing + expired appeal deadline", "12. Select 'Write Off' from the Triage Disposition dropdown and submit", "13. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then add a triage note documenting: co-29 timely filing denial, claim submitted ~200 days after service (exceeds 180-day limit), no proof of earlier submission, appeal deadline also expired, write-off is appropriate" ], "evals": [ { "id": "denial-medium-20-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the denial detail page for DEN-003", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-20-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent clicked the Remittance Image tab to review the EOB", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-20-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected the correct disposition: Write Off", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Write Off", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-20-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent added a triage note documenting the timely filing investigation", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-medium-20-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Triage note mentions CO-29 timely filing denial code", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention denial code CO-29 (timely filing limit expired)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-20-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Triage note identifies the late submission exceeding the 180-day filing limit", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that the claim was submitted approximately 200 days after the service date (2024-08-15), exceeding the 180-day filing deadline? Score 1.0 if the note mentions the late submission timing or that it exceeded the filing limit, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-20-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Triage note notes the expired appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that the appeal deadline (2026-02-01) has already passed or expired? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-medium-20-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Triage note explains write-off due to both late filing and expired appeal deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that a write-off is appropriate because both the original filing was late AND the appeal deadline has expired, making the claim unrecoverable? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Write Off" }, "rationale": "CO-29 timely filing denial from Blue Cross Blue Shield. The service date was 2024-08-15 and the claim was not submitted until approximately 2025-03-03 (~200 days later), exceeding the 180-day timely filing deadline. Review of the claim history and submission records shows no proof of an earlier timely submission. Furthermore, the appeal deadline of 2026-02-01 has already passed, so even if there were grounds for appeal, it is too late to file one. This claim is unrecoverable on two fronts: the original filing was genuinely late, and the window to appeal has also closed. Write-off is the only appropriate disposition." }, { "id": "denial-hard-1", "title": "Auth Mismatch Investigation - Wrong CPT on Existing Auth", "instruction": "For denial DEN-026 (Rivera, Marcus), investigate the CO-197 no-authorization denial. Review the denial details, remittance image, and patient inquiry in the EMR. Check eligibility on the Payer A portal (provider@payera.com / demo123). Based on all evidence gathered, determine the appropriate resolution - do not assume the obvious action is correct. Execute the resolution on the payer portal and return to EMR to document your complete investigation, findings, and rationale for the action you chose in a triage note.", "goal": "For denial DEN-026 (Rivera, Marcus), investigate the CO-197 no-authorization denial. Review the denial details, remittance image, and patient inquiry in the EMR. Check eligibility on the Payer A portal (provider@payera.com / demo123). Based on all evidence gathered, determine the appropriate resolution - do not assume the obvious action is correct. Execute the resolution on the payer portal and return to EMR to document your complete investigation, findings, and rationale for the action you chose in a triage note.", "difficulty": "hard", "category": "ambiguous_decision", "challengeType": "denial_triage", "possible": true, "points": 18, "configTaskId": "denial_hard_1", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Rivera, Marcus" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-026 for Rivera, Marcus - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. On the denial detail page, note: CO-197 denial code, $4,200 amount, CPT 43239 (EGD with biopsy), diagnoses K21.0/K22.0", "3. Note service date 2025-11-20, appeal deadline 2026-03-20, patient DOB 1974-05-12, member ID AET234567890", "4. Note in Notes History: 'Prior auth on file - verify coverage scope before appeal'", "5. [REQUIRED - DO THIS STEP] Click the 'Remittance Image' tab (second tab next to Retest). Review the EOB showing CO-197 no-authorization denial. Record CARC/RARC codes. This step is evaluated.", "6. [REQUIRED - BEFORE LEAVING EMR] Scroll down to find the Documents section. Click the 'View ->' button on the RIGHT side of the 'Clinical Notes - EGD with Biopsy.pdf' row - do NOT click the document name/title (that does nothing). This opens the document viewer page. Then click 'Download'. Note alarm symptoms: dysphagia, weight loss, Grade B esophagitis. This document is required for the appeal attachment.", "After downloading, click '< Back' ONCE to return to the denial detail page. Do NOT click back a second time - that takes you to the worklist and will cause you to lose your place. If you accidentally end up on the worklist, use navigate_to to go directly to the denial detail page.", "7. Note key alarm symptoms for the appeal rationale: dysphagia, weight loss of 6 lbs over 2 months, Grade B esophagitis", "9. Click the patient name 'Rivera, Marcus' link in the top banner - navigates to Patient Inquiry page (/patient/MRN89012876)", "10. On Patient Inquiry, find the Authorizations section: AUTH-2025-92001, Status: Active, Expires: 2026-04-20", "11. Note the auth note: 'Auth covers CPT 99214 (office visit) only. Does NOT cover CPT 43239 (EGD with biopsy).' - this is the key finding", "12. Click the 'Start Appeal' button in the Actions panel on the right sidebar - this navigates to the Payer A portal login page", "13. Log in with provider@payera.com / demo123", "14. Click 'Member eligibility' in the nav bar - this step is required to verify the patient's plan details", "15. Fill in required fields: Member ID AET234567890, Last Name 'Rivera', First Name 'Marcus', DOB '1974-05-12'. Click Submit", "16. Confirm and note results: plan type is Aetna PPO, Status: Active, Group: Bay Area Medical Group (GRP-44821), PCP: Dr. Angela Torres, Auth Required: Yes", "17. Click 'Appeals' in the navigation bar to go to the Appeals / Claim Status Inquiry page", "18. Enter member ID AET234567890 in the Member ID search field and click Search", "19. In the search results, click on the claim row for CLM-2025-00026 to view the Claim Status Detail", "20. Click the 'Dispute Claim' button", "21. Fill in the Contact Name field with 'Dr. Angela Torres' (the provider listed on the denial)", "22. In the Supporting Rationale field, write: (a) auth AUTH-2025-92001 exists but covers CPT 99214 not 43239 - this is a CPT mismatch, (b) EGD was medically necessary due to alarm symptoms (dysphagia, weight loss), (c) retro auth is not the correct resolution", "23. [REQUIRED] Scroll to 'Available Documents from EMR' - click '+ Attach' next to the downloaded clinical notes document. The evaluator checks that at least 1 document is attached.", "24. Click 'Submit Appeal'. Note the confirmation number (format APL-AET-XXXXXXXX)", "25. Click 'Return to EMR' to go back to the denials workqueue", "26. You are now on the WORKLIST (/emr/denied). To re-open the DEN-026 detail page, click on the PATIENT NAME 'Rivera, Marcus' (the purple underlined name button in that row) - single-clicking the row body only selects it (turns green) but does NOT navigate. If the single click on the name does not navigate, double-click anywhere on the Rivera, Marcus row. As a last resort use navigate_to('/emr/denied/DEN-026') to go directly.", "27. Once on the DEN-026 detail page (/emr/denied/DEN-026), go to the Actions panel on the right. Click the Triage Disposition dropdown and select 'Route to Clinical Appeals'.", "28. IMMEDIATELY after selecting disposition, click the Triage Note field and type your note WITHOUT clicking any other tabs - clicking tabs clears the note field.", "29. Write triage note documenting: auth AUTH-2025-92001 found for wrong CPT (99214 vs 43239), CPT mismatch explained, clinical rationale for EGD (alarm symptoms - dysphagia, weight loss), Aetna PPO plan type confirmed via eligibility check, why retro auth is incorrect approach (auth exists but for wrong CPT), appeal confirmation number (APL-AET-XXXXXXXX), investigation steps taken", "30. Click 'Submit Disposition' to finalize - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-1-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-026", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded/viewed clinical documentation", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry to find existing auth", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked eligibility on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.checkedEligibility", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted appeal on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-7", "type": "jmespath", "category": "Document Handling", "description": "Agent attached clinical documentation to the appeal", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAttachmentCount > `0`", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale explains auth/CPT mismatch", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_a_state. full_state. appealActions) explain the authorization mismatch - that AUTH-2025-92001 covers CPT 99214 (office visit) but the billed procedure was CPT 43239 (EGD with biopsy)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-1-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale includes clinical justification for EGD", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_a_state. full_state. appealActions) provide clinical justification for the EGD, referencing alarm symptoms such as dysphagia or weight loss? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-1-eval-10", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected Route to Clinical Appeals disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-11", "type": "jmespath", "category": "Documentation", "description": "Agent documented in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-1-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note references auth number AUTH-2025-92001", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the specific auth number AUTH-2025-92001? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-1-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note identifies CPT mismatch (99214 vs 43239)", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that the existing auth covers CPT 99214 (office visit), not the billed CPT 43239 (EGD with biopsy)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-1-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Note mentions dysphagia as clinical justification", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the patient's alarm symptoms of dysphagia (difficulty swallowing)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-1-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Note mentions weight loss as clinical justification", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the patient's weight loss as a clinical finding supporting medical necessity? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-1-eval-16", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include an appeal confirmation number (format APL-AET-XXXXXXXX)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-1-eval-17", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains why retro auth is wrong approach", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why filing a retroactive authorization would be incorrect - because an auth already exists, so the issue is a mismatch, not a missing auth? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-1-eval-18", "type": "llm_judge", "category": "Documentation", "description": "Note mentions Aetna PPO plan type", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the patient's Aetna PPO plan type? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-197 no-authorization denial for EGD with biopsy (CPT 43239). Investigation reveals existing auth AUTH-2025-92001, but it covers CPT 99214 (office visit), not the billed CPT 43239. The CPT mismatch means the auth does not cover the procedure. Appeal should explain the mismatch and provide clinical rationale for EGD based on alarm symptoms (dysphagia, weight loss)." }, { "id": "denial-hard-2", "title": "Expired Deadline Investigation - Emergency Craniotomy", "instruction": "For denial DEN-027 (Chen, Grace), investigate the CO-50 medical necessity denial for a $15,800 emergency craniotomy. Review the denial details and remittance image. Download the clinical documentation. Check the claim status on the Payer B portal (provider@payerb.com / demo123). Based on ALL the evidence gathered, determine the appropriate course of action. Select the appropriate disposition and document your complete analysis and recommendation in a triage note.", "goal": "For denial DEN-027 (Chen, Grace), investigate the CO-50 medical necessity denial for a $15,800 emergency craniotomy. Review the denial details and remittance image. Download the clinical documentation. Check the claim status on the Payer B portal (provider@payerb.com / demo123). Based on ALL the evidence gathered, determine the appropriate course of action. Select the appropriate disposition and document your complete analysis and recommendation in a triage note.", "difficulty": "hard", "category": "ambiguous_decision", "challengeType": "denial_triage", "possible": true, "points": 12, "configTaskId": "denial_hard_2", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Chen, Grace" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate DEN-027 for Chen, Grace - click on the patient's NAME (purple underlined button) to open the denial detail page. Single-clicking the row body only selects it; click the name text specifically or double-click the row.", "2. On the denial detail page, note: CO-50 denial code, $15,800 amount, CPT 61312 (emergency craniotomy), diagnoses S06.5X0A/S06.6X0A", "3. Note service date 2025-08-15, denial date 2025-10-10, patient DOB 1968-03-22, member ID ANT567890234", "4. CRITICAL: Note the appeal deadline is 2026-01-10 - this is EXPIRED", "5. Note in Notes History: 'ALERT: Appeal deadline 2026-01-10 has EXPIRED. Standard appeal window closed.' and 'Strong clinical case - emergency craniotomy for acute subdural hematoma'", "6. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab - review the EOB showing CO-50 medical necessity denial from Anthem Blue Cross This step is evaluated.", "7. Observe on the remittance: CARC CO-50 - 'These are non-covered services because this is not deemed a medical necessity by the payer'", "8. Scroll down to the Documents section. Click the 'View ->' button on the RIGHT side of the 'Operative Report - Emergency Craniotomy.pdf' row (NOT the document name - that does nothing). This opens the document viewer page.", "9. Note the strong clinical evidence: GCS 8 on arrival, large acute left-sided subdural hematoma with 12mm midline shift and early uncal herniation, emergent surgery required to prevent brainstem compression and death", "10. Click 'Download' on the operative report to save clinical documentation", "10b. After downloading, click '< Back' ONCE to return to the DEN-027 detail page. Do NOT click back a second time - that takes you to the worklist. If you end up on the worklist by mistake, use navigate_to('/emr/denied/DEN-027') to return directly.", "11. Click the 'Start Appeal' button in the Actions panel - this navigates to the Payer B portal login page", "12. Log in with provider@payerb.com / demo123", "13. Click 'Appeals' in the navigation bar to access the claims/appeals search", "14. Enter member ID ANT567890234 in the search field. Click Search", "15. Locate the claim for Chen, Grace (CLM-2025-00027, $15,800, CO-50). Note the appeal deadline 2026-01-10 is expired", "16. Recognize that filing an appeal through the portal is NOT viable - the deadline has passed and the portal will not accept a late submission", "17. Click 'Return to EMR' to go back to the denials workqueue", "18. You are now on the WORKLIST. Click the PATIENT NAME button in that row to re-open the DEN-027 detail page - single-clicking the row body only selects it. If the name click does not navigate, double-click the row. As a last resort use navigate_to('/emr/denied/DEN-027') to go directly.", "19. In the Actions panel (right side), select 'Escalate to Supervisor' from the Triage Disposition dropdown (NOT appeal - deadline expired).", "20. IMMEDIATELY after selecting disposition, click the Triage Note field and type your note WITHOUT clicking any other tabs - clicking tabs clears the note. Write: (a) CO-50 medical necessity denial for emergency craniotomy ($15,800), (b) appeal deadline 2026-01-10 has EXPIRED, (c) strong clinical evidence - life-threatening subdural hematoma, GCS 8, emergent surgery, (d) standard appeal cannot be filed due to expired deadline, (e) recommend supervisor review for potential late appeal exception given high dollar value and compelling clinical case", "21. Click 'Submit Disposition' to finalize the escalation - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-2-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-027", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-2-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-2-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-2-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent accessed Payer B portal to check claim status", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-2-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected Escalate to Supervisor disposition (NOT appeal, since deadline is expired)", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Escalate to Supervisor", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-2-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent documented analysis in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-2-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note identifies expired deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that the appeal deadline (2026-01-10) has EXPIRED? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-2-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note acknowledges strong clinical case", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note acknowledge the strong clinical evidence for the emergency craniotomy (life-threatening subdural hematoma, GCS 8, emergent surgery)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-2-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains standard appeal not possible", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why a standard appeal cannot be filed (deadline expired) despite the strong clinical evidence? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-2-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note recommends supervisor review for late exception", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend supervisor review for potential late appeal exception, mentioning the $15,800 value or high-dollar amount? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-2-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note references specific expired deadline date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the specific expired deadline date of 2026-01-10 (or January 10, 2026)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-2-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note explains why standard portal appeal is not viable", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why submitting a standard appeal through the payer portal is not viable due to the expired deadline? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Escalate to Supervisor" }, "rationale": "CO-50 medical necessity denial for emergency craniotomy ($15,800). Strong clinical case (life-threatening subdural hematoma, GCS 8, emergent surgery), but the appeal deadline (2026-01-10) has expired. Standard appeal cannot be filed. Must escalate to supervisor for potential late appeal exception given the high dollar value and strong clinical evidence." }, { "id": "denial-hard-3", "title": "Double Rejection Investigation - Misrouted Claim", "instruction": "For denial DEN-028 (Patel, Raj), investigate the N418 misrouted claim denial. Review the denial details, remittance image, and patient inquiry for coverage and delegation details. Determine the correct course of action based on all available evidence - do not assume the standard resolution for this denial code applies. Select the appropriate disposition and document your investigation and recommendation in a triage note.", "goal": "For denial DEN-028 (Patel, Raj), investigate the N418 misrouted claim denial. Review the denial details, remittance image, and patient inquiry for coverage and delegation details. Determine the correct course of action based on all available evidence - do not assume the standard resolution for this denial code applies. Select the appropriate disposition and document your investigation and recommendation in a triage note.", "difficulty": "hard", "category": "ambiguous_decision", "challengeType": "denial_triage", "possible": true, "points": 10, "configTaskId": "denial_hard_3", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Patel, Raj" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), scroll down the list to find DEN-028 for Patel, Raj. Click DIRECTLY on the row (patient name or denial ID text) to open the denial detail page. Do NOT click navigation buttons at the top of the browser.", "2. The denial detail page opens on the Retest tab by default. In the Errors section (orange border at top), note: N418 - 'Claim submitted to incorrect payer. Services under delegated capitation arrangement.'", "3. In the Claim/Invoice Header section below the Errors, note: CLM-2025-00028, Pacific Health Alliance, $3,100 remaining balance, Delegated Group: Valley Medical Group", "4. Note the appeal deadline 2026-03-15 shown in the title bar at top right", "5. Scroll down to the Notes History section (bottom of the Retest tab). Note: 'N418 misrouted - previously rerouted to Valley Medical Group per delegation. Check submission history for outcome.'", "6. Scroll up to the Claim History section (below the Diagnosis Codes, above Documents). This table shows submission history", "7. Note first row: 2025-10-11, Original, CLM-2025-00028, $3,410.00, status 'Denied - N418 Misrouted'", "8. KEY FINDING - Note second row: 2025-12-10, Corrected, CLM-2025-00028-R1, $3,410.00, status 'Rejected - Not under capitation agreement' - Valley Medical Group also rejected the claim", "9. Recognize that the standard resolution (rerouting to Valley Medical Group) will NOT work because Valley has already rejected the claim - this is a double rejection", "10. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab (second tab, next to Retest). Review the EOB document showing N418 denial This step is evaluated.", "11. On the remittance, observe: CARC N418 - 'Claim submitted to incorrect payer. Services under delegated capitation arrangement.' Also note DEL GRP: Valley Medical Group on the patient/claim info line", "12. Click the patient name 'Patel, Raj' in the Patient Info Banner (light blue bar below the purple header) - this links to the Patient Inquiry page", "13. On Patient Inquiry, review the Coverage section: Pacific Health Alliance, Medicaid Managed Care plan, member ID PHA678901234, Status: Active", "14. Note there is NO payer portal for Pacific Health Alliance (no portal credentials in the right panel) - resolution must happen through EMR disposition", "15. You are now on the Patient Inquiry page. Click '< Back' (top-left of the page header) to return to the DEN-028 detail page. Alternatively use navigate_to('/emr/denied/DEN-028') to go directly.", "16. Once on the DEN-028 detail page (/emr/denied/DEN-028), go to the Actions panel on the right. Select 'Escalate to Supervisor' from the Triage Disposition dropdown.", "17. IMMEDIATELY after selecting disposition, click the Triage Note field and type your note WITHOUT clicking any other tabs - clicking tabs clears the note. In the Triage Note text area write: (a) N418 misrouted denial from Pacific Health Alliance for $3,100, (b) Pacific Health rejected claim and directed it to delegated entity Valley Medical Group, (c) Valley Medical Group ALSO rejected the claim - double rejection, (d) standard reroute to Valley Medical Group will not resolve the issue because they already rejected, (e) recommend supervisor intervention for payer dispute resolution between Pacific Health Alliance and Valley Medical Group", "18. Click 'Submit Disposition' button to finalize the escalation - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-3-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-028", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-3-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-3-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry for coverage and delegation details", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-3-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected Escalate to Supervisor disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Escalate to Supervisor", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-3-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent documented investigation in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-3-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note identifies N418 misrouted denial from Pacific Health Alliance", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the N418 misrouted denial from Pacific Health Alliance? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-3-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note explains Pacific Health rejected as misrouted to Valley Medical Group", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that Pacific Health rejected the claim as misrouted to Valley Medical Group? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-3-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note explains Valley Medical Group also rejected the claim", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that Valley Medical Group ALSO rejected the claim - neither entity accepted? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-3-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends supervisor intervention for payer dispute resolution", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend supervisor intervention or escalation for payer dispute resolution since both entities rejected? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-3-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains why standard reroute is insufficient", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why a standard reroute to Valley Medical Group would NOT resolve this denial (because Valley already rejected the claim)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Escalate to Supervisor" }, "rationale": "N418 misrouted claim denial. Pacific Health Alliance says claim should go to Valley Medical Group (delegated entity), but Valley Medical Group has also rejected the claim. Neither entity will accept responsibility. Standard appeal or resubmission will not resolve the issue. Must escalate to supervisor for payer dispute resolution." }, { "id": "denial-hard-4", "title": "Multiple Modifier Errors -- Per-Line Coding Correction", "instruction": "For denial DEN-029 (Kim, Sophia), investigate the CO-4 coding error denial. Review the denial details and remittance image carefully, examining each line item individually. Some line items may need corrections, others may be correctly coded -- do not assume all lines have the same issue. For each line, determine independently whether a correction is needed and what specific correction applies. Select the appropriate disposition and document a detailed per-line analysis explaining your determination for each CPT code in your triage note.", "goal": "For denial DEN-029 (Kim, Sophia), investigate the CO-4 coding error denial. Review the denial details and remittance image carefully, examining each line item individually. Some line items may need corrections, others may be correctly coded -- do not assume all lines have the same issue. For each line, determine independently whether a correction is needed and what specific correction applies. Select the appropriate disposition and document a detailed per-line analysis explaining your determination for each CPT code in your triage note.", "difficulty": "hard", "category": "ambiguous_decision", "challengeType": "denial_triage", "possible": true, "points": 10, "configTaskId": "denial_hard_4", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Kim, Sophia" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate DEN-029 for Kim, Sophia - click on the patient's NAME (purple underlined button) to open the denial detail page. Single-clicking the row body only selects it; click the name text specifically or double-click the row.", "2. The denial detail page opens on the Retest tab. In the Errors section (orange border at top), note: CO-4 - 'The procedure code is inconsistent with the modifier used or a required modifier is missing.'", "3. In the Claim/Invoice Header section, note: CLM-2025-00029, Aetna, $2,750 remaining balance, service date 2025-11-10, appeal deadline 2026-03-01", "4. Review the three Line Item Cards below the header. Each card shows a CPT code with its billed/paid/adjustment amounts: Ln 1: 99214 (office visit E/M), Ln 2: 93000 (EKG), Ln 3: 36415 (venipuncture). All three show CO-4 denial reason and remark codes MA130, N657", "5. Click on each Line Item Card to expand it - review the Allowed, Denied, Patient Resp, and Remark Codes for each line", "6. Scroll down to the Notes History section. Note: 'CO-4 coding error - review each line item individually'", "7. [REQUIRED - DO THIS STEP] Click the 'Remittance Image' tab (second tab next to Retest). Review the EOB showing CO-4 denial. This step is evaluated.", "8. On the remittance, examine the claim detail table - each service line shows PROC/CPT, BILLED, ALLOWED, PROV PD, and CARC columns. All three lines show CARC CO-4 and RARC MA130", "9. Note the ADJUSTMENT REASON CODES section at bottom of the remittance: CO-4 - 'The procedure code is inconsistent with the modifier used or a required modifier is missing.' and MA130 - 'Your claim contains incomplete and/or invalid information'", "10. Analyze CPT 99214: this is an E/M service billed on the same day as a procedure (93000 EKG). Per CMS billing rules, same-day E/M requires modifier -25 to indicate a separately identifiable service", "11. Analyze CPT 93000: this is an EKG. If the facility only performed the technical component (tracing), it should be billed with modifier -TC. Without -TC, it bills as the global service (technical + professional)", "12. Analyze CPT 36415: this is venipuncture (blood draw). This is a standalone service that does not require a modifier - it is correctly coded", "13. Determine the correction plan: add -25 to 99214, add -TC to 93000, leave 36415 as-is", "14. In the right panel, scroll to the Actions section. Select 'Corrected Claim - Resubmit' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 15. In the Triage Note text area below the dropdown, write a detailed note documenting: per-line analysis for all 3 CPTs, specific modifier needed for each (99214 needs -25, 93000 needs -TC, 36415 is correct), explanation of the billing rules violated (same-day E/M rule, technical component rule), and the corrected claim resubmission plan", "16. Click 'Submit Disposition' to finalize - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-4-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-029", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-4-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image to analyze per-line remark codes", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-4-eval-3", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected Corrected Claim - Resubmit disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Corrected Claim - Resubmit", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-4-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent documented correction plan in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-4-eval-5", "type": "llm_judge", "category": "Documentation", "description": "Note identifies -25 modifier needed for CPT 99214", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that modifier -25 is needed for CPT 99214 (same-day E/M with procedure)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-4-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note identifies -TC modifier needed for CPT 93000", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that modifier -TC is needed for CPT 93000 (technical component for EKG)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-4-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note correctly identifies 36415 does not need correction", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note correctly identify that CPT 36415 (venipuncture) is correctly coded and does not need correction? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-4-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains same-day E/M billing rule", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the same-day E/M + procedure billing rule requiring -25 modifier? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-4-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains technical component billing rule", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the technical component billing requiring -TC modifier? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-4-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note provides corrected claim resubmission plan", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note provide a corrected claim resubmission plan with specific modifiers for each line? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Corrected Claim - Resubmit" }, "rationale": "CO-4 coding error denial with multiple CPT codes on same date of service. CPT 99214 needs -25 modifier (same-day E/M with procedure), CPT 93000 needs -TC modifier (technical component for EKG), and CPT 36415 (venipuncture) is correctly coded. Corrected claim should be resubmitted with proper modifiers." }, { "id": "denial-hard-5", "title": "COB Decision - Dual Coverage Physical Therapy Appeal", "instruction": "For denial DEN-030 (Foster, James), investigate the CO-50 medical necessity denial from Anthem ($7,500 for physical therapy post-ACL repair). Review the denial details, remittance image, and patient inquiry. Pay careful attention to ALL insurance coverage information. Download the clinical documentation. Based on your findings, determine the best course of action, navigate to the appropriate payer portal to take action, and set a follow-up task. Return to EMR and document your complete analysis including all coverage findings in a triage note.", "goal": "For denial DEN-030 (Foster, James), investigate the CO-50 medical necessity denial from Anthem ($7,500 for physical therapy post-ACL repair). Review the denial details, remittance image, and patient inquiry. Pay careful attention to ALL insurance coverage information. Download the clinical documentation. Based on your findings, determine the best course of action, navigate to the appropriate payer portal to take action, and set a follow-up task. Return to EMR and document your complete analysis including all coverage findings in a triage note.", "difficulty": "hard", "category": "ambiguous_decision", "challengeType": "denial_triage", "possible": true, "points": 17, "configTaskId": "denial_hard_5", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Foster, James" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate DEN-030 for Foster, James - click on the patient's NAME (purple underlined button) to open the denial detail page. Single-clicking the row body only selects it; click the name text specifically or double-click the row.", "2. The denial detail page opens on the Retest tab. In the Errors section (orange border at top), note: CO-50 - 'These are non-covered services because this is not deemed a medical necessity by the payer.'", "3. In the Claim/Invoice Header section, note: CLM-2025-00030, Anthem Blue Cross, $7,500 remaining balance. Review the Line Item Cards for CPTs 97110/97140/97530 (physical therapy). Note appeal deadline 2026-02-28 in the title bar", "4. Scroll down to the Notes History section. Note: 'Review all coverage information before determining action'", "5. [REQUIRED - DO THIS STEP] Click the 'Remittance Image' tab (second tab next to Retest). Review the EOB showing CO-50 medical necessity denial. This step is evaluated - do NOT skip it.", "6. Click the patient name 'Foster, James' link in the Patient Info Banner (light blue bar below the purple header) - navigates to Patient Inquiry page", "7. On Patient Inquiry, review the Coverage section: Anthem Blue Cross, PPO plan, member ID ANT678901345, Status: Active", "8. KEY FINDING: Note the dual coverage - patient has secondary insurance through Aetna (member ID AET445566778) via spouse employer plan. Anthem Blue Cross is primary, Aetna is secondary", "9. You are now on the Patient Inquiry page. Click '< Back' (top-left of the page header) to return to the DEN-030 detail page. Alternatively use navigate_to('/emr/denied/DEN-030') to go directly.", "10. On the Retest tab, scroll down to the Documents section. Click the 'View ->' button on the RIGHT side of the 'Clinical Notes - PT Post-ACL Repair.pdf' row (NOT the document name - that does nothing). This opens the document viewer. Note: 6 weeks post ACL reconstruction, structured PT required per AAOS/APTA guidelines.", "11. Click 'Download' on the clinical notes (will be needed for the appeal)", "11b. After downloading, click '< Back' ONCE to return to the DEN-030 detail page. Do NOT click back a second time - that takes you to the worklist. If you end up on the worklist by mistake, use navigate_to('/emr/denied/DEN-030') to return directly.", "12. In the right panel, note the Portal Access section showing Payer B portal credentials (provider@payerb.com / demo123). Click the 'Start Appeal' button in the Actions section to navigate to the Payer B portal", "13. On the Payer B login page, enter provider@payerb.com and demo123, then click Login", "14. Click 'Appeals' in the sidebar navigation to access the Claim Status Inquiry page", "15. Enter member ID ANT678901345 in the Member ID field and click Search", "16. In the search results, click on the claim row for CLM-2025-00030 to view the Claim Status Detail", "17. Click the 'Dispute Claim' button", "18. Fill in the Contact Name field with 'Dr. Michael Chen'", "19. In the Supporting Rationale field, write: (a) PT post-ACL repair is standard of care per AAOS/APTA guidelines, (b) patient is 6 weeks post-surgery requiring 12-16 weeks structured PT, (c) strong clinical progress documented, (d) appealing primary (Anthem) first per COB rules before pursuing secondary (Aetna)", "20. Scroll to the 'Available Documents from EMR' section - the downloaded clinical notes should appear. Click '+ Attach' next to 'Clinical Notes - PT Post-ACL Repair.pdf'.", "21. Click 'Submit Appeal'. Note the confirmation number (format APL-ANT-XXXXXXXX)", "22. Click 'Return to EMR' to go back to the denials workqueue. You are now on the WORKLIST. Click the PATIENT NAME button in that row to re-open the DEN-030 detail page - single-clicking the row body only selects it. If the name click does not navigate, double-click the row. As a last resort use navigate_to('/emr/denied/DEN-030') to go directly.", "23. In the right panel, scroll to the Actions section. Click 'Add Follow-up Task'. Set a follow-up date (e.g., 30 days out), select reason 'Awaiting payer response', and click 'Schedule Follow-up'", "24. In the Actions section, select 'Appeal Filed' from the Triage Disposition dropdown.", "25. IMMEDIATELY after selecting disposition, click the Triage Note field and type your note WITHOUT clicking any other tabs - clicking tabs clears the note. Write a comprehensive note. Start with: CO-50 medical necessity denial for PT post-ACL repair ($7,500)", "26. Continue in the triage note: state that dual coverage was identified - Anthem Blue Cross is the primary insurer (member ID ANT678901345). Aetna is the secondary insurer (member ID AET445566778), obtained through the patient's spouse's employer plan. Explicitly mention that Aetna coverage is via the spouse", "27. Continue in the triage note: explain the decision to appeal the primary insurer (Anthem) first. State that PT post-ACL repair is standard of care per AAOS/APTA guidelines, the clinical evidence is strong, and therefore appealing the primary denial has a high likelihood of success. Per COB rules, the primary insurer must be resolved before billing the secondary", "28. Continue in the triage note: include the appeal confirmation number from the Payer B portal (format APL-ANT-XXXXXXXX)", "29. Continue in the triage note: explain why billing the secondary (Aetna) directly without first appealing the primary (Anthem) would be incorrect per COB rules", "30. Continue in the triage note: include a follow-up plan for secondary billing - if the primary appeal with Anthem fails, the next step is to bill the remaining balance to Aetna as secondary insurance. Note the follow-up task has been set to track this", "31. Click 'Submit Disposition' to finalize - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-5-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-030", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry for coverage details", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-4", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent accessed payer portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent filed appeal on Payer B (Anthem)", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-7", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references PT post-ACL as standard of care", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_b_state. full_state. appealActions) reference that physical therapy post-ACL repair is standard of care? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-5-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale mentions COB rules for primary appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_b_state. full_state. appealActions) mention appealing the primary insurer (Anthem) first per coordination of benefits rules? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-5-eval-9", "type": "jmespath", "category": "Task Resolution", "description": "Agent added follow-up task", "points": 1, "query": "full_state.agentActions.addedFollowUpTask", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-10", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-11", "type": "jmespath", "category": "Documentation", "description": "Agent documented analysis in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-5-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note identifies Anthem Blue Cross as primary insurer", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify Anthem Blue Cross as the primary insurer (member ID ANT678901345)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-5-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note identifies Aetna as secondary insurer via spouse", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify Aetna as the secondary insurer obtained through the spouse's employer plan? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-5-eval-14", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains primary appeal decision", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the decision to appeal primary (Anthem) first based on strong clinical case for PT post-ACL repair? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-5-eval-15", "type": "llm_judge", "category": "Documentation", "description": "Note includes secondary billing follow-up", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include a follow-up plan for secondary billing through Aetna if primary appeal fails? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-5-eval-16", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains why skipping primary appeal is incorrect", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why billing the secondary (Aetna) directly without first appealing the primary (Anthem) would be incorrect per COB rules? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-5-eval-17", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include an appeal confirmation number from the Payer B portal (e.g., APL-ANT-XXXXXXXX)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "CO-50 medical necessity denial for physical therapy post-ACL repair ($7,500). Patient has dual coverage: Anthem Blue Cross (primary, ANT678901345) and Aetna (secondary, AET445566778 via spouse). Appeal primary (Anthem) first with strong clinical case for PT. Set follow-up for secondary billing through Aetna if primary appeal fails." }, { "id": "denial-hard-6", "title": "Same-Patient Hospital Stay - Batch Denial Analysis", "instruction": "In the denials workqueue, identify all 4 denials for Adams, Victoria from the list view - noting Error Codes, Paid Amounts, and Invoice (claim IDs). These are from a single hospital stay. Click into the priority denial (CO-50, $8,500, Account: Adams, Victoria) to review details and remittance in depth, then navigate to Payer B to file the medical necessity appeal. Optionally review the CO-197 denial details to confirm the auth routing. Set a follow-up task. Return to EMR and document a unified strategy covering all four denials - including the relationship between them, per-denial action plans, and total dollars at risk - in a triage note.", "goal": "In the denials workqueue, identify all 4 denials for Adams, Victoria from the list view - noting Error Codes, Paid Amounts, and Invoice (claim IDs). These are from a single hospital stay. Click into the priority denial (CO-50, $8,500, Account: Adams, Victoria) to review details and remittance in depth, then navigate to Payer B to file the medical necessity appeal. Optionally review the CO-197 denial details to confirm the auth routing. Set a follow-up task. Return to EMR and document a unified strategy covering all four denials - including the relationship between them, per-denial action plans, and total dollars at risk - in a triage note.", "difficulty": "hard", "category": "batch_analysis", "challengeType": "denial_triage", "possible": true, "points": 18, "configTaskId": "denial_hard_6", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Adams, Victoria" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), scan the list view and locate all rows where the Account column shows 'Adams, Victoria'", "2. For the first Adams row, note the Error Code, Paid Amount, and Invoice (claim ID) from the table columns", "3. For the second Adams row, note the Error Code, Paid Amount, and Invoice from the table columns", "4. For the third Adams row, note the Error Code, Paid Amount, and Invoice from the table columns", "5. For the fourth Adams row, note the Error Code, Paid Amount, and Invoice from the table columns", "6. Confirm all 4 Adams denials from the list: CO-50 / $8,500 (DEN-036), CO-197 / $12,300 (DEN-037), CO-4 / $950 (DEN-038), CO-50 / $3,200 (DEN-039)", "7. Calculate total dollars at risk from the Paid Amount column: $8,500 + $12,300 + $950 + $3,200 = $24,950", "8. Single-click the CO-50 / $8,500 row (DEN-036) to select it and reveal the bottom 4-panel grid", "9. In the Payment Details panel (top-left), confirm the Payer is Anthem Blue Cross and review the Billed/Paid amounts and Invoice", "10. In the Payment Errors panel (top-right), review the CO-50 error code and remark codes", "11. Single-click the CO-197 / $12,300 row (DEN-037) to see its bottom panels", "12. In the Payment Details panel, confirm the Payer is Anthem Blue Cross", "13. In the Payment Errors panel, note CO-197 (precertification/authorization not obtained) - this is a prior auth issue, not an appeal", "14. Double-click the CO-50 / $8,500 row (DEN-036) to open the denial detail page", "15. On the Retest tab (default), review the Errors section (orange border) confirming CO-50 (medical necessity)", "16. In the Claim/Invoice Header, confirm: ER visit, $8,500 denied amount, Anthem Blue Cross", "17. Review the Line Item Card for CPT 99285", "18. Note the appeal deadline in the title bar", "19. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab This step is evaluated.", "20. Review the remittance advice document noting the CO-50 reason and billed/allowed amounts", "21. (Optional) Click the '<- Back to Denials' button to return to the workqueue", "22. (Optional) Double-click DEN-037 to open its detail page", "23. (Optional) On the Retest tab, confirm CO-197 for emergent surgery ($12,300) - confirming this should be routed to prior auth team for retroactive authorization", "24. (Optional) Click '<- Back to Denials' to return to the workqueue", "25. (Optional) Double-click DEN-036 to return to its detail page", "26. On DEN-036's detail page, in the right panel note the Portal Access section showing Payer B credentials (provider@payerb.com / demo123)", "27. Click the 'Start Appeal' button to navigate to the Payer B portal", "28. On the Payer B login page, enter provider@payerb.com in the username field", "29. Enter demo123 in the password field", "30. Click the Login button", "31. Click 'Appeals' in the sidebar navigation", "32. On the Claim Status Inquiry page, enter member ID ANT901234678 in the Member ID field and click Search", "33. In the search results, click on claim CLM-2025-00036 (Adams, Victoria, $8,500, CO-50) to view the Claim Status Detail", "34. Click the 'Dispute Claim' button to open the dispute form", "35. Fill in the Contact Name field with 'Dr. James Liu' (the provider listed on the denial)", "36. In Supporting Rationale, explain that this is a medical necessity appeal for an ER visit that was part of a connected episode of care (hospital stay 10/28-11/02). Reference the denial code (CO-50), claim details (DEN-036, $8,500 denied amount, patient Adams, Victoria), state that the ER visit was medically necessary and led to emergent surgery, and indicate that clinical documentation is attached. Reference related claims in the episode: CLM-2025-00037, CLM-2025-00038, CLM-2025-00039", "37. (Optional) If any EMR documents appear in the Supporting Documentation section, click '+ Attach' next to each one to attach them. If the section shows 'No documents attached', documents are optional — you do NOT need to go back to EMR. Proceed to submit.", "38. CRITICAL: Scroll down if needed to ensure the Submit Appeal button is FULLY VISIBLE before clicking. The Submit button is enabled as long as Contact Name and Supporting Rationale are filled in — do NOT go back to EMR looking for documents. Click 'Submit Appeal'.", "39. Note the confirmation number displayed on screen", "40. Click 'Return to EMR' to go back to the denials workqueue", "41. Double-click DEN-036 to navigate to the denial detail page", "42. In the right panel, scroll to the Actions section", "43. Click 'Add Follow-up Task' to set a follow-up for the appeal response (e.g., 30 days out, reason: Awaiting payer response for DEN-036 appeal)", "44. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown", "45. In the Triage Note text area, start by stating that all 4 denials (DEN-036, DEN-037, DEN-038, DEN-039) are from the same hospital stay for Adams, Victoria (dates 10/28-11/02). Argue that these should be treated as a connected episode of care: the ER visit (DEN-036) led to emergent surgery (DEN-037), which required labs (DEN-038) and follow-up care (DEN-039)", "46. Continue in the triage note: calculate total dollars at risk: $8,500 + $12,300 + $950 + $3,200 = $24,950", "47. Continue in the triage note: recommend a medical necessity appeal for DEN-036 (CO-50, ER visit, $8,500) - note the appeal has been filed on Payer B and include the appeal confirmation number", "48. Continue in the triage note: recommend a medical necessity appeal for DEN-039 (CO-50, follow-up, $3,200) referencing the same episode of care", "49. Continue in the triage note: for DEN-037 (CO-197, emergent surgery, $12,300), recommend routing to the prior auth team for retroactive authorization - this is a prior auth issue, NOT an appeals issue. The surgery was emergent so retro auth should be approved", "50. Continue in the triage note: for DEN-038 (CO-4, lab work, $950), identify that a modifier correction is needed and recommend resubmission with the corrected modifier", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 51. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then continue in the triage note: provide a unified sequenced action plan with timeline - step 1: den-036 co-50 appeal already submitted (confirmation #), awaiting response (follow-up in 30 days); step 2: route den-037 to prior auth team for retro auth immediately; step 3: file den-039 co-50 appeal referencing same episode of care (within 1 week); step 4: resubmit den-038 with corrected modifier (within 1 week)", "52. Continue in the triage note (do NOT click any tabs): note that DEN-037 has been routed to the prior auth team - once they approve the retroactive authorization, use that approval as evidence when filing the DEN-039 CO-50 appeal, since it proves the emergent surgery was authorized and the follow-up was part of a valid episode of care. In other words, hold off on filing DEN-039's appeal until the prior auth team resolves DEN-037's retro auth, because that approval strengthens the medical necessity argument for the entire episode", "53. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-6-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-6-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-6-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent accessed Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-6-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent filed appeal on Payer B for ER visit medical necessity denial", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-6-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-6-eval-6", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Appeal rationale references medical necessity and episode of care", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_b_state. full_state. appealActions) reference the ER visit as medically necessary and mention the connected hospital stay or related claims? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-6-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent added follow-up task to track appeal response", "points": 1, "query": "full_state.agentActions.addedFollowUpTask", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-6-eval-8", "type": "jmespath", "category": "Documentation", "description": "Agent documented unified strategy in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-6-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies all 4 denials from same hospital stay", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that all 4 denials (DEN-036, DEN-037, DEN-038, DEN-039) are from the same hospital stay for Adams, Victoria (10/28-11/02)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note calculates total dollars at risk", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note calculate the total dollars at risk as approximately $24,950 ($8,500 + $12,300 + $950 + $3,200)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends medical necessity appeal for DEN-036", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend a medical necessity appeal for DEN-036 (CO-50, ER visit, $8,500)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-12", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends medical necessity appeal for DEN-039", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend a medical necessity appeal for DEN-039 (CO-50, follow-up, $3,200)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-13", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends routing CO-197 to prior auth team for retro auth", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend routing the CO-197 denial (DEN-037 emergent surgery) to the prior auth team for retroactive authorization? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Note identifies modifier correction for CO-4 denial", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify modifier correction needed for the CO-4 denial (DEN-038 lab work)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-15", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note argues episode-of-care connection", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note argue that the denials should be appealed as a connected episode of care (ER visit led to emergent surgery, which required labs and follow-up)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-16", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note correctly routes CO-197 to prior auth team instead of appealing", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note correctly identify that DEN-037 (CO-197, emergent surgery) should be routed to the prior auth team for retroactive authorization rather than handled as an appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-17", "type": "llm_judge", "category": "Documentation", "description": "Note includes DEN-036 appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number from the Payer B portal for the DEN-036 medical necessity appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-6-eval-18", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note sequences DEN-039 appeal after DEN-037 retro auth", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note state that the DEN-039 CO-50 appeal should wait until after DEN-037's authorization (CO-197) is approved? Accept any phrasing that conveys this dependency (e.g. 'hold DEN-039 appeal pending auth approval', 'file DEN-039 after CO-197 resolves'). Score 1.0 if the dependency is stated, 0.0 if not.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "Four denials from a single hospital stay (10/28-11/02) for Adams, Victoria. Total at risk: ~$24,950. A&D specialist files medical necessity appeal for DEN-036 ($8,500 ER visit, CO-50) on Payer B with episode-of-care argument. DEN-037 ($12,300 emergent surgery, CO-197) is routed to the prior auth team for retro auth - not an appeals issue. DEN-039 ($3,200 follow-up, CO-50) also needs medical necessity appeal. DEN-038 ($950 lab, CO-4) needs modifier correction and resubmission." }, { "id": "denial-hard-7", "title": "Deadline Triage - Retro Auth for Actionable Denial, Escalate Expired Denial", "instruction": "Scan the denials workqueue to identify denials with approaching or expired appeal deadlines. Open the actionable non-expired denial - Reyes, Carmen (DEN-041) - and review its details and remittance image. Navigate to Payer B to file a retro authorization request for this CO-197 no-auth denial before the deadline expires. Also check the Brooks denial (DEN-040) to confirm its deadline status. Return to EMR and document your deadline triage findings and the retro auth action taken in a triage note on DEN-041.", "goal": "Scan the denials workqueue to identify denials with approaching or expired appeal deadlines. Open the actionable non-expired denial - Reyes, Carmen (DEN-041) - and review its details and remittance image. Navigate to Payer B to file a retro authorization request for this CO-197 no-auth denial before the deadline expires. Also check the Brooks denial (DEN-040) to confirm its deadline status. Return to EMR and document your deadline triage findings and the retro auth action taken in a triage note on DEN-041.", "difficulty": "hard", "category": "deadline_triage", "challengeType": "denial_triage", "possible": true, "points": 11, "configTaskId": "denial_hard_7", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Reyes, Carmen" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), scan the list to identify denials. Click on each row to view details in the bottom 4-panel grid", "2. Double-click the Brooks row (DEN-040) to open its denial detail page. Note the appeal deadline in the title bar: 2026-01-05 - this is EXPIRED", "3. Click the '<- Back to Denials' button to return to the workqueue", "4. Double-click the Reyes row (DEN-041) to open its denial detail page", "5. Note the appeal deadline in the title bar: 2026-05-15 - only deadline 2026-05-15. This is the actionable denial with a valid deadline", "6. On the Retest tab (default), review the Errors section (orange border) confirming CO-197 (no prior authorization)", "7. In the Claim/Invoice Header, note the denied amount ($4,800), procedure details, and payer (Anthem Blue Cross)", "8. [REQUIRED - DO NOT SKIP] IMPORTANT: Click the 'Remittance Image' tab (second tab next to Retest) - you MUST view the remittance image before proceeding. Review the remittance advice and note the CARC/RARC codes This step is evaluated.", "9. In the right panel, note the Portal Access section showing Payer B credentials (provider@payerb.com / demo123). Click 'Start Appeal' to navigate to the Payer B portal", "10. On the Payer B login page, enter provider@payerb.com and demo123, then click Login", "11. Click 'Auth Inquiry' in the sidebar navigation to check if any authorization exists for this patient", "12. Search for member ID ANT012345789 to check authorization status", "13. Note the auth inquiry findings - confirm no valid authorization exists for this service", "14. Click 'Appeals' in the sidebar navigation", "15. On the Claim Status Inquiry page, enter member ID ANT012345789 and click Search", "16. In the search results, click on claim CLM-2025-00041 (Reyes, $4,800, CO-197) to view the Claim Status Detail", "17. Click the 'Dispute Claim' button to open the dispute form", "18. Fill in the Contact Name field with 'Dr. Daniel Adams' (the provider listed on the denial)", "19. In Supporting Rationale, reference the denial code (CO-197), claim details (DEN-041, $4,800, patient Reyes), the deadline (2026-05-15), and request retroactive authorization review. Use admin-appropriate language", "20. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal' to submit. note the confirmation number", "21. Click 'Return to EMR' to go back to the denials workqueue", "22. Double-click the Reyes row (DEN-041) to navigate to its denial detail page", "23. [REQUIRED - DO NOT SKIP] In the right panel, scroll to the Actions section. In the Triage Note text area, write: DEN-041 (Reyes, $4,800, CO-197) - appeal deadline 2026-05-15 (actionable). This is a no-auth denial. Reviewed denial details and remittance image. Navigated to Payer B and submitted retro authorization request. Include the confirmation number This step is evaluated.", "24. Continue in the triage note: Also identified during workqueue scan - DEN-040 (Brooks, $6,100, CO-50) has an EXPIRED appeal deadline of 2026-01-05. This denial cannot be appealed through standard channels. Recommend escalating DEN-040 to supervisor for late appeal exception review given the $6,100 amount", "25. Continue in the triage note: PRIORITY ACTION PLAN - (1) DEN-041 retro auth submitted, monitor for response before 2026-05-15 deadline, (2) Escalate DEN-040 to supervisor immediately for expired deadline review", "26. Select 'Escalate to Supervisor' from the Triage Disposition dropdown", "27. Click 'Submit Disposition' to save the triage note and disposition" ], "evals": [ { "id": "denial-hard-7-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-7-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-7-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent accessed Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-7-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Escalate to Supervisor", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-7-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent documented deadline triage in EMR triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-7-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note identifies DEN-041 with its deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify DEN-041 (Reyes, $4,800) as a denial requiring action, mentioning its deadline of 2026-05-15? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-7-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note flags DEN-040 expired deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify DEN-040 (Brooks, $6,100) as having an EXPIRED deadline (2026-01-05)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-7-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note documents CO-197 denial and retro auth action", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention that DEN-041 is a CO-197 no-auth denial and that retro authorization was requested or action was taken on Payer B? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-7-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note reports payer portal findings", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note report findings from the Payer B portal visit (e. g., claim status, auth inquiry results, or retro auth submission)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-7-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note recommends escalation for expired DEN-040", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend escalating DEN-040 (Brooks) to a supervisor for late appeal exception review due to the expired deadline? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-7-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note provides prioritized action plan", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note provide a prioritized action plan that addresses DEN-041 (retro auth submitted, monitor before deadline) and flags DEN-040 for supervisor review? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Escalate to Supervisor" }, "rationale": "DEN-041 (Reyes, $4,800, CO-197) has deadline 2026-05-15 - only actionable deadline. This is the actionable denial with a valid deadline. It's a no-auth denial requiring retro authorization on Payer B. DEN-040 (Brooks, $6,100, CO-50) has deadline 2026-01-05 which is expired - cannot be appealed normally, needs supervisor escalation. The disposition is 'Escalate to Supervisor' because the overall situation includes an expired high-value denial that requires supervisor intervention." }, { "id": "denial-hard-8", "title": "Aetna CO-50 Priority Triage - Identify Top 3, Appeal Highest-Value, Flag Expired Deadline", "instruction": "Filter the denials workqueue to show only Aetna denials. Scan the filtered list to identify the three highest-value CO-50 (medical necessity) denials. Open the highest-value CO-50 denial, review its details and remittance image, and file an appeal on the Payer A portal with clinical documentation. Then check the third-highest CO-50 denial to review its appeal deadline. Write a triage note on the highest-value denial documenting your investigation, the appeal filed, and prioritized next steps for the other high-value CO-50 denials.", "goal": "Filter the denials workqueue to show only Aetna denials. Scan the filtered list to identify the three highest-value CO-50 (medical necessity) denials. Open the highest-value CO-50 denial, review its details and remittance image, and file an appeal on the Payer A portal with clinical documentation. Then check the third-highest CO-50 denial to review its appeal deadline. Write a triage note on the highest-value denial documenting your investigation, the appeal filed, and prioritized next steps for the other high-value CO-50 denials.", "difficulty": "hard", "category": "batch_analysis", "challengeType": "denial_triage", "possible": true, "points": 12, "configTaskId": "denial_hard_8", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "O'Brien, Margaret" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), click the 'Filter' button in the toolbar to reveal the filter dropdowns", "2. Select 'Aetna' from the payer filter dropdown to isolate all Aetna denials", "3. Scan the filtered list. The table columns show Error Code, Error Message, Paid Amount, Account (patient name), Account Type, Batch No, Batch Date, Check Number, and Invoice (claim ID). Click on each row to view additional details in the bottom 4-panel grid", "4. Identify all CO-50 denials in the list by checking the Error Code column. Note each CO-50 denial's Paid Amount and Account name", "5. Determine the three highest-value CO-50 denials: DEN-031 (O'Brien, $22,000), DEN-044 (Price, $18,500), DEN-040 (Brooks, $6,100)", "6. Double-click DEN-031 (O'Brien, $22,000) - the highest-value CO-50 - to open its detail page", "7. On the Retest tab (default), review the Errors section (orange border) confirming the CO-50 medical necessity denial", "8. In the Claim/Invoice Header, note the denied amount ($22,000), procedure description (cardiac rehabilitation), and appeal deadline (2026-03-10)", "9. Check the Documents section for clinical documentation on file", "10. [REQUIRED - DO NOT SKIP] IMPORTANT: Click the 'Remittance Image' tab (second tab next to Retest) - you MUST view the remittance image before proceeding. Review the remittance advice and note the CARC/RARC codes and payer remarks This step is evaluated.", "11. In the right panel, note the Portal Access section showing Payer A credentials (provider@payera.com / demo123). Click 'Start Appeal' to navigate to the Payer A portal", "12. On the Payer A login page, enter provider@payera.com and demo123, then click Login", "13. On the Claim Status Inquiry page, enter member ID AET678901543 in the Member ID field and click Search", "14. In the search results, click on claim CLM-2025-00031 (O'Brien, $22,000, CO-50) to view the Claim Status Detail", "15. Click the 'Dispute Claim' button to open the dispute form", "16. Fill in the Contact Name field with 'Dr. David Williams' (the provider listed on the denial)", "17. In Supporting Rationale, explicitly write that this is a CO-50 denial for DEN-031, patient O'Brien, denied amount $22,000, and that supporting clinical documentation is attached. Use admin-appropriate language - do NOT write clinical justifications, and do NOT omit any of those four elements.", "18. Click 'Attach Document' to attach supporting clinical documentation", "19. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal' to submit the dispute. WAIT for the confirmation screen/banner to appear before leaving the portal, and copy the exact confirmation number displayed on screen.", "20. Click 'Return to EMR' to go back to the denials workqueue", "21. On the workqueue, double-click DEN-040 (Brooks, $6,100, CO-50) to open its detail page", "22. In the Claim/Invoice Header, check the appeal deadline - it is 2026-01-05, which is EXPIRED. This denial cannot be appealed through standard channels", "23. Click the back button or navigate back to the workqueue, then double-click DEN-031 (O'Brien) to return to its detail page", "24. [REQUIRED - DO NOT SKIP] In the right panel, scroll to the Actions section. In the Triage Note text area, write: DEN-031 (O'Brien, $22,000, CO-50) is the highest-value Aetna medical necessity denial. Reviewed denial details, remittance image, and clinical documentation. Filed appeal on Payer A - include the exact confirmation number from step 19. This step is evaluated.", "25. Continue in the triage note: DEN-044 (Price, $18,500, CO-50) is the second-highest-value Aetna CO-50 denial and should be triaged and appealed next", "26. Continue in the triage note: DEN-040 (Brooks, $6,100, CO-50) has an EXPIRED appeal deadline of 2026-01-05. This denial cannot be appealed through standard channels. Recommend escalation to supervisor for late appeal exception review or write-off", "27. Continue in the triage note: PRIORITY NEXT STEPS - (1) Appeal DEN-044 (Price, $18,500) immediately as deadline is 2026-03-08, (2) Escalate DEN-040 (Brooks, $6,100) to supervisor for expired deadline review", "28. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown", "29. Click 'Submit Disposition' to save the triage note and disposition" ], "evals": [ { "id": "denial-hard-8-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-8-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-8-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent accessed Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-8-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent filed appeal on Payer A for DEN-031", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-8-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-8-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent documented triage note in EMR", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-8-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Appeal rationale references CO-50, claim details, and clinical documentation", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_a_state. full_state. appealActions) reference the denial code (CO-50), include claim details (DEN-031, denied amount ~$22,000, patient O'Brien), and indicate that clinical documentation was attached? The rationale should use admin-appropriate language - NOT clinical justifications. Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-8-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note identifies DEN-031 as highest-value Aetna CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify DEN-031 (O'Brien, ~$22,000) as the highest-value Aetna CO-50 denial? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-8-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies DEN-044 as second-highest CO-50 needing follow-up", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention DEN-044 (Price, ~$18,500) as another high-value Aetna CO-50 denial needing follow-up triage? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-8-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note flags DEN-040 expired deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note note that DEN-040 (Brooks, ~$6,100) has an expired appeal deadline (2026-01-05) and therefore cannot be appealed through standard channels? Score 1.0 if the note mentions the expired deadline, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-8-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number for DEN-031? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-8-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note recommends prioritized next steps for remaining CO-50 denials", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend prioritized next steps - e. g., appeal DEN-044 next, and escalate or write off DEN-040 due to expired deadline? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "Filter workqueue by Aetna, scan for CO-50 denials. The three highest-value are DEN-031 (O'Brien, $22,000), DEN-044 (Price, $18,500), and DEN-040 (Brooks, $6,100). Open DEN-031 for full investigation - CO-50 medical necessity denial for cardiac rehab, clinical documentation on file. File appeal on Payer A with clinical docs attached. Check DEN-040 and discover its appeal deadline (2026-01-05) is expired - cannot be appealed normally, recommend escalation or write-off. DEN-044 is the next priority for appeal. Write triage note on DEN-031's record documenting the investigation, appeal confirmation, and prioritized next steps." }, { "id": "denial-hard-9", "title": "Anthem CO-50 Priority Triage - Appeal Highest-Value, Flag Urgent Deadline", "instruction": "Filter the denials workqueue to show only Anthem Blue Cross denials. Identify the highest-value CO-50 (medical necessity) denial. Open it, review its details and remittance image, and file an appeal on the Payer B portal with clinical documentation. Also identify other high-priority Anthem denials during your scan - particularly any with urgent deadlines or requiring different action types. Write a triage note on the highest-value denial documenting your investigation, the appeal filed, and prioritized next steps for other Anthem denials you identified.", "goal": "Filter the denials workqueue to show only Anthem Blue Cross denials. Identify the highest-value CO-50 (medical necessity) denial. Open it, review its details and remittance image, and file an appeal on the Payer B portal with clinical documentation. Also identify other high-priority Anthem denials during your scan - particularly any with urgent deadlines or requiring different action types. Write a triage note on the highest-value denial documenting your investigation, the appeal filed, and prioritized next steps for other Anthem denials you identified.", "difficulty": "hard", "category": "batch_analysis", "challengeType": "denial_triage", "possible": true, "points": 12, "configTaskId": "denial_hard_9", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Bailey, Christina" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), click the 'Filter' button in the toolbar to reveal the filter dropdowns", "2. Select 'Anthem Blue Cross' from the Payer filter dropdown to isolate all Anthem denials", "3. Scan the filtered list. The table columns show Error Code, Error Message, Paid Amount, Account (patient name), Account Type, Batch No, Batch Date, Check Number, and Invoice (claim ID). Click on each row to view additional details in the bottom 4-panel grid", "4. Identify all CO-50 denials in the list by checking the Error Code column. Note each CO-50 denial's Paid Amount and Account name", "5. Determine that DEN-047 (Bailey, $9,200) is the highest-value Anthem CO-50 denial", "6. Also note DEN-030 (Foster, $7,500, CO-50) and DEN-049 (Howard, $5,600, CO-197) as other high-value Anthem denials", "7. Double-click DEN-047 (Bailey, $9,200) to open its detail page", "8. On the Retest tab (default), review the Errors section (orange border) confirming CO-50 (medical necessity)", "9. In the Claim/Invoice Header, note the denied amount ($9,200), procedure description (biologic infusion), and appeal deadline (2026-03-10)", "10. Check the Documents section for clinical documentation on file", "11. [REQUIRED - DO THIS STEP] Click the 'Remittance Image' tab (second tab next to Retest). You MUST click this tab and review it before proceeding to the payer portal. Note the CARC/RARC codes and payer remarks. This step is evaluated.", "12. In the right panel, note the Portal Access section showing Payer B credentials (provider@payerb.com / demo123). Click 'Start Appeal' to navigate to the Payer B portal", "13. On the Payer B login page, enter provider@payerb.com and demo123, then click Login", "14. Click 'Appeals' in the sidebar navigation", "15. On the Claim Status Inquiry page, enter member ID ANT123456890 in the Member ID field and click Search", "16. In the search results, click on claim CLM-2025-00047 (Bailey, Christina, $9,200, CO-50) to view the Claim Status Detail", "17. Click the 'Dispute Claim' button to open the dispute form", "18. Fill in the Contact Name field with 'Dr. Elizabeth Park' (the provider listed on the denial)", "19. In Supporting Rationale, explicitly write that this is a CO-50 denial for DEN-047, patient Bailey, denied amount $9,200, and that supporting clinical documentation is attached. Use admin-appropriate language - do NOT write clinical justifications, and do NOT omit any of those four elements.", "20. Click 'Attach Document' to attach supporting clinical documentation", "21. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal' to submit the dispute. WAIT for the confirmation screen/banner to appear before leaving the portal, and copy the exact confirmation number displayed on screen.", "22. Click 'Return to EMR' to go back to the denials workqueue", "23. Double-click DEN-030 (Foster) to briefly check its detail page. Note the appeal deadline: 2026-02-28 - only 3 DAYS remaining. This is URGENT", "24. Click the '<- Back to Denials' button to return to the workqueue", "25. Double-click DEN-047 (Bailey) to return to its denial detail page", "26. [REQUIRED - DO NOT SKIP] In the right panel, scroll to the Actions section. In the Triage Note text area, write: DEN-047 (Bailey, $9,200, CO-50) is the highest-value Anthem medical necessity denial. Reviewed denial details, remittance image, and clinical documentation. Filed appeal on Payer B - include the exact confirmation number from step 21. This step is evaluated.", "27. Continue in the triage note: DEN-030 (Foster, $7,500, CO-50) has an URGENT deadline of 2026-02-28 (3 days remaining). This denial must be triaged and appealed immediately as the next priority", "28. Continue in the triage note: DEN-049 (Howard, $5,600, CO-197) is a no-auth denial. Recommend routing to the prior auth team for retroactive authorization - this is a prior auth issue, not an appeals issue", "29. Continue in the triage note: PRIORITY NEXT STEPS - (1) Appeal DEN-030 (Foster, $7,500) immediately before 2026-02-28 deadline, (2) Route DEN-049 (Howard, $5,600) to prior auth team for retro auth", "30. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown", "31. Click 'Submit Disposition' to save the triage note and disposition" ], "evals": [ { "id": "denial-hard-9-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-9-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-9-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent accessed Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-9-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent filed appeal on Payer B for highest-value CO-50 denial", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-9-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-9-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent documented triage note in EMR", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-9-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Appeal rationale references CO-50, claim details, and clinical documentation", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_b_state. full_state. appealActions) reference the denial code (CO-50), include claim details (DEN-047, denied amount ~$9,200, patient Bailey), and indicate that clinical documentation was attached? The rationale should use admin-appropriate language - NOT clinical justifications. Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-9-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note identifies DEN-047 as highest-value Anthem CO-50", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify DEN-047 (Bailey, ~$9,200) as the highest-value Anthem CO-50 denial? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-9-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note flags DEN-030 urgent deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention DEN-030 (Foster, ~$7,500, CO-50) with its urgent deadline of 2026-02-28 (3 days remaining)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-9-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number for DEN-047? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-9-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note identifies DEN-049 as CO-197 needing retro auth", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify DEN-049 (Howard, ~$5,600, CO-197) as a no-auth denial needing routing to the prior auth team for retro authorization? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-9-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note recommends prioritized next steps", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend prioritized next steps - e. g., appeal DEN-030 immediately due to urgent deadline, route DEN-049 to prior auth team? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "Filter workqueue by Anthem Blue Cross. Identify DEN-047 (Bailey, $9,200, CO-50) as the highest-value CO-50 denial. File appeal on Payer B with clinical documentation. Also note DEN-030 (Foster, $7,500, CO-50) has urgent deadline 2026-02-28 (3 days) - must be triaged next. DEN-049 (Howard, $5,600, CO-197) is a no-auth denial that should go to prior auth team. Write triage note on DEN-047's record documenting the appeal and prioritized next steps." }, { "id": "denial-hard-10", "title": "Coding Error Investigation - Laterality Modifier Correction with Payer Dispute", "instruction": "Open denial DEN-046 for Cooper, Frank. This is a CO-4 coding error denial for a knee arthroscopy. Review all available information including the remittance image to identify the specific modifier correction needed. Navigate to Payer A (provider@payera.com / demo123) to check claim status and file a dispute with the corrected coding rationale. Return to EMR and document your investigation, the specific coding correction, and the dispute filed in a triage note.", "goal": "Open denial DEN-046 for Cooper, Frank. This is a CO-4 coding error denial for a knee arthroscopy. Review all available information including the remittance image to identify the specific modifier correction needed. Navigate to Payer A (provider@payera.com / demo123) to check claim status and file a dispute with the corrected coding rationale. Return to EMR and document your investigation, the specific coding correction, and the dispute filed in a triage note.", "difficulty": "hard", "category": "coding_correction", "challengeType": "denial_triage", "possible": true, "points": 10, "configTaskId": "denial_hard_10", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Cooper, Frank" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate DEN-046 for Cooper, Frank - click on the patient's NAME (purple underlined button) to open the denial detail page - single-clicking the row body only selects it; click the name text specifically or double-click the row", "2. Double-click the Cooper row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming CO-4 (coding error - modifier inconsistency)", "4. In the Claim/Invoice Header, note the denied amount ($1,650), procedure description (knee arthroscopy), and appeal deadline (2026-03-10)", "5. Review the Line Item Cards - note CPT 29881 (knee arthroscopy, meniscectomy) was denied for missing laterality modifier", "6. [REQUIRED - DO NOT SKIP] IMPORTANT: Click the 'Remittance Image' tab (second tab next to Retest) - you MUST view the remittance image before proceeding. Review the remittance advice and confirm the CO-4 reason code and remark codes indicating the missing laterality modifier This step is evaluated.", "7. Identify the specific correction needed: -LT modifier must be added to CPT 29881 to indicate the procedure was performed on the left knee", "8. In the right panel, note the Portal Access section showing Payer A credentials (provider@payera.com / demo123). Click 'Start Appeal' to navigate to the Payer A portal", "9. On the Payer A login page, enter provider@payera.com and demo123, then click Login", "10. Click 'Claims' in the sidebar navigation", "11. Enter member ID AET012345789 in the Member ID field and click Search", "12. In the search results, click on claim CLM-2025-00046 (Cooper, $1,650, CO-4) to view the Claim Status Detail", "13. Click the 'Dispute Claim' button to open the dispute form", "14. Fill in the Contact Name field with 'Dr. Catherine Lee' (the provider listed on the denial)", "15. In Supporting Rationale, explicitly reference the denial code (CO-4), explain that the laterality modifier -LT was missing from CPT 29881, include claim details (DEN-046, $1,650, patient Cooper), and state the corrected claim should include modifier -LT on CPT 29881. Use admin-appropriate language and do NOT omit CO-4, -LT, DEN-046, amount, or patient name.", "16. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal' to submit the dispute. note the confirmation number displayed on screen", "17. Click 'Return to EMR' to go back to the denials workqueue", "18. Double-click the Cooper row (DEN-046) to navigate to its denial detail page", "19. In the right panel, scroll to the Actions section. In the Triage Note text area, write: DEN-046 (Cooper, $1,650, CO-4) - coding error denial for CPT 29881 (knee arthroscopy, meniscectomy). The -LT laterality modifier was missing. CPT guidelines require laterality modifiers (-LT for left, -RT for right) on procedures performed on paired anatomic structures such as knees", "20. Continue in the triage note: Reviewed claim CLM-2025-00046 on Payer A and filed dispute with corrected coding rationale. Include the confirmation number from step 16", "21. Continue in the triage note: CORRECTION - Add modifier -LT to CPT 29881 and resubmit corrected claim. This is an administrative coding correction, not a clinical dispute", "22. Select 'Corrected Claim - Resubmit' from the Triage Disposition dropdown", "23. Click 'Submit Disposition' to save the triage note and disposition" ], "evals": [ { "id": "denial-hard-10-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-046", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-10-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-10-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent accessed Payer A portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-10-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent filed dispute on Payer A for DEN-046", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-10-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Corrected Claim - Resubmit", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-10-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent documented triage note in EMR", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-10-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Dispute rationale references CO-4, -LT modifier, and claim details", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's appeal rationale (submittedRationale in payer_a_state. full_state. appealActions) reference the denial code (CO-4), the missing laterality modifier (-LT), and include claim details (DEN-046, amount, patient Cooper)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{payer_a_state.full_state.appealActions.submittedRationale}}\n{{payer_b_state.full_state.appealActions.submittedRationale}}", "studentAnswerContext": "appeal rationale text" }, { "id": "denial-hard-10-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note identifies -LT modifier needed for CPT 29881", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that DEN-046 needs the -LT laterality modifier added to CPT 29881? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-10-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note explains laterality modifier coding rule", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the coding rule - that CPT guidelines require laterality modifiers (-LT/-RT) on procedures performed on paired anatomic structures like knees? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-10-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note includes dispute confirmation number", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the dispute confirmation number from Payer A? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Corrected Claim - Resubmit" }, "rationale": "CO-4 denial for CPT 29881 (knee arthroscopy, meniscectomy) - missing -LT laterality modifier to indicate left knee. The procedure was performed on a paired anatomic structure (knee) and CPT guidelines require laterality modifiers. Check claim status on Payer A, file dispute with corrected coding rationale explaining the -LT modifier, then document in EMR." }, { "id": "denial-hard-11", "title": "Timely Filing Investigation - Clearinghouse Evidence", "instruction": "For denial DEN-033 (Santos, Elena), investigate the CO-29 timely filing denial ($5,400). Review the denial details and remittance image. Critically examine the submission history for this claim - look for clearinghouse acknowledgment records and calculate the number of days between the service date and the initial submission. Check the patient inquiry for coverage details. Based on your investigation of the filing timeline evidence, determine whether this should be written off or if there is a basis for appeal. Select the appropriate disposition and document your complete analysis with specific evidence in a triage note.", "goal": "For denial DEN-033 (Santos, Elena), investigate the CO-29 timely filing denial ($5,400). Review the denial details and remittance image. Critically examine the submission history for this claim - look for clearinghouse acknowledgment records and calculate the number of days between the service date and the initial submission. Check the patient inquiry for coverage details. Based on your investigation of the filing timeline evidence, determine whether this should be written off or if there is a basis for appeal. Select the appropriate disposition and document your complete analysis with specific evidence in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 11, "configTaskId": "denial_hard_11", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Santos, Elena" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate the row for Santos, Elena - note the Error Code, Paid Amount, and Invoice (claim ID) from the table columns", "2. Double-click the Santos row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming the CO-29 timely filing denial for $5,400", "4. In the Claim/Invoice Header, note the service date, denial date, and appeal deadline", "5. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB and note the CO-29 reason and payer remarks This step is evaluated.", "6. Click the 'Retest' tab to return to the default view", "7. Scroll down to the 'Claim History' section on the detail page - this table shows submission records with columns: Date, Type, Claim #, Amount, Status, Clearinghouse, and Response Date", "8. Examine the submission records for a clearinghouse acknowledgment - note the submission date and the acknowledgment ID", "9. Calculate the number of days between the service date and the initial submission date shown in the Claim History", "10. Click the patient name link in the banner to navigate to the Patient Inquiry page to check coverage details", "11. Compare the calculated filing days against the payer's 180-day timely filing limit", "12. Return to the denial detail page", "13. In the right panel, scroll to the Actions section", "14. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 15. In the Triage Note text area, document: CO-29 identification, the clearinghouse acknowledgment ID, the day calculation showing filing was within the 180-day limit, and recommendation to appeal with proof of timely submission", "16. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-11-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-033", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-11-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-11-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-11-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-11-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-11-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note identifies CO-29 timely filing denial", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify this as a CO-29 timely filing denial? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-11-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note finds clearinghouse acknowledgment within deadline", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the clearinghouse acknowledgment showing submission was received within the 180-day deadline (service date 04/10/2025, submitted 10/05/2025 = 178 days)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-11-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note calculates filing within 180-day limit", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note calculate or mention that the filing was within the 180-day limit (178 days vs 180 day limit)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-11-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note references acknowledgment ID", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the specific acknowledgment ID (ACK-CLH-20251005-7892)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-11-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends appeal not write-off", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend appeal rather than write-off, stating there is proof of timely filing? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-11-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends appeal with proof", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the appeal should include the clearinghouse proof of timely submission as supporting evidence? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-29 timely filing denial for $5,400. Investigation reveals clearinghouse acknowledgment (ACK-CLH-20251005-7892) showing the claim was submitted on 10/05/2025, which is 178 days after the 04/10/2025 service date - within the 180-day filing limit. This is not a write-off; the appeal should include the clearinghouse proof of timely submission." }, { "id": "denial-hard-12", "title": "Emergency OON Exception - Prudent Layperson Standard", "instruction": "For denial DEN-034 (Washington, Derek), investigate the PR-242 out-of-network denial ($3,800) for an emergency appendectomy. Review the denial details and remittance image. Download the clinical documentation (ER notes). Navigate to the Payer A portal (provider@payera.com / demo123) to check patient's eligibility - note the plan type and OON coverage. Based on all the evidence (plan type, service type, clinical scenario), determine the correct course of action. File the appropriate action on Payer A portal. Return to EMR and document your complete analysis explaining why you chose the action you did in a triage note.", "goal": "For denial DEN-034 (Washington, Derek), investigate the PR-242 out-of-network denial ($3,800) for an emergency appendectomy. Review the denial details and remittance image. Download the clinical documentation (ER notes). Navigate to the Payer A portal (provider@payera.com / demo123) to check patient's eligibility - note the plan type and OON coverage. Based on all the evidence (plan type, service type, clinical scenario), determine the correct course of action. File the appropriate action on Payer A portal. Return to EMR and document your complete analysis explaining why you chose the action you did in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 11, "configTaskId": "denial_hard_12", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Washington, Derek" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-034 for Washington, Derek - click on the patient's NAME (purple underlined button) to open the denial detail page. Single-clicking the row body only selects it; click the name text specifically or double-click the row.", "2. Review the denial details - note PR-242 OON denial for $3,800, Member ID AET567891234.", "3. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB. This step is evaluated.", "4. [REQUIRED - BEFORE LEAVING EMR] On the Retest tab, scroll down to the Documents section. Click the 'View ->' button on the RIGHT side of the ER notes document row (NOT the document name - that does nothing). This opens the document viewer. Then click 'Download'.", "5. After downloading, click '< Back' ONCE to return to the DEN-034 detail page. Do NOT click back a second time - that takes you to the worklist. If you end up on the worklist by mistake, use navigate_to('/emr/denied/DEN-034') to return directly.", "6. Click the 'Start Appeal' button in the Actions panel - this navigates to the Payer A portal login page.", "7. Log in with provider@payera.com / demo123.", "8. Click 'Member eligibility' in the nav bar. Fill in: Member ID AET567891234, Last Name 'Washington', First Name 'Derek', DOB '1985-04-20'. Click Search.", "9. KEY FINDING: Note the plan type is HMO with no OON benefits. However, this was an emergency appendectomy - the emergency service exception applies regardless of HMO OON restrictions.", "10. Click 'Appeals' in the navigation bar. Enter member ID AET567891234 in the Member ID field and click Search.", "11. In the search results, click on the claim row for CLM-2025-00034 to view the Claim Status Detail.", "12. Click the 'Dispute Claim' button.", "13. Fill in the Contact Name field with 'Dr. Robert Kim' (the provider listed on the denial).", "14. In the Supporting Rationale field, write the appeal referencing: (a) PR-242 OON denial code, (b) service was an emergency appendectomy, (c) emergency service exception applies - patient cannot be held responsible for OON charges for emergency services, (d) clinical documentation (ER notes) attached.", "15. Scroll to the 'Available Documents from EMR' section - the downloaded ER notes should appear. Click '+ Attach' next to the document.", "16. Click 'Submit Appeal'. Note the confirmation number (format APL-AET-XXXXXXXX).", "17. Click 'Return to EMR' to go back to the denials workqueue.", "18. You are now on the WORKLIST. Click the PATIENT NAME 'Washington, Derek' to re-open DEN-034 detail page, or use navigate_to('/emr/denied/DEN-034') to go directly.", "19. Once on the DEN-034 detail page, select 'Route to Clinical Appeals' from the Triage Disposition dropdown.", "20. IMMEDIATELY after selecting disposition, click the Triage Note field and type your note WITHOUT clicking any other tabs - clicking tabs clears the note field. Document: PR-242 OON denial code, HMO plan with no OON benefits identified, emergency appendectomy qualifies for emergency service exception, appeal filed on Payer A with confirmation number, rationale for appeal versus transferring balance to patient.", "21. Click 'Submit Disposition' - do NOT scroll or click any other element between typing the note and clicking Submit." ], "evals": [ { "id": "denial-hard-12-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-034", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-12-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-12-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation (ER notes)", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-12-eval-4", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted appeal on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-12-eval-5", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-12-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-12-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note identifies PR-242 OON denial", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify this as a PR-242 out-of-network denial? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-12-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note confirms HMO with no OON", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note confirm the patient has an HMO plan with no out-of-network benefits? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-12-eval-9", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recognizes emergency exception applies", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recognize that this was an EMERGENCY appendectomy and therefore qualifies for an emergency exception? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-12-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note references emergency exception applicability to OON denial", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference that the service was an emergency and that emergency exceptions may apply to OON denials? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-12-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains emergency nature supports appeal over patient transfer", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the emergency nature of the service supports filing an appeal rather than transferring the balance to the patient? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "PR-242 out-of-network denial for emergency appendectomy ($3,800). Patient has an HMO plan with no OON benefits. However, this was an emergency appendectomy - the emergency service exception applies. The patient should not be held responsible for OON charges for emergency services. Appeal should cite the emergency service exception with clinical documentation attached." }, { "id": "denial-hard-13", "title": "Duplicate vs Corrected Claim Investigation", "instruction": "For denial DEN-035 (Murphy, Colleen), investigate the CO-18 duplicate claim denial ($2,200). Review the denial details and remittance image. This claim was classified as a duplicate, but examine the related claims and submission history carefully. Check the payment posting tab for transaction history. Determine whether this is truly a duplicate or if there is a different explanation. Select the appropriate disposition and document your investigation findings and recommended action in a triage note.", "goal": "For denial DEN-035 (Murphy, Colleen), investigate the CO-18 duplicate claim denial ($2,200). Review the denial details and remittance image. This claim was classified as a duplicate, but examine the related claims and submission history carefully. Check the payment posting tab for transaction history. Determine whether this is truly a duplicate or if there is a different explanation. Select the appropriate disposition and document your investigation findings and recommended action in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 11, "configTaskId": "denial_hard_13", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Murphy, Colleen" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate the row for Murphy, Colleen - note the Error Code, Paid Amount, and Invoice (claim ID) from the table columns", "2. Double-click the Murphy row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming CO-18 duplicate claim denial for $2,200", "4. In the Claim/Invoice Header, note the service date, denial date, and appeal deadline", "5. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB and note the CO-18 reason and payer remarks This step is evaluated.", "6. Click the 'Payment Posting' tab to review the transaction history - note the Total Billed, Total Paid, and Balance Due amounts", "7. Click the 'Retest' tab to return to the default view", "8. In the right panel, review the Related Claims cards - locate the original claim (CLM-2025-00035-ORIG) and note its status and relationship type", "9. Scroll down to the 'Claim History' section on the detail page - review the submission records table showing the original-to-corrected submission sequence with dates, types, and statuses", "10. Note the submission type of the current claim (corrected) and the denial reason on the original claim (CO-16, missing information)", "11. In the right panel, scroll to the Actions section", "12. Select 'Corrected Claim - Resubmit' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 13. In the Triage Note text area, document: CO-18 duplicate denial identification, the original claim (CLM-2025-00035-ORIG) was denied with CO-16 for missing referring provider NPI, this claim is a corrected resubmission (not a true duplicate), submission history evidence showing original-to-corrected sequence, and recommendation to dispute the duplicate classification", "14. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-13-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-035", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-13-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-13-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed payment posting tab", "points": 1, "query": "full_state.agentActions.viewedPaymentPosting", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-13-eval-4", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-13-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Corrected Claim - Resubmit", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-13-eval-6", "type": "llm_judge", "category": "Documentation", "description": "Note identifies CO-18 duplicate denial", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify this as a CO-18 duplicate claim denial? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-13-eval-7", "type": "llm_judge", "category": "Documentation", "description": "Note finds original was denied with CO-16", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note discover that the original claim (CLM-2025-00035-ORIG) was denied with CO-16 (missing information), NOT paid? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-13-eval-8", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains corrected claim not duplicate", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that this is a CORRECTED claim resubmission, not a true duplicate? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-13-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note references submission history", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the submission history showing original to corrected sequence? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-13-eval-10", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends disputing duplicate", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend disputing or appealing the duplicate classification? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-13-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note identifies corrected information", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify what specific missing information was corrected (referring provider NPI)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Corrected Claim - Resubmit" }, "rationale": "CO-18 duplicate claim denial for $2,200. Investigation reveals the original claim (CLM-2025-00035-ORIG) was denied with CO-16 for missing information (referring provider NPI). This claim is a corrected resubmission, not a true duplicate. The submission history shows the original to corrected sequence. The duplicate classification should be disputed." }, { "id": "denial-hard-14", "title": "Auth CPT Mismatch - Cardiac Rehab", "instruction": "For denial DEN-031 (O'Brien, Margaret), investigate the CO-50 medical necessity denial for $22,000 cardiac rehabilitation. Review the denial details, remittance image, and patient inquiry. Note that there is an EXISTING authorization on file. Navigate to the Payer A portal (provider@payera.com / demo123) to check eligibility and search for the claim details. Investigate whether the authorization truly covers the services billed. File an appeal based on your findings. Set a follow-up task for tracking. Return to EMR and document your complete auth mismatch analysis and appeal details in a triage note.", "goal": "For denial DEN-031 (O'Brien, Margaret), investigate the CO-50 medical necessity denial for $22,000 cardiac rehabilitation. Review the denial details, remittance image, and patient inquiry. Note that there is an EXISTING authorization on file. Navigate to the Payer A portal (provider@payera.com / demo123) to check eligibility and search for the claim details. Investigate whether the authorization truly covers the services billed. File an appeal based on your findings. Set a follow-up task for tracking. Return to EMR and document your complete auth mismatch analysis and appeal details in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 13, "configTaskId": "denial_hard_14", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "O'Brien, Margaret" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-031 for O'Brien, Margaret - click on the patient's NAME (purple underlined button) to open the denial detail page. Single-clicking the row body only selects it; click the name text specifically or double-click the row.", "2. Review the denial details - note CO-50 medical necessity denial for $22,000 cardiac rehab, CPT 93797, Member ID AET678901543.", "3. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB. This step is evaluated.", "4. Click '< Back' or navigate back to the Retest tab to return to the denial detail.", "5. Click the patient name 'O'Brien, Margaret' link in the banner to navigate to the Patient Inquiry page. Find existing auth AUTH-2025-93100: it covers CPT 93798 (comprehensive cardiac rehab) while the claim billed CPT 93797 - this is the CPT mismatch.", "6. Click '< Back' ONCE to return to the DEN-031 detail page. Do NOT click back a second time - that would take you to the worklist. If you end up on the worklist by mistake, use navigate_to('/emr/denied/DEN-031') to return directly to the detail page.", "7. [REQUIRED - BEFORE LEAVING EMR] On the Retest tab, scroll down to the Documents section. Click the 'View ->' button on the RIGHT side of the 'Clinical Notes - Cardiac Rehabilitation.pdf' row (NOT the document name - that does nothing). This opens the document viewer page. Then click 'Download'.", "8. After downloading, click '< Back' ONCE to return to the DEN-031 detail page. Do NOT click back a second time. If you end up on the worklist, use navigate_to('/emr/denied/DEN-031') to return.", "9. Confirm you are on the DEN-031 detail page (URL should be /emr/denied/DEN-031). Then click the 'Start Appeal' button in the right Actions panel to navigate to the Payer A portal login page.", "10. Log in with provider@payera.com / demo123.", "11. After signing in you will land on the Claim Status Inquiry page. STOP - do NOT type anything in the search form yet. You MUST first check eligibility. Click 'Member eligibility' in the navigation bar at the top of the page to navigate to the Member Eligibility page.", "12. [REQUIRED - THIS STEP IS EVALUATED] On the Member Eligibility page, fill in: Member ID AET678901543, Last Name 'O'Brien', First Name 'Margaret', DOB '1960-01-28'. Click Search. Confirm Active Aetna PPO coverage. This is evaluated - do not skip.", "13. Now click 'Appeals' in the top navigation bar to go to the Claim Status Inquiry page. Enter member ID AET678901543 and click Search.", "14. In the search results, click on the claim row for CLM-2025-00031 to view the Claim Status Detail.", "15. You are now on the Claim Status Detail page. You MUST file the appeal before leaving. Scroll up if needed to find the 'Dispute Claim' button. Click 'Dispute Claim'. DO NOT click 'Return to EMR' until AFTER you have filled and submitted the appeal form.", "16. Fill in the Contact Name field with 'Dr. David Williams' (the provider listed on the denial).", "17. In the Supporting Rationale field, write the appeal referencing: (a) CPT mismatch - auth AUTH-2025-93100 covers CPT 93798 but claim billed CPT 93797, (b) cardiac rehab was medically necessary, (c) clinical documentation attached as supporting evidence.", "18. Scroll to the 'Available Documents from EMR' section - the downloaded clinical notes should appear. Click '+ Attach' next to 'Clinical Notes - Cardiac Rehabilitation.pdf'.", "19. Click 'Submit Appeal'. Wait for the appeal confirmation screen showing a confirmation number (format APL-AET-XXXXXXXX). Note this number.", "20. Only AFTER seeing the appeal confirmation, click 'Return to EMR' to go back to the denials workqueue.", "21. You are now on the WORKLIST (/emr/denied). DO NOT try to click on the list. Use navigate_to('/emr/denied/DEN-031') - use this exact path format, do NOT include the full domain URL or query parameters.", "21. [REQUIRED - THIS STEP IS EVALUATED] Once on the DEN-031 detail page, go to the Actions panel on the right and click the 'Add Follow-up Task' button. A form will appear: type a valid follow-up date in the date field (YYYY-MM-DD), leave reason as 'Awaiting payer response', then click the 'Schedule Follow-up' button. IMPORTANT: remember this follow-up timing/date and include it in the triage note as the follow-up plan. This is evaluated and must be done before submitting the triage disposition.", "22. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown.", "23. IMMEDIATELY after selecting disposition, click the Triage Note field and type your note WITHOUT clicking any other tabs - clicking tabs clears the note field.", "24. Document triage note with: auth AUTH-2025-93100 identified (covers CPT 93798 not 93797 - CPT mismatch), CO-50 denial for $22,000 cardiac rehab, appeal filed on Payer A with confirmation number, clinical documentation attached, and an explicit follow-up plan with timeline/date (for example, follow up on the scheduled date or within a stated number of business days).", "25. Click 'Submit Disposition' - do NOT scroll or click any other element between typing the note and clicking Submit." ], "evals": [ { "id": "denial-hard-14-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-031", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry for auth history", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked eligibility on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.checkedEligibility", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed claim detail on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted appeal on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent added a follow-up task for tracking", "points": 1, "query": "full_state.agentActions.addedFollowUpTask", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-8", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-9", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-14-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note identifies existing auth for cardiac rehab", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that an existing auth (AUTH-2025-93100) was approved for cardiac rehab? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-14-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note explains CPT mismatch", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the CPT mismatch - auth covers 93798 (comprehensive cardiac rehab) but claim billed 93797 (physician services for cardiac rehab)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-14-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note references clinical documentation attached to support appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the CPT mismatch and reference that clinical documentation was attached to support the appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-14-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal and follow-up plan", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation and a follow-up plan with timeline? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-50 medical necessity denial for $22,000 cardiac rehabilitation. Investigation reveals existing auth AUTH-2025-93100 approved for cardiac rehab, but it covers CPT 93798 (comprehensive cardiac rehab) while the claim billed CPT 93797 (physician services for cardiac rehab). Appeal should reference the CPT mismatch and include clinical documentation attaching as supporting evidence." }, { "id": "denial-hard-15", "title": "Partial Unbundling Analysis - NCCI Edit vs Independent CPT", "instruction": "For denial DEN-032 (Nakamura, Kenji), analyze the CO-97 bundling denial ($1,890) which has 3 CPT codes (97597, 97602, 97610) all denied under CO-97. Review the denial details and remittance image. Download the clinical documentation. Determine which CPT codes are a legitimate NCCI edit pair and which is independently billable. Navigate to the Payer B portal (provider@payerb.com / demo123) to file an appeal with per-line justification. Return to EMR and document your complete unbundling analysis in a triage note.", "goal": "For denial DEN-032 (Nakamura, Kenji), analyze the CO-97 bundling denial ($1,890) which has 3 CPT codes (97597, 97602, 97610) all denied under CO-97. Review the denial details and remittance image. Download the clinical documentation. Determine which CPT codes are a legitimate NCCI edit pair and which is independently billable. Navigate to the Payer B portal (provider@payerb.com / demo123) to file an appeal with per-line justification. Return to EMR and document your complete unbundling analysis in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 12, "configTaskId": "denial_hard_15", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Nakamura, Kenji" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate the row for Nakamura, Kenji - note the Error Code (CO-97), Paid Amount, and Invoice (claim ID) from the table columns", "2. Double-click the Nakamura row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming CO-97 bundling denial for $1,890", "4. In the Line Item Cards, review all 3 denied CPT codes: 97597 (selective debridement), 97602 (non-selective debridement), and 97610 (ultrasound wound therapy) - note the billed amount, allowed amount, and adjustment code for each line", "5. In the Claim/Invoice Header, note the service date, denial date, and appeal deadline", "6. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB - note the CO-97 reason and any remark codes per line This step is evaluated.", "7. Click the 'Retest' tab to return to the default view", "8. In the Documents section, download the clinical documentation (wound care notes)", "9. Analyze which CPT codes form an NCCI edit pair: 97597 (selective debridement) and 97602 (non-selective debridement) are column 1/column 2 codes in the NCCI table. 97610 (ultrasound wound therapy) is a separate therapeutic modality not in the same NCCI edit pair", "10. Click the 'Start Appeal' button in the Actions panel - this navigates to the Payer B portal login page", "11. Log in with provider@payerb.com / demo123", "12. Click 'Appeals' in the sidebar navigation to access the Claim Status Inquiry page", "13. Enter member ID ANT789012456 in the Member ID field and click Search", "14. In the search results, click on the claim row for CLM-2025-00032 (Nakamura, $1,890, CO-97) to view the Claim Status Detail", "15. Click the 'Dispute Claim' button to open the dispute form", "16. Fill in the Contact Name field with 'Dr. Sarah Mitchell' (the provider listed on the denial)", "17. In the Supporting Rationale field, write per-line justification: 97597 and 97602 are an NCCI edit pair that were performed on separate wound sites (modifier 59/XS applicable); 97610 is a distinct therapeutic modality (ultrasound wound therapy) not bundled with debridement under NCCI edits. Make the action plan explicit line-by-line: rebill 97597/97602 with modifier 59/XS, and separately appeal the denial of 97610. Clinical documentation attached.", "18. Click 'Attach Document' to attach the wound care notes as supporting evidence", "19. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal'. note the confirmation number displayed on screen", "20. Click 'Return to EMR' to go back to the denials workqueue", "21. Double-click the Nakamura row (DEN-032) to navigate to its denial detail page", "22. In the right panel, scroll to the Actions section", "23. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 24. In the Triage Note text area, document: CO-97 bundling denial for 3 CPTs (97597, 97602, 97610); NCCI edit pair identified (97597/97602 - selective and non-selective debridement); modifier 59/XS recommended to distinguish separate wound sites; 97610 (ultrasound wound therapy) is independently billable and should not be bundled with debridement; and an explicit per-line action plan: rebill 97597/97602 with modifier 59/XS, separately appeal 97610; include appeal confirmation number", "25. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-15-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-032", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-15-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-15-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-15-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-15-eval-5", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted appeal on Payer B", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-15-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-15-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-15-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note identifies NCCI edit pair", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the NCCI edit pair (97597 and 97602 - selective and non-selective debridement)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-15-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies independently billable CPT", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify CPT 97610 (ultrasound wound therapy) as independently billable and NOT part of the NCCI pair? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-15-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note recommends modifier 59/XS", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend modifier 59 or XS for the NCCI edit pair to distinguish separate services? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-15-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains why 97610 is separate", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why CPT 97610 should not be bundled with the debridement codes? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-15-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note includes per-line action plan", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include a per-line action plan (rebill NCCI pair with modifier, appeal separate denial of 97610)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-97 bundling denial for $1,890 with 3 CPT codes (97597, 97602, 97610). Analysis reveals 97597 (selective debridement) and 97602 (non-selective debridement) are a legitimate NCCI edit pair that can be billed separately with modifier 59/XS. CPT 97610 (ultrasound wound therapy) is independently billable and should not be bundled with the debridement codes. Per-line action: rebill NCCI pair with modifier, appeal separate denial of 97610." }, { "id": "denial-hard-16", "title": "Expired Auth Complication - High-Value Spinal Fusion", "instruction": "For denial DEN-044 (Price, Samuel), investigate this high-value CO-50 medical necessity denial ($18,500) for a spinal fusion procedure. Review the denial details and remittance image. Check the patient inquiry for authorization history. Navigate to the Payer A portal (provider@payera.com / demo123) to check the patient's eligibility and review the claim. Based on ALL evidence gathered - the denial reason, authorization status, clinical documentation, and timeline - determine the correct course of action. Select the appropriate disposition and document your complete analysis, explaining why you chose the action you did, in a triage note.", "goal": "For denial DEN-044 (Price, Samuel), investigate this high-value CO-50 medical necessity denial ($18,500) for a spinal fusion procedure. Review the denial details and remittance image. Check the patient inquiry for authorization history. Navigate to the Payer A portal (provider@payera.com / demo123) to check the patient's eligibility and review the claim. Based on ALL evidence gathered - the denial reason, authorization status, clinical documentation, and timeline - determine the correct course of action. Select the appropriate disposition and document your complete analysis, explaining why you chose the action you did, in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 14, "configTaskId": "denial_hard_16", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Price, Samuel" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate the row for Price, Samuel - note the Error Code (CO-50), Paid Amount ($18,500), and Invoice (claim ID) from the table columns", "2. Double-click the Price row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming CO-50 medical necessity denial for $18,500", "4. In the Claim/Invoice Header, note the service date (2025-10-10), denial date, and appeal deadline", "5. Review the Line Item Cards for CPT 22612 (spinal fusion), 22840 (instrumentation), and 20930 (bone allograft) - note the billed and denied amounts", "6. In the Documents section, click 'View' on the Operative Report - Spinal Fusion.pdf to review the clinical documentation supporting the procedure", "7. Note the existing authorization section showing AUTH-2025-22612 - note its status (Expired) and expiration date (2025-09-30)", "8. Note that the surgery date (2025-10-10) is AFTER the auth expiration date (2025-09-30) - a 10-day gap", "9. Review the notes mentioning the surgery was delayed due to pre-op cardiac clearance requirement", "10. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB - note the CO-50 reason and any remark codes This step is evaluated.", "11. Click the patient name link in the banner to navigate to the Patient Inquiry page - review auth history confirming AUTH-2025-22612 details", "12. Return to the denial detail page", "13. Click the 'Start Appeal' button in the Actions panel - this navigates to the Payer A portal login page", "14. Log in with provider@payera.com / demo123", "15. Go to Member eligibility page. Fill in required fields: Member ID AET890123567, Patient Last Name 'Price', Patient First Name 'Samuel', DOB '1963-06-25' (visible on denial detail page banner). Click Search", "16. Review the eligibility results - note the plan type, coverage status, and authorization requirements", "17. Go to the Claim Status Inquiry page, enter member ID AET890123567 in the Member ID field and click Search", "18. In the search results, click on the claim row for CLM-2025-00044 (Price, $18,500, CO-50) to view the Claim Status Detail", "19. Review the claim detail - note the denial reason, appeal deadline, and any payer remarks about the auth status", "20. Click 'Return to EMR' to go back to the denials workqueue", "21. Double-click the Price row (DEN-044) to navigate to its denial detail page", "22. In the right panel, scroll to the Actions section", "23. Select 'Escalate to Supervisor' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 24. In the Triage Note text area, document: CO-50 denial for $18,500 spinal fusion; auth AUTH-2025-22612 was approved for this procedure but expired 2025-09-30 (10 days before surgery on 2025-10-10); surgery delayed due to pre-op cardiac clearance; this is not a simple medical necessity appeal - the expired auth complicates the situation; recommend escalation to coordinate retro auth extension with prior auth team before filing medical necessity appeal; strong clinical case details including failed conservative management and Grade II spondylolisthesis; eligibility verification findings from Payer A portal", "25. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-16-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-044", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-16-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-16-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry for auth history", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-16-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked eligibility on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.checkedEligibility", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-16-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed claim detail on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-16-eval-6", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-16-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Escalate to Supervisor", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-16-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note identifies existing auth for spinal fusion", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the existing authorization AUTH-2025-22612 for spinal fusion? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-16-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies auth expired before surgery date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that the auth EXPIRED on 2025-09-30, BEFORE the surgery date of 2025-10-10? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-16-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note explains reason for surgery delay", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain WHY the surgery was delayed past the auth window (pre-op cardiac clearance requirement)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-16-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recognizes expired auth complicates standard appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recognize that this is NOT a simple medical necessity appeal - the expired auth complicates the situation and requires auth extension or retro auth before a standard appeal can proceed? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-16-eval-12", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends retro auth extension rather than standard appeal", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend escalation or routing to the prior auth team for a retroactive auth extension, rather than filing a standard medical necessity appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-16-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note mentions value and clinical strength", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention the $18,500 value and the strong clinical case (failed conservative management, Grade II spondylolisthesis)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-16-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Note includes eligibility verification findings", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include eligibility verification findings from the Payer A portal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Escalate to Supervisor" }, "rationale": "CO-50 medical necessity denial for $18,500 spinal fusion. At first glance this looks like a standard medical necessity appeal with strong clinical evidence (failed conservative management x12 months, Grade II spondylolisthesis). But investigation reveals auth AUTH-2025-22612 was approved for this exact procedure - however it EXPIRED on 2025-09-30, and the surgery was performed on 2025-10-10 (10 days late). The surgery was delayed due to a pre-op cardiac clearance requirement. This is NOT a simple medical necessity appeal - the expired auth changes the strategy. The agent should escalate to supervisor because: (1) a retro auth extension is needed to cover the 10-day gap, (2) the medical necessity denial may be secondary to the auth expiration, and (3) a $18,500 case with this complexity warrants supervisor review to coordinate between prior auth team and appeals." }, { "id": "denial-hard-17", "title": "Auth Laterality Mismatch - Right Auth vs Left Surgery", "instruction": "For denial DEN-049 (Howard, Lisa), investigate this CO-197 no-authorization denial ($5,600) for a shoulder arthroscopy. Review the denial details, remittance image, and clinical documentation. Check the patient inquiry for any auth history - pay attention to what you find. Navigate to the Payer B portal (provider@payerb.com / demo123) and use the Auth Inquiry page to search for any existing authorization for member ANT345678012. Compare the auth details with what was actually billed. Based on your investigation, determine the correct course of action and select the appropriate disposition. Document your complete analysis - including what you found, the discrepancy, and why it matters - in a triage note.", "goal": "For denial DEN-049 (Howard, Lisa), investigate this CO-197 no-authorization denial ($5,600) for a shoulder arthroscopy. Review the denial details, remittance image, and clinical documentation. Check the patient inquiry for any auth history - pay attention to what you find. Navigate to the Payer B portal (provider@payerb.com / demo123) and use the Auth Inquiry page to search for any existing authorization for member ANT345678012. Compare the auth details with what was actually billed. Based on your investigation, determine the correct course of action and select the appropriate disposition. Document your complete analysis - including what you found, the discrepancy, and why it matters - in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 13, "configTaskId": "denial_hard_17", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Howard, Lisa" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate the row for Howard, Lisa - note the Error Code (CO-197), Paid Amount ($5,600), and Invoice (claim ID) from the table columns", "2. Double-click the Howard row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming CO-197 no-authorization denial for $5,600", "4. In the Claim/Invoice Header, note the service date (2025-10-22), the CPT code (29827), and the diagnosis code (M75.112 - LEFT shoulder)", "5. Note the existing authorization section showing AUTH-ANT-2025-29827 - note its status (Active) and the note describing what it covers (RIGHT shoulder, 29827-RT, M75.111)", "6. Compare: the auth is for the RIGHT shoulder but the claim was billed for the LEFT shoulder - this is a laterality mismatch", "7. In the Documents section, download the operative report (Left Shoulder Arthroscopy) and review it - confirm the surgery was performed on the LEFT shoulder based on updated imaging", "8. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB - note the CO-197 reason and payer remarks This step is evaluated.", "9. Click the patient name link in the banner to navigate to the Patient Inquiry page - review auth history confirming AUTH-ANT-2025-29827 details and laterality", "10. Click the 'Start Appeal' button in the Actions panel - this navigates to the Payer B portal login page", "11. Log in with provider@payerb.com / demo123", "12. Navigate to the 'Home' tab", "13. Navigate to the 'Authorizations and Referrals' tab", "14. On the Auth/Referral Inquiry page, enter member ID ANT345678012 in the Member ID field and click Search", "15. Review the search results - find AUTH-ANT-2025-29827 showing 'Approved' for 'Shoulder arthroscopy with rotator cuff repair (CPT 29827-RT) - RIGHT shoulder'", "16. Confirm the laterality mismatch: auth approved for RIGHT shoulder, claim billed for LEFT shoulder", "17. Click 'Return to EMR' to go back to the denials workqueue", "18. Double-click the Howard row (DEN-049) to navigate to its denial detail page", "19. In the right panel, scroll to the Actions section", "20. Select 'Escalate to Supervisor' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 21. In the Triage Note text area, document: CO-197 denial for $5,600 LEFT shoulder arthroscopy (29827-LT, M75.112); auth AUTH-ANT-2025-29827 exists but approved for RIGHT shoulder (29827-RT, M75.111); operative report confirms surgery was correctly on LEFT shoulder per updated imaging; auth was originally requested for wrong side; this requires auth laterality amendment - not a simple retro auth or standard appeal; recommend supervisor escalation to coordinate with prior auth team and payer auth department", "22. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-17-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-049", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-17-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-17-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry for auth history", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-17-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-17-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched auth inquiry on Payer B", "points": 1, "query": "payer_b_state.full_state.appealActions.searchedAuthInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-17-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Escalate to Supervisor", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-17-eval-7", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-17-eval-8", "type": "llm_judge", "category": "Documentation", "description": "Note identifies existing approved auth", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that an existing auth AUTH-ANT-2025-29827 was found and is APPROVED? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-17-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies laterality mismatch (right auth vs left billed)", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the LATERALITY MISMATCH - the auth is for the RIGHT shoulder (29827-RT, M75.111) but the claim was billed for the LEFT shoulder (29827-LT, M75.112)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-17-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note confirms surgery was correctly on left shoulder per op report", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the clinical documentation (operative report) confirming the surgery was correctly performed on the LEFT shoulder? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-17-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains auth was requested for wrong side", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the auth was originally requested for the wrong side, and the clinical situation changed based on updated imaging? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-17-eval-12", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends escalation for auth laterality amendment", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recommend escalation because a simple retro auth or standard appeal won't resolve this - the auth needs to be amended to the correct laterality, which requires supervisor coordination? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-17-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note documents laterality codes (ICD-10 or modifiers)", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention both the ICD-10 codes (M75.111 for right on auth, M75.112 for left on claim) or the modifier codes (RT vs LT) to document the laterality discrepancy? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Escalate to Supervisor" }, "rationale": "CO-197 denial for $5,600 shoulder arthroscopy. Investigation reveals an auth (AUTH-ANT-2025-29827) WAS approved - but for the RIGHT shoulder (29827-RT, M75.111). The claim was billed for the LEFT shoulder (29827-LT, M75.112). The operative report confirms surgery was correctly performed on the LEFT shoulder based on updated imaging. This is a laterality mismatch - the auth was originally requested for the wrong side. A simple retro auth request or standard appeal won't work because an approved auth exists (just for the wrong side). The auth needs to be amended to the correct laterality, which requires supervisor coordination between the prior auth team and the payer's auth department." }, { "id": "denial-hard-18", "title": "Denied Auth Due to Missing Step Therapy Documentation", "instruction": "For denial DEN-047 (Bailey, Christina), investigate this CO-50 medical necessity denial ($9,200) for a biologic infusion with multiple CPT codes. Review the denial details, remittance image, and clinical documentation - note the step therapy history. Check the Payer B portal (provider@payerb.com / demo123) auth inquiry for any existing authorization for member ANT123456890. Pay close attention to the auth status and reason - it may change your approach. Based on ALL evidence gathered, determine the correct course of action. File an appeal on the Payer B portal with appropriate documentation. Return to EMR and document your complete analysis - including what you found, the disconnect between the clinical docs and the auth, and why it matters - in a triage note.", "goal": "For denial DEN-047 (Bailey, Christina), investigate this CO-50 medical necessity denial ($9,200) for a biologic infusion with multiple CPT codes. Review the denial details, remittance image, and clinical documentation - note the step therapy history. Check the Payer B portal (provider@payerb.com / demo123) auth inquiry for any existing authorization for member ANT123456890. Pay close attention to the auth status and reason - it may change your approach. Based on ALL evidence gathered, determine the correct course of action. File an appeal on the Payer B portal with appropriate documentation. Return to EMR and document your complete analysis - including what you found, the disconnect between the clinical docs and the auth, and why it matters - in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 13, "configTaskId": "denial_hard_18", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Bailey, Christina" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate the row for Bailey, Christina - note the Error Code (CO-50), Paid Amount ($9,200), and Invoice (claim ID) from the table columns", "2. Double-click the Bailey row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming CO-50 medical necessity denial for $9,200", "4. In the Claim/Invoice Header, note the service date (2025-10-25), the CPT codes (96413 infusion, J1745 infliximab, 99214 E/M), and diagnosis codes (M05.79, M06.09)", "5. Note the existing authorization section showing AUTH-ANT-2025-47100 - note its status (Denied) and the reason (step therapy documentation not submitted)", "6. In the Documents section, download the clinical documentation (Clinical Notes - Biologic Infusion) - review the step therapy history showing failed methotrexate, leflunomide, sulfasalazine, and hydroxychloroquine", "7. Recognize the disconnect: clinical docs prove step therapy was completed, but the auth says documentation wasn't submitted with the auth request", "8. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB - note the CO-50 reason and payer remarks This step is evaluated.", "9. Click the 'Start Appeal' button in the Actions panel - this navigates to the Payer B portal login page", "10. Log in with provider@payerb.com / demo123", "11. Click the 'Authorizations and Referrals' tab", "12. On the Auth/Referral Inquiry page, enter member ID ANT123456890 in the Member ID field and click Search", "13. Review the search results - find AUTH-ANT-2025-47100 showing 'Denied' with reason 'Step therapy documentation not submitted with authorization request'", "14. Confirm the key finding: the auth was denied NOT because step therapy wasn't completed, but because the documentation proving it wasn't included in the auth submission", "15. Click 'Appeals' in the sidebar navigation to access the Claim Status Inquiry page", "16. Enter member ID ANT123456890 in the Member ID field and click Search", "17. In the search results, click on the claim row for CLM-2025-00047 (Bailey, $9,200, CO-50) to view the Claim Status Detail", "18. Click the 'Dispute Claim' button to open the dispute form", "19. Fill in the Contact Name field with 'Dr. Elizabeth Park' (the provider listed on the denial)", "20. In the Supporting Rationale field, write: CO-50 denial for biologic infusion (96413, J1745, 99214). Auth AUTH-ANT-2025-47100 was denied due to missing step therapy documentation. Clinical records confirm step therapy was completed - patient failed methotrexate (6 months, inadequate response), leflunomide (4 months, hepatotoxicity), sulfasalazine (3 months, GI intolerance), and hydroxychloroquine (4 months, inadequate response). Step therapy documentation is attached to this appeal. Biologic therapy with infliximab is indicated per ACR 2021 guidelines for RA patients who have failed >=2 conventional DMARDs.", "21. Click 'Attach Document' to attach the clinical documentation (step therapy history) as supporting evidence", "22. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal'. note the confirmation number (format apl-ant-xxxxxxxx)", "23. Click 'Return to EMR' to go back to the denials workqueue", "24. Double-click the Bailey row (DEN-047) to navigate to its denial detail page", "25. In the right panel, scroll to the Actions section", "26. Select 'Route to Clinical Appeals' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 27. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then in the triage note text area, document: co-50 denial for $9,200 biologic infusion (96413, j1745, 99214); auth auth-ant-2025-47100 was submitted but denied - step therapy documentation not included in auth request; clinical documentation confirms step therapy was completed (failed methotrexate, leflunomide, sulfasalazine, hydroxychloroquine per acr guidelines); the co-50 denial is downstream of the failed auth; appeal filed on payer b with step therapy documentation attached; appeal confirmation [number]; the clinical evidence directly addresses the reason the auth was denied", "28. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-18-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-047", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-3", "type": "jmespath", "category": "Document Handling", "description": "Agent downloaded clinical documentation", "points": 1, "query": "full_state.agentActions.downloadedSupportingDoc", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent searched auth inquiry on Payer B", "points": 1, "query": "payer_b_state.full_state.appealActions.searchedAuthInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted appeal on Payer B", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-8", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-18-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies existing auth was DENIED", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that auth AUTH-ANT-2025-47100 was found but its status is DENIED? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-18-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note identifies auth denied due to missing step therapy documentation", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that the auth was denied because step therapy DOCUMENTATION was not submitted with the authorization request (not because step therapy wasn't completed)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-18-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note confirms step therapy was completed per clinical docs", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note recognize that step therapy WAS actually completed per clinical documentation (failed methotrexate, leflunomide, sulfasalazine, hydroxychloroquine)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-18-eval-12", "type": "llm_judge", "category": "Documentation", "description": "Note explains documentation gap between clinical completion and auth submission", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain the disconnect - step therapy was clinically completed but the proof/documentation was not included in the prior auth submission? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-18-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal confirmation and documentation attachment", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number and note that the step therapy documentation was attached to the appeal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-50 medical necessity denial for $9,200 biologic infusion. Clinical documentation clearly shows completed step therapy (failed methotrexate, leflunomide, sulfasalazine, hydroxychloroquine). However, auth inquiry on Payer B reveals auth AUTH-ANT-2025-47100 was DENIED - reason: step therapy documentation not submitted with authorization request. The key insight: the step therapy was clinically completed, but the proof wasn't included when the auth was requested. The CO-50 denial is downstream of this failed auth. The agent should file an appeal with the step therapy documentation attached - the clinical evidence exists and directly addresses the reason the auth was denied. Route to Clinical Appeals is correct because the standard appeal process can resolve this by providing the missing documentation." }, { "id": "denial-hard-19", "title": "Auth Body Region Mismatch - Cervical Auth vs Lumbar MRI", "instruction": "For denial DEN-045 (Reed, Janet), investigate this CO-197 no-authorization denial ($3,400) for an MRI lumbar spine. Review the denial details, remittance image, and clinical documentation. Check the patient inquiry for any auth history - pay attention to what you find. Navigate to the Payer A portal (provider@payera.com / demo123) to check eligibility for member AET901234678 and review the claim status. Compare the auth details with what was actually billed. Based on your investigation, file an appeal on the Payer A portal with clinical progression documentation. Return to EMR and document your complete analysis - including what you found, the discrepancy, and why it matters - in a triage note.", "goal": "For denial DEN-045 (Reed, Janet), investigate this CO-197 no-authorization denial ($3,400) for an MRI lumbar spine. Review the denial details, remittance image, and clinical documentation. Check the patient inquiry for any auth history - pay attention to what you find. Navigate to the Payer A portal (provider@payera.com / demo123) to check eligibility for member AET901234678 and review the claim status. Compare the auth details with what was actually billed. Based on your investigation, file an appeal on the Payer A portal with clinical progression documentation. Return to EMR and document your complete analysis - including what you found, the discrepancy, and why it matters - in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 14, "configTaskId": "denial_hard_19", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Reed, Janet" }, "stepByStep": [ "1. On the Denials Workqueue, locate DEN-045 for Reed, Janet - click on the patient's NAME (purple underlined button) to open the denial detail page. Single-clicking the row body only selects it; click the name text specifically. If you cannot find the row by scrolling, use navigate_to('/emr/denied/DEN-045') to go directly.", "2. Review the Errors section confirming CO-197 no-authorization denial for $3,400, service date 2025-11-01, CPT 72148 (MRI lumbar spine), diagnosis codes M54.5 and M51.16.", "3. Note the existing authorization AUTH-AET-2025-45200 - it is Active but covers MRI CERVICAL spine (CPT 72156, M54.2), NOT lumbar spine. This is a body region mismatch.", "4. [REQUIRED - BEFORE LEAVING EMR] In the Documents section on the Retest tab, scroll down and find the Cervical MRI Report. Click the 'View ->' button on the RIGHT side of that document row (NOT the document name - that does nothing). This opens the document viewer. Click 'Download'.", "5. After downloading, click '< Back' ONCE to return to the DEN-045 detail page. Do NOT click back a second time - that takes you to the worklist. If you end up on the worklist, use navigate_to('/emr/denied/DEN-045') to return.", "6. [REQUIRED - BEFORE LEAVING EMR] Now click 'View ->' for the Lumbar MRI Order document row. Download it.", "7. After downloading, click '< Back' ONCE to return to the DEN-045 detail page. Do NOT click back a second time. If you end up on the worklist, use navigate_to('/emr/denied/DEN-045') to return.", "8. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB - note the CO-197 reason and payer remarks. This step is evaluated.", "9. Click the patient name 'Reed, Janet' link in the banner to navigate to the Patient Inquiry page - confirm AUTH-AET-2025-45200 covers cervical spine (CPT 72156), not lumbar.", "10. Click '< Back' ONCE to return to the DEN-045 detail page. If you end up on the worklist, use navigate_to('/emr/denied/DEN-045').", "11. Confirm you are on the DEN-045 detail page (URL: /emr/denied/DEN-045). Click the 'Start Appeal' button in the Actions panel to navigate to the Payer A portal login page.", "12. Log in with provider@payera.com / demo123.", "13. [REQUIRED - THIS STEP IS EVALUATED] Click 'Member eligibility' in the top navigation bar BEFORE going to Appeals. Fill in: Member ID AET901234678, Last Name 'Reed', First Name 'Janet', DOB '1970-09-12'. Click Search and wait for the eligibility result to load. Do NOT skip or substitute this with the claim search; the eligibility search itself is evaluated. Confirm the plan requires prior authorization.", "14. Click 'Appeals' in the top navigation bar to go to the Claim Status Inquiry page. Enter member ID AET901234678 and click Search.", "15. In the search results, click on the claim row for CLM-2025-00045 (Reed, $3,400, CO-197) to view the Claim Status Detail.", "16. Click the 'Dispute Claim' button to open the dispute form.", "17. Fill in the Contact Name field with 'Dr. Daniel Adams' (the provider listed on the denial).", "18. In the Supporting Rationale field, write: CO-197 denial for MRI lumbar spine (CPT 72148). Auth AUTH-AET-2025-45200 was approved for MRI cervical spine (CPT 72156). Cervical MRI 2025-10-15 revealed incidental finding of L4-L5 disc herniation - dedicated lumbar MRI was clinically indicated by cervical MRI findings. Cervical MRI report and lumbar MRI order attached documenting clinical progression.", "19. Scroll to the 'Available Documents from EMR' section - both downloaded documents should appear. Click '+ Attach' next to each document.", "20. Click 'Submit Appeal'. Note the confirmation number (format APL-AET-XXXXXXXX).", "21. Click 'Return to EMR' to go back to the denials workqueue.", "22. You are now on the WORKLIST (/emr/denied). DO NOT try to click on the list - the worklist may be scrolled and you will click the wrong patient. Use navigate_to('/emr/denied/DEN-045') to go directly to the correct denial detail page.", "23. Once on the DEN-045 detail page, select 'Route to Clinical Appeals' from the Triage Disposition dropdown.", "24. IMMEDIATELY after selecting disposition, click the Triage Note field and type your note WITHOUT clicking any other tabs - clicking tabs clears the note field.", "25. Document triage note: CO-197 denial for $3,400 MRI lumbar spine (CPT 72148, M51.16); auth AUTH-AET-2025-45200 exists but covers MRI CERVICAL spine (CPT 72156, M54.2) - body region mismatch; cervical MRI 2025-10-15 showed C5-C6 herniation with incidental finding of L4-L5 disc herniation; lumbar MRI clinically indicated by cervical findings but no separate auth obtained; appeal filed on Payer A with clinical progression documentation attached; appeal confirmation [number].", "26. Click 'Submit Disposition' - do NOT scroll or click any other element between typing the note and clicking Submit." ], "evals": [ { "id": "denial-hard-19-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-045", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry for auth history", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked eligibility on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.checkedEligibility", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed claim detail on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted appeal on Payer A", "points": 1, "query": "payer_a_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Route to Clinical Appeals", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-8", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-19-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies existing auth for wrong body region", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that auth AUTH-AET-2025-45200 exists/was found, and that it covers MRI CERVICAL spine (CPT 72156), not MRI LUMBAR spine (CPT 72148)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-19-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note identifies body region mismatch (cervical auth vs lumbar billed)", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify the BODY REGION MISMATCH - the auth covers cervical spine MRI but the claim was billed for lumbar spine MRI? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-19-eval-11", "type": "llm_judge", "category": "Documentation", "description": "Note references cervical MRI findings indicating lumbar MRI", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note reference the cervical MRI findings that clinically indicated the lumbar MRI (incidental finding of lumbar disc herniation at L4-L5 on the cervical MRI)? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-19-eval-12", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains clinical progression from cervical to lumbar", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the auth was obtained for the initial presentation (cervical radiculopathy) but the clinical situation evolved - cervical MRI findings revealed lumbar pathology requiring a separate lumbar MRI? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-19-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal confirmation and clinical progression documentation", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number and mention that clinical progression documentation (cervical MRI report showing lumbar findings) was attached? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-19-eval-14", "type": "llm_judge", "category": "Documentation", "description": "Note documents body region codes (CPT or region names)", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note mention both CPT codes (72156 cervical on auth, 72148 lumbar on claim) or the body regions (cervical vs lumbar) to document the mismatch? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Route to Clinical Appeals" }, "rationale": "CO-197 no-authorization denial for $3,400 MRI lumbar spine (CPT 72148). Investigation reveals auth AUTH-AET-2025-45200 IS active - but it covers MRI CERVICAL spine (CPT 72156, M54.2 cervical radiculopathy), not MRI LUMBAR spine (CPT 72148, M51.16 lumbar disc herniation). The cervical MRI was performed 2025-10-15 and incidentally discovered lumbar disc herniation at L4-L5, prompting the lumbar MRI order. The auth was obtained for the initial cervical presentation, but the clinical situation evolved based on imaging findings. This is a body region mismatch - the auth covers the wrong anatomical region. The agent should file an appeal with the clinical progression documentation (cervical MRI report showing incidental lumbar findings) to justify the lumbar MRI as a clinically indicated follow-up. The standard appeal process can handle this because the evidence clearly links the lumbar MRI to findings from the authorized cervical study." }, { "id": "denial-hard-20", "title": "Delegation Terminated - N418 Reroute Would Be Incorrect", "instruction": "For denial DEN-048 (Ross, Daniel), investigate this N418 misrouted claim denial ($2,100). The EMR notes suggest rerouting to the delegated medical group - but don't take that at face value. Review the denial details, remittance image, and all notes carefully. Check the patient inquiry for coverage history. Navigate to the Payer B portal (provider@payerb.com / demo123) and review the claim status detail for member ANT234567901. Pay close attention to the payer remarks. Based on ALL evidence gathered, determine whether rerouting is actually the correct action. Select the appropriate disposition and document your complete analysis - including what the EMR suggested, what you actually found, and why it changes the approach - in a triage note.", "goal": "For denial DEN-048 (Ross, Daniel), investigate this N418 misrouted claim denial ($2,100). The EMR notes suggest rerouting to the delegated medical group - but don't take that at face value. Review the denial details, remittance image, and all notes carefully. Check the patient inquiry for coverage history. Navigate to the Payer B portal (provider@payerb.com / demo123) and review the claim status detail for member ANT234567901. Pay close attention to the payer remarks. Based on ALL evidence gathered, determine whether rerouting is actually the correct action. Select the appropriate disposition and document your complete analysis - including what the EMR suggested, what you actually found, and why it changes the approach - in a triage note.", "difficulty": "hard", "category": "complex_investigation", "challengeType": "denial_triage", "possible": true, "points": 13, "configTaskId": "denial_hard_20", "website": { "name": "EMR Denials Portal", "url": "https://emrportal.vercel.app/denied" }, "patient": { "name": "Ross, Daniel" }, "stepByStep": [ "1. On the Denials Workqueue (/denied), locate the row for Ross, Daniel - note the Error Code (N418), Paid Amount ($2,100), and Invoice (claim ID) from the table columns", "2. Double-click the Ross row to open the denial detail page", "3. On the Retest tab (default), review the Errors section (orange border) confirming N418 misrouted denial for $2,100", "4. In the Claim/Invoice Header, note the service date (2025-10-18), CPT codes (99214, 90471), and diagnosis codes (E11.9, Z23)", "5. Note the delegation info showing Bay Area Medical Group", "6. Review the notes - note the first two suggest rerouting, but the third note says 'Coverage update: Patient plan changed from Anthem HMO to Anthem PPO effective 2025-10-01 - verify current delegation status before rerouting'", "7. Note the insurance section shows current plan is PPO - but the N418 denial references capitation delegation, which is an HMO concept", "8. [REQUIRED - DO NOT SKIP] Click the 'Remittance Image' tab to review the EOB - note the N418 reason and payer remarks This step is evaluated.", "9. Click the patient name link in the banner to navigate to the Patient Inquiry page - review coverage history confirming the plan change from HMO to PPO", "10. Click the 'Start Appeal' button in the Actions panel - this navigates to the Payer B portal login page", "11. Log in with provider@payerb.com / demo123", "12. Click 'Appeals' in the sidebar navigation to access the Claim Status Inquiry page", "13. Enter member ID ANT234567901 in the Member ID field and click Search", "14. In the search results, click on the claim row for CLM-2025-00048 (Ross, $2,100, N418) to view the Claim Status Detail", "15. Review the claim detail - note the denial reason includes: 'Bay Area Medical Group capitation arrangement terminated effective 2025-09-30 per member plan change from HMO to PPO. Claims for dates of service after 09/30/2025 should be processed by Anthem Blue Cross directly under PPO benefits.'", "16. Confirm the key finding: the delegation ended 2025-09-30, service date is 2025-10-18 - rerouting to Bay Area would be incorrect", "17. Click the 'Dispute Claim' button to open the dispute form", "18. Fill in the Contact Name field with 'Dr. Sandra Phillips' (the provider listed on the denial)", "19. In the Supporting Rationale field, write: N418 misroute denial references Bay Area Medical Group delegation. However, member's plan changed from Anthem HMO to Anthem PPO effective 2025-10-01. Bay Area Medical Group capitation arrangement terminated 2025-09-30. Service date 2025-10-18 falls under PPO plan with no delegation arrangement. Claim should be reprocessed by Anthem Blue Cross directly under PPO benefits. Rerouting to Bay Area Medical Group is not appropriate as delegation was no longer active on the date of service.", "20. CRITICAL: Scroll down if needed to ensure the Submit button is FULLY VISIBLE before clicking. Then click 'submit appeal'. note the confirmation number (format apl-ant-xxxxxxxx)", "21. Click 'Return to EMR' to go back to the denials workqueue", "22. Double-click the Ross row (DEN-048) to navigate to its denial detail page", "23. In the right panel, scroll to the Actions section", "24. Select 'Appeal Filed' from the Triage Disposition dropdown", "IMMEDIATELY after selecting disposition, type the triage note WITHOUT clicking any other tabs - clicking tabs clears the note field. 25. In the Triage Note text area, document: N418 misrouted denial for $2,100; EMR notes suggested rerouting to Bay Area Medical Group but this would be INCORRECT; patient plan changed from Anthem HMO to Anthem PPO effective 2025-10-01; Bay Area Medical Group capitation arrangement terminated 2025-09-30; service date 2025-10-18 is after delegation ended; N418 was issued based on stale delegation data from prior HMO plan; claim should be reprocessed by Anthem directly under PPO benefits; appeal filed on Payer B requesting reprocessing; appeal confirmation [number]", "26. Click 'Submit Disposition' to save the triage note - do NOT scroll or click any other element between typing the note and clicking Submit" ], "evals": [ { "id": "denial-hard-20-eval-1", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed denial details for DEN-048", "points": 1, "query": "full_state.agentActions.viewedDenialDetails", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-2", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed remittance image", "points": 1, "query": "full_state.agentActions.viewedRemittanceImage", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-3", "type": "jmespath", "category": "Information Retrieval", "description": "Agent checked patient inquiry for coverage history", "points": 1, "query": "full_state.agentActions.viewedPatientInquiry", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-4", "type": "jmespath", "category": "Information Retrieval", "description": "Agent navigated to the Payer B portal", "points": 1, "query": "full_state.agentActions.accessedPayerPortalForDenial", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-5", "type": "jmespath", "category": "Information Retrieval", "description": "Agent viewed claim detail on Payer B", "points": 1, "query": "payer_b_state.full_state.appealActions.viewedClaimDetail", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-6", "type": "jmespath", "category": "Task Resolution", "description": "Agent submitted appeal on Payer B", "points": 1, "query": "payer_b_state.full_state.appealActions.submittedAppeal", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-7", "type": "jmespath", "category": "Task Resolution", "description": "Agent selected correct triage disposition", "points": 1, "query": "full_state.agentActions.selectedDisposition", "expectedValue": "Appeal Filed", "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-8", "type": "jmespath", "category": "Documentation", "description": "Agent documented in triage note", "points": 1, "query": "full_state.agentActions.documentedAppealInEpic", "expectedValue": true, "rubric": null, "studentAnswer": null, "studentAnswerContext": null }, { "id": "denial-hard-20-eval-9", "type": "llm_judge", "category": "Documentation", "description": "Note identifies plan change from HMO to PPO before service date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that the patient's plan changed from HMO to PPO effective 2025-10-01, BEFORE the service date of 2025-10-18? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-20-eval-10", "type": "llm_judge", "category": "Documentation", "description": "Note identifies delegation terminated before service date", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note identify that the Bay Area Medical Group delegation/capitation arrangement terminated effective 2025-09-30 due to the plan change? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-20-eval-11", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note explains why rerouting is incorrect", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain why rerouting to Bay Area Medical Group would be INCORRECT - the delegation was no longer active on the date of service? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-20-eval-12", "type": "llm_judge", "category": "Clinical Reasoning", "description": "Note recommends reprocessing under PPO", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note explain that the claim should be reprocessed by Anthem directly under PPO benefits, not rerouted to the delegated group? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" }, { "id": "denial-hard-20-eval-13", "type": "llm_judge", "category": "Documentation", "description": "Note includes appeal confirmation", "points": 1, "query": null, "expectedValue": null, "rubric": "Did the agent's triage note include the appeal confirmation number from the Payer B portal? Score 1.0 if yes, 0.0 if no.", "studentAnswer": "{{full_state.triageNotes}}", "studentAnswerContext": "triage note" } ], "expectedOutcome": { "expected_disposition": "Appeal Filed" }, "rationale": "N418 misrouted denial for $2,100. EMR notes say 'reroute to Bay Area Medical Group' - but this would be WRONG. Investigation reveals the patient's plan changed from Anthem HMO to Anthem PPO effective 2025-10-01. The Bay Area Medical Group capitation arrangement terminated with the HMO plan on 2025-09-30. The service date is 2025-10-18 - after the delegation ended. The N418 was issued based on stale delegation data from the prior HMO plan. Under the current PPO plan, there is no delegation - Anthem should process the claim directly. The correct action is to file an appeal on Payer B requesting reprocessing under PPO benefits, not reroute to Bay Area Medical Group." } ] } ] }, "citation": { "bibtex": { "type": "article", "key": "healthadminbench", "title": "HealthAdminBench: A Benchmark for Evaluating LLMs on Solving Administrative Healthcare Tasks", "authors": [ "Suhana Bedi", "Ryan Welch", "Ethan Steinberg", "Michael Wornow", "Taeil Matthew Kim", "Haroun Ahmed", "Peter Sterling", "Bravim Purohit", "Qurat Akram", "Angelic Acosta", "Esther Nubla", "Pritika Sharma", "Mike Pfeffer", "Sanmi Koyejo", "Nigam Shah" ], "year": 2026 }, "raw": "@article{healthadminbench,\n title={HealthAdminBench: A Benchmark for Evaluating LLMs on Solving Administrative Healthcare Tasks},\n author={Suhana Bedi and Ryan Welch and Ethan Steinberg and Michael Wornow and Taeil Matthew Kim and Haroun Ahmed and Peter Sterling and Bravim Purohit and Qurat Akram and Angelic Acosta and Esther Nubla and Pritika Sharma and Mike Pfeffer and Sanmi Koyejo and Nigam Shah},\n year={2026}\n}" } }