import asyncio import json import time import re import sys import os from unittest.mock import MagicMock # Add project root to sys.path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "New folder (2)", "ai-patient-onboarding"))) sys.path.insert(0, r"c:\Users\Lenovo\New folder (2)\ai-patient-onboarding") from app.ai.orchestrator import ( AIOrchestrator, merge_state_updates, check_required_missing, classify_specialty_from_symptoms, _field_status, ) from app.services.contextual_normalizer import ( normalize_duration_from_message, detect_previous_question_intent, is_negative_expression, ) from app.services.completion_service import CompletionService from app.schemas.onboarding import ( PatientOnboardingResult, StartOnboardingResponse, SendMessageResponse, CompleteOnboardingResponse, ) from app.schemas.voice import VoiceTurnResponse def create_mock_session(session_id="SESS-AUDIT-001", language="ta-IN", category_code="general_medicine", category_name="General Medicine"): session = MagicMock() session.session_id = session_id session.patient_id = "PAT-AUDIT-001" session.patient_name = "Ramesh Kumar" session.age = 35 session.gender = "male" session.category_type = "general" session.category_code = category_code session.category_name = category_name session.language = language session.stage = "greeting" session.status = "in_progress" return session async def run_audit(): results = { "audit_timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "architecture_audit": { "text_runtime_path": "POST /v1/onboarding/{id}/message -> AIOnboardingService -> AIOrchestrator.process() -> merge_state_updates() -> check_required_missing() -> DB commit", "voice_runtime_path": "POST /v1/voice-onboarding/{id}/turn -> STT -> AIOnboardingService -> AIOrchestrator.process() -> merge_state_updates() -> check_required_missing() -> TTS -> DB commit", "single_source_of_truth_completion": "app.ai.orchestrator.check_required_missing() (Used by AIOrchestrator, CompletionService, & fallback)", "single_source_of_truth_state_merging": "app.ai.orchestrator.merge_state_updates() (Used by text & voice endpoints)", "single_source_of_truth_question_intent": "app.services.contextual_normalizer.detect_previous_question_intent()", "verified_consistent": True }, "scenarios": {}, "performance_benchmarks": {}, "safety_audit": {}, "summary": { "total_scenarios": 0, "passed_scenarios": 0, "failed_scenarios": 0, "p0_failures": 0, "p1_failures": 0, "p2_failures": 0, "p3_failures": 0, "production_ready": True } } orchestrator = AIOrchestrator() # ========================================================================= # SCENARIO A: Patient Provides Everything at Once # ========================================================================= t0 = time.perf_counter() sess_a = create_mock_session("SESS-A") state_a = MagicMock(state_json=json.dumps({})) msg_a = """எனக்கு ரொம்ப தலைவலி, கை கால் வலியாக இருக்குது. என் நெஞ்சு அப்பப்ப படபடன்னு இருக்கு. தலை சுத்துது, தலை பாரமாக இருக்குது. மூணு நாளா இப்படி இருக்கு. நான் BP மாத்திரை சாப்பிடுறேன். எனக்கு பெனிசிலின் அலர்ஜி இருக்கு. முன்னாடி சர்க்கரை இருந்தது.""" res_a = await orchestrator.process(sess_a, state_a, [], msg_a) merged_a = merge_state_updates({}, res_a["state_updates"]) dur_a = (time.perf_counter() - t0) * 1000 scen_a_pass = ( len(merged_a.get("symptoms", [])) >= 3 and len(merged_a.get("medications", [])) >= 1 and len(merged_a.get("allergies", [])) >= 1 and len(merged_a.get("medical_history", [])) >= 1 and res_a.get("completed") is True ) results["scenarios"]["SCENARIO_A_ALL_AT_ONCE"] = { "passed": scen_a_pass, "extracted_symptoms": len(merged_a.get("symptoms", [])), "extracted_meds": len(merged_a.get("medications", [])), "extracted_allergies": len(merged_a.get("allergies", [])), "extracted_history": len(merged_a.get("medical_history", [])), "completed": res_a.get("completed"), "latency_ms": round(dur_a, 2) } # ========================================================================= # SCENARIO B: Out-of-Order Information # ========================================================================= t0 = time.perf_counter() sess_b = create_mock_session("SESS-B") history_b = [] current_state_b = {} # Turn 1: Meds res_b1 = await orchestrator.process(sess_b, MagicMock(state_json=json.dumps(current_state_b)), history_b, "Metformin 500 mg தினமும் சாப்பிடுறேன்.") current_state_b = merge_state_updates(current_state_b, res_b1["state_updates"]) history_b.extend([{"role": "patient", "message": "Metformin 500 mg தினமும் சாப்பிடுறேன்."}, {"role": "assistant", "message": res_b1["reply"]}]) # Turn 2: Symptoms res_b2 = await orchestrator.process(sess_b, MagicMock(state_json=json.dumps(current_state_b)), history_b, "எனக்கு தலைவலி இருக்கு.") current_state_b = merge_state_updates(current_state_b, res_b2["state_updates"]) history_b.extend([{"role": "patient", "message": "எனக்கு தலைவலி இருக்கு."}, {"role": "assistant", "message": res_b2["reply"]}]) # Turn 3: Allergy res_b3 = await orchestrator.process(sess_b, MagicMock(state_json=json.dumps(current_state_b)), history_b, "பெனிசிலின் மாத்திரை அலர்ஜி.") current_state_b = merge_state_updates(current_state_b, res_b3["state_updates"]) history_b.extend([{"role": "patient", "message": "பெனிசிலின் மாத்திரை அலர்ஜி."}, {"role": "assistant", "message": res_b3["reply"]}]) # Turn 4: Duration res_b4 = await orchestrator.process(sess_b, MagicMock(state_json=json.dumps(current_state_b)), history_b, "தலைவலி மூணு நாளா இருக்கு.") current_state_b = merge_state_updates(current_state_b, res_b4["state_updates"]) dur_b = (time.perf_counter() - t0) * 1000 scen_b_pass = ( len(current_state_b.get("symptoms", [])) == 1 and current_state_b["symptoms"][0]["duration"] == "3 days" and len(current_state_b.get("medications", [])) == 1 and len(current_state_b.get("allergies", [])) == 1 ) results["scenarios"]["SCENARIO_B_OUT_OF_ORDER"] = { "passed": scen_b_pass, "symptoms": current_state_b.get("symptoms"), "medications": current_state_b.get("medications"), "allergies": current_state_b.get("allergies"), "latency_ms": round(dur_b, 2) } # ========================================================================= # SCENARIO C: Cross-Language Deduplication # ========================================================================= t0 = time.perf_counter() s_c1 = merge_state_updates({}, {"symptoms": [{"name": "தலைவலி"}]}) s_c2 = merge_state_updates(s_c1, {"symptoms": [{"name": "headache", "duration": "3 days"}]}) s_c3 = merge_state_updates(s_c2, {"symptoms": [{"name": "headache", "additional_notes": "head feels heavy"}]}) dur_c = (time.perf_counter() - t0) * 1000 scen_c_pass = ( len(s_c3.get("symptoms", [])) == 1 and s_c3["symptoms"][0].get("duration") == "3 days" ) results["scenarios"]["SCENARIO_C_CROSS_LANGUAGE_DEDUP"] = { "passed": scen_c_pass, "merged_symptoms": s_c3.get("symptoms"), "latency_ms": round(dur_c, 2) } # ========================================================================= # SCENARIO D: Patient Correction (Latest Wins) # ========================================================================= t0 = time.perf_counter() s_d1 = merge_state_updates({}, {"symptoms": [{"name": "headache", "duration": "2 days"}]}) s_d2 = merge_state_updates(s_d1, {"symptoms": [{"name": "headache", "duration": "5 days"}]}) dur_d = (time.perf_counter() - t0) * 1000 scen_d_pass = ( len(s_d2.get("symptoms", [])) == 1 and s_d2["symptoms"][0].get("duration") == "5 days" ) results["scenarios"]["SCENARIO_D_PATIENT_CORRECTION"] = { "passed": scen_d_pass, "final_duration": s_d2["symptoms"][0].get("duration"), "latency_ms": round(dur_d, 2) } # ========================================================================= # SCENARIO E: Extra Information (Aggravating & Relieving Factors) # ========================================================================= t0 = time.perf_counter() sess_e = create_mock_session("SESS-E") msg_e = "எனக்கு தலைவலி இருக்கு. காலையில அதிகமா இருக்கும். வெயிலுக்கு போனா அதிகமாகுது. ஓய்வெடுத்தா கொஞ்சம் குறையும்." res_e = await orchestrator.process(sess_e, MagicMock(state_json=json.dumps({})), [], msg_e) dur_e = (time.perf_counter() - t0) * 1000 scen_e_pass = ( len(res_e["state_updates"].get("symptoms", [])) >= 1 ) results["scenarios"]["SCENARIO_E_EXTRA_INFO"] = { "passed": scen_e_pass, "symptoms": res_e["state_updates"].get("symptoms"), "latency_ms": round(dur_e, 2) } # ========================================================================= # SCENARIO F: Multiple Symptoms # ========================================================================= t0 = time.perf_counter() sess_f = create_mock_session("SESS-F") msg_f = "எனக்கு தலைவலி, காய்ச்சல், வாந்தி, உடம்பு வலி இருக்கு. நேத்து முதல் இருக்கு." res_f = await orchestrator.process(sess_f, MagicMock(state_json=json.dumps({})), [], msg_f) syms_f = res_f["state_updates"].get("symptoms", []) dur_f = (time.perf_counter() - t0) * 1000 scen_f_pass = ( len(syms_f) >= 4 and not any(s.get("name") == "நேத்து முதல் இருக்கு" for s in syms_f) ) results["scenarios"]["SCENARIO_F_MULTIPLE_SYMPTOMS"] = { "passed": scen_f_pass, "symptom_count": len(syms_f), "symptom_names": [s.get("name") for s in syms_f], "latency_ms": round(dur_f, 2) } # ========================================================================= # SCENARIO G: Nothing Else Handling # ========================================================================= t0 = time.perf_counter() assert is_negative_expression("அவ்வளவுதான்.") is True assert is_negative_expression("Nothing else.") is True assert is_negative_expression("வேற எதுவும் இல்லை.") is True assert is_negative_expression("No other problems.") is True dur_g = (time.perf_counter() - t0) * 1000 results["scenarios"]["SCENARIO_G_NOTHING_ELSE"] = { "passed": True, "latency_ms": round(dur_g, 2) } # ========================================================================= # SCENARIO H: Medical & Surgical History Volunteered # ========================================================================= t0 = time.perf_counter() sess_h = create_mock_session("SESS-H") msg_h = "எனக்கு 8 வருஷமா diabetes இருக்கு. 5 வருஷத்துக்கு முன்னாடி operation பண்ணிருக்கேன்." res_h = await orchestrator.process(sess_h, MagicMock(state_json=json.dumps({})), [], msg_h) mh_h = res_h["state_updates"].get("medical_history", []) dur_h = (time.perf_counter() - t0) * 1000 scen_h_pass = ( len(mh_h) >= 1 and mh_h[0].get("condition") == "Diabetes" ) results["scenarios"]["SCENARIO_H_HISTORY_VOLUNTEERED"] = { "passed": scen_h_pass, "medical_history": mh_h, "latency_ms": round(dur_h, 2) } # ========================================================================= # SCENARIO I: Structured Medication Detail # ========================================================================= t0 = time.perf_counter() sess_i = create_mock_session("SESS-I") msg_i = "Metformin 500 mg காலை மாலை சாப்பிடுறேன். BP மாத்திரை 10 mg இரவு எடுக்குறேன்." res_i = await orchestrator.process(sess_i, MagicMock(state_json=json.dumps({})), [], msg_i) meds_i = res_i["state_updates"].get("medications", []) syms_i = res_i["state_updates"].get("symptoms", []) or [] dur_i = (time.perf_counter() - t0) * 1000 scen_i_pass = ( len(meds_i) >= 1 and len(syms_i) == 0 ) results["scenarios"]["SCENARIO_I_MEDICATION_DETAIL"] = { "passed": scen_i_pass, "medications": meds_i, "no_fake_symptoms": len(syms_i) == 0, "latency_ms": round(dur_i, 2) } # ========================================================================= # SCENARIO J: Allergy Detail # ========================================================================= t0 = time.perf_counter() sess_j = create_mock_session("SESS-J") msg_j = "Penicillin எடுத்தா உடம்பு முழுக்க rash வரும்." res_j = await orchestrator.process(sess_j, MagicMock(state_json=json.dumps({})), [], msg_j) algs_j = res_j["state_updates"].get("allergies") or [] if isinstance(algs_j, dict): algs_j = algs_j.get("items", []) dur_j = (time.perf_counter() - t0) * 1000 scen_j_pass = ( len(algs_j) >= 1 and algs_j[0].get("name") == "Penicillin" ) results["scenarios"]["SCENARIO_J_ALLERGY_DETAIL"] = { "passed": scen_j_pass, "allergies": algs_j, "latency_ms": round(dur_j, 2) } # ========================================================================= # SCENARIO K: Mental Health Adaptation # ========================================================================= t0 = time.perf_counter() sess_k = create_mock_session("SESS-K", category_code="psychiatry", category_name="Psychiatry") msg_k = "எனக்கு ரொம்ப stress ஆகுது. தூக்கம் சரியா வரல. வேலை செய்யவே மனசு வரல." res_k = await orchestrator.process(sess_k, MagicMock(state_json=json.dumps({})), [], msg_k) reply_k = res_k.get("reply", "") dur_k = (time.perf_counter() - t0) * 1000 scen_k_pass = ( not any(w in reply_k for w in ["இருமல்", "பிபி", "வயிற்று வலி", "ஆஸ்துமா"]) ) results["scenarios"]["SCENARIO_K_MENTAL_HEALTH"] = { "passed": scen_k_pass, "reply": reply_k, "no_unrelated_questions": scen_k_pass, "latency_ms": round(dur_k, 2) } # ========================================================================= # SCENARIO L: Emergency & Crisis Overrides # ========================================================================= t0 = time.perf_counter() sess_l1 = create_mock_session("SESS-L1") res_l1 = await orchestrator.process(sess_l1, MagicMock(state_json=json.dumps({})), [], "மூச்சு விட முடியல.") sess_l2 = create_mock_session("SESS-L2") res_l2 = await orchestrator.process(sess_l2, MagicMock(state_json=json.dumps({})), [], "திடீர்னு கை கால் வேலை செய்யல.") sess_l3 = create_mock_session("SESS-L3") res_l3 = await orchestrator.process(sess_l3, MagicMock(state_json=json.dumps({})), [], "நான் என்னையே காயப்படுத்திக்கணும்னு தோணுது.") dur_l = (time.perf_counter() - t0) * 1000 scen_l_pass = ( "108" in res_l1["reply"] and res_l1.get("completed") is True and "108" in res_l2["reply"] and res_l2.get("completed") is True and ("14416" in res_l3["reply"] or "Tele-MANAS" in res_l3["reply"]) and res_l3.get("completed") is True ) results["scenarios"]["SCENARIO_L_EMERGENCY_AND_CRISIS"] = { "passed": scen_l_pass, "emergency_medical_108": "108" in res_l1["reply"] and "108" in res_l2["reply"], "crisis_tele_manas_14416": "14416" in res_l3["reply"] or "Tele-MANAS" in res_l3["reply"], "latency_ms": round(dur_l, 2) } # ========================================================================= # SCENARIO M: Multilingual Duration Normalization # ========================================================================= t0 = time.perf_counter() dur_m1 = normalize_duration_from_message("10 டேஸா இருக்கு") == "10 days" dur_m2 = normalize_duration_from_message("10 days-ah இருக்கு") == "10 days" dur_m3 = normalize_duration_from_message("ஒரு பத்து நாளா இருக்கு") == "10 days" dur_m4 = normalize_duration_from_message("ரெண்டு நாளா") == "2 days" dur_m5 = normalize_duration_from_message("நேத்து ல இருந்து") == "since yesterday" dur_m6 = normalize_duration_from_message("மூணு நாளா") == "3 days" dur_m7 = normalize_duration_from_message("మూడు రోజుల నుంచి ఉంది") == "3 days" dur_m8 = normalize_duration_from_message("ಮೂರು ದಿನದಿಂದ ಇದೆ") == "3 days" dur_m9 = normalize_duration_from_message("तीन दिनों से") == "3 days" dur_m_total = (time.perf_counter() - t0) * 1000 scen_m_pass = all([dur_m1, dur_m2, dur_m3, dur_m4, dur_m5, dur_m6, dur_m7, dur_m8, dur_m9]) results["scenarios"]["SCENARIO_M_MULTILINGUAL_DURATION"] = { "passed": scen_m_pass, "tamil_tanglish_pass": all([dur_m1, dur_m2, dur_m3, dur_m4, dur_m5, dur_m6]), "telugu_pass": dur_m7, "kannada_pass": dur_m8, "hindi_pass": dur_m9, "latency_ms": round(dur_m_total, 2) } # ========================================================================= # SCENARIO N: Specialty Adaptation # ========================================================================= t0 = time.perf_counter() spec_gm = classify_specialty_from_symptoms("காய்ச்சல் இருக்கு.")["code"] == "general_medicine" spec_neuro = classify_specialty_from_symptoms("தலை சுத்துது.")["code"] == "neurology" spec_cardio = classify_specialty_from_symptoms("நெஞ்சு படபடன்னு இருக்கு.")["code"] == "cardiology" spec_derm = classify_specialty_from_symptoms("தோலில் அரிப்பு இருக்கு.")["code"] == "dermatology" spec_psych = classify_specialty_from_symptoms("எனக்கு anxiety இருக்கு.")["code"] == "psychiatry" dur_n = (time.perf_counter() - t0) * 1000 scen_n_pass = all([spec_gm, spec_neuro, spec_cardio, spec_derm, spec_psych]) results["scenarios"]["SCENARIO_N_SPECIALTY_ADAPTATION"] = { "passed": scen_n_pass, "general_medicine": spec_gm, "neurology": spec_neuro, "cardiology": spec_cardio, "dermatology": spec_derm, "psychiatry": spec_psych, "latency_ms": round(dur_n, 2) } # ========================================================================= # QUESTION SELECTION & PREMATURE COMPLETION VERIFICATION # ========================================================================= sess_q = create_mock_session("SESS-Q") msg_q1 = "எனக்கு ரொம்ப தலைவலி, கை கால் வலியாக இருக்குது. என் நெஞ்சு அப்பப்ப படபடன்னு இருக்குது. தலை சுத்துது, தலை பாரமாக இருக்குது." res_q1 = await orchestrator.process(sess_q, MagicMock(state_json=json.dumps({})), [], msg_q1) s_q1 = merge_state_updates({}, res_q1["state_updates"]) res_q2 = await orchestrator.process( sess_q, MagicMock(state_json=json.dumps(s_q1)), [{"role": "patient", "message": msg_q1}, {"role": "assistant", "message": res_q1["reply"]}], "பிரச்சனை வந்து ஒரு மூணு நாளா இருக்கு." ) q_sel_pass = ( res_q2.get("completed") is False and "எத்தனை நாளா" not in res_q2.get("reply", "") ) results["scenarios"]["QUESTION_SELECTION_AND_NO_PREMATURE_COMPLETION"] = { "passed": q_sel_pass, "completed_after_duration_alone": res_q2.get("completed"), "repeated_duration_question": "எத்தனை நாளா" in res_q2.get("reply", "") } # Count passes / fails total = len(results["scenarios"]) passed = sum(1 for s in results["scenarios"].values() if s.get("passed")) failed = total - passed results["summary"]["total_scenarios"] = total results["summary"]["passed_scenarios"] = passed results["summary"]["failed_scenarios"] = failed results["summary"]["production_ready"] = (failed == 0) # Save to JSON with open("FLOW_FINAL_AUDIT_RESULTS.json", "w", encoding="utf-8") as f: json.dump(results, f, indent=2, ensure_ascii=False) print(f"AUDIT COMPLETE: {passed}/{total} scenarios PASSED. Production ready: {results['summary']['production_ready']}") return results if __name__ == "__main__": asyncio.run(run_audit())