Gokulvarthan-Ramesh/aionboarding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement AI-orchestrated result extraction and service connectivity tests

Browse Source
This commit is contained in:
Gokulvarthan-Ramesh
2026-08-27 17:45:07 +05:30
parent 59c1372fe3
commit 3d2596ced1
11 changed files with 546 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai-patient-onboarding/app/ai/orchestrator.py
+67 -17
View File
@@ -3,7 +3,7 @@ import json
from app.ai.openai_client import OpenAIClient
from app.ai.prompts import build_system_prompt
from app.ai.validator import validate_ai_response
from app.config import OPENAI_API_KEY
from app.config import OPENAI_API_KEY, GEMINI_API_KEY, GEMINI_MODEL, ANTHROPIC_API_KEY, ANTHROPIC_MODEL
from app.specialties.registry import get_specialty_config
@@ -49,7 +49,22 @@ class AIOrchestrator:
prompt = build_system_prompt(context)
# Real OpenAI Call if valid API key is set
# 1. Primary: Google Gemini (High-Speed Multilingual Indic Reasoning ~0.8s)
if GEMINI_API_KEY and GEMINI_API_KEY.strip() not in ["your_gemini_key", ""]:
try:
import google.generativeai as genai
genai.configure(api_key=GEMINI_API_KEY)
model = genai.GenerativeModel(GEMINI_MODEL or "gemini-3.6-flash")
full_prompt = f"{prompt}\n\nPATIENT MESSAGE: {user_message}\n\nRespond strictly with JSON matching the schema above."
res = await model.generate_content_async(full_prompt)
raw_text = res.text.replace("```json", "").replace("```", "").strip()
parsed = json.loads(raw_text)
validated = validate_ai_response(parsed)
return validated.model_dump()
except Exception as e:
print(f"[Gemini Orchestrator Error]: {e}")
# 2. Secondary: OpenAI Call if valid API key is set
if OPENAI_API_KEY and OPENAI_API_KEY.strip() not in ["your_openai_api_key", "your_openai_key", ""]:
try:
raw_response = await self.openai.generate_response(
@@ -62,7 +77,25 @@ class AIOrchestrator:
except Exception as e:
print(f"[OpenAI Orchestrator Error]: {e}")
# Fallback simulator for offline development with concise clinical flow
# 3. Tertiary: Anthropic Claude
if ANTHROPIC_API_KEY and ANTHROPIC_API_KEY.strip() not in ["your_anthropic_key", ""]:
try:
from anthropic import AsyncAnthropic
client = AsyncAnthropic(api_key=ANTHROPIC_API_KEY)
res = await client.messages.create(
model=ANTHROPIC_MODEL or "claude-sonnet-4-5-20250929",
max_tokens=600,
system=prompt,
messages=[{"role": "user", "content": user_message}],
)
raw_text = res.content[0].text.replace("```json", "").replace("```", "").strip()
parsed = json.loads(raw_text)
validated = validate_ai_response(parsed)
return validated.model_dump()
except Exception as e:
print(f"[Claude Orchestrator Error]: {e}")
# 4. Fallback simulator for offline development with localized language support
parsed = self._simulate_specialty_response(session, specialty, user_message, len(conversation_history))
validated = validate_ai_response(parsed)
return validated.model_dump()
@@ -70,13 +103,36 @@ class AIOrchestrator:
def _simulate_specialty_response(self, session, specialty, user_message, turn_count=0):
stage = session.stage
code = specialty.get("code", "general")
lang = (session.language or "ta-IN").lower()
msg_lower = user_message.lower().strip()
is_negative = msg_lower in ["no", "no more", "nothing", "nothing else", "know nothing", "none", "illness none"]
is_negative = msg_lower in ["no", "no more", "nothing", "nothing else", "know nothing", "none", "illness none", "இல்லை", "ஒன்றுமில்லை", "नहीं"]
# Fast conclusion if patient says "No" or already had 4 turns
# Localized clinical phrases
if lang.startswith("ta"):
conclusion_text = "நன்றி. உங்கள் உடல்நல விவரங்கள் பதிவு செய்யப்பட்டுள்ளன. மருத்துவர் விரைவில் பரிசீலிப்பார்."
symptom_q = "இந்த அறிகுறிகள் எவ்வளவு நாட்களாக இருக்கிறது, மற்றும் ஏதேனும் மாத்திரைகள் சாப்பிட்டீர்களா?"
med_q = "நன்றி. உங்களுக்கு ஏதேனும் மருந்து ஒவ்வாமை (Allergy) அல்லது வேறு உடல்நல பிரச்சனைகள் உள்ளதா?"
elif lang.startswith("hi"):
conclusion_text = "धन्यवाद। आपकी स्वास्थ्य जानकारी दर्ज कर ली गई है। डॉक्टर जल्द ही समीक्षा करेंगे।"
symptom_q = "आपको यह समस्या कितने दिनों से है और क्या आपने कोई दवाई ली है?"
med_q = "धन्यवाद। क्या आपको किसी दवाई से एलर्जी या कोई अन्य बीमारी है?"
elif lang.startswith("te"):
conclusion_text = "ధన్యవాదాలు. మీ ఆరోగ్య వివరాలు నమోదు చేయబడ్డాయి."
symptom_q = "ఈ సమస్య మీకు ఎన్ని రోజుల నుండి ఉంది మరియు ఏవైనా మందులు వాడారా?"
med_q = "ధన్యవాదాలు. మీకు ఏవైనా మందుల పడనితనం (Allergy) ఉందా?"
elif lang.startswith("kn"):
conclusion_text = "ಧನ್ಯವಾದಗಳು. ನಿಮ್ಮ ಆರೋಗ್ಯದ ಮಾಹಿತಿ ದಾಖಲಾಗಿದೆ."
symptom_q = "ಈ ಸಮಸ್ಯೆ ನಿಮಗೆ ಎಷ್ಟು ದಿನಗಳಿಂದ ಇದೆ ಮತ್ತು ಯಾವುದೇ ಮಾತ್ರೆಗಳನ್ನು ತೆಗೆದುಕೊಂಡಿದ್ದೀರಾ?"
med_q = "ಧನ್ಯವಾದಗಳು. ನಿಮಗೆ ಯಾವುದೇ ಔಷಧಿ ಅಲರ್ಜಿ ಇದೆಯೇ?"
else:
conclusion_text = "Thank you. I have recorded your intake details for the doctor to review."
symptom_q = "How long have you had these symptoms, and have you taken any medicines?"
med_q = "Thank you. Do you have any known drug allergies or existing health conditions?"
# Fast conclusion if patient says "No" or already had 6 turns
if is_negative or turn_count >= 6:
return {
"reply": "Thank you. I have recorded your intake details for the doctor to review.",
"reply": conclusion_text,
"next_stage": "confirmation",
"state_updates": {
"reason_for_visit": None,
@@ -89,12 +145,8 @@ class AIOrchestrator:
}
if stage == "greeting":
if code == "cardiology":
reply = "Could you please share what cardiac or chest symptoms you are experiencing?"
else:
reply = "Could you please describe what primary symptoms you are experiencing today?"
return {
"reply": reply,
"reply": symptom_q,
"next_stage": "reason_for_visit",
"state_updates": {
"reason_for_visit": user_message,
@@ -106,9 +158,8 @@ class AIOrchestrator:
"completed": False
}
elif stage == "reason_for_visit":
reply = "How long have you had these symptoms, and have you taken any medicines?"
return {
"reply": reply,
"reply": med_q,
"next_stage": "medications",
"state_updates": {
"reason_for_visit": user_message,
@@ -120,10 +171,9 @@ class AIOrchestrator:
"completed": False
}
elif stage == "medications":
reply = "Thank you. Do you have any known drug allergies or existing health conditions?"
return {
"reply": reply,
"next_stage": "allergies",
"reply": conclusion_text,
"next_stage": "confirmation",
"state_updates": {
"reason_for_visit": None,
"symptoms": [],
@@ -135,7 +185,7 @@ class AIOrchestrator:
}
else:
return {
"reply": "Thank you. I have recorded your intake details for the doctor to review.",
"reply": conclusion_text,
"next_stage": "confirmation",
"state_updates": {
"reason_for_visit": None,
ai-patient-onboarding/app/ai/sarvam_client.py
+2 -2
View File
@@ -55,7 +55,7 @@ class SarvamClient:
return None
lang_prefix = language_code.lower()[:2]
speaker = "meera" if lang_prefix == "ta" else "arvind"
speaker = "anushka" if lang_prefix in ["ta", "te", "hi"] else "karun"
client = self._get_client()
headers = {
@@ -66,7 +66,7 @@ class SarvamClient:
"inputs": [text],
"target_language_code": language_code,
"speaker": speaker,
"model": "bulbul:v1",
"model": "bulbul:v2",
}
try:
res = await client.post(
ai-patient-onboarding/app/api/voice_onboarding.py
+34 -12
View File
@@ -130,9 +130,14 @@ async def voice_turn(
detail="Voice onboarding session is not active",
)
# 2. Transcribe speech if audio provided with multi-model Indic recognition
# 2. Fast-path transcript: use real-time Web Speech text_fallback first for zero-latency processing
patient_transcript = ""
if audio:
if text_fallback and text_fallback.strip():
from app.services.medical_speech_normalizer import normalize_medical_speech
patient_transcript = normalize_medical_speech(text_fallback.strip(), language_code=session.language)
# If text_fallback was not available, run server-side multi-model STT
if not patient_transcript and audio:
audio_bytes = await audio.read()
if len(audio_bytes) > 0:
patient_transcript = await voice_service.speech_to_text(
@@ -147,18 +152,35 @@ async def voice_turn(
},
)
# If backend STT is empty, use frontend real-time transcript fallback
if (not patient_transcript or not patient_transcript.strip()) and text_fallback and text_fallback.strip():
from app.services.medical_speech_normalizer import normalize_medical_speech
patient_transcript = normalize_medical_speech(text_fallback.strip(), language_code=session.language)
# If STILL empty, user spoke nothing or silence was recorded.
# Do NOT substitute fake default text or auto-fill clinical fields!
if not patient_transcript or not patient_transcript.strip():
if session.language.startswith("ta"):
patient_transcript = "எனக்கு உடல்நலக்குறைவு உள்ளது."
elif session.language.startswith("hi"):
patient_transcript = "मुझे स्वास्थ्य संबंधी समस्या है।"
lang = session.language.lower()
if lang.startswith("ta"):
no_speech_msg = "மன்னிக்கவும், உங்கள் குரல் கேட்கவில்லை. தயவுசெய்து உங்கள் உடல்நலப் பிரச்சினை பற்றி பேசவும்."
elif lang.startswith("hi"):
no_speech_msg = "क्षमा करें, मुझे आपकी आवाज़ सुनाई नहीं दी। कृपया अपनी स्वास्थ्य समस्या के बारे में बोलें।"
elif lang.startswith("te"):
no_speech_msg = "క్షమించండి, మీ ధ్వని వినిపించలేదు. దయచేసి మీ ఆరోగ్య సమస్య గురించి మాట్లాడండి."
elif lang.startswith("kn"):
no_speech_msg = "ಕ್ಷಮಿಸಿ, ನಿಮ್ಮ ಧ್ವನಿ ಕೇಳಿಸಲಿಲ್ಲ. ದಯವಿಟ್ಟು ನಿಮ್ಮ ಆರೋಗ್ಯದ ತೊಂದರೆ ಬಗ್ಗೆ ಮಾತನಾಡಿ."
else:
patient_transcript = "I am experiencing symptoms."
no_speech_msg = "I didn't hear anything. Please speak into your microphone and describe your health symptoms."
audio_id, _ = await voice_service.text_to_speech(
text=no_speech_msg,
language_code=session.language,
)
return VoiceTurnResponse(
success=True,
session_id=session_id,
patient_transcript="",
ai_text=no_speech_msg,
stage=session.stage,
completed=False,
audio_url=f"/v1/voice-onboarding/audio/{audio_id}",
)
ai-patient-onboarding/app/config.py
+2 -2
View File
@@ -14,10 +14,10 @@ OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")
ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-3-5-sonnet-20241022")
ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-5-20250929")
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")
GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-1.5-flash")
GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-3.6-flash")
SARVAM_API_KEY = os.getenv("SARVAM_API_KEY", "")
ai-patient-onboarding/app/services/result_extraction_service.py
+36 -4
View File
@@ -95,10 +95,42 @@ Return ONLY JSON conforming to this exact structure:
validated = PatientOnboardingResult.model_validate(parsed)
return validated.model_dump()
except Exception as e:
print(f"[Result Extraction Error]: {e}, falling back to structured state")
print(f"[OpenAI Result Extraction Error]: {e}")
# Secondary: Google Gemini extraction
from app.config import GEMINI_API_KEY, GEMINI_MODEL, ANTHROPIC_API_KEY, ANTHROPIC_MODEL
if GEMINI_API_KEY and GEMINI_API_KEY.strip() not in ["your_gemini_key", ""]:
try:
import google.generativeai as genai
genai.configure(api_key=GEMINI_API_KEY)
model = genai.GenerativeModel(GEMINI_MODEL or "gemini-3.6-flash")
res = await model.generate_content_async(prompt)
cleaned = res.text.replace("```json", "").replace("```", "").strip()
parsed = json.loads(cleaned)
validated = PatientOnboardingResult.model_validate(parsed)
return validated.model_dump()
except Exception as e:
print(f"[Gemini Result Extraction Error]: {e}")
# Fallback simulator from state
# Tertiary: Anthropic Claude extraction
if ANTHROPIC_API_KEY and ANTHROPIC_API_KEY.strip() not in ["your_anthropic_key", ""]:
try:
from anthropic import AsyncAnthropic
client = AsyncAnthropic(api_key=ANTHROPIC_API_KEY)
res = await client.messages.create(
model=ANTHROPIC_MODEL or "claude-sonnet-4-5-20250929",
max_tokens=800,
system=prompt,
messages=[{"role": "user", "content": "Generate structured onboarding result"}],
)
cleaned = res.content[0].text.replace("```json", "").replace("```", "").strip()
parsed = json.loads(cleaned)
validated = PatientOnboardingResult.model_validate(parsed)
return validated.model_dump()
except Exception as e:
print(f"[Claude Result Extraction Error]: {e}")
# Fallback simulator from state (strictly without fake dummy strings)
chief_complaint = current_state.get("reason_for_visit")
if isinstance(chief_complaint, dict):
chief_complaint = chief_complaint.get("value")
@@ -126,8 +158,8 @@ Return ONLY JSON conforming to this exact structure:
"code": session.category_code,
"name": session.category_name,
},
"chief_complaint": chief_complaint or "Consultation Intake",
"symptoms": symptoms_list or [{"name": chief_complaint or "Reported discomfort", "duration": "recent", "severity": "mild", "description": None}],
"chief_complaint": chief_complaint or None,
"symptoms": symptoms_list,
"medical_history": history_list,
"medications": meds_list,
"allergies": allergies_list,
ai-patient-onboarding/app/services/voice_service.py
+10 -86
View File
@@ -57,7 +57,7 @@ class VoiceService:
lang_prefix = language_code.lower()[:2]
raw_transcript = ""
# 1. Sarvam AI Saaras v2 for Native Indian Languages
# 1. Primary: Sarvam AI Saaras v2 for Native Indian Languages (Ultra-fast ~0.9s)
if (
self.sarvam_key
and self.sarvam_key.strip()
@@ -65,7 +65,7 @@ class VoiceService:
and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"]
):
try:
async with httpx2.AsyncClient(timeout=8.0) as client:
async with httpx2.AsyncClient(timeout=2.5) as client:
files = {
"file": (filename, audio_bytes, "audio/webm")
}
@@ -91,86 +91,7 @@ class VoiceService:
except Exception as e:
print(f"[Sarvam STT Error]: {e}")
# 2. Google Gemini Multimodal Audio (Excellent with Tanglish/Hinglish & Medical terms)
if (
not raw_transcript
and self.gemini_key
and self.gemini_key.strip()
and self.gemini_key not in ["your_gemini_key", ""]
):
try:
import google.generativeai as genai
genai.configure(api_key=self.gemini_key)
model = genai.GenerativeModel(self.gemini_model or "gemini-1.5-flash")
mime_type = "audio/webm"
if filename.endswith(".wav"):
mime_type = "audio/wav"
elif filename.endswith(".mp3"):
mime_type = "audio/mp3"
audio_part = {
"mime_type": mime_type,
"data": audio_bytes,
}
audio_prompt = f"""
You are an expert medical transcription engine for Indian hospitals.
Accurately transcribe the spoken audio into text.
Language context: {language_code}. Specialty: {specialty_code}.
Handle code-switching (e.g. Tamil/Hindi mixed with English medical words like BP, sugar, chest pain, tablets, fever, vomiting).
Return ONLY the exact verbatim transcription without explanation, punctuation commentary, or markdown quotes.
"""
response = await model.generate_content_async([audio_part, audio_prompt])
text = response.text.strip().replace('"', '').replace("```", "")
if text and len(text) > 1 and "unable to transcribe" not in text.lower():
raw_transcript = text
except Exception as e:
print(f"[Gemini Audio STT Error]: {e}")
# 3. OpenAI Whisper-1 with Medical Specialty Context Prompting
if (
not raw_transcript
and self.openai_key
and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]
):
client = AsyncOpenAI(api_key=self.openai_key, timeout=8.0)
try:
audio_file = io.BytesIO(audio_bytes)
audio_file.name = filename if filename.endswith((".wav", ".mp3", ".webm", ".m4a")) else "audio.webm"
specialty_vocab = {
"cardiology": "Cardiology, chest pain, palpitations, ECG, hypertension, BP, heart attack, shortness of breath, Metformin, Telmisartan, Amlodipine, Atorvastatin, Sorbitrate",
"orthopedics": "Orthopedics, joint pain, knee pain, fracture, swelling, arthritis, Paracetamol, Tramadol, Calcium, Vitamin D, physiotherapy",
"general": "General medicine, fever, cough, cold, headache, diabetes, high blood pressure, vomiting, loose motion, Paracetamol 650, Dolo 650, Azithromycin, Pantoprazole",
"pediatrics": "Pediatrics, baby fever, vaccination, cough, cold, vomiting, appetite loss, Paracetamol drops",
"dermatology": "Dermatology, skin rash, itching, redness, fungal infection, Cetirizine, ointment, allergy",
}
spec_terms = specialty_vocab.get(specialty_code, specialty_vocab["general"])
lang_prompts = {
"ta": f"மருத்துவ ஆலோசனை, நெஞ்சு வலி, சர்க்கரை வியாதி, பிபி, மாத்திரை, காய்ச்சல், இருமல், வலி, {spec_terms}",
"hi": f"चिकित्सा, सीने में दर्द, बीपी, शुगर, बुखार, दवाई, सिरदर्द, खांसी, {spec_terms}",
"te": f"వైద్యం, ఛాతీ నొప్పి, షుగర్, బీపీ, మందులు, జ్వరం, {spec_terms}",
"kn": f"ವೈದ್ಯಕೀಯ, ಎದೆ ನೋವು, ಬಿಪಿ, ಸಕ್ಕರೆ ಕಾಯಿಲೆ, ಮಾತ್ರೆಗಳು, ಜ್ವರ, {spec_terms}",
"en": f"Medical symptoms, chest pain, diabetes, blood pressure, fever, cough, {spec_terms}",
}
med_prompt = lang_prompts.get(lang_prefix, lang_prompts["en"])
transcript = await client.audio.transcriptions.create(
model="whisper-1",
file=audio_file,
language=lang_prefix if lang_prefix in ["ta", "hi", "te", "kn", "en"] else "en",
prompt=med_prompt,
temperature=0.0,
)
if transcript.text and len(transcript.text.strip()) > 1:
raw_transcript = transcript.text.strip()
except Exception as e:
print(f"[OpenAI Whisper Error]: {e}")
finally:
await client.close()
# 4. Apply Medical Speech Normalizer to eliminate phonetic mishearings
# Apply Medical Speech Normalizer to eliminate phonetic mishearings
normalized = normalize_medical_speech(raw_transcript, language_code=language_code)
return normalized
@@ -200,7 +121,8 @@ Return ONLY the exact verbatim transcription without explanation, punctuation co
and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"]
):
try:
async with httpx2.AsyncClient(timeout=6.0) as client:
speaker = "anushka" if lang_prefix in ["ta", "hi", "te", "kn"] else "karun"
async with httpx2.AsyncClient(timeout=3.0) as client:
headers = {
"api-subscription-key": self.sarvam_key,
"Content-Type": "application/json",
@@ -208,8 +130,8 @@ Return ONLY the exact verbatim transcription without explanation, punctuation co
payload = {
"inputs": [cleaned_text],
"target_language_code": language_code,
"speaker": "meera",
"model": "bulbul:v1",
"speaker": speaker,
"model": "bulbul:v2",
}
res = await client.post(
"https://api.sarvam.ai/text-to-speech",
@@ -224,12 +146,14 @@ Return ONLY the exact verbatim transcription without explanation, punctuation co
file_path = AUDIO_DIR / f"{audio_id}.wav"
file_path.write_bytes(audio_data)
return audio_id, audio_data
else:
print(f"[Sarvam TTS HTTP Error]: {res.status_code} - {res.text}")
except Exception as e:
print(f"[Sarvam TTS Error]: {e}")
# 2. Use OpenAI TTS
if self.openai_key and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]:
client = AsyncOpenAI(api_key=self.openai_key, timeout=6.0)
client = AsyncOpenAI(api_key=self.openai_key, timeout=1.5)
try:
response = await client.audio.speech.create(
model="tts-1",