feat: implement AI-orchestrated result extraction and service connectivity tests
This commit is contained in:
@@ -95,10 +95,42 @@ Return ONLY JSON conforming to this exact structure:
|
||||
validated = PatientOnboardingResult.model_validate(parsed)
|
||||
return validated.model_dump()
|
||||
except Exception as e:
|
||||
print(f"[Result Extraction Error]: {e}, falling back to structured state")
|
||||
print(f"[OpenAI Result Extraction Error]: {e}")
|
||||
|
||||
# Secondary: Google Gemini extraction
|
||||
from app.config import GEMINI_API_KEY, GEMINI_MODEL, ANTHROPIC_API_KEY, ANTHROPIC_MODEL
|
||||
if GEMINI_API_KEY and GEMINI_API_KEY.strip() not in ["your_gemini_key", ""]:
|
||||
try:
|
||||
import google.generativeai as genai
|
||||
genai.configure(api_key=GEMINI_API_KEY)
|
||||
model = genai.GenerativeModel(GEMINI_MODEL or "gemini-3.6-flash")
|
||||
res = await model.generate_content_async(prompt)
|
||||
cleaned = res.text.replace("```json", "").replace("```", "").strip()
|
||||
parsed = json.loads(cleaned)
|
||||
validated = PatientOnboardingResult.model_validate(parsed)
|
||||
return validated.model_dump()
|
||||
except Exception as e:
|
||||
print(f"[Gemini Result Extraction Error]: {e}")
|
||||
|
||||
# Fallback simulator from state
|
||||
# Tertiary: Anthropic Claude extraction
|
||||
if ANTHROPIC_API_KEY and ANTHROPIC_API_KEY.strip() not in ["your_anthropic_key", ""]:
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
client = AsyncAnthropic(api_key=ANTHROPIC_API_KEY)
|
||||
res = await client.messages.create(
|
||||
model=ANTHROPIC_MODEL or "claude-sonnet-4-5-20250929",
|
||||
max_tokens=800,
|
||||
system=prompt,
|
||||
messages=[{"role": "user", "content": "Generate structured onboarding result"}],
|
||||
)
|
||||
cleaned = res.content[0].text.replace("```json", "").replace("```", "").strip()
|
||||
parsed = json.loads(cleaned)
|
||||
validated = PatientOnboardingResult.model_validate(parsed)
|
||||
return validated.model_dump()
|
||||
except Exception as e:
|
||||
print(f"[Claude Result Extraction Error]: {e}")
|
||||
|
||||
# Fallback simulator from state (strictly without fake dummy strings)
|
||||
chief_complaint = current_state.get("reason_for_visit")
|
||||
if isinstance(chief_complaint, dict):
|
||||
chief_complaint = chief_complaint.get("value")
|
||||
@@ -126,8 +158,8 @@ Return ONLY JSON conforming to this exact structure:
|
||||
"code": session.category_code,
|
||||
"name": session.category_name,
|
||||
},
|
||||
"chief_complaint": chief_complaint or "Consultation Intake",
|
||||
"symptoms": symptoms_list or [{"name": chief_complaint or "Reported discomfort", "duration": "recent", "severity": "mild", "description": None}],
|
||||
"chief_complaint": chief_complaint or None,
|
||||
"symptoms": symptoms_list,
|
||||
"medical_history": history_list,
|
||||
"medications": meds_list,
|
||||
"allergies": allergies_list,
|
||||
|
||||
@@ -57,7 +57,7 @@ class VoiceService:
|
||||
lang_prefix = language_code.lower()[:2]
|
||||
raw_transcript = ""
|
||||
|
||||
# 1. Sarvam AI Saaras v2 for Native Indian Languages
|
||||
# 1. Primary: Sarvam AI Saaras v2 for Native Indian Languages (Ultra-fast ~0.9s)
|
||||
if (
|
||||
self.sarvam_key
|
||||
and self.sarvam_key.strip()
|
||||
@@ -65,7 +65,7 @@ class VoiceService:
|
||||
and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"]
|
||||
):
|
||||
try:
|
||||
async with httpx2.AsyncClient(timeout=8.0) as client:
|
||||
async with httpx2.AsyncClient(timeout=2.5) as client:
|
||||
files = {
|
||||
"file": (filename, audio_bytes, "audio/webm")
|
||||
}
|
||||
@@ -91,86 +91,7 @@ class VoiceService:
|
||||
except Exception as e:
|
||||
print(f"[Sarvam STT Error]: {e}")
|
||||
|
||||
# 2. Google Gemini Multimodal Audio (Excellent with Tanglish/Hinglish & Medical terms)
|
||||
if (
|
||||
not raw_transcript
|
||||
and self.gemini_key
|
||||
and self.gemini_key.strip()
|
||||
and self.gemini_key not in ["your_gemini_key", ""]
|
||||
):
|
||||
try:
|
||||
import google.generativeai as genai
|
||||
genai.configure(api_key=self.gemini_key)
|
||||
model = genai.GenerativeModel(self.gemini_model or "gemini-1.5-flash")
|
||||
|
||||
mime_type = "audio/webm"
|
||||
if filename.endswith(".wav"):
|
||||
mime_type = "audio/wav"
|
||||
elif filename.endswith(".mp3"):
|
||||
mime_type = "audio/mp3"
|
||||
|
||||
audio_part = {
|
||||
"mime_type": mime_type,
|
||||
"data": audio_bytes,
|
||||
}
|
||||
audio_prompt = f"""
|
||||
You are an expert medical transcription engine for Indian hospitals.
|
||||
Accurately transcribe the spoken audio into text.
|
||||
Language context: {language_code}. Specialty: {specialty_code}.
|
||||
Handle code-switching (e.g. Tamil/Hindi mixed with English medical words like BP, sugar, chest pain, tablets, fever, vomiting).
|
||||
Return ONLY the exact verbatim transcription without explanation, punctuation commentary, or markdown quotes.
|
||||
"""
|
||||
response = await model.generate_content_async([audio_part, audio_prompt])
|
||||
text = response.text.strip().replace('"', '').replace("```", "")
|
||||
if text and len(text) > 1 and "unable to transcribe" not in text.lower():
|
||||
raw_transcript = text
|
||||
except Exception as e:
|
||||
print(f"[Gemini Audio STT Error]: {e}")
|
||||
|
||||
# 3. OpenAI Whisper-1 with Medical Specialty Context Prompting
|
||||
if (
|
||||
not raw_transcript
|
||||
and self.openai_key
|
||||
and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]
|
||||
):
|
||||
client = AsyncOpenAI(api_key=self.openai_key, timeout=8.0)
|
||||
try:
|
||||
audio_file = io.BytesIO(audio_bytes)
|
||||
audio_file.name = filename if filename.endswith((".wav", ".mp3", ".webm", ".m4a")) else "audio.webm"
|
||||
|
||||
specialty_vocab = {
|
||||
"cardiology": "Cardiology, chest pain, palpitations, ECG, hypertension, BP, heart attack, shortness of breath, Metformin, Telmisartan, Amlodipine, Atorvastatin, Sorbitrate",
|
||||
"orthopedics": "Orthopedics, joint pain, knee pain, fracture, swelling, arthritis, Paracetamol, Tramadol, Calcium, Vitamin D, physiotherapy",
|
||||
"general": "General medicine, fever, cough, cold, headache, diabetes, high blood pressure, vomiting, loose motion, Paracetamol 650, Dolo 650, Azithromycin, Pantoprazole",
|
||||
"pediatrics": "Pediatrics, baby fever, vaccination, cough, cold, vomiting, appetite loss, Paracetamol drops",
|
||||
"dermatology": "Dermatology, skin rash, itching, redness, fungal infection, Cetirizine, ointment, allergy",
|
||||
}
|
||||
|
||||
spec_terms = specialty_vocab.get(specialty_code, specialty_vocab["general"])
|
||||
lang_prompts = {
|
||||
"ta": f"மருத்துவ ஆலோசனை, நெஞ்சு வலி, சர்க்கரை வியாதி, பிபி, மாத்திரை, காய்ச்சல், இருமல், வலி, {spec_terms}",
|
||||
"hi": f"चिकित्सा, सीने में दर्द, बीपी, शुगर, बुखार, दवाई, सिरदर्द, खांसी, {spec_terms}",
|
||||
"te": f"వైద్యం, ఛాతీ నొప్పి, షుగర్, బీపీ, మందులు, జ్వరం, {spec_terms}",
|
||||
"kn": f"ವೈದ್ಯಕೀಯ, ಎದೆ ನೋವು, ಬಿಪಿ, ಸಕ್ಕರೆ ಕಾಯಿಲೆ, ಮಾತ್ರೆಗಳು, ಜ್ವರ, {spec_terms}",
|
||||
"en": f"Medical symptoms, chest pain, diabetes, blood pressure, fever, cough, {spec_terms}",
|
||||
}
|
||||
med_prompt = lang_prompts.get(lang_prefix, lang_prompts["en"])
|
||||
|
||||
transcript = await client.audio.transcriptions.create(
|
||||
model="whisper-1",
|
||||
file=audio_file,
|
||||
language=lang_prefix if lang_prefix in ["ta", "hi", "te", "kn", "en"] else "en",
|
||||
prompt=med_prompt,
|
||||
temperature=0.0,
|
||||
)
|
||||
if transcript.text and len(transcript.text.strip()) > 1:
|
||||
raw_transcript = transcript.text.strip()
|
||||
except Exception as e:
|
||||
print(f"[OpenAI Whisper Error]: {e}")
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
# 4. Apply Medical Speech Normalizer to eliminate phonetic mishearings
|
||||
# Apply Medical Speech Normalizer to eliminate phonetic mishearings
|
||||
normalized = normalize_medical_speech(raw_transcript, language_code=language_code)
|
||||
return normalized
|
||||
|
||||
@@ -200,7 +121,8 @@ Return ONLY the exact verbatim transcription without explanation, punctuation co
|
||||
and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"]
|
||||
):
|
||||
try:
|
||||
async with httpx2.AsyncClient(timeout=6.0) as client:
|
||||
speaker = "anushka" if lang_prefix in ["ta", "hi", "te", "kn"] else "karun"
|
||||
async with httpx2.AsyncClient(timeout=3.0) as client:
|
||||
headers = {
|
||||
"api-subscription-key": self.sarvam_key,
|
||||
"Content-Type": "application/json",
|
||||
@@ -208,8 +130,8 @@ Return ONLY the exact verbatim transcription without explanation, punctuation co
|
||||
payload = {
|
||||
"inputs": [cleaned_text],
|
||||
"target_language_code": language_code,
|
||||
"speaker": "meera",
|
||||
"model": "bulbul:v1",
|
||||
"speaker": speaker,
|
||||
"model": "bulbul:v2",
|
||||
}
|
||||
res = await client.post(
|
||||
"https://api.sarvam.ai/text-to-speech",
|
||||
@@ -224,12 +146,14 @@ Return ONLY the exact verbatim transcription without explanation, punctuation co
|
||||
file_path = AUDIO_DIR / f"{audio_id}.wav"
|
||||
file_path.write_bytes(audio_data)
|
||||
return audio_id, audio_data
|
||||
else:
|
||||
print(f"[Sarvam TTS HTTP Error]: {res.status_code} - {res.text}")
|
||||
except Exception as e:
|
||||
print(f"[Sarvam TTS Error]: {e}")
|
||||
|
||||
# 2. Use OpenAI TTS
|
||||
if self.openai_key and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]:
|
||||
client = AsyncOpenAI(api_key=self.openai_key, timeout=6.0)
|
||||
client = AsyncOpenAI(api_key=self.openai_key, timeout=1.5)
|
||||
try:
|
||||
response = await client.audio.speech.create(
|
||||
model="tts-1",
|
||||
|
||||
Reference in New Issue
Block a user