Gokulvarthan-Ramesh/aionboarding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete real-time voice-ai engine upgrades and frontend integration guide

Browse Source
This commit is contained in:
Gokulvarthan-Ramesh
2026-09-03 12:34:55 +05:30
parent c23d979338
commit 37440993f7
56 changed files with 2521 additions and 3040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai-patient-onboarding/app/services/voice_service.py
+37 -11
View File
@@ -57,26 +57,23 @@ class VoiceService:
lang_prefix = language_code.lower()[:2]
raw_transcript = ""
# 1. Primary: Sarvam AI Saaras v3 for Native Indian Languages (Ultra-fast Indic STT)
# 1. Primary: Sarvam AI Saaras v3 for Indic Code-Switching (Tanglish, Hinglish, Telglish, Kanglish)
if (
self.sarvam_key
and self.sarvam_key.strip()
and self.sarvam_key not in ["your_sarvam_api_key", "your_sarvam_key", ""]
and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"]
):
try:
async with httpx2.AsyncClient(timeout=5.0) as client:
files = {
"file": (filename, audio_bytes, "audio/webm")
}
# Use 'unknown' or requested code for auto code-switching language detection
target_lang = "unknown" if language_code in ["auto", "unknown", "code_switching", "multilingual"] else language_code
async with httpx2.AsyncClient(timeout=6.0) as client:
files = {"file": (filename, audio_bytes, "audio/webm")}
data = {
"model": "saaras:v3",
"language_code": language_code,
"language_code": target_lang,
"with_diacritics": "true",
}
headers = {
"api-subscription-key": self.sarvam_key
}
headers = {"api-subscription-key": self.sarvam_key}
res = await client.post(
"https://api.sarvam.ai/speech-to-text",
files=files,
@@ -91,7 +88,23 @@ class VoiceService:
except Exception as e:
print(f"[Sarvam STT Error]: {e}")
# Apply Medical Speech Normalizer to eliminate phonetic mishearings
# 2. Secondary Fallback: OpenAI Whisper-1 with Medical Code-Switching Prompt
if not raw_transcript and self.openai_key and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]:
try:
client = AsyncOpenAI(api_key=self.openai_key, timeout=7.0)
file_obj = ("audio.webm", audio_bytes, "audio/webm")
transcription = await client.audio.transcriptions.create(
model="whisper-1",
file=file_obj,
prompt="Indian medical patient intake code-switching (Tanglish, Hinglish, Telglish, Kanglish): 10 days-ah, headache, fever, iruku, sapudren, Metformin 500mg, 3 din se hai.",
temperature=0.0
)
if transcription and transcription.text:
raw_transcript = transcription.text.strip()
except Exception as e:
print(f"[Whisper STT Fallback Error]: {e}")
# Apply Medical Speech Normalizer to handle multi-dialect expressions
normalized = normalize_medical_speech(raw_transcript, language_code=language_code)
return normalized
@@ -100,11 +113,23 @@ class VoiceService:
text: str,
language_code: str = "en-IN",
audio_id: str | None = None,
patient_age: int | None = None,
) -> tuple[str, bytes]:
lang_prefix = language_code.lower()[:2]
if not audio_id:
audio_id = f"aud_{uuid4().hex[:12]}"
# Calculate Age-Adaptive Voice Speed & Tone Pace Parameters
pace = 1.0
openai_speed = 1.0
if patient_age is not None:
if patient_age > 60:
pace = 0.85 # Slow down speech rate by 15% for elderly patients
openai_speed = 0.85
elif patient_age < 12:
pace = 0.95 # Warm, reassuring speech pace for pediatric/parents
openai_speed = 0.95
# Clean punctuation and brackets before synthesis
cleaned_text = (
text.replace("(", "")
@@ -134,6 +159,7 @@ class VoiceService:
"target_language_code": language_code,
"speaker": speaker,
"model": "bulbul:v3",
"pace": pace,
}
res = await client.post(
"https://api.sarvam.ai/text-to-speech",