271 lines
13 KiB
Python
271 lines
13 KiB
Python
import base64
|
|
import io
|
|
import os
|
|
import wave
|
|
import struct
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
import httpx2
|
|
from openai import AsyncOpenAI
|
|
|
|
from app.config import OPENAI_API_KEY, SARVAM_API_KEY, GEMINI_API_KEY, GEMINI_MODEL
|
|
from app.services.medical_speech_normalizer import normalize_medical_speech
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
|
AUDIO_DIR = BASE_DIR / "data" / "audio"
|
|
AUDIO_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
class VoiceService:
|
|
|
|
def __init__(self):
|
|
self.openai_key = OPENAI_API_KEY
|
|
self.sarvam_key = SARVAM_API_KEY
|
|
self.gemini_key = GEMINI_API_KEY
|
|
self.gemini_model = GEMINI_MODEL
|
|
|
|
def get_initial_greeting(self, patient_name: str | None, language: str) -> str:
|
|
name = patient_name or "அவர்களே"
|
|
lang_lower = language.lower()
|
|
|
|
if lang_lower.startswith("ta"):
|
|
return f"வணக்கம் {name}, CureSelect மருத்துவமனைக்கு நல்வரவு. இன்று தங்களுக்கு ஏற்பட்டுள்ள உடல்நல பாதிப்பு பற்றி கூற முடியுமா?"
|
|
elif lang_lower.startswith("hi"):
|
|
return f"नमस्ते {name} जी, CureSelect अस्पताल में आपका स्वागत है। आज आप किस स्वास्थ्य समस्या के लिए परामर्श लेना चाहते हैं?"
|
|
elif lang_lower.startswith("te"):
|
|
return f"నమస్కారం {name} గారు, CureSelect హాస్పిటల్కు స్వాగతం. ఈరోజు మీకున్న ఆరోగ్య సమస్య గురించి చెప్పగలరా?"
|
|
elif lang_lower.startswith("kn"):
|
|
return f"ನಮಸ್ಕಾರ {name} ಅವರೇ, CureSelect ಆಸ್ಪತ್ರೆಗೆ ಸುಸ್ವಾಗತ. ಇಂದು ನಿಮ್ಮ ಆರೋಗ್ಯದ ತೊಂದರೆ ಬಗ್ಗೆ ತಿಳಿಸಬಹುದೇ?"
|
|
else:
|
|
return f"Hello {name}, welcome to CureSelect Healthcare. Could you please share what health symptoms you are experiencing today?"
|
|
|
|
async def speech_to_text(
|
|
self,
|
|
audio_bytes: bytes,
|
|
filename: str = "audio.webm",
|
|
language_code: str = "en-IN",
|
|
specialty_code: str = "general",
|
|
patient_context: dict | None = None,
|
|
) -> str:
|
|
"""
|
|
High-precision multi-model Indian speech transcription pipeline:
|
|
1. Sarvam AI Saaras v2 (Specialized Indic Speech Engine)
|
|
2. Google Gemini Multimodal Audio (Multilingual Code-Switching & Acoustic Context)
|
|
3. OpenAI Whisper-1 (Medical Specialty Context Boosted)
|
|
4. Clinical & Phonetic Post-Processing Normalizer
|
|
"""
|
|
lang_prefix = language_code.lower()[:2]
|
|
raw_transcript = ""
|
|
|
|
# 1. Sarvam AI Saaras v2 for Native Indian Languages
|
|
if (
|
|
self.sarvam_key
|
|
and self.sarvam_key.strip()
|
|
and self.sarvam_key not in ["your_sarvam_api_key", "your_sarvam_key", ""]
|
|
and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"]
|
|
):
|
|
try:
|
|
async with httpx2.AsyncClient(timeout=8.0) as client:
|
|
files = {
|
|
"file": (filename, audio_bytes, "audio/webm")
|
|
}
|
|
data = {
|
|
"model": "saaras:v2",
|
|
"language_code": language_code,
|
|
"with_diacritics": "true",
|
|
}
|
|
headers = {
|
|
"api-subscription-key": self.sarvam_key
|
|
}
|
|
res = await client.post(
|
|
"https://api.sarvam.ai/speech-to-text",
|
|
files=files,
|
|
data=data,
|
|
headers=headers,
|
|
)
|
|
if res.status_code == 200:
|
|
res_json = res.json()
|
|
transcript = res_json.get("transcript", "").strip()
|
|
if transcript:
|
|
raw_transcript = transcript
|
|
except Exception as e:
|
|
print(f"[Sarvam STT Error]: {e}")
|
|
|
|
# 2. Google Gemini Multimodal Audio (Excellent with Tanglish/Hinglish & Medical terms)
|
|
if (
|
|
not raw_transcript
|
|
and self.gemini_key
|
|
and self.gemini_key.strip()
|
|
and self.gemini_key not in ["your_gemini_key", ""]
|
|
):
|
|
try:
|
|
import google.generativeai as genai
|
|
genai.configure(api_key=self.gemini_key)
|
|
model = genai.GenerativeModel(self.gemini_model or "gemini-1.5-flash")
|
|
|
|
mime_type = "audio/webm"
|
|
if filename.endswith(".wav"):
|
|
mime_type = "audio/wav"
|
|
elif filename.endswith(".mp3"):
|
|
mime_type = "audio/mp3"
|
|
|
|
audio_part = {
|
|
"mime_type": mime_type,
|
|
"data": audio_bytes,
|
|
}
|
|
audio_prompt = f"""
|
|
You are an expert medical transcription engine for Indian hospitals.
|
|
Accurately transcribe the spoken audio into text.
|
|
Language context: {language_code}. Specialty: {specialty_code}.
|
|
Handle code-switching (e.g. Tamil/Hindi mixed with English medical words like BP, sugar, chest pain, tablets, fever, vomiting).
|
|
Return ONLY the exact verbatim transcription without explanation, punctuation commentary, or markdown quotes.
|
|
"""
|
|
response = await model.generate_content_async([audio_part, audio_prompt])
|
|
text = response.text.strip().replace('"', '').replace("```", "")
|
|
if text and len(text) > 1 and "unable to transcribe" not in text.lower():
|
|
raw_transcript = text
|
|
except Exception as e:
|
|
print(f"[Gemini Audio STT Error]: {e}")
|
|
|
|
# 3. OpenAI Whisper-1 with Medical Specialty Context Prompting
|
|
if (
|
|
not raw_transcript
|
|
and self.openai_key
|
|
and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]
|
|
):
|
|
client = AsyncOpenAI(api_key=self.openai_key, timeout=8.0)
|
|
try:
|
|
audio_file = io.BytesIO(audio_bytes)
|
|
audio_file.name = filename if filename.endswith((".wav", ".mp3", ".webm", ".m4a")) else "audio.webm"
|
|
|
|
specialty_vocab = {
|
|
"cardiology": "Cardiology, chest pain, palpitations, ECG, hypertension, BP, heart attack, shortness of breath, Metformin, Telmisartan, Amlodipine, Atorvastatin, Sorbitrate",
|
|
"orthopedics": "Orthopedics, joint pain, knee pain, fracture, swelling, arthritis, Paracetamol, Tramadol, Calcium, Vitamin D, physiotherapy",
|
|
"general": "General medicine, fever, cough, cold, headache, diabetes, high blood pressure, vomiting, loose motion, Paracetamol 650, Dolo 650, Azithromycin, Pantoprazole",
|
|
"pediatrics": "Pediatrics, baby fever, vaccination, cough, cold, vomiting, appetite loss, Paracetamol drops",
|
|
"dermatology": "Dermatology, skin rash, itching, redness, fungal infection, Cetirizine, ointment, allergy",
|
|
}
|
|
|
|
spec_terms = specialty_vocab.get(specialty_code, specialty_vocab["general"])
|
|
lang_prompts = {
|
|
"ta": f"மருத்துவ ஆலோசனை, நெஞ்சு வலி, சர்க்கரை வியாதி, பிபி, மாத்திரை, காய்ச்சல், இருமல், வலி, {spec_terms}",
|
|
"hi": f"चिकित्सा, सीने में दर्द, बीपी, शुगर, बुखार, दवाई, सिरदर्द, खांसी, {spec_terms}",
|
|
"te": f"వైద్యం, ఛాతీ నొప్పి, షుగర్, బీపీ, మందులు, జ్వరం, {spec_terms}",
|
|
"kn": f"ವೈದ್ಯಕೀಯ, ಎದೆ ನೋವು, ಬಿಪಿ, ಸಕ್ಕರೆ ಕಾಯಿಲೆ, ಮಾತ್ರೆಗಳು, ಜ್ವರ, {spec_terms}",
|
|
"en": f"Medical symptoms, chest pain, diabetes, blood pressure, fever, cough, {spec_terms}",
|
|
}
|
|
med_prompt = lang_prompts.get(lang_prefix, lang_prompts["en"])
|
|
|
|
transcript = await client.audio.transcriptions.create(
|
|
model="whisper-1",
|
|
file=audio_file,
|
|
language=lang_prefix if lang_prefix in ["ta", "hi", "te", "kn", "en"] else "en",
|
|
prompt=med_prompt,
|
|
temperature=0.0,
|
|
)
|
|
if transcript.text and len(transcript.text.strip()) > 1:
|
|
raw_transcript = transcript.text.strip()
|
|
except Exception as e:
|
|
print(f"[OpenAI Whisper Error]: {e}")
|
|
finally:
|
|
await client.close()
|
|
|
|
# 4. Apply Medical Speech Normalizer to eliminate phonetic mishearings
|
|
normalized = normalize_medical_speech(raw_transcript, language_code=language_code)
|
|
return normalized
|
|
|
|
async def text_to_speech(
|
|
self,
|
|
text: str,
|
|
language_code: str = "en-IN",
|
|
) -> tuple[str, bytes]:
|
|
lang_prefix = language_code.lower()[:2]
|
|
audio_id = f"aud_{uuid4().hex[:12]}"
|
|
|
|
# Clean punctuation and brackets before synthesis
|
|
cleaned_text = (
|
|
text.replace("(", "")
|
|
.replace(")", "")
|
|
.replace("*", "")
|
|
.replace("#", "")
|
|
.replace("- ", "")
|
|
.strip()
|
|
)
|
|
|
|
# 1. Use Sarvam AI TTS for Indian Languages (Native Indian Accent)
|
|
if (
|
|
self.sarvam_key
|
|
and self.sarvam_key.strip()
|
|
and self.sarvam_key not in ["your_sarvam_api_key", "your_sarvam_key", ""]
|
|
and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"]
|
|
):
|
|
try:
|
|
async with httpx2.AsyncClient(timeout=6.0) as client:
|
|
headers = {
|
|
"api-subscription-key": self.sarvam_key,
|
|
"Content-Type": "application/json",
|
|
}
|
|
payload = {
|
|
"inputs": [cleaned_text],
|
|
"target_language_code": language_code,
|
|
"speaker": "meera",
|
|
"model": "bulbul:v1",
|
|
}
|
|
res = await client.post(
|
|
"https://api.sarvam.ai/text-to-speech",
|
|
json=payload,
|
|
headers=headers,
|
|
)
|
|
if res.status_code == 200:
|
|
res_json = res.json()
|
|
audios = res_json.get("audios", [])
|
|
if audios:
|
|
audio_data = base64.b64decode(audios[0])
|
|
file_path = AUDIO_DIR / f"{audio_id}.wav"
|
|
file_path.write_bytes(audio_data)
|
|
return audio_id, audio_data
|
|
except Exception as e:
|
|
print(f"[Sarvam TTS Error]: {e}")
|
|
|
|
# 2. Use OpenAI TTS
|
|
if self.openai_key and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]:
|
|
client = AsyncOpenAI(api_key=self.openai_key, timeout=6.0)
|
|
try:
|
|
response = await client.audio.speech.create(
|
|
model="tts-1",
|
|
voice="shimmer",
|
|
input=cleaned_text,
|
|
speed=1.0,
|
|
)
|
|
audio_data = response.content
|
|
file_path = AUDIO_DIR / f"{audio_id}.mp3"
|
|
file_path.write_bytes(audio_data)
|
|
return audio_id, audio_data
|
|
except Exception as e:
|
|
print(f"[OpenAI TTS Error]: {e}")
|
|
finally:
|
|
await client.close()
|
|
|
|
# 3. Fallback
|
|
audio_data = self._generate_fallback_wav()
|
|
file_path = AUDIO_DIR / f"{audio_id}.wav"
|
|
file_path.write_bytes(audio_data)
|
|
return audio_id, audio_data
|
|
|
|
def _generate_fallback_wav(self) -> bytes:
|
|
buf = io.BytesIO()
|
|
with wave.open(buf, "wb") as wf:
|
|
wf.setnchannels(1)
|
|
wf.setsampwidth(2)
|
|
wf.setframerate(16000)
|
|
data = [int(1000 * (i % 32 < 16)) for i in range(4000)]
|
|
wf.writeframes(struct.pack(f"<{len(data)}h", *data))
|
|
return buf.getvalue()
|
|
|
|
def get_audio_path(self, audio_id: str) -> Path | None:
|
|
for ext in [".wav", ".mp3", ".webm", ".m4a"]:
|
|
p = AUDIO_DIR / f"{audio_id}{ext}"
|
|
if p.exists():
|
|
return p
|
|
return None
|