import base64 import io import os import wave import struct from pathlib import Path from uuid import uuid4 import httpx2 from openai import AsyncOpenAI from app.config import OPENAI_API_KEY, SARVAM_API_KEY, GEMINI_API_KEY, GEMINI_MODEL from app.services.medical_speech_normalizer import normalize_medical_speech BASE_DIR = Path(__file__).resolve().parent.parent.parent AUDIO_DIR = BASE_DIR / "data" / "audio" AUDIO_DIR.mkdir(parents=True, exist_ok=True) class VoiceService: def __init__(self): self.openai_key = OPENAI_API_KEY self.sarvam_key = SARVAM_API_KEY self.gemini_key = GEMINI_API_KEY self.gemini_model = GEMINI_MODEL def get_initial_greeting(self, patient_name: str | None, language: str) -> str: name = patient_name or "அவர்களே" lang_lower = language.lower() if lang_lower.startswith("ta"): return f"வணக்கம் {name}, CureSelect மருத்துவமனைக்கு நல்வரவு. இன்று தங்களுக்கு ஏற்பட்டுள்ள உடல்நல பாதிப்பு பற்றி கூற முடியுமா?" elif lang_lower.startswith("hi"): return f"नमस्ते {name} जी, CureSelect अस्पताल में आपका स्वागत है। आज आप किस स्वास्थ्य समस्या के लिए परामर्श लेना चाहते हैं?" elif lang_lower.startswith("te"): return f"నమస్కారం {name} గారు, CureSelect హాస్పిటల్‌కు స్వాగతం. ఈరోజు మీకున్న ఆరోగ్య సమస్య గురించి చెప్పగలరా?" elif lang_lower.startswith("kn"): return f"ನಮಸ್ಕಾರ {name} ಅವರೇ, CureSelect ಆಸ್ಪತ್ರೆಗೆ ಸುಸ್ವಾಗತ. ಇಂದು ನಿಮ್ಮ ಆರೋಗ್ಯದ ತೊಂದರೆ ಬಗ್ಗೆ ತಿಳಿಸಬಹುದೇ?" else: return f"Hello {name}, welcome to CureSelect Healthcare. Could you please share what health symptoms you are experiencing today?" async def speech_to_text( self, audio_bytes: bytes, filename: str = "audio.webm", language_code: str = "en-IN", specialty_code: str = "general", patient_context: dict | None = None, ) -> str: """ High-precision multi-model Indian speech transcription pipeline: 1. Sarvam AI Saaras v2 (Specialized Indic Speech Engine) 2. Google Gemini Multimodal Audio (Multilingual Code-Switching & Acoustic Context) 3. OpenAI Whisper-1 (Medical Specialty Context Boosted) 4. Clinical & Phonetic Post-Processing Normalizer """ lang_prefix = language_code.lower()[:2] raw_transcript = "" # 1. Sarvam AI Saaras v2 for Native Indian Languages if ( self.sarvam_key and self.sarvam_key.strip() and self.sarvam_key not in ["your_sarvam_api_key", "your_sarvam_key", ""] and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"] ): try: async with httpx2.AsyncClient(timeout=8.0) as client: files = { "file": (filename, audio_bytes, "audio/webm") } data = { "model": "saaras:v2", "language_code": language_code, "with_diacritics": "true", } headers = { "api-subscription-key": self.sarvam_key } res = await client.post( "https://api.sarvam.ai/speech-to-text", files=files, data=data, headers=headers, ) if res.status_code == 200: res_json = res.json() transcript = res_json.get("transcript", "").strip() if transcript: raw_transcript = transcript except Exception as e: print(f"[Sarvam STT Error]: {e}") # 2. Google Gemini Multimodal Audio (Excellent with Tanglish/Hinglish & Medical terms) if ( not raw_transcript and self.gemini_key and self.gemini_key.strip() and self.gemini_key not in ["your_gemini_key", ""] ): try: import google.generativeai as genai genai.configure(api_key=self.gemini_key) model = genai.GenerativeModel(self.gemini_model or "gemini-1.5-flash") mime_type = "audio/webm" if filename.endswith(".wav"): mime_type = "audio/wav" elif filename.endswith(".mp3"): mime_type = "audio/mp3" audio_part = { "mime_type": mime_type, "data": audio_bytes, } audio_prompt = f""" You are an expert medical transcription engine for Indian hospitals. Accurately transcribe the spoken audio into text. Language context: {language_code}. Specialty: {specialty_code}. Handle code-switching (e.g. Tamil/Hindi mixed with English medical words like BP, sugar, chest pain, tablets, fever, vomiting). Return ONLY the exact verbatim transcription without explanation, punctuation commentary, or markdown quotes. """ response = await model.generate_content_async([audio_part, audio_prompt]) text = response.text.strip().replace('"', '').replace("```", "") if text and len(text) > 1 and "unable to transcribe" not in text.lower(): raw_transcript = text except Exception as e: print(f"[Gemini Audio STT Error]: {e}") # 3. OpenAI Whisper-1 with Medical Specialty Context Prompting if ( not raw_transcript and self.openai_key and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""] ): client = AsyncOpenAI(api_key=self.openai_key, timeout=8.0) try: audio_file = io.BytesIO(audio_bytes) audio_file.name = filename if filename.endswith((".wav", ".mp3", ".webm", ".m4a")) else "audio.webm" specialty_vocab = { "cardiology": "Cardiology, chest pain, palpitations, ECG, hypertension, BP, heart attack, shortness of breath, Metformin, Telmisartan, Amlodipine, Atorvastatin, Sorbitrate", "orthopedics": "Orthopedics, joint pain, knee pain, fracture, swelling, arthritis, Paracetamol, Tramadol, Calcium, Vitamin D, physiotherapy", "general": "General medicine, fever, cough, cold, headache, diabetes, high blood pressure, vomiting, loose motion, Paracetamol 650, Dolo 650, Azithromycin, Pantoprazole", "pediatrics": "Pediatrics, baby fever, vaccination, cough, cold, vomiting, appetite loss, Paracetamol drops", "dermatology": "Dermatology, skin rash, itching, redness, fungal infection, Cetirizine, ointment, allergy", } spec_terms = specialty_vocab.get(specialty_code, specialty_vocab["general"]) lang_prompts = { "ta": f"மருத்துவ ஆலோசனை, நெஞ்சு வலி, சர்க்கரை வியாதி, பிபி, மாத்திரை, காய்ச்சல், இருமல், வலி, {spec_terms}", "hi": f"चिकित्सा, सीने में दर्द, बीपी, शुगर, बुखार, दवाई, सिरदर्द, खांसी, {spec_terms}", "te": f"వైద్యం, ఛాతీ నొప్పి, షుగర్, బీపీ, మందులు, జ్వరం, {spec_terms}", "kn": f"ವೈದ್ಯಕೀಯ, ಎದೆ ನೋವು, ಬಿಪಿ, ಸಕ್ಕರೆ ಕಾಯಿಲೆ, ಮಾತ್ರೆಗಳು, ಜ್ವರ, {spec_terms}", "en": f"Medical symptoms, chest pain, diabetes, blood pressure, fever, cough, {spec_terms}", } med_prompt = lang_prompts.get(lang_prefix, lang_prompts["en"]) transcript = await client.audio.transcriptions.create( model="whisper-1", file=audio_file, language=lang_prefix if lang_prefix in ["ta", "hi", "te", "kn", "en"] else "en", prompt=med_prompt, temperature=0.0, ) if transcript.text and len(transcript.text.strip()) > 1: raw_transcript = transcript.text.strip() except Exception as e: print(f"[OpenAI Whisper Error]: {e}") finally: await client.close() # 4. Apply Medical Speech Normalizer to eliminate phonetic mishearings normalized = normalize_medical_speech(raw_transcript, language_code=language_code) return normalized async def text_to_speech( self, text: str, language_code: str = "en-IN", ) -> tuple[str, bytes]: lang_prefix = language_code.lower()[:2] audio_id = f"aud_{uuid4().hex[:12]}" # Clean punctuation and brackets before synthesis cleaned_text = ( text.replace("(", "") .replace(")", "") .replace("*", "") .replace("#", "") .replace("- ", "") .strip() ) # 1. Use Sarvam AI TTS for Indian Languages (Native Indian Accent) if ( self.sarvam_key and self.sarvam_key.strip() and self.sarvam_key not in ["your_sarvam_api_key", "your_sarvam_key", ""] and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"] ): try: async with httpx2.AsyncClient(timeout=6.0) as client: headers = { "api-subscription-key": self.sarvam_key, "Content-Type": "application/json", } payload = { "inputs": [cleaned_text], "target_language_code": language_code, "speaker": "meera", "model": "bulbul:v1", } res = await client.post( "https://api.sarvam.ai/text-to-speech", json=payload, headers=headers, ) if res.status_code == 200: res_json = res.json() audios = res_json.get("audios", []) if audios: audio_data = base64.b64decode(audios[0]) file_path = AUDIO_DIR / f"{audio_id}.wav" file_path.write_bytes(audio_data) return audio_id, audio_data except Exception as e: print(f"[Sarvam TTS Error]: {e}") # 2. Use OpenAI TTS if self.openai_key and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]: client = AsyncOpenAI(api_key=self.openai_key, timeout=6.0) try: response = await client.audio.speech.create( model="tts-1", voice="shimmer", input=cleaned_text, speed=1.0, ) audio_data = response.content file_path = AUDIO_DIR / f"{audio_id}.mp3" file_path.write_bytes(audio_data) return audio_id, audio_data except Exception as e: print(f"[OpenAI TTS Error]: {e}") finally: await client.close() # 3. Fallback audio_data = self._generate_fallback_wav() file_path = AUDIO_DIR / f"{audio_id}.wav" file_path.write_bytes(audio_data) return audio_id, audio_data def _generate_fallback_wav(self) -> bytes: buf = io.BytesIO() with wave.open(buf, "wb") as wf: wf.setnchannels(1) wf.setsampwidth(2) wf.setframerate(16000) data = [int(1000 * (i % 32 < 16)) for i in range(4000)] wf.writeframes(struct.pack(f"<{len(data)}h", *data)) return buf.getvalue() def get_audio_path(self, audio_id: str) -> Path | None: for ext in [".wav", ".mp3", ".webm", ".m4a"]: p = AUDIO_DIR / f"{audio_id}{ext}" if p.exists(): return p return None