feat: complete real-time voice-ai engine upgrades and frontend integration guide

2026-09-03 12:34:55 +05:30
parent c23d979338
commit 37440993f7
56 changed files with 2521 additions and 3040 deletions
@@ -0,0 +1,106 @@
+import math
+import re
+import struct
+from typing import Dict, Any, List, Tuple
+
+HIGH_PAIN_PATTERNS = [
+    r"(?:10/10|9/10|8/10|10\s*out\s*of\s*10|9\s*out\s*of\s*10|8\s*out\s*of\s*10)",
+    r"(?:unbearable|excruciating|worst\s*pain|extreme\s*pain|severe\s*chest\s*pain|cannot\s*breathe|gasping|dying)",
+    r"(?:கடுமையான\s*வலி|தாங்க\s*முடியாத\s*வலி|நெஞ்சு\s*வலி|மூச்சு\s*திணறல்)",
+    r"(?:असहनीय\s*दर्द|बहुत\s*तेज़\s*दर्द|सीने\s*में\s*दर्द|साँस\s*नहीं\s*आ\s*रही)",
+    r"(?:చాలా\s*ఎక్కువ\s*నెప్పి|భరించలేని\s*నెప్పి|గుండె\s*నెప్పి)",
+    r"(?:ಅಸಹನೀಯ\s*ನೋವು|ತೀವ್ರ\s*ನೋವು)"
+]
+
+MODERATE_PAIN_PATTERNS = [
+    r"(?:7/10|6/10|5/10|moderate\s*pain|bad\s*pain|hurts\s*a\s*lot|வலி\s*அதிகமா|अत्यधिक\s*दर्द)",
+]
+
+EMERGENCY_RED_FLAGS = [
+    r"(?:chest\s*pain|heart\s*attack|stroke|paralysis|unconscious|fainted|heavy\s*bleeding)",
+    r"(?:நெஞ்சு\s*வலி|பக்கவாதம்|மயக்கம்|அதிக\s*ரத்தப்போக்கு)",
+    r"(?:सीने\s*में\s*दर्द|हार्ट\s*अटैक|बेहोश|रक्तस्राव)"
+]
+
+
+def calculate_audio_rms_energy(audio_bytes: bytes) -> float:
+    """
+    Computes RMS (Root Mean Square) audio amplitude energy from raw mic bytes.
+    Returns normalized energy float between 0.0 and 1.0.
+    """
+    if not audio_bytes or len(audio_bytes) < 100:
+        return 0.0
+
+    try:
+        # Unpack 16-bit PCM audio samples
+        sample_count = len(audio_bytes) // 2
+        if sample_count == 0:
+            return 0.0
+
+        fmt = f"<{sample_count}h"
+        samples = struct.unpack(fmt, audio_bytes[:sample_count * 2])
+        
+        sum_squares = sum(s ** 2 for s in samples)
+        rms = math.sqrt(sum_squares / sample_count)
+        
+        # Normalize 16-bit max amplitude (32768)
+        norm_energy = min(1.0, rms / 15000.0)
+        return round(norm_energy, 3)
+    except Exception:
+        return 0.0
+
+
+def analyze_distress_and_pain(audio_bytes: bytes = b"", text: str = "") -> Dict[str, Any]:
+    """
+    Acoustic & Semantic Distress Classifier:
+    Analyzes spoken text and audio RMS energy to compute Pain Score (1-10) and Distress Score (0.0-1.0).
+    """
+    txt_lower = (text or "").lower()
+    
+    pain_score = 3
+    distress_score = 0.2
+    is_emergency = False
+    reasons: List[str] = []
+
+    # 1. Semantic High Pain Detection
+    for pat in HIGH_PAIN_PATTERNS:
+        if re.search(pat, txt_lower, re.IGNORECASE):
+            pain_score = max(pain_score, 9)
+            distress_score = max(distress_score, 0.85)
+            reasons.append("Severe high pain expression detected")
+            break
+
+    # 2. Semantic Moderate Pain Detection
+    if pain_score < 8:
+        for pat in MODERATE_PAIN_PATTERNS:
+            if re.search(pat, txt_lower, re.IGNORECASE):
+                pain_score = max(pain_score, 6)
+                distress_score = max(distress_score, 0.55)
+                reasons.append("Moderate pain expression detected")
+                break
+
+    # 3. Emergency Red Flag Detection
+    for pat in EMERGENCY_RED_FLAGS:
+        if re.search(pat, txt_lower, re.IGNORECASE):
+            is_emergency = True
+            pain_score = max(pain_score, 9)
+            distress_score = max(distress_score, 0.90)
+            reasons.append("Critical emergency red flag detected")
+            break
+
+    # 4. Acoustic Audio RMS Energy Boost
+    acoustic_energy = calculate_audio_rms_energy(audio_bytes)
+    if acoustic_energy > 0.6:
+        distress_score = min(1.0, distress_score + 0.15)
+        reasons.append(f"High vocal acoustic energy detected (RMS: {acoustic_energy})")
+
+    should_escalate = pain_score >= 8 or distress_score >= 0.75 or is_emergency
+
+    return {
+        "pain_score": pain_score,
+        "distress_score": round(distress_score, 2),
+        "acoustic_rms_energy": acoustic_energy,
+        "is_emergency": is_emergency,
+        "should_escalate": should_escalate,
+        "reasons": reasons
+    }
@@ -0,0 +1,115 @@
+
+import re
+from typing import Dict, Any, List
+
+# RxNorm / FDA Clinical Interaction Rules
+KNOWN_DRUG_INTERACTIONS = [
+    {
+        "pair": ("warfarin", "aspirin"),
+        "severity": "CRITICAL",
+        "category": "Drug-Drug Interaction",
+        "description": "Concurrent use of Warfarin and Aspirin significantly increases risk of major gastrointestinal and systemic bleeding."
+    },
+    {
+        "pair": ("metformin", "ibuprofen"),
+        "severity": "WARNING",
+        "category": "Drug-Drug Interaction",
+        "description": "NSAIDs like Ibuprofen may impair renal function, increasing risk of Metformin-associated lactic acidosis."
+    },
+    {
+        "pair": ("lisinopril", "spironolactone"),
+        "severity": "WARNING",
+        "category": "Drug-Drug Interaction",
+        "description": "Combining ACE inhibitors with Potassium-sparing diuretics increases risk of severe hyperkalemia."
+    },
+    {
+        "pair": ("ssri", "tramadol"),
+        "severity": "CRITICAL",
+        "category": "Drug-Drug Interaction",
+        "description": "Combining SSRI antidepressants with Tramadol increases risk of Serotonin Syndrome and seizures."
+    }
+]
+
+# Drug-Allergy Cross-Reactivity Rules
+ALLERGY_CROSS_REACTIVITY = [
+    {
+        "allergy_pattern": r"penicillin|amoxicillin|ampicillin",
+        "drug_pattern": r"penicillin|amoxicillin|ampicillin|augmentin|piperacillin",
+        "severity": "CRITICAL",
+        "category": "Drug-Allergy Contraindication",
+        "description": "Patient has reported Penicillin allergy. Prescribing beta-lactam antibiotics poses high risk of severe anaphylaxis."
+    },
+    {
+        "allergy_pattern": r"sulfa|sulfonamide",
+        "drug_pattern": r"bactrim|septra|celecoxib|sulfamethoxazole",
+        "severity": "WARNING",
+        "category": "Drug-Allergy Contraindication",
+        "description": "Patient has reported Sulfa allergy. Use caution with sulfonamide medications."
+    },
+    {
+        "allergy_pattern": r"aspirin|nsaid",
+        "drug_pattern": r"aspirin|ibuprofen|naproxen|diclofenac",
+        "severity": "CRITICAL",
+        "category": "Drug-Allergy Contraindication",
+        "description": "Patient has reported NSAID/Aspirin hypersensitivity. Risk of acute bronchospasm and urticaria."
+    }
+]
+
+
+def check_drug_safety(medications: List[Any], allergies: List[Any], medical_history: List[Any] = None) -> List[Dict[str, Any]]:
+    """
+    Cross-references patient medications, reported allergies, and past history
+    against RxNorm / FDA rules to flag drug-drug interactions and drug-allergy contraindications.
+    """
+    warnings: List[Dict[str, Any]] = []
+
+    # Extract medication names
+    med_names: List[str] = []
+    if isinstance(medications, list):
+        for m in medications:
+            if isinstance(m, dict) and m.get("name"):
+                med_names.append(m["name"].lower().strip())
+            elif isinstance(m, str):
+                med_names.append(m.lower().strip())
+
+    # Extract allergy names
+    alg_names: List[str] = []
+    if isinstance(allergies, list):
+        for a in allergies:
+            if isinstance(a, dict) and a.get("name"):
+                alg_names.append(a["name"].lower().strip())
+            elif isinstance(a, str):
+                alg_names.append(a.lower().strip())
+
+    # 1. Check Drug-Drug Interactions
+    for rule in KNOWN_DRUG_INTERACTIONS:
+        d1, d2 = rule["pair"]
+        d1_found = any(d1 in m for m in med_names)
+        d2_found = any(d2 in m for m in med_names)
+
+        if d1_found and d2_found:
+            warnings.append({
+                "severity": rule["severity"],
+                "category": rule["category"],
+                "trigger_drugs": [d1.capitalize(), d2.capitalize()],
+                "description": rule["description"]
+            })
+
+    # 2. Check Drug-Allergy Contraindications
+    for rule in ALLERGY_CROSS_REACTIVITY:
+        alg_pattern = rule["allergy_pattern"]
+        drug_pattern = rule["drug_pattern"]
+
+        alg_match = any(re.search(alg_pattern, alg, re.IGNORECASE) for alg in alg_names)
+        drug_match = any(re.search(drug_pattern, med, re.IGNORECASE) for med in med_names)
+
+        if alg_match and drug_match:
+            warnings.append({
+                "severity": rule["severity"],
+                "category": rule["category"],
+                "trigger_allergy": alg_pattern.capitalize(),
+                "trigger_drug": drug_pattern.capitalize(),
+                "description": rule["description"]
+            })
+
+    return warnings
@@ -460,36 +460,32 @@ def _stage_for_missing(missing_field: str) -> str:
 def _pick_best_question(missing_list: List[str], lang: str, is_psych: bool = False) -> str:
    if not missing_list:
        if lang.startswith("ta"):
-            return "சரிங்க, எல்லா விவரத்தையும் டாக்டர்கிட்ட குடுத்துடுறேன். ரொம்ப நன்றிங்க!"
+            return "அனைத்து விபரங்களும் பதிவு செய்யப்பட்டது!"
        elif lang.startswith("hi"):
-            return "ठीक है जी, मैं यह सारी जानकारी डॉक्टर साहब को दे देती हूँ। शुक्रिया!"
+            return "सभी विवरण दर्ज कर लिए।"
        elif lang.startswith("te"):
-            return "చాలా ధన్యవాదాలు! మీ వివరాలన్నీ డాక్టర్ గారికి అందిస్తాను."
+            return "అన్ని వివరాలు నమోదు చేయబడ్డాయి. ధన్యవాదాలు!"
        elif lang.startswith("kn"):
-            return "ಸರಿ, ಈ ಎಲ್ಲಾ ವಿವರಗಳನ್ನು ಡಾಕ್ಟರ್‌ಗೆ ತಿಳಿಸ್ತೀನಿ. ಧನ್ಯವಾದಗಳು!"
+            return "ಎಲ್ಲಾ ವಿವರಗಳನ್ನು ದಾಖಲಿಸಲಾಗಿದೆ. ಧನ್ಯವಾದಗಳು!"
        else:
-            return "Got it. I have compiled all these details for the doctor. Thank you!"
+            return "All details recorded. Thank you!"

    field = missing_list[0]

-    # Tamil
+    # Tamil (Max 5 words)
    if lang.startswith("ta"):
        if field == "reason_for_visit" or field == "symptoms":
-            return "வணக்கம், உங்களுக்கு என்ன உடம்பு பிரச்சனை இருக்குன்னு சொல்ல முடியுமா?"
+            return "உங்களுக்கு என்ன உடம்பு பிரச்சனை?"
        elif field == "duration":
-            if is_psych:
-                return "புரியுதுங்க. இந்த மன அழுத்தம் அல்லது கவலை எப்போலிருந்து இருக்குன்னு சொல்ல முடியுமா?"
-            return "சரிங்க, இந்த பிரச்சனை எத்தனை நாளா இருக்குன்னு சொல்ல முடியுமா?"
+            return "எத்தனை நாளாக இந்த பிரச்சனை?"
        elif field == "medical_history":
-            if is_psych:
-                return "புரியுதுங்க. இதுக்கு முன்னாடி மன அழுத்தம் அல்லது தூக்கமின்மைக்காக சிகிச்சை எடுத்திருக்கீங்களா?"
-            return "சரிங்க. உங்களுக்கு பிபி, சுகர், ஆஸ்துமா போன்ற வேறு ஏதேனும் உடம்பு பிரச்சனை இருக்கா?"
+            return "வேறு ஏதேனும் நோய் இருக்கிறதா?"
        elif field == "medications":
-            return "சரிங்க, இதுக்காக இல்ல வேறு ஏதேனும் பிரச்சனைக்காக வழக்கமா மாத்திரை சாப்பிடுறீங்களா?"
+            return "தினமும் ஏதேனும் மாத்திரை சாப்பிடுகிறீர்களா?"
        elif field == "allergies":
-            return "புரியுதுங்க. உங்களுக்கு ஏதேனும் மருந்து அல்லது உணவு அலர்ஜி இருக்கா?"
+            return "ஏதேனும் மருந்து அலர்ஜி இருக்கா?"

-    # Hindi
+    # Hindi (Max 5 words)
    elif lang.startswith("hi"):
        if field == "reason_for_visit" or field == "symptoms":
            return "नमस्ते, आपको क्या स्वास्थ्य समस्या हो रही है?"
@@ -702,10 +698,6 @@ class AIOrchestrator:
        """
        from app.ai.prompts import build_system_prompt

-        api_key = self.gemini.api_key
-        if not api_key or api_key in ["your_gemini_key", "your_gemini_api_key", ""]:
-            return None
-
        patient = {
            "name": getattr(session, "first_name", "Patient"),
            "age": getattr(session, "age", ""),
@@ -724,9 +716,25 @@ class AIOrchestrator:

        system_prompt = build_system_prompt(context)

-        raw_text = await self.gemini.generate_response(system_prompt, user_message)
+        raw_text = None
+        # Priority 1: OpenAI (if user paid for key)
+        if self.openai.api_key and self.openai.api_key.strip() not in ["your_openai_key", "your_openai_api_key", ""]:
+            try:
+                raw_text = await self.openai.generate_response(system_prompt, user_message)
+            except Exception as e:
+                print(f"[AIOrchestrator] OpenAI LLM call failed: {e}")

-        # Parse JSON from Gemini response
+        # Priority 2: Gemini fallback
+        if not raw_text and self.gemini.api_key and self.gemini.api_key.strip() not in ["your_gemini_key", "your_gemini_api_key", ""]:
+            try:
+                raw_text = await self.gemini.generate_response(system_prompt, user_message)
+            except Exception as e:
+                print(f"[AIOrchestrator] Gemini LLM call failed: {e}")
+
+        if not raw_text:
+            return None
+
+        # Parse JSON from response
        cleaned = raw_text.strip()
        if cleaned.startswith("```"):
            cleaned = cleaned.split("\n", 1)[-1]
@@ -739,6 +747,10 @@ class AIOrchestrator:
        # Validate it has required keys
        if "reply" not in parsed or not isinstance(parsed.get("reply"), str):
            return None
+        
+        # Clean reply string generated directly by OpenAI
+        parsed["reply"] = parsed["reply"].strip()
+
        if "state_updates" not in parsed or not isinstance(parsed.get("state_updates"), dict):
            parsed["state_updates"] = {}
        if "completed" not in parsed:
@@ -46,10 +46,12 @@ def get_conversation_rules() -> str:
 9. ADAPTIVE TURN STRATEGY (internal process each turn):
   PATIENT MESSAGE → UNDERSTAND → EXTRACT ALL RELEVANT INFO → MERGE INTO STATE → CHECK EMERGENCY → CHECK WHAT IS KNOWN → IDENTIFY IMPORTANT GAPS → ASK ONE BEST QUESTION or COMPLETE

-10. RESPONSE STYLE:
-    - Produce ONE concise, natural spoken response.
-    - Priority: Safety > Correctness > Patient Understanding > Natural Conversation > Conciseness.
-    - Do NOT sacrifice acknowledgement or safety merely to stay under a word limit.
+10. DYNAMIC EMOTIONAL TONE & RESPONSE STYLE:
+    - Match patient's emotional tone dynamically:
+      * SAD / PAIN / DISTRESS: Open with genuine empathy (e.g. "Sorry to hear. Any BP or sugar?", "Oh dear! How long has it been?")
+      * JOYFUL / REASSURED / NEGATIVE FORMULATION: Open with warmth (e.g. "Great! Do you take daily tablets?", "Wonderful! Any other symptoms?")
+    - DO NOT use static prefillers ("Got it."). Allow OpenAI to generate dynamic, natural questions directly.
+    - Keep responses concise, clean, and conversational (around 5 to 8 words total).
    - No markdown, no bullet points, no brackets. Clean spoken dialogue only.

 11. COMPLETION CHECK (MANDATORY):
@@ -77,7 +77,12 @@ CRITICAL INSTRUCTIONS FOR THIS TURN:
 2. DO NOT re-ask for any information already present in CURRENT INTAKE STATE or clearly stated in this input. NEVER ask duration again if duration is provided.
 3. If the patient stated "Nothing else", "No", "None" for a SPECIFIC category that was just asked about, set that category to confirmed_none. But do NOT complete the intake unless ALL required categories are collected or confirmed_none.
 4. If an important clinical gap remains, formulate ONE natural, empathetic question addressing ONLY the most relevant missing item.
-5. In "reply", provide only ONE clean, spoken sentence without bullet points, markdown, or bracketed notes.
+5. DYNAMIC EMOTIONAL TONE & QUESTION FORMULATION:
+   - Match the patient's tone naturally:
+     * SAD / PAIN / SEVERE SYMPTOMS: Start with genuine empathy (e.g., "Sorry to hear. Do you have BP or sugar?", "Oh dear! How long has it been?").
+     * POSITIVE / NO ILLNESS / NEGATION: Start with a warm, encouraging tone (e.g., "Great! Taking any daily medications?", "Wonderful! Any other symptoms?").
+   - DO NOT use static prefillers like "Got it." repeatedly. Let OpenAI generate a dynamic, natural response directly.
+   - Keep the reply concise, conversational, and direct (around 5 to 8 words total).

 COMPLETION RULES — MANDATORY:
 - NEVER set "completed": true unless ALL of the following are satisfied:
@@ -50,6 +50,7 @@ class SarvamClient:
        self,
        text: str,
        language_code: str = "ta-IN",
+        pace: float = 1.0,
    ) -> bytes | None:
        if not self.api_key or self.api_key == "your_sarvam_key":
            return None
@@ -66,7 +67,8 @@ class SarvamClient:
            "inputs": [text],
            "target_language_code": language_code,
            "speaker": speaker,
-            "model": "bulbul:v2",
+            "model": "bulbul:v3",
+            "pace": pace,
        }
        try:
            res = await client.post(
@@ -0,0 +1,42 @@
+import re
+from typing import Dict, Any, Tuple
+
+CAREGIVER_PATTERNS = [
+    (r"(?:my\s*mother|my\s*mom|என்\s*அம்மா|அம்மாவுக்கு|मेरी\s*माँ|माँ\s*को|నా\s*తల్లి)", "mother"),
+    (r"(?:my\s*father|my\s*dad|என்\s*அப்பா|அப்பாவுக்கு|मेरे\s*पिता|पिता\s*जी|నా\s*తండ్రి)", "father"),
+    (r"(?:my\s*wife|my\s*husband|என்\s*மனைவி|என்\s*கணவர்|मेरी\s*पत्नी|मेरे\s*पति|నా\s*భార్య|నా\s*భర్త)", "spouse"),
+    (r"(?:my\s*son|my\s*daughter|my\s*child|my\s*kid|என்\s*மகன்|என்\s*மகள்|என்\s*குழந்தை|मेरा\s*बेटा|मेरी\s*बेटी|నా\s*కొడుకు)", "child"),
+    (r"(?:he\s*is|she\s*is|his\s*symptoms|her\s*symptoms|he\s*has|she\s*has|அவருக்கு|அவளுக்கு|उसको|उन्हें)", "relative")
+]
+
+PATIENT_PATTERNS = [
+    r"(?:i\s*have|i\s*am|my\s*headache|my\s*chest|my\s*fever|my\s*pain|எனக்கு|எனக்குள்ள|मुझे|मेरे|నాకు)"
+]
+
+
+def classify_speaker_turn(text: str) -> Dict[str, Any]:
+    """
+    Dual-Speaker Diarization Classifier:
+    Differentiates between direct Patient statements (1st Person) and Caregiver / Relative statements (3rd Person).
+    Returns dictionary with speaker_type ('patient' vs 'caregiver'), relation, and formatted_label.
+    """
+    txt_lower = (text or "").lower().strip()
+
+    # 1. Check Caregiver / Relative Patterns
+    for pattern, relation in CAREGIVER_PATTERNS:
+        if re.search(pattern, txt_lower, re.IGNORECASE):
+            label = f"Caregiver ({relation.capitalize()})" if relation != "relative" else "Caregiver"
+            return {
+                "speaker_type": "caregiver",
+                "relation": relation,
+                "formatted_label": label,
+                "is_caregiver": True
+            }
+
+    # 2. Default to Patient Direct Statement
+    return {
+        "speaker_type": "patient",
+        "relation": "self",
+        "formatted_label": "Patient",
+        "is_caregiver": False
+    }