feat: complete production hardening implementation, specialty canonicalization, google-genai migration, voice reliability, sqlite concurrency, and test suite consolidation

ai-patient-onboarding/app/ai/extraction/__init__.py

@@ -0,0 +1,21 @@
+"""
+Extraction package.
+"""
+
+from app.ai.extraction.multilingual_normalizer import normalize_text, detect_language
+from app.ai.extraction.symptom_extractor import extract_symptoms
+from app.ai.extraction.medication_extractor import extract_medications
+from app.ai.extraction.allergy_extractor import extract_allergies
+from app.ai.extraction.history_extractor import extract_history
+from app.ai.extraction.negation_engine import is_negated, extract_family_subject
+
+__all__ = [
+    "normalize_text",
+    "detect_language",
+    "extract_symptoms",
+    "extract_medications",
+    "extract_allergies",
+    "extract_history",
+    "is_negated",
+    "extract_family_subject",
+]

ai-patient-onboarding/app/ai/extraction/allergy_extractor.py

@@ -0,0 +1,56 @@
+"""
+Allergy extraction engine distinguishing drug/food allergies from side effects.
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Union
+
+ALLERGEN_PATTERNS = [
+    (r"(penicillin)", "Penicillin"),
+    (r"(sulfa)", "Sulfa drugs"),
+    (r"(aspirin)", "Aspirin"),
+    (r"(dust)", "Dust allergy"),
+    (r"(peanuts)", "Peanuts"),
+    (r"(milk|dairy)", "Milk / Dairy"),
+]
+
+
+def extract_allergies(
+    message: str,
+    normalized_msg: str,
+    last_ai_message: Optional[str] = None
+) -> Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]:
+    """
+    Extracts structured allergies or confirmed_none container.
+    """
+    msg_lower = normalized_msg.lower()
+
+    # Check for explicit negation of allergies
+    neg_phrases = [
+        "no allergies", "no allergy", "no drug allergy", "no food allergy", "not allergic",
+        "nkda", "no allergic reaction", "அலர்ஜி எதுவும் இல்லை", "எந்த அலர்ஜியும் இல்லை",
+        "कोई एलर्जी नहीं", "कोई समस्या नहीं", "ఏ అలర్జీ లేదు", "ಯಾವ ಅಲರ್ಜಿ ಇಲ್ಲ"
+    ]
+    if any(neg in msg_lower for neg in neg_phrases):
+        return {"status": "confirmed_none", "items": []}
+
+    allergies: List[Dict[str, Any]] = []
+    has_allergen = any(re.search(pat, msg_lower) for pat, _ in ALLERGEN_PATTERNS)
+    has_alg_indicator = any(k in msg_lower or k in message for k in ["allergic", "allergy", "அலர்ஜி", "ஒவ்வாமை", "एलर्जी", "rash", "reaction", "side effect", "தடிப்பு", "எடுத்தா"])
+
+    # Side effect distinction: "Paracetamol didn't reduce fever" is NOT an allergy
+    if "didn't reduce" in msg_lower or "didn't work" in msg_lower or "no effect" in msg_lower:
+        return None
+
+    if has_allergen or has_alg_indicator:
+        for pat, allergen in ALLERGEN_PATTERNS:
+            if re.search(pat, msg_lower):
+                alg_obj: Dict[str, Any] = {"name": allergen}
+                if "rash" in msg_lower or "தடிப்பு" in message or "खुजली" in message:
+                    alg_obj["reaction"] = "Rash / Skin reaction"
+                allergies.append(alg_obj)
+
+        if not allergies and has_alg_indicator:
+            allergies.append({"name": "Reported allergy", "type": "unspecified"})
+
+    return allergies if allergies else None

ai-patient-onboarding/app/ai/extraction/history_extractor.py

@@ -0,0 +1,83 @@
+"""
+Medical, surgical, family history, and previous episode extraction engine.
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Tuple, Union
+
+from app.ai.extraction.negation_engine import extract_family_subject
+
+HISTORY_KEYWORDS = [
+    ("diabetes", "Diabetes"), ("sugar", "Diabetes"), ("diabetic", "Diabetes"),
+    ("hypertension", "Hypertension (High BP)"), ("high bp", "Hypertension (High BP)"), ("bp", "Hypertension (High BP)"),
+    ("asthma", "Asthma"), ("thyroid", "Thyroid disorder"),
+    ("சர்க்கரை நோய்", "Diabetes"), ("சுகர்", "Diabetes"), ("பிபி", "Hypertension"),
+    ("डायबिटीज", "Diabetes"), ("शुगर", "Diabetes"), ("बीपी", "Hypertension"),
+]
+
+
+def extract_history(
+    message: str,
+    normalized_msg: str,
+    duration: Optional[str] = None
+) -> Dict[str, Any]:
+    """
+    Extracts medical_history, family_history, surgical_history, previous_episodes, and additional_information.
+    """
+    msg_lower = normalized_msg.lower()
+    updates: Dict[str, Any] = {}
+
+    # Check for explicit negation of medical history for self
+    neg_history_phrases = [
+        "no past history", "no medical history", "no health conditions", "no bp no diabetes",
+        "no chronic illness", "don't have diabetes", "no diabetes", "no bp",
+        "எந்த நோயும் இல்லை", "வேற ஒன்னும் இல்லை", "பிபி சுகர் எதுவும் இல்ல",
+        "कोई बीमारी नहीं", "बीमारी नहीं है", "ఏ సమస్య లేదు", "ಯಾವ ಕಾಯಿಲೆ ಇಲ್ಲ"
+    ]
+
+    # Check if patient mentions family member (e.g., "my father has diabetes")
+    is_fam, relation, _ = extract_family_subject(message)
+
+    if is_fam:
+        fam_list: List[Dict[str, Any]] = []
+        for kw, cond in HISTORY_KEYWORDS:
+            if re.search(rf"\b{re.escape(kw)}\b", msg_lower) or kw in message:
+                fam_list.append({"relation": relation, "condition": cond})
+        if fam_list:
+            updates["family_history"] = fam_list
+
+    if any(neg in msg_lower for neg in neg_history_phrases):
+        updates["medical_history"] = {"status": "confirmed_none", "items": []}
+    elif not is_fam:
+        med_history: List[Dict[str, Any]] = []
+        for kw, cond in HISTORY_KEYWORDS:
+            if re.search(rf"\b{re.escape(kw)}\b", msg_lower) or kw in message:
+                h_obj: Dict[str, Any] = {"condition": cond}
+                h_dur_m = re.search(r"(?:for|since)\s+(\d+\s+(?:years|months|yrs))", msg_lower)
+                if h_dur_m:
+                    h_obj["duration"] = h_dur_m.group(1).strip()
+                if not any(x.get("condition") == cond for x in med_history):
+                    med_history.append(h_obj)
+        if med_history:
+            updates["medical_history"] = med_history
+
+    # Surgical History & Previous Episodes
+    if "surgery" in msg_lower or "operation" in msg_lower or "ஆபரேஷன்" in message:
+        updates["surgical_history"] = [{"procedure": "Prior surgery reported", "approximate_date": duration or "Past"}]
+
+    if "same problem" in msg_lower or "had this before" in msg_lower or "மூன்று மாதங்களுக்கு முன்" in message:
+        updates["previous_episodes"] = [{"description": "Similar symptoms previously", "time": duration or "Past"}]
+
+    # Additional Information / Functional Impact
+    add_info: List[Dict[str, Any]] = []
+    if "couldn't eat" in msg_lower or "not eaten" in msg_lower or "சாப்பிட முடியல" in message:
+        add_info.append({"category": "functional_impact", "value": "Loss of appetite / difficulty eating"})
+    if "couldn't sleep" in msg_lower or "haven't slept" in msg_lower or "தூங்க முடியல" in message:
+        add_info.append({"category": "functional_impact", "value": "Sleep disruption"})
+    if "weak" in msg_lower or "tired" in msg_lower or "ரொம்ப டயர்டா இருக்கு" in message:
+        add_info.append({"category": "functional_impact", "value": "Severe weakness / fatigue"})
+
+    if add_info:
+        updates["additional_information"] = add_info
+
+    return updates

ai-patient-onboarding/app/ai/extraction/medication_extractor.py

@@ -0,0 +1,68 @@
+"""
+Medication extraction engine supporting structured dosage, frequency, and confirmed_none status.
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Union
+
+KNOWN_MED_PATTERNS = [
+    (r"(metformin(?:\s+\d+\s*mg)?(?:\s+twice\s+a\s+day)?)", "Metformin"),
+    (r"(paracetamol(?:\s+\d+\s*mg)?)", "Paracetamol"),
+    (r"(dolo(?:\s*650)?)", "Dolo 650 (Paracetamol)"),
+    (r"(pantocid|pan-d|pan 40|omez|omeprazole)", "Pantoprazole / Antacid"),
+    (r"(telmisartan|telma\s*\d+)", "Telmisartan (BP)"),
+    (r"(amlodipine|amlong\s*\d+)", "Amlodipine (BP)"),
+    (r"((?:bp|பிபி|blood\s*pressure)\s*(?:மாத்திரை|medicine|tablet|tablets|மருந்து))", "BP Medication"),
+    (r"((?:sugar|சுகர்|சர்க்கரை)\s*(?:மாத்திரை|medicine|tablet|tablets|மருந்து))", "Diabetes Medication"),
+    (r"((?:thyroid|தைராய்டு)\s*(?:மாத்திரை|medicine|tablet|tablets|மருந்து))", "Thyroid Medication"),
+]
+
+
+def extract_medications(
+    message: str,
+    normalized_msg: str,
+    last_ai_message: Optional[str] = None,
+    is_allergy_utterance: bool = False
+) -> Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]:
+    """
+    Extracts structured medications or confirmed_none container.
+    """
+    msg_lower = normalized_msg.lower()
+
+    # Check for explicit negation of medicines
+    neg_phrases = [
+        "no medicines", "no medication", "don't take any medicine", "don't take medicines",
+        "no tablets", "not taking any medicine", "no daily medicines", "no regular medicines",
+        "மாத்திரை எதுவும் இல்லை", "எந்த மாத்திரையும் சாப்பிடல", "மருந்து எதுவும் இல்லை",
+        "कोई दवा नहीं", "दवाई नहीं लेते", "दवा नहीं लेता", "ఏ మందులు లేవు", "ಯಾವ ಮಾತ್ರೆ ಇಲ್ಲ"
+    ]
+    if any(neg in msg_lower for neg in neg_phrases):
+        return {"status": "confirmed_none", "items": []}
+
+    meds: List[Dict[str, Any]] = []
+
+    # Known medicine matchers
+    for pat, med_label in KNOWN_MED_PATTERNS:
+        m_match = re.search(pat, msg_lower)
+        if m_match:
+            matched_str = m_match.group(1).strip()
+            med_obj: Dict[str, Any] = {"name": med_label}
+
+            dosage_m = re.search(r"(\d+\s*mg|\d+\s*ml)", matched_str)
+            if dosage_m:
+                med_obj["dosage"] = dosage_m.group(1).strip()
+
+            freq_m = re.search(r"(twice\s+a\s+day|once\s+a\s+day|\d+\s+times\s+a\s+day|daily|twice)", matched_str)
+            if freq_m:
+                med_obj["frequency"] = freq_m.group(1).strip()
+
+            if not any(x.get("name") == med_obj["name"] for x in meds):
+                meds.append(med_obj)
+
+    # Generic medicine indicator matchers (only if not an allergy utterance)
+    if not meds and not is_allergy_utterance:
+        generic_pat = r"(tablets|goli|mathirai|மாத்திரை|மருந்து|दवाई|दवा|మాత్రలు|ಮಾತ್ರೆಗಳು)"
+        if re.search(generic_pat, msg_lower) and any(verb in msg_lower for verb in ["take", "taking", "சாப்பிடுறேன்", "சாப்பிட்டேன்", "வாங்குறேன்", "लेता", "लेती", "వాడుతున్నాను"]):
+            meds.append({"name": "Unspecified medication"})
+
+    return meds if meds else None

ai-patient-onboarding/app/ai/extraction/multilingual_normalizer.py

@@ -0,0 +1,71 @@
+"""
+Multilingual normalization layer for clinical messages.
+Preserves raw input while generating normalized, canonical representations.
+"""
+
+import re
+import unicodedata
+from typing import Dict, Any, Tuple
+
+from app.services.contextual_normalizer import (
+    normalize_duration_from_message,
+    detect_previous_question_intent,
+    is_duration_expression,
+    is_severity_expression,
+    is_location_expression,
+    is_negative_expression,
+    is_casual_filler,
+    extract_severity_from_message,
+    extract_location_from_message,
+)
+
+
+def normalize_text(text: str) -> str:
+    """
+    Cleans Unicode, strips extra spaces, standardizes quotes and punctuation.
+    """
+    if not text:
+        return ""
+    # Unicode Normalization Form C
+    normalized = unicodedata.normalize("NFC", text)
+    # Replace non-breaking spaces and smart quotes
+    normalized = normalized.replace("\xa0", " ").replace("’", "'").replace("“", '"').replace("”", '"')
+    # Collapse multiple whitespaces
+    normalized = re.sub(r"\s+", " ", normalized).strip()
+    return normalized
+
+
+def detect_language(text: str, default_lang: str = "en") -> str:
+    """
+    Detects script/language (ta, hi, te, kn, en).
+    """
+    if not text:
+        return default_lang
+
+    # Script character ranges
+    has_ta = bool(re.search(r"[\u0B80-\u0BFF]", text))
+    has_hi = bool(re.search(r"[\u0900-\u097F]", text))
+    has_te = bool(re.search(r"[\u0C00-\u0C7F]", text))
+    has_kn = bool(re.search(r"[\u0C80-\u0CFF]", text))
+
+    if has_ta:
+        return "ta-IN"
+    if has_hi:
+        return "hi-IN"
+    if has_te:
+        return "te-IN"
+    if has_kn:
+        return "kn-IN"
+
+    # Transliterated language keywords
+    msg_lower = text.lower()
+    if any(w in msg_lower for w in ["irukku", "thalaivali", "romba", "mudiyala", "saapudren"]):
+        return "ta-IN"
+    if any(w in msg_lower for w in ["dard", "ho raha", "hoga", "le raha"]):
+        return "hi-IN"
+    if any(w in msg_lower for w in ["noppi", "undi", "thala noppi"]):
+        return "te-IN"
+    if any(w in msg_lower for w in ["novu", "ide", "thale novu"]):
+        return "kn-IN"
+
+    return default_lang or "en-IN"

ai-patient-onboarding/app/ai/extraction/negation_engine.py

@@ -0,0 +1,54 @@
+"""
+Contextual negation and scope resolution engine.
+"""
+
+import re
+from typing import Dict, Any, Tuple
+
+
+# Family relations
+FAMILY_RELATIONS = [
+    ("father", "father"), ("dad", "father"), ("appa", "father"), ("அப்பா", "father"), ("पिता", "father"), ("नाना", "grandfather"),
+    ("mother", "mother"), ("mom", "mother"), ("amma", "mother"), ("அம்மா", "mother"), ("माँ", "mother"), ("माता", "mother"),
+    ("brother", "brother"), ("annaa", "brother"), ("தம்பி", "brother"), ("अண்ணன்", "brother"), ("भाई", "brother"),
+    ("sister", "sister"), ("akka", "sister"), ("அக்கா", "sister"), ("தங்கை", "sister"), ("बहन", "sister"),
+    ("parent", "parent"), ("grand-father", "grandfather"), ("grandfather", "grandfather"), ("grandmother", "grandmother"),
+]
+
+# Negation indicators
+NEGATION_PHRASES = [
+    "don't have", "do not have", "no history of", "no symptoms", "no allergies", "no allergy", "no medicine",
+    "no medications", "don't take", "do not take", "never had", "without", "denies", "absent",
+    "இல்லை", "எதுவும் இல்லை", "இல்லவே இல்லை", "மாத்திரை எதுவும் இல்லை", "நோயும் இல்லை",
+    "नहीं है", "कोई बीमारी नहीं", "कोई एलर्जी नहीं", "कोई दवा नहीं",
+    "లేవు", "ఏమీ లేదు", "ఇబ్బంది లేదు",
+    "ಇಲ್ಲ", "ಯಾವುದೇ ಇಲ್ಲ",
+]
+
+
+def is_negated(text: str) -> bool:
+    """Returns True if the text expresses explicit negation."""
+    if not text:
+        return False
+    msg_lower = text.lower().strip()
+    if msg_lower in ["no", "none", "nothing", "no more", "nil", "illness none", "nkda"]:
+        return True
+    return any(neg in msg_lower for neg in NEGATION_PHRASES)
+
+
+def extract_family_subject(text: str) -> Tuple[bool, str, str]:
+    """
+    Checks if text refers to family member rather than the patient.
+    Returns: (is_family_history, relation, condition_text)
+    """
+    if not text:
+        return False, "", text
+
+    msg_lower = text.lower().strip()
+    for rel_kw, rel_canonical in FAMILY_RELATIONS:
+        pattern = rf"\b{re.escape(rel_kw)}\b"
+        if re.search(pattern, msg_lower) or rel_kw in text:
+            # Check if patient explicitly negates disease for self in same turn
+            return True, rel_canonical, text
+
+    return False, "", text

ai-patient-onboarding/app/ai/extraction/symptom_extractor.py

@@ -0,0 +1,117 @@
+"""
+Layered symptom extraction engine supporting attribute mapping per symptom.
+"""
+
+import re
+from typing import Dict, Any, List, Optional, Set, Tuple
+
+from app.services.contextual_normalizer import (
+    normalize_duration_from_message,
+    extract_severity_from_message,
+    extract_location_from_message,
+    is_duration_expression,
+    is_severity_expression,
+    is_location_expression,
+    is_casual_filler,
+    is_negative_expression,
+)
+
+
+SYMPTOM_DICTIONARY = [
+    # General & Respiratory
+    ("fever", "fever"), ("feverish", "fever"), ("காய்ச்சல்", "fever"), ("बुखार", "fever"), ("జ్వరం", "fever"), ("ಜ್ವರ", "fever"),
+    ("cough", "cough"), ("coughing", "cough"), ("இருமல்", "cough"), ("खांसी", "cough"), ("దగ్గు", "cough"), ("ಕೆಮ್ಮು", "cough"),
+    ("cold", "cold"), ("sneezing", "cold"), ("சளி", "cold"), ("सर्दी", "cold"), ("జలుబు", "cold"), ("ನೆಗಡಿ", "cold"),
+    ("breathlessness", "breathlessness"), ("shortness of breath", "breathlessness"), ("difficulty breathing", "breathlessness"),
+    ("மூச்சு திணறல்", "breathlessness"), ("மூச்சு வாங்குது", "breathlessness"), ("மூச்சு வாங்குறேன்", "breathlessness"), ("சாப்பிடும்போது மூச்சு", "breathlessness"), ("சாப்பிடும் போது மூச்சு", "breathlessness"), ("सांस फूलना", "breathlessness"),
+    ("palpitations", "palpitations"), ("படபடன்னு", "palpitations"), ("நெஞ்சு படபடன்னு", "palpitations"), ("படபடப்பு", "palpitations"),
+    # Pain & Head
+    ("headache", "headache"), ("head ache", "headache"), ("head pain", "headache"), ("migraine", "headache"),
+    ("தலைவலி", "headache"), ("தலை பாரம்", "headache"), ("सिरदर्द", "headache"), ("తలనొప్పి", "headache"), ("ತಲೆನೋವು", "headache"),
+    ("chest pain", "chest pain"), ("chest pressure", "chest pain"), ("நெஞ்சு வலி", "chest pain"), ("छाती में दर्द", "chest pain"),
+    ("stomach pain", "stomach pain"), ("abdominal pain", "stomach pain"), ("stomach ache", "stomach pain"),
+    ("வயிற்று வலி", "stomach pain"), ("पेट दर्द", "stomach pain"), ("కడుపు నొప్పి", "stomach pain"), ("<ctrl42> ಹೊಟ್ಟೆ ನೋವು", "stomach pain"),
+    ("back pain", "back pain"), ("முதுகு வலி", "back pain"), ("पीठ दर्द", "back pain"),
+    ("joint pain", "joint pain"), ("knee pain", "knee pain"), ("மூட்டு வலி", "joint pain"), ("घुटने का दर्द", "knee pain"),
+    # Neurological / ENT
+    ("dizziness", "dizziness"), ("giddiness", "dizziness"), ("dizzy", "dizziness"),
+    ("தலை சுத்துது", "dizziness"), ("தலைசுற்றல்", "dizziness"), ("மயக்கம்", "dizziness"), ("चक्कर", "dizziness"),
+    ("sore throat", "sore throat"), ("throat pain", "sore throat"), ("தொண்டை வலி", "sore throat"), ("गले में खराश", "sore throat"),
+    # GI
+    ("vomiting", "vomiting"), ("vomit", "vomiting"), ("vomited", "vomiting"), ("nausea", "vomiting"),
+    ("வாந்தி", "vomiting"), ("உल्टी", "vomiting"), ("వాంతులు", "vomiting"), ("ವಾಂತಿ", "vomiting"),
+    ("loose motion", "diarrhea"), ("diarrhea", "diarrhea"), ("loose stools", "diarrhea"),
+    ("வயிற்றுப்போக்கு", "diarrhea"), ("दस्त", "diarrhea"),
+    # Dermatological
+    ("itching", "itching"), ("rash", "rash"), ("skin rash", "rash"),
+    ("அரிப்பு", "itching"), ("தழும்பு", "rash"), ("खुजली", "itching"),
+]
+
+
+def extract_symptoms(
+    message: str,
+    normalized_msg: str,
+    current_state: Optional[Dict[str, Any]] = None
+) -> Tuple[List[Dict[str, Any]], Optional[str]]:
+    """
+    Extracts symptoms and maps attributes (duration, severity, location, frequency) specifically.
+    Returns: (symptoms_list, primary_reason_for_visit)
+    """
+    msg_lower = normalized_msg.lower()
+    found_symptoms: List[Dict[str, Any]] = []
+    found_names: Set[str] = set()
+
+    # Extract global context attributes
+    duration = normalize_duration_from_message(message)
+    severity = extract_severity_from_message(message)
+    location = extract_location_from_message(message)
+
+    # Frequency extraction (e.g. "twice", "2 times")
+    freq_match = re.search(r"(\b\d+\s*times\b|\btwice\b|\bonce\b|\bmultiple times\b)", msg_lower)
+    frequency = freq_match.group(1) if freq_match else None
+
+    # Scan for symptoms
+    for kw, canonical in SYMPTOM_DICTIONARY:
+        pattern = rf"\b{re.escape(kw)}\b"
+        if re.search(pattern, msg_lower) or kw in message:
+            if canonical not in found_names:
+                found_names.add(canonical)
+                # Store canonical name as 'name' if kw is an English inflection like 'vomit', else keep kw if non-English
+                stored_name = canonical if kw in ["vomit", "vomited", "coughing", "feverish"] else kw
+                sym_obj: Dict[str, Any] = {"name": stored_name}
+                if duration:
+                    sym_obj["duration"] = duration
+                if severity:
+                    sym_obj["severity"] = severity
+                if location:
+                    sym_obj["location"] = location
+                if frequency and ("vomit" in canonical or "வாந்தி" in kw or "உल्टी" in kw):
+                    sym_obj["frequency"] = frequency
+                found_symptoms.append(sym_obj)
+
+    primary_rfv = list(found_names)[0] if found_names else None
+
+    # Check for medication or non-symptom statements to prevent fake symptom fallback
+    is_med_statement = any(k in msg_lower or k in message for k in ["metformin", "dolo", "paracetamol", "pantocid", "telmisartan", "amlodipine", "tablets", "medicine", "pill", "pills", "goli", "mathirai", "மாத்திரை", "மருந்து", "சாப்பிடுறேன்", "சாப்பிட்டேன்", "வாங்குறேன்", "दवाई", "दवा"])
+
+    # Fallback: single phrase short symptom message
+    if not found_symptoms and message and len(message.split()) < 10 and not is_med_statement:
+        if not (
+            is_duration_expression(message)
+            or is_severity_expression(message)
+            or is_location_expression(message)
+            or is_casual_filler(message)
+            or is_negative_expression(message)
+        ):
+            clean_symptom = message.strip()
+            sym_obj = {"name": clean_symptom}
+            if duration:
+                sym_obj["duration"] = duration
+            if severity:
+                sym_obj["severity"] = severity
+            if location:
+                sym_obj["location"] = location
+            found_symptoms.append(sym_obj)
+            primary_rfv = clean_symptom
+
+    return found_symptoms, primary_rfv