feat: complete production hardening implementation, specialty canonicalization, google-genai migration, voice reliability, sqlite concurrency, and test suite consolidation

2026-09-07 12:57:54 +05:30
parent c23d979338
commit 92fa71ae4a
99 changed files with 5167 additions and 3550 deletions
@@ -0,0 +1,71 @@
+"""
+Multilingual normalization layer for clinical messages.
+Preserves raw input while generating normalized, canonical representations.
+"""
+
+import re
+import unicodedata
+from typing import Dict, Any, Tuple
+
+from app.services.contextual_normalizer import (
+    normalize_duration_from_message,
+    detect_previous_question_intent,
+    is_duration_expression,
+    is_severity_expression,
+    is_location_expression,
+    is_negative_expression,
+    is_casual_filler,
+    extract_severity_from_message,
+    extract_location_from_message,
+)
+
+
+def normalize_text(text: str) -> str:
+    """
+    Cleans Unicode, strips extra spaces, standardizes quotes and punctuation.
+    """
+    if not text:
+        return ""
+    # Unicode Normalization Form C
+    normalized = unicodedata.normalize("NFC", text)
+    # Replace non-breaking spaces and smart quotes
+    normalized = normalized.replace("\xa0", " ").replace("’", "'").replace("“", '"').replace("”", '"')
+    # Collapse multiple whitespaces
+    normalized = re.sub(r"\s+", " ", normalized).strip()
+    return normalized
+
+
+def detect_language(text: str, default_lang: str = "en") -> str:
+    """
+    Detects script/language (ta, hi, te, kn, en).
+    """
+    if not text:
+        return default_lang
+
+    # Script character ranges
+    has_ta = bool(re.search(r"[\u0B80-\u0BFF]", text))
+    has_hi = bool(re.search(r"[\u0900-\u097F]", text))
+    has_te = bool(re.search(r"[\u0C00-\u0C7F]", text))
+    has_kn = bool(re.search(r"[\u0C80-\u0CFF]", text))
+
+    if has_ta:
+        return "ta-IN"
+    if has_hi:
+        return "hi-IN"
+    if has_te:
+        return "te-IN"
+    if has_kn:
+        return "kn-IN"
+
+    # Transliterated language keywords
+    msg_lower = text.lower()
+    if any(w in msg_lower for w in ["irukku", "thalaivali", "romba", "mudiyala", "saapudren"]):
+        return "ta-IN"
+    if any(w in msg_lower for w in ["dard", "ho raha", "hoga", "le raha"]):
+        return "hi-IN"
+    if any(w in msg_lower for w in ["noppi", "undi", "thala noppi"]):
+        return "te-IN"
+    if any(w in msg_lower for w in ["novu", "ide", "thale novu"]):
+        return "kn-IN"
+
+    return default_lang or "en-IN"