""" Multilingual normalization layer for clinical messages. Preserves raw input while generating normalized, canonical representations. """ import re import unicodedata from typing import Dict, Any, Tuple from app.services.contextual_normalizer import ( normalize_duration_from_message, detect_previous_question_intent, is_duration_expression, is_severity_expression, is_location_expression, is_negative_expression, is_casual_filler, extract_severity_from_message, extract_location_from_message, ) def normalize_text(text: str) -> str: """ Cleans Unicode, strips extra spaces, standardizes quotes and punctuation. """ if not text: return "" # Unicode Normalization Form C normalized = unicodedata.normalize("NFC", text) # Replace non-breaking spaces and smart quotes normalized = normalized.replace("\xa0", " ").replace("’", "'").replace("“", '"').replace("”", '"') # Collapse multiple whitespaces normalized = re.sub(r"\s+", " ", normalized).strip() return normalized def detect_language(text: str, default_lang: str = "en") -> str: """ Detects script/language (ta, hi, te, kn, en). """ if not text: return default_lang # Script character ranges has_ta = bool(re.search(r"[\u0B80-\u0BFF]", text)) has_hi = bool(re.search(r"[\u0900-\u097F]", text)) has_te = bool(re.search(r"[\u0C00-\u0C7F]", text)) has_kn = bool(re.search(r"[\u0C80-\u0CFF]", text)) if has_ta: return "ta-IN" if has_hi: return "hi-IN" if has_te: return "te-IN" if has_kn: return "kn-IN" # Transliterated language keywords msg_lower = text.lower() if any(w in msg_lower for w in ["irukku", "thalaivali", "romba", "mudiyala", "saapudren"]): return "ta-IN" if any(w in msg_lower for w in ["dard", "ho raha", "hoga", "le raha"]): return "hi-IN" if any(w in msg_lower for w in ["noppi", "undi", "thala noppi"]): return "te-IN" if any(w in msg_lower for w in ["novu", "ide", "thale novu"]): return "kn-IN" return default_lang or "en-IN"