""" Contextual Normalizer for Clinical Intake. Provides: - Tamil, Tanglish, Hindi, and English number & duration normalization. - Contextual intent recognition from previous AI dialogue turns. - Non-symptom expression filters (duration, severity, location, negative phrases, filler). - Deduplication and deterministic clinical attribute extraction. """ import re from typing import Optional, Tuple, Dict, Any, List # 1. Spoken Number Mappings across Tamil, Tanglish, Hindi, English TAMIL_NUMBER_MAP: Dict[str, int] = { "ஒரு": 1, "ஒண்ணு": 1, "ஒன்று": 1, "ஒன்னு": 1, "ரெண்டு": 2, "இரண்டு": 2, "மூணு": 3, "மூன்று": 3, "நாலு": 4, "நான்கு": 4, "அஞ்சு": 5, "ஐந்து": 5, "ஆறு": 6, "ஏழு": 7, "எட்டு": 8, "ஒன்பது": 9, "பத்து": 10, "பதினொன்னு": 11, "பதினொன்று": 11, "பன்னிரண்டு": 12, "பன்னிரெண்டு": 12, "பதின்மூன்று": 13, "பதினாலு": 14, "பதினான்கு": 14, "பதினைந்து": 15, "பதினாஞ்சு": 15, "பதினாறு": 16, "பதினேழு": 17, "பதினெட்டு": 18, "பத்தொன்பது": 19, "இருபது": 20, "இருபத்தஞ்சு": 25, "முப்பது": 30, "நாற்பது": 40, "ஐம்பது": 50, } TANGLISH_NUMBER_MAP: Dict[str, int] = { "oru": 1, "onnu": 1, "ondru": 1, "rendu": 2, "irandu": 2, "moonu": 3, "moonru": 3, "naalu": 4, "nangu": 4, "anju": 5, "ainthu": 5, "aaru": 6, "yezhu": 7, "ezhu": 7, "elu": 7, "ettu": 8, "onbathu": 9, "ombadhu": 9, "pathu": 10, "padhu": 10, "pathinonnu": 11, "pannirendu": 12, "pathinaanchu": 15, "pathinainthu": 15, "irubathu": 20, "irupathu": 20, "muppathu": 30, } HINDI_NUMBER_MAP: Dict[str, int] = { "एक": 1, "दो": 2, "तीन": 3, "चार": 4, "पांच": 5, "पाँच": 5, "छह": 6, "सात": 7, "आठ": 8, "नौ": 9, "दस": 10, "ग्यारह": 11, "बारह": 12, "पंद्रह": 15, "बीस": 20, "तीस": 30, "चालीस": 40, "पचास": 50, } TELUGU_NUMBER_MAP: Dict[str, int] = { "ఒకటి": 1, "ఒక": 1, "రెండు": 2, "మూడు": 3, "నాలుగు": 4, "ఐదు": 5, "ఆరు": 6, "ఏడు": 7, "ఎనిమిది": 8, "తొమ్మిది": 9, "పది": 10, } KANNADA_NUMBER_MAP: Dict[str, int] = { "ಒಂದು": 1, "ಒಂದ": 1, "ಎರಡು": 2, "ಮೂರು": 3, "ನಾಲ್ಕು": 4, "ಐದು": 5, "ಆರು": 6, "ಏಳು": 7, "ಎಂಟು": 8, "ಒಂಬತ್ತು": 9, "ಹತ್ತು": 10, } ENGLISH_NUMBER_MAP: Dict[str, int] = { "a": 1, "an": 1, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, "eleven": 11, "twelve": 12, "thirteen": 13, "fourteen": 14, "fifteen": 15, "sixteen": 16, "seventeen": 17, "eighteen": 18, "nineteen": 19, "twenty": 20, "twenty-five": 25, "twenty five": 25, "thirty": 30, "forty": 40, "fifty": 50, } def normalize_duration_from_message(message: str) -> Optional[str]: """ Detects and canonicalizes duration phrases across English, Tamil, Tanglish, and Mixed forms. Returns canonical strings such as: '10 days', '2 days', '1 week', '2 months', 'since yesterday', 'since last week', 'since Monday', 'for a few days', 'for many months'. Returns None if no duration is recognized. """ if not message or not message.strip(): return None msg = message.strip() msg_lower = msg.lower() # 1. Check relative date / anchor phrases rel_patterns = [ (r"(?:நேற்றிலிருந்து|நேத்து\s*ல\s*இருந்து|நேத்துல\s*இருந்து|நேத்து\s*இருந்து|நேற்று\s*முதல்|since\s*yesterday|कल\s*से|निन्नೆಯಿಂದ|నిన్నటి\s*నుండి)", "since yesterday"), (r"(?:இன்னைக்கு\s*காலையிலிருந்து|காலையிலிருந்து|today\s*morning|since\s*morning|आज\s*सुबह\s*से)", "since morning"), (r"(?:since\s*last\s*night|நேத்து\s*ராத்திரி\s*இருந்து|நேத்து\s*நைட்டிலிருந்து)", "since last night"), (r"(?:since\s*last\s*week|போன\s*வாரத்திலிருந்து|पिछले\s*हफ़्ते\s*से)", "since last week"), (r"(?:since\s*(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday))", lambda m: m.group(0).strip()), (r"(?:for\s*a\s*few\s*days|few\s*days|சில\s*நாட்களா|கொஞ்ச\s*நாளா|कुछ\s*दिनों\s*से)", "for a few days"), (r"(?:for\s*many\s*months|ரொம்ப\s*மாசமா|பல\s*மாதங்களா|कई\s*महीनों\s*से)", "for many months"), ] for pat, canonical in rel_patterns: m = re.search(pat, msg, re.IGNORECASE) or re.search(pat, msg_lower, re.IGNORECASE) if m: if callable(canonical): return canonical(m) return canonical # 2. Number word or digit + Unit detection all_tamil_nums = "|".join(sorted(TAMIL_NUMBER_MAP.keys(), key=len, reverse=True)) all_tanglish_nums = "|".join(sorted(TANGLISH_NUMBER_MAP.keys(), key=len, reverse=True)) all_hindi_nums = "|".join(sorted(HINDI_NUMBER_MAP.keys(), key=len, reverse=True)) all_telugu_nums = "|".join(sorted(TELUGU_NUMBER_MAP.keys(), key=len, reverse=True)) all_kannada_nums = "|".join(sorted(KANNADA_NUMBER_MAP.keys(), key=len, reverse=True)) all_eng_nums = "|".join(sorted(ENGLISH_NUMBER_MAP.keys(), key=len, reverse=True)) num_regex_part = rf"(\d+|{all_tamil_nums}|{all_tanglish_nums}|{all_hindi_nums}|{all_telugu_nums}|{all_kannada_nums}|{all_eng_nums})" unit_regex_part = ( r"(?:days|day|டேஸா|டேஸ்|டே|நாட்களா|நாட்களாக|நாளா|நாளாக|நாளு|நாட்கள்|நாள்|" r"weeks|week|வாரமா|வாரமாக|வாரம்|வாரங்கள்|" r"months|month|மாசம்|மாசமா|மாதமா|மாதம்|மாதங்களாக|" r"years|year|வருஷம்|வருசமா|வருடம்|வருடங்கள்|" r"hours|hour|hrs|மணி\s*நேரம்|மணி|" r"naala|naal\s*ah|naal|nala|vaarama|vaaram|maasama|maasam|varusham|varushama|" r"दिनों|दिन|हफ़्ते|हफ्ता|महीने|साल|घंटे|" r"రోజుల\s*నుండి|రోజుల\s*నుంచి|రోజులుగా|రోజుల|రోజులు|వారం|వారాలు|" r"ದಿನದಿಂದ|ದಿನಗಳಿಂದ|ದಿನಗಳು|ದಿನ|ವಾರದಿಂದ|ವಾರಗಳು)" ) # Prefix qualifiers: "ஒரு 10 டேஸா", "about 10 days", "for 10 days", "இன்னும் ஒரு பத்து நாளா" full_pattern = ( rf"(?:(?:எனக்கு\s*வந்து\s*)?(?:இன்னும்\s*)?(?:ஒரு|oru|about|around|for\s+the\s+last|for|approx|approximately)\s+)?" rf"{num_regex_part}\s*[-]?\s*{unit_regex_part}" rf"(?:[-]?\s*(?:ah|a|ஆ|இருக்கு|ஆச்சு|ஆகியது|ஆகப்\s*போகுது|உంది|ಇದೆ|now))?" ) match = re.search(full_pattern, msg, re.IGNORECASE) or re.search(full_pattern, msg_lower, re.IGNORECASE) if match: raw_num_str = match.group(1).lower().strip() matched_text = match.group(0).lower() # Parse number val = None if raw_num_str.isdigit(): val = int(raw_num_str) elif raw_num_str in TAMIL_NUMBER_MAP: val = TAMIL_NUMBER_MAP[raw_num_str] elif raw_num_str in TANGLISH_NUMBER_MAP: val = TANGLISH_NUMBER_MAP[raw_num_str] elif raw_num_str in HINDI_NUMBER_MAP: val = HINDI_NUMBER_MAP[raw_num_str] elif raw_num_str in TELUGU_NUMBER_MAP: val = TELUGU_NUMBER_MAP[raw_num_str] elif raw_num_str in KANNADA_NUMBER_MAP: val = KANNADA_NUMBER_MAP[raw_num_str] elif raw_num_str in ENGLISH_NUMBER_MAP: val = ENGLISH_NUMBER_MAP[raw_num_str] # Determine unit unit = "days" if any(w in matched_text for w in ["week", "வாரம்", "வார", "vaaram", "vaarama", "हफ़्ते", "हफ्ता", "వారం", "ವಾರ"]): unit = "week" if val == 1 else "weeks" elif any(w in matched_text for w in ["month", "மாச", "மாத", "maasam", "maasama", "महीने", "నెలా", "ತಿಂಗಳು"]): unit = "month" if val == 1 else "months" elif any(w in matched_text for w in ["year", "வருஷ", "வருட", "varusham", "varushama", "साल", "సంవత్సరం", "ವರ್ಷ"]): unit = "year" if val == 1 else "years" elif any(w in matched_text for w in ["hour", "மணி", "hrs", "घंटे", "గంటలు", "ಗಂಟೆ"]): unit = "hour" if val == 1 else "hours" else: unit = "day" if val == 1 else "days" if val is not None: return f"{val} {unit}" # 3. Fallback loose check for standalone Tanglish/Tamil tokens (e.g. "10 days-ah", "10 naala", "2 days-ah") loose_match = re.search(r"(\d+)\s*[-]?\s*(?:days?|naal[a]?|nal[a]?|weeks?|months?|years?|டேஸா?|டேஸ்|நாளா?|நாட்களா?)(?:[-]?\s*(?:ah|a|இருக்கு))?", msg_lower) if loose_match: val = int(loose_match.group(1)) unit = "day" if val == 1 else "days" if "week" in loose_match.group(0): unit = "week" if val == 1 else "weeks" elif "month" in loose_match.group(0): unit = "month" if val == 1 else "months" elif "year" in loose_match.group(0): unit = "year" if val == 1 else "years" return f"{val} {unit}" return None def detect_previous_question_intent(last_ai_message: str) -> Optional[str]: """ Identifies what clinical category the previous AI message asked about. Returns one of: 'duration', 'severity', 'location', 'onset', 'frequency', 'triggers', 'relieving_factors', 'aggravating_factors', 'medications', 'allergies', 'medical_history', 'surgical_history', 'family_history', 'previous_episodes', or None. """ if not last_ai_message or not last_ai_message.strip(): return None msg = last_ai_message.lower() # Duration indicators if any(k in msg for k in [ "எத்தனை நாளா", "எப்போலிருந்து", "எத்தனை நாள்", "எப்போ இருந்து", "கால அளவு", "how long", "since when", "how many days", "how many weeks", "duration", "कितने दिनों से", "कब से", "कितने दिन", "ఎన్ని రోజుల", "ఎప్పటి నుండి", "ఎన్ని రోజులు", "ಎಷ್ಟು ದಿನದಿಂದ", "ಯಾವಾಗಿನಿಂದ", "ಎಷ್ಟು ದಿನ", ]): return "duration" # Severity indicators if any(k in msg for k in [ "எவ்வளவு தீவிரமா", "ரொம்ப வலியா", "தீவிரம்", "தாங்க முடியாத", "எப்படி இருக்கு வலி", "how severe", "how bad", "mild or severe", "severity", "scale of 1", "कितना तेज", "तीव्रता", "ఎంత తీవ్రంగా", "ಎಷ್ಟು ತೀವ್ರ", ]): return "severity" # Location indicators if any(k in msg for k in [ "எந்த இடத்துல", "எந்த பக்கத்துல", "எங்கே வலிக்குது", "எங்கு", "எந்த பகுதி", "where does it hurt", "which side", "location of the pain", "where is", "कहाँ दर्द", "किस जगह", "ఎక్కడ నొప్పి", "ಯಾವ ಜಾಗದಲ್ಲಿ", ]): return "location" # Onset indicators if any(k in msg for k in [ "திடீர்னு", "படிப்படியா", "எப்படி ஆரம்பிச்சது", "onset", "sudden", "gradual", "how did it start", "अचानक", "धीरे-धीरे", ]): return "onset" # Frequency indicators if any(k in msg for k in [ "எத்தனை வாட்டி", "எத்தனை முறை", "அடிக்கடி", "frequency", "how often", "how many times", "कितनी बार", "ఎన్ని సార్లు", "ಎಷ್ಟು ಬಾರಿ", ]): return "frequency" # Triggers / Aggravating / Relieving indicators if any(k in msg for k in [ "எப்போ அதிகமாகுது", "எப்போ குறையுது", "சாப்பிட்ட பிறகு", "trigger", "worse", "better", "relieve", "aggravate", ]): return "triggers" # Combined Medications + Allergies indicators (asked together in one question) has_med_kw = any(k in msg for k in [ "மாத்திரை", "மருந்து", "மெடிசின்", "medication", "medicine", "tablets", "taking anything", "taken any", "दवाई", "दवा", "మందులు", "ಔಷಧ", ]) has_alg_kw = any(k in msg for k in [ "அலர்ஜி", "ஒவ்வாமை", "allergy", "allergies", "allergic", "एलर्जी", "అలర్జీ", "ಅಲರ್ಜಿ", ]) if has_med_kw and has_alg_kw: return "medications_and_allergies" # Medications indicators if has_med_kw or any(k in msg for k in [ "டேப்லெட்", "எடுத்துக்குறீங்களா", "சாப்பிட்டீங்களா", "சாப்பிடுறீங்களா", "prescribed", "மாத்திரை சாப்பிடுறீங்களா", "మాత్ర", "ಮಾತ್ರೆ", ]): return "medications" # Allergies indicators if has_alg_kw or any(k in msg for k in [ "ஏதேனும் அலர்ஜி", "மாத்திரை அலர்ஜி", "reaction to any medicine", ]): return "allergies" # Medical History indicators if any(k in msg for k in [ "பிபி", "சுகர்", "ஆஸ்துமா", "முந்தைய", "உடம்பு பிரச்சனை", "வேறு ஏதேனும்", "medical history", "chronic condition", "past illness", "diabetes", "hypertension", "पुरानी बीमारी", "बीपी", "शुगर", "పాత ఆరోగ్య సమస్యలు", "హಳೆಯ ತೊಂದರೆ", ]): return "medical_history" # Surgical History indicators if any(k in msg for k in [ "ஆபரேஷன்", "அறுவை சிகிச்சை", "surgery", "operation", "surgical history", "ऑपरेशन", ]): return "surgical_history" # Family History indicators if any(k in msg for k in [ "குடும்பத்துல", "குடும்ப பின்னணி", "family history", "family members", "parivarik", ]): return "family_history" # Previous Episodes indicators if any(k in msg for k in [ "இதுக்கு முன்னாடி", "முன்னாடி வந்திருக்கா", "previous episode", "had this before", "same problem before", ]): return "previous_episodes" return None def is_duration_expression(text: str) -> bool: """Check if the text is primarily a duration statement rather than a symptom name.""" if not text: return False norm = normalize_duration_from_message(text) if norm is not None: clean = re.sub(r"[^\w\s\u0B80-\u0BFF]", "", text.lower()).strip() words = clean.split() if len(words) <= 8: return True filler_words = {"எனக்கு", "வந்து", "ஒரு", "இன்னும்", "இருக்கு", "ஆச்சு", "ஆகுது", "ஆ", "for", "me", "it", "is", "about", "around", "since", "from", "now"} non_filler = [w for w in words if w not in filler_words] if len(non_filler) <= 3: return True return False def is_severity_expression(text: str) -> bool: """Check if the text is primarily a severity descriptor.""" if not text: return False t_lower = text.lower().strip() patterns = [ "mild", "moderate", "severe", "very severe", "unbearable", "high", "ரொம்ப severe", "ரொம்ப அதிகம்", "மிதமான", "தாங்க முடியாத", "அதிகமா இருக்கு", "ரொம்ப வலி", "बहुत तेज", "हल्का", "कम", "చాలా తీవ్రంగా", "తక్కువగా", "ತುಂಬಾ ಜಾಸ್ತಿ", ] return any(p in t_lower for p in patterns) and len(text.split()) <= 6 def is_location_expression(text: str) -> bool: """Check if the text is primarily a body location descriptor.""" if not text: return False t_lower = text.lower().strip() patterns = [ "right side", "left side", "front", "back", "lower abdomen", "upper abdomen", "both sides", "வலது பக்கம்", "இடது பக்கம்", "வலது பக்கத்துல", "இடது பக்கத்துல", "முன் பக்கம்", "பின் பக்கம்", "அடிவயிறு", "दाएं तरफ", "बाएं तरफ", "పీట", "కుడి వైపు", "ఎడమ వైపు", "ಬಲಗಡೆ", "ಎಡಗಡೆ", ] return any(p in t_lower for p in patterns) and len(text.split()) <= 6 def is_negative_expression(text: str) -> bool: """Check if text is an explicit negation / nothing else.""" if not text: return False t_clean = re.sub(r"[^\w\s\u0B80-\u0BFF\u0900-\u097F\u0C00-\u0C7F\u0C80-\u0CFF]", "", text.lower()).strip() words = t_clean.split() if not words: return False negative_keywords = [ "no", "no more", "nothing", "nothing else", "none", "no illness", "no problem", "that's all", "thats all", "இல்லை", "இல்லைங்க", "இல்ல", "ஒன்றுமில்லை", "ஒன்னுமில்ல", "ஏதுமில்லை", "அவ்வளவுதான்", "வேற எதுவும் இல்ல", "எனக்கு வேற எதுவும் இல்ல", "வேறு எதுவும் இல்லை", "வேறு எதுவும் இல்ல", "வேற எதுவும் இல்லை", "नहीं", "कुछ नहीं", "कोई नहीं", "बस यही है", "और कुछ नहीं", "లేదు", "ఏమీ లేదు", "లేదండి", "ఇంతేనండి", "ఇంకేమీ లేదు", "ಇಲ್ಲ", "ಇಲ್ಲವೇ ಇಲ್ಲ", "ಏನೂ ಇಲ್ಲ", "ಅಷ್ಟೇ", "ഇല്ല", "ഒന്നുമില്ല", ] for k in negative_keywords: k_clean = re.sub(r"[^\w\s\u0B80-\u0BFF\u0900-\u097F\u0C00-\u0C7F\u0C80-\u0CFF]", "", k.lower()).strip() if k_clean == t_clean: return True if len(words) <= 6: pattern = rf"(?:^|\s){re.escape(k_clean)}(?:\s|$)" if re.search(pattern, t_clean): return True return False def is_casual_filler(text: str) -> bool: """Identify casual filler statements that should not be extracted as clinical entities.""" if not text or not str(text).strip(): return False t_lower = str(text).lower().strip() filler_patterns = [ r"^(?:சரி\s*சார்\s*சரி\s*சார்|சரி\s*சார்|சரிங்க|ஓகே\s*சார்|ஓகேங்க|ok\s*sir|ok\s*mam|okay|yes|yeah|sure)$", r"^(?:நான்\s*bus-?ல\s*வந்தேன்|auto-?ல\s*வந்தேன்|நடந்து\s*வந்தேன்)$", r"^(?:hello|hi|vanakkam|namaste)$", ] return any(re.search(p, t_lower) for p in filler_patterns) def extract_severity_from_message(msg: str) -> Optional[str]: """Extract normalized severity string.""" m_lower = msg.lower() if any(k in m_lower for k in ["very severe", "unbearable", "ரொம்ப அதிகம்", "தாங்க முடியாத", "बहुत तेज"]): return "severe" if any(k in m_lower for k in ["severe", "ரொம்ப severe", "அதிகமா", "तेज"]): return "severe" if any(k in m_lower for k in ["moderate", "medium", "மிதமான", "मध्यम"]): return "moderate" if any(k in m_lower for k in ["mild", "light", "லேசா", "हल्का"]): return "mild" return None def extract_location_from_message(msg: str) -> Optional[str]: """Extract normalized location string.""" m_lower = msg.lower() if any(k in m_lower for k in ["right side", "வலது பக்கம்", "வலது பக்கத்துல", "दाएं"]): return "right side" if any(k in m_lower for k in ["left side", "இடது பக்கம்", "இடது பக்கத்துல", "बाएं"]): return "left side" if any(k in m_lower for k in ["front", "முன் பக்கம்"]): return "front" if any(k in m_lower for k in ["back", "பின் பக்கம்"]): return "back" if any(k in m_lower for k in ["lower abdomen", "அடிவயிறு"]): return "lower abdomen" return None