72 lines
2.1 KiB
Python
72 lines
2.1 KiB
Python
"""
|
||
Multilingual normalization layer for clinical messages.
|
||
Preserves raw input while generating normalized, canonical representations.
|
||
"""
|
||
|
||
import re
|
||
import unicodedata
|
||
from typing import Dict, Any, Tuple
|
||
|
||
from app.services.contextual_normalizer import (
|
||
normalize_duration_from_message,
|
||
detect_previous_question_intent,
|
||
is_duration_expression,
|
||
is_severity_expression,
|
||
is_location_expression,
|
||
is_negative_expression,
|
||
is_casual_filler,
|
||
extract_severity_from_message,
|
||
extract_location_from_message,
|
||
)
|
||
|
||
|
||
def normalize_text(text: str) -> str:
|
||
"""
|
||
Cleans Unicode, strips extra spaces, standardizes quotes and punctuation.
|
||
"""
|
||
if not text:
|
||
return ""
|
||
# Unicode Normalization Form C
|
||
normalized = unicodedata.normalize("NFC", text)
|
||
# Replace non-breaking spaces and smart quotes
|
||
normalized = normalized.replace("\xa0", " ").replace("’", "'").replace("“", '"').replace("”", '"')
|
||
# Collapse multiple whitespaces
|
||
normalized = re.sub(r"\s+", " ", normalized).strip()
|
||
return normalized
|
||
|
||
|
||
def detect_language(text: str, default_lang: str = "en") -> str:
|
||
"""
|
||
Detects script/language (ta, hi, te, kn, en).
|
||
"""
|
||
if not text:
|
||
return default_lang
|
||
|
||
# Script character ranges
|
||
has_ta = bool(re.search(r"[\u0B80-\u0BFF]", text))
|
||
has_hi = bool(re.search(r"[\u0900-\u097F]", text))
|
||
has_te = bool(re.search(r"[\u0C00-\u0C7F]", text))
|
||
has_kn = bool(re.search(r"[\u0C80-\u0CFF]", text))
|
||
|
||
if has_ta:
|
||
return "ta-IN"
|
||
if has_hi:
|
||
return "hi-IN"
|
||
if has_te:
|
||
return "te-IN"
|
||
if has_kn:
|
||
return "kn-IN"
|
||
|
||
# Transliterated language keywords
|
||
msg_lower = text.lower()
|
||
if any(w in msg_lower for w in ["irukku", "thalaivali", "romba", "mudiyala", "saapudren"]):
|
||
return "ta-IN"
|
||
if any(w in msg_lower for w in ["dard", "ho raha", "hoga", "le raha"]):
|
||
return "hi-IN"
|
||
if any(w in msg_lower for w in ["noppi", "undi", "thala noppi"]):
|
||
return "te-IN"
|
||
if any(w in msg_lower for w in ["novu", "ide", "thale novu"]):
|
||
return "kn-IN"
|
||
|
||
return default_lang or "en-IN"
|