From 59c1372fe37742feed1ef401de65864c58bbb6a7 Mon Sep 17 00:00:00 2001 From: Gokulvarthan Ramesh Date: Wed, 26 Aug 2026 11:07:19 +0530 Subject: [PATCH] feat: AI patient onboarding backend and documentation --- .gitignore | 45 ++ BACKEND_API_DOCUMENTATION.md | 532 ++++++++++++++++++ ai-patient-onboarding/.gitignore | 17 + ai-patient-onboarding/API_DOCUMENTATION.md | 532 ++++++++++++++++++ ai-patient-onboarding/app/__init__.py | 1 + ai-patient-onboarding/app/ai/__init__.py | 1 + ai-patient-onboarding/app/ai/claude_client.py | 69 +++ ai-patient-onboarding/app/ai/context.py | 33 ++ ai-patient-onboarding/app/ai/gemini_client.py | 73 +++ ai-patient-onboarding/app/ai/openai_client.py | 39 ++ ai-patient-onboarding/app/ai/orchestrator.py | 149 +++++ ai-patient-onboarding/app/ai/prompts.py | 89 +++ ai-patient-onboarding/app/ai/sarvam_client.py | 84 +++ ai-patient-onboarding/app/ai/validator.py | 12 + ai-patient-onboarding/app/api/__init__.py | 1 + ai-patient-onboarding/app/api/documents.py | 84 +++ ai-patient-onboarding/app/api/onboarding.py | 325 +++++++++++ ai-patient-onboarding/app/api/voice.py | 4 + .../app/api/voice_onboarding.py | 343 +++++++++++ ai-patient-onboarding/app/config.py | 26 + .../app/database/__init__.py | 1 + .../app/database/database.py | 47 ++ ai-patient-onboarding/app/database/models.py | 162 ++++++ ai-patient-onboarding/app/main.py | 52 ++ ai-patient-onboarding/app/schemas/__init__.py | 1 + .../app/schemas/documents.py | 20 + .../app/schemas/onboarding.py | 100 ++++ ai-patient-onboarding/app/schemas/voice.py | 31 + .../app/services/__init__.py | 1 + .../app/services/ai_onboarding_service.py | 21 + .../app/services/completion_service.py | 52 ++ .../app/services/conversation_engine.py | 75 +++ .../app/services/medical_speech_normalizer.py | 65 +++ .../app/services/onboarding_service.py | 103 ++++ .../services/patient_submission_service.py | 13 + .../app/services/result_extraction_service.py | 139 +++++ .../app/services/validation_service.py | 32 ++ .../app/services/voice_service.py | 270 +++++++++ .../app/specialties/__init__.py | 1 + .../app/specialties/cardiology.py | 40 ++ .../app/specialties/diabetology.py | 36 ++ .../app/specialties/general.py | 33 ++ .../app/specialties/mental_health.py | 32 ++ .../app/specialties/registry.py | 18 + ai-patient-onboarding/app/voice/__init__.py | 1 + .../app/voice/audio_manager.py | 25 + .../app/voice/speech_to_text.py | 65 +++ .../app/voice/text_to_speech.py | 67 +++ ai-patient-onboarding/requirements.txt | 50 ++ 49 files changed, 4012 insertions(+) create mode 100644 .gitignore create mode 100644 BACKEND_API_DOCUMENTATION.md create mode 100644 ai-patient-onboarding/.gitignore create mode 100644 ai-patient-onboarding/API_DOCUMENTATION.md create mode 100644 ai-patient-onboarding/app/__init__.py create mode 100644 ai-patient-onboarding/app/ai/__init__.py create mode 100644 ai-patient-onboarding/app/ai/claude_client.py create mode 100644 ai-patient-onboarding/app/ai/context.py create mode 100644 ai-patient-onboarding/app/ai/gemini_client.py create mode 100644 ai-patient-onboarding/app/ai/openai_client.py create mode 100644 ai-patient-onboarding/app/ai/orchestrator.py create mode 100644 ai-patient-onboarding/app/ai/prompts.py create mode 100644 ai-patient-onboarding/app/ai/sarvam_client.py create mode 100644 ai-patient-onboarding/app/ai/validator.py create mode 100644 ai-patient-onboarding/app/api/__init__.py create mode 100644 ai-patient-onboarding/app/api/documents.py create mode 100644 ai-patient-onboarding/app/api/onboarding.py create mode 100644 ai-patient-onboarding/app/api/voice.py create mode 100644 ai-patient-onboarding/app/api/voice_onboarding.py create mode 100644 ai-patient-onboarding/app/config.py create mode 100644 ai-patient-onboarding/app/database/__init__.py create mode 100644 ai-patient-onboarding/app/database/database.py create mode 100644 ai-patient-onboarding/app/database/models.py create mode 100644 ai-patient-onboarding/app/main.py create mode 100644 ai-patient-onboarding/app/schemas/__init__.py create mode 100644 ai-patient-onboarding/app/schemas/documents.py create mode 100644 ai-patient-onboarding/app/schemas/onboarding.py create mode 100644 ai-patient-onboarding/app/schemas/voice.py create mode 100644 ai-patient-onboarding/app/services/__init__.py create mode 100644 ai-patient-onboarding/app/services/ai_onboarding_service.py create mode 100644 ai-patient-onboarding/app/services/completion_service.py create mode 100644 ai-patient-onboarding/app/services/conversation_engine.py create mode 100644 ai-patient-onboarding/app/services/medical_speech_normalizer.py create mode 100644 ai-patient-onboarding/app/services/onboarding_service.py create mode 100644 ai-patient-onboarding/app/services/patient_submission_service.py create mode 100644 ai-patient-onboarding/app/services/result_extraction_service.py create mode 100644 ai-patient-onboarding/app/services/validation_service.py create mode 100644 ai-patient-onboarding/app/services/voice_service.py create mode 100644 ai-patient-onboarding/app/specialties/__init__.py create mode 100644 ai-patient-onboarding/app/specialties/cardiology.py create mode 100644 ai-patient-onboarding/app/specialties/diabetology.py create mode 100644 ai-patient-onboarding/app/specialties/general.py create mode 100644 ai-patient-onboarding/app/specialties/mental_health.py create mode 100644 ai-patient-onboarding/app/specialties/registry.py create mode 100644 ai-patient-onboarding/app/voice/__init__.py create mode 100644 ai-patient-onboarding/app/voice/audio_manager.py create mode 100644 ai-patient-onboarding/app/voice/speech_to_text.py create mode 100644 ai-patient-onboarding/app/voice/text_to_speech.py create mode 100644 ai-patient-onboarding/requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f78d4f --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# Ignore Frontend Directory +frontend/ + +# Dependencies +node_modules/ +venv/ +env/ +.venv/ + +# Environment Variables & Secrets +.env +.env.local +.env.*.local +*.env + +# Python Cache & Databases +__pycache__/ +*.py[cod] +*$py.class +*.sqlite +*.sqlite3 +*.db +data/*.db +data/audio/ +data/audio_temp/ +data/reports/ +data/transcripts/ +*.mp3 +*.wav +*.ogg +*.m4a +.pytest_cache/ + +# Build & Output +dist/ +dist-ssr/ +build/ +*.local +.cache/ + +# IDE & OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db diff --git a/BACKEND_API_DOCUMENTATION.md b/BACKEND_API_DOCUMENTATION.md new file mode 100644 index 0000000..fd918fc --- /dev/null +++ b/BACKEND_API_DOCUMENTATION.md @@ -0,0 +1,532 @@ +# AI Patient Onboarding — Backend API Documentation + +This document provides a comprehensive guide for integrating with the **AI Patient Onboarding FastAPI Backend**. It covers all REST API endpoints, request/response models, voice integration flows, medical document processing, and EHR integration. + +--- + +## Table of Contents +1. [Architecture & Server Overview](#1-architecture--server-overview) +2. [Environment Configuration (`.env`)](#2-environment-configuration-env) +3. [Running the Backend Locally](#3-running-the-backend-locally) +4. [Voice Onboarding API (`/v1/voice-onboarding`)](#4-voice-onboarding-api-v1voice-onboarding) + - [4.1 Start Voice Session](#41-start-voice-session) + - [4.2 Send Voice / Speech Turn](#42-send-voice--speech-turn) + - [4.3 Fetch Synthesized Voice Audio](#43-fetch-synthesized-voice-audio) + - [4.4 Complete & Confirm Session](#44-complete--confirm-session) +5. [Text Onboarding API (`/v1/onboarding`)](#5-text-onboarding-api-v1onboarding) + - [5.1 Start Text Session](#51-start-text-session) + - [5.2 Send Text Message](#52-send-text-message) + - [5.3 Complete Text Onboarding](#53-complete-text-onboarding) +6. [Medical Document & Vision API (`/v1/documents`)](#6-medical-document--vision-api-v1documents) +7. [System Health Check (`/health`)](#7-system-health-check-health) +8. [Connecting External Hospital EMR / EHR System](#8-connecting-external-hospital-emr--ehr-system) +9. [Multi-Language & Clinical Specialties](#9-multi-language--clinical-specialties) +10. [Client Integration Code Samples](#10-client-integration-code-samples) + +--- + +## 1. Architecture & Server Overview + +- **Framework:** FastAPI (Python 3.10+) with SQLAlchemy ORM +- **Default Base URL:** `http://127.0.0.1:8000` +- **Interactive Swagger Documentation:** `http://127.0.0.1:8000/docs` +- **OpenAPI JSON Spec:** `http://127.0.0.1:8000/openapi.json` +- **CORS:** Pre-configured to allow all origins (`*`) for easy React web and mobile client integration. + +``` ++------------------+ +-------------------------------+ +-----------------------+ +| Client Device | | FastAPI Backend Engine | | Hospital EHR / Database| ++------------------+ +-------------------------------+ +-----------------------+ + | | | + | --- POST /start ----------------> | (Initialize Session + State) | + | <-- Greeting Text & Audio URL --- | (Synthesized TTS Audio) | + | | | + | --- POST /turn (Audio / Text) --> | (Sarvam STT -> GPT-4o -> State) | + | <-- Next Question & Audio URL --- | | + | | | + | --- POST /upload (Prescription) > | (Gemini Vision OCR -> Extract Meds) | + | <-- Extracted Findings ---------- | | + | | | + | --- POST /complete (confirmed) -> | (Extract EHR JSON) ---------------->| POST /patients + | <-- Final Success Status -------- | <--- 200 OK Record Saved -----------| +``` + +--- + +## 2. Environment Configuration (`.env`) + +Create or update the `.env` file in the `ai-patient-onboarding/` directory: + +```ini +APP_NAME="AI Patient Onboarding" +APP_ENV=development +APP_VERSION=1.0.0 + +# Database URL +DATABASE_URL="sqlite:///./data/onboarding.db" + +# AI Provider API Keys +OPENAI_API_KEY="sk-..." +OPENAI_MODEL="gpt-4o-mini" + +ANTHROPIC_API_KEY="sk-ant-..." +ANTHROPIC_MODEL="claude-3-5-sonnet-20241022" + +GEMINI_API_KEY="AIzaSy..." +GEMINI_MODEL="gemini-1.5-flash" + +SARVAM_API_KEY="your_sarvam_key" + +# External Destination Hospital EHR / Patient API (Optional) +PATIENT_API_BASE_URL="https://your-hospital-emr.com/api/v1" +PATIENT_API_TOKEN="your_secure_bearer_token" +``` + +--- + +## 3. Running the Backend Locally + +```bash +# Navigate to the backend directory +cd ai-patient-onboarding + +# Activate Python Virtual Environment +# Windows PowerShell: +.\venv\Scripts\Activate.ps1 +# Linux / macOS: +# source venv/bin/activate + +# Install dependencies if not already installed +pip install -r requirements.txt + +# Start FastAPI development server +uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload +``` + +--- + +## 4. Voice Onboarding API (`/v1/voice-onboarding`) + +### 4.1 Start Voice Session + +Creates a new voice onboarding session, initializes dynamic state, and returns localized assistant greeting with audio. + +- **URL:** `POST /v1/voice-onboarding/start` +- **Headers:** `Content-Type: application/json` +- **Request Body:** +```json +{ + "patient": { + "id": "PAT-9842", + "name": "Arun Kumar", + "age": 42, + "gender": "male", + "abha_id": "91-1234-5678-9012", + "phone": "+91 98765 43210", + "insurance_scheme": "PM-JAY", + "dietary_habits": "Vegetarian" + }, + "category": { + "type": "speciality", + "code": "cardio", + "name": "Cardiology" + }, + "language": "en-IN" +} +``` + +- **Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "patient_id": "PAT-9842", + "category": { + "type": "speciality", + "code": "cardio", + "name": "Cardiology" + }, + "language": "en-IN", + "status": "in_progress", + "stage": "greeting", + "greeting_text": "Hello Arun Kumar, welcome to CureSelect Cardiology department. What symptoms or discomfort are you experiencing today?", + "audio_url": "/v1/voice-onboarding/audio/voice_greeting_89bf1a.wav" +} +``` + +--- + +### 4.2 Send Voice / Speech Turn + +Submits the patient's recorded audio or transcribed speech. The backend runs multi-model Indic speech recognition (Sarvam/Whisper), extracts medical entities, updates conversational state, and generates the next question with audio TTS. + +- **URL:** `POST /v1/voice-onboarding/{session_id}/turn` +- **Headers:** `Content-Type: multipart/form-data` +- **Form Data Parameters:** + - `audio` *(Optional File)*: WebM, WAV, or MP3 recorded audio file from the user's microphone. + - `text_fallback` *(Optional String)*: Real-time browser speech recognition transcript fallback. + +- **Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "patient_transcript": "I have had severe chest pain and breathlessness since yesterday evening.", + "ai_text": "I understand. Does the chest pain radiate to your left arm or jaw, and are you currently taking any blood pressure or heart medications?", + "stage": "symptoms_inquiry", + "completed": false, + "audio_url": "/v1/voice-onboarding/audio/speech_reply_30ef2.wav" +} +``` + +--- + +### 4.3 Fetch Synthesized Voice Audio + +Streams audio file generated for assistant responses. + +- **URL:** `GET /v1/voice-onboarding/audio/{audio_id}` +- **Response:** `audio/wav` or `audio/mpeg` binary stream. + +--- + +### 4.4 Complete & Confirm Session + +Extracts complete structured clinical results. Supports a two-step confirmation workflow (review preview $\rightarrow$ permanent submission). + +- **URL:** `POST /v1/voice-onboarding/{session_id}/complete` +- **Headers:** `Content-Type: application/json` + +#### Step A: Request Review Preview (`confirmed: false`) +```json +{ + "confirmed": false +} +``` +**Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "status": "awaiting_confirmation", + "result": { + "patient_id": "PAT-9842", + "abha_id": "91-1234-5678-9012", + "phone": "+91 98765 43210", + "insurance_scheme": "PM-JAY", + "category": { + "type": "speciality", + "code": "cardio", + "name": "Cardiology" + }, + "chief_complaint": "Acute onset retrosternal chest pain and dyspnea", + "symptoms": [ + { + "name": "Chest tightness", + "duration": "1 day", + "severity": "Moderate to Severe", + "description": "Retrosternal pressure aggravated by exertion" + } + ], + "medical_history": ["Hypertension (5 years)"], + "medications": ["Amlodipine 5mg OD"], + "allergies": ["Penicillin"], + "dietary_habits": "Vegetarian", + "additional_information": "Non-smoker", + "language": "en-IN", + "onboarding_completed": true + }, + "message": "Please review the collected information and confirm by voice or tap." +} +``` + +#### Step B: Confirm & Submit to EHR (`confirmed: true`) +```json +{ + "confirmed": true +} +``` +**Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "status": "submitted_and_cleaned", + "result": { ... }, + "message": "Voice onboarding successfully saved to patient record." +} +``` + +--- + +## 5. Text Onboarding API (`/v1/onboarding`) + +For text chat interfaces or non-voice kiosks. + +### 5.1 Start Text Session +- **URL:** `POST /v1/onboarding/start` +- **Body:** Same as Voice Start (`patient`, `category`, `language`). +- **Response:** Returns `session_id`, `greeting_message`, and initial state. + +### 5.2 Send Text Message +- **URL:** `POST /v1/onboarding/{session_id}/message` +- **Body:** +```json +{ + "message": "I've had knee pain for 3 days after a run." +} +``` +- **Response:** +```json +{ + "success": true, + "session_id": "ONB-84A12B1C", + "stage": "symptoms_inquiry", + "message": "Is the knee swollen or locking when you walk?", + "status": "in_progress" +} +``` + +### 5.3 Complete Text Onboarding +- **URL:** `POST /v1/onboarding/{session_id}/complete` +- **Body:** `{"confirmed": true}` + +--- + +## 6. Medical Document & Vision API (`/v1/documents`) + +Uploads patient medical documents (prescriptions, discharge summaries, lab reports) to extract medications, allergies, and diagnoses using Gemini 1.5 Flash Vision, automatically injecting findings into the active session state. + +- **URL:** `POST /v1/documents/{session_id}/upload` +- **Headers:** `Content-Type: multipart/form-data` +- **Form Field:** `file` *(Binary image or PDF file)* + +- **Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "document_type": "prescription", + "summary": "Prescription from Cardiologist dated 15-Aug-2024", + "findings": ["Ejection Fraction 55%", "Normal Sinus Rhythm"], + "medications_found": ["Atorvastatin 20mg", "Metoprolol 25mg"], + "allergies_found": ["Sulfa drugs"], + "message": "Medical document analyzed and integrated into onboarding session." +} +``` + +--- + +## 7. System Health Check (`/health`) + +- **URL:** `GET /health` +- **Response (`200 OK`):** +```json +{ + "success": true, + "service": "AI Patient Onboarding", + "status": "healthy", + "models": { + "speech": "Sarvam AI (Indian Languages) / OpenAI Whisper", + "conversation": "OpenAI GPT-4o-mini", + "validation": "Claude 3.5 Sonnet", + "vision_documents": "Gemini 1.5 Flash" + } +} +``` + +--- + +## 8. Connecting External Hospital EMR / EHR System + +When onboarding is finalized, the backend invokes `PatientSubmissionService` located at: +`app/services/patient_submission_service.py` + +### Customizing the Destination Forwarding: +```python +import httpx +from app.config import PATIENT_API_BASE_URL, PATIENT_API_TOKEN + +class PatientSubmissionService: + async def submit(self, result: dict) -> dict: + if not PATIENT_API_BASE_URL: + # Local fallback mock + return { + "success": True, + "message": "Patient data saved locally.", + "patient_id": result.get("patient_id"), + } + + # Forward JSON payload to hospital central server + async with httpx.AsyncClient(timeout=15.0) as client: + headers = { + "Authorization": f"Bearer {PATIENT_API_TOKEN}", + "Content-Type": "application/json", + } + response = await client.post( + f"{PATIENT_API_BASE_URL}/patients/{result.get('patient_id')}/onboarding", + json=result, + headers=headers, + ) + + if response.status_code in [200, 201]: + return {"success": True, "data": response.json()} + + return { + "success": False, + "error": f"Hospital API error: {response.text}", + } +``` + +--- + +## 9. Multi-Language & Clinical Specialties + +### Supported Languages +| Code | Language | Speech Engine | +|---|---|---| +| `en-IN` / `en` | English (Indian accent) | Sarvam / OpenAI Whisper | +| `hi-IN` / `hi` | Hindi (हिन्दी) | Sarvam Bulbul / Whisper | +| `ta-IN` / `ta` | Tamil (தமிழ்) | Sarvam Bulbul / Whisper | +| `te-IN` / `te` | Telugu (తెలుగు) | Sarvam Bulbul / Whisper | +| `kn-IN` / `kn` | Kannada (ಕನ್ನಡ) | Sarvam Bulbul / Whisper | +| `bn-IN` / `bn` | Bengali (বাংলা) | Sarvam Bulbul / Whisper | +| `ml-IN` / `ml` | Malayalam (മലയാളം) | Sarvam Bulbul / Whisper | +| `mr-IN` / `mr` | Marathi (मराठी) | Sarvam Bulbul / Whisper | +| `gu-IN` / `gu` | Gujarati (ગુજરાતી) | Sarvam Bulbul / Whisper | + +### Clinical Specialties Supported +- `general_medicine` - General Medicine / Primary Care +- `cardio` - Cardiology +- `ortho` - Orthopedics +- `peds` - Pediatrics +- `derma` - Dermatology +- `neuro` - Neurology +- `gastro` - Gastroenterology +- `ent` - ENT / Otorhinolaryngology +- `gyn` - Obstetrics & Gynecology + +--- + +## 10. Client Integration Code Samples + +### 10.1 Python Client (`httpx`) +```python +import httpx +import asyncio + +API_BASE = "http://127.0.0.1:8000" + +async def main(): + async with httpx.AsyncClient() as client: + # 1. Start Session + start_res = await client.post(f"{API_BASE}/v1/voice-onboarding/start", json={ + "patient": {"id": "P-1001", "name": "Deepak Sharma", "age": 45, "gender": "male"}, + "category": {"type": "speciality", "code": "cardio", "name": "Cardiology"}, + "language": "hi-IN" + }) + session = start_res.json() + session_id = session["session_id"] + print(f"Session started: {session_id}") + print(f"Greeting: {session['greeting_text']}") + + # 2. Send Turn + turn_res = await client.post( + f"{API_BASE}/v1/voice-onboarding/{session_id}/turn", + data={"text_fallback": "मुझे सीने में भारीपन महसूस हो रहा है"} + ) + turn_data = turn_res.json() + print(f"AI Response: {turn_data['ai_text']}") + + # 3. Finalize & Submit + complete_res = await client.post( + f"{API_BASE}/v1/voice-onboarding/{session_id}/complete", + json={"confirmed": True} + ) + print(f"EHR Saved: {complete_res.json()}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### 10.2 JavaScript / TypeScript (`fetch`) +```typescript +const API_BASE = "http://127.0.0.1:8000"; + +// Start voice onboarding +export async function startVoiceOnboarding(patient: any, category: any, language = "en-IN") { + const res = await fetch(`${API_BASE}/v1/voice-onboarding/start`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ patient, category, language }) + }); + return res.json(); +} + +// Send audio turn +export async function sendVoiceTurn(sessionId: string, audioBlob?: Blob, textFallback?: string) { + const formData = new FormData(); + if (audioBlob) formData.append("audio", audioBlob, "speech.webm"); + if (textFallback) formData.append("text_fallback", textFallback); + + const res = await fetch(`${API_BASE}/v1/voice-onboarding/${sessionId}/turn`, { + method: "POST", + body: formData + }); + return res.json(); +} + +// Upload medical report +export async function uploadPrescription(sessionId: string, file: File) { + const formData = new FormData(); + formData.append("file", file); + + const res = await fetch(`${API_BASE}/v1/documents/${sessionId}/upload`, { + method: "POST", + body: formData + }); + return res.json(); +} + +// Complete onboarding +export async function completeOnboarding(sessionId: string, confirmed: boolean) { + const res = await fetch(`${API_BASE}/v1/voice-onboarding/${sessionId}/complete`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ confirmed }) + }); + return res.json(); +} +``` + +### 10.3 cURL Commands + +#### Start Session: +```bash +curl -X POST "http://127.0.0.1:8000/v1/voice-onboarding/start" \ + -H "Content-Type: application/json" \ + -d '{ + "patient": {"id": "P-101", "name": "Arun Kumar", "age": 40, "gender": "male"}, + "category": {"type": "general", "code": "general_medicine", "name": "General Medicine"}, + "language": "en-IN" + }' +``` + +#### Send Voice Turn (with text fallback): +```bash +curl -X POST "http://127.0.0.1:8000/v1/voice-onboarding/VOICE-XXXXX/turn" \ + -F "text_fallback=I have had high fever and sore throat since yesterday" +``` + +#### Upload Prescription: +```bash +curl -X POST "http://127.0.0.1:8000/v1/documents/VOICE-XXXXX/upload" \ + -F "file=@/path/to/prescription.pdf" +``` + +#### Confirm & Submit: +```bash +curl -X POST "http://127.0.0.1:8000/v1/voice-onboarding/VOICE-XXXXX/complete" \ + -H "Content-Type: application/json" \ + -d '{"confirmed": true}' +``` diff --git a/ai-patient-onboarding/.gitignore b/ai-patient-onboarding/.gitignore new file mode 100644 index 0000000..18f2784 --- /dev/null +++ b/ai-patient-onboarding/.gitignore @@ -0,0 +1,17 @@ +venv/ +.env +__pycache__/ +*.pyc +.pytest_cache/ +data/audio/ +data/audio_temp/ +data/reports/ +data/transcripts/ +*.db +*.sqlite +*.sqlite3 +*.mp3 +*.wav +*.ogg +*.m4a + diff --git a/ai-patient-onboarding/API_DOCUMENTATION.md b/ai-patient-onboarding/API_DOCUMENTATION.md new file mode 100644 index 0000000..fd918fc --- /dev/null +++ b/ai-patient-onboarding/API_DOCUMENTATION.md @@ -0,0 +1,532 @@ +# AI Patient Onboarding — Backend API Documentation + +This document provides a comprehensive guide for integrating with the **AI Patient Onboarding FastAPI Backend**. It covers all REST API endpoints, request/response models, voice integration flows, medical document processing, and EHR integration. + +--- + +## Table of Contents +1. [Architecture & Server Overview](#1-architecture--server-overview) +2. [Environment Configuration (`.env`)](#2-environment-configuration-env) +3. [Running the Backend Locally](#3-running-the-backend-locally) +4. [Voice Onboarding API (`/v1/voice-onboarding`)](#4-voice-onboarding-api-v1voice-onboarding) + - [4.1 Start Voice Session](#41-start-voice-session) + - [4.2 Send Voice / Speech Turn](#42-send-voice--speech-turn) + - [4.3 Fetch Synthesized Voice Audio](#43-fetch-synthesized-voice-audio) + - [4.4 Complete & Confirm Session](#44-complete--confirm-session) +5. [Text Onboarding API (`/v1/onboarding`)](#5-text-onboarding-api-v1onboarding) + - [5.1 Start Text Session](#51-start-text-session) + - [5.2 Send Text Message](#52-send-text-message) + - [5.3 Complete Text Onboarding](#53-complete-text-onboarding) +6. [Medical Document & Vision API (`/v1/documents`)](#6-medical-document--vision-api-v1documents) +7. [System Health Check (`/health`)](#7-system-health-check-health) +8. [Connecting External Hospital EMR / EHR System](#8-connecting-external-hospital-emr--ehr-system) +9. [Multi-Language & Clinical Specialties](#9-multi-language--clinical-specialties) +10. [Client Integration Code Samples](#10-client-integration-code-samples) + +--- + +## 1. Architecture & Server Overview + +- **Framework:** FastAPI (Python 3.10+) with SQLAlchemy ORM +- **Default Base URL:** `http://127.0.0.1:8000` +- **Interactive Swagger Documentation:** `http://127.0.0.1:8000/docs` +- **OpenAPI JSON Spec:** `http://127.0.0.1:8000/openapi.json` +- **CORS:** Pre-configured to allow all origins (`*`) for easy React web and mobile client integration. + +``` ++------------------+ +-------------------------------+ +-----------------------+ +| Client Device | | FastAPI Backend Engine | | Hospital EHR / Database| ++------------------+ +-------------------------------+ +-----------------------+ + | | | + | --- POST /start ----------------> | (Initialize Session + State) | + | <-- Greeting Text & Audio URL --- | (Synthesized TTS Audio) | + | | | + | --- POST /turn (Audio / Text) --> | (Sarvam STT -> GPT-4o -> State) | + | <-- Next Question & Audio URL --- | | + | | | + | --- POST /upload (Prescription) > | (Gemini Vision OCR -> Extract Meds) | + | <-- Extracted Findings ---------- | | + | | | + | --- POST /complete (confirmed) -> | (Extract EHR JSON) ---------------->| POST /patients + | <-- Final Success Status -------- | <--- 200 OK Record Saved -----------| +``` + +--- + +## 2. Environment Configuration (`.env`) + +Create or update the `.env` file in the `ai-patient-onboarding/` directory: + +```ini +APP_NAME="AI Patient Onboarding" +APP_ENV=development +APP_VERSION=1.0.0 + +# Database URL +DATABASE_URL="sqlite:///./data/onboarding.db" + +# AI Provider API Keys +OPENAI_API_KEY="sk-..." +OPENAI_MODEL="gpt-4o-mini" + +ANTHROPIC_API_KEY="sk-ant-..." +ANTHROPIC_MODEL="claude-3-5-sonnet-20241022" + +GEMINI_API_KEY="AIzaSy..." +GEMINI_MODEL="gemini-1.5-flash" + +SARVAM_API_KEY="your_sarvam_key" + +# External Destination Hospital EHR / Patient API (Optional) +PATIENT_API_BASE_URL="https://your-hospital-emr.com/api/v1" +PATIENT_API_TOKEN="your_secure_bearer_token" +``` + +--- + +## 3. Running the Backend Locally + +```bash +# Navigate to the backend directory +cd ai-patient-onboarding + +# Activate Python Virtual Environment +# Windows PowerShell: +.\venv\Scripts\Activate.ps1 +# Linux / macOS: +# source venv/bin/activate + +# Install dependencies if not already installed +pip install -r requirements.txt + +# Start FastAPI development server +uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload +``` + +--- + +## 4. Voice Onboarding API (`/v1/voice-onboarding`) + +### 4.1 Start Voice Session + +Creates a new voice onboarding session, initializes dynamic state, and returns localized assistant greeting with audio. + +- **URL:** `POST /v1/voice-onboarding/start` +- **Headers:** `Content-Type: application/json` +- **Request Body:** +```json +{ + "patient": { + "id": "PAT-9842", + "name": "Arun Kumar", + "age": 42, + "gender": "male", + "abha_id": "91-1234-5678-9012", + "phone": "+91 98765 43210", + "insurance_scheme": "PM-JAY", + "dietary_habits": "Vegetarian" + }, + "category": { + "type": "speciality", + "code": "cardio", + "name": "Cardiology" + }, + "language": "en-IN" +} +``` + +- **Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "patient_id": "PAT-9842", + "category": { + "type": "speciality", + "code": "cardio", + "name": "Cardiology" + }, + "language": "en-IN", + "status": "in_progress", + "stage": "greeting", + "greeting_text": "Hello Arun Kumar, welcome to CureSelect Cardiology department. What symptoms or discomfort are you experiencing today?", + "audio_url": "/v1/voice-onboarding/audio/voice_greeting_89bf1a.wav" +} +``` + +--- + +### 4.2 Send Voice / Speech Turn + +Submits the patient's recorded audio or transcribed speech. The backend runs multi-model Indic speech recognition (Sarvam/Whisper), extracts medical entities, updates conversational state, and generates the next question with audio TTS. + +- **URL:** `POST /v1/voice-onboarding/{session_id}/turn` +- **Headers:** `Content-Type: multipart/form-data` +- **Form Data Parameters:** + - `audio` *(Optional File)*: WebM, WAV, or MP3 recorded audio file from the user's microphone. + - `text_fallback` *(Optional String)*: Real-time browser speech recognition transcript fallback. + +- **Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "patient_transcript": "I have had severe chest pain and breathlessness since yesterday evening.", + "ai_text": "I understand. Does the chest pain radiate to your left arm or jaw, and are you currently taking any blood pressure or heart medications?", + "stage": "symptoms_inquiry", + "completed": false, + "audio_url": "/v1/voice-onboarding/audio/speech_reply_30ef2.wav" +} +``` + +--- + +### 4.3 Fetch Synthesized Voice Audio + +Streams audio file generated for assistant responses. + +- **URL:** `GET /v1/voice-onboarding/audio/{audio_id}` +- **Response:** `audio/wav` or `audio/mpeg` binary stream. + +--- + +### 4.4 Complete & Confirm Session + +Extracts complete structured clinical results. Supports a two-step confirmation workflow (review preview $\rightarrow$ permanent submission). + +- **URL:** `POST /v1/voice-onboarding/{session_id}/complete` +- **Headers:** `Content-Type: application/json` + +#### Step A: Request Review Preview (`confirmed: false`) +```json +{ + "confirmed": false +} +``` +**Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "status": "awaiting_confirmation", + "result": { + "patient_id": "PAT-9842", + "abha_id": "91-1234-5678-9012", + "phone": "+91 98765 43210", + "insurance_scheme": "PM-JAY", + "category": { + "type": "speciality", + "code": "cardio", + "name": "Cardiology" + }, + "chief_complaint": "Acute onset retrosternal chest pain and dyspnea", + "symptoms": [ + { + "name": "Chest tightness", + "duration": "1 day", + "severity": "Moderate to Severe", + "description": "Retrosternal pressure aggravated by exertion" + } + ], + "medical_history": ["Hypertension (5 years)"], + "medications": ["Amlodipine 5mg OD"], + "allergies": ["Penicillin"], + "dietary_habits": "Vegetarian", + "additional_information": "Non-smoker", + "language": "en-IN", + "onboarding_completed": true + }, + "message": "Please review the collected information and confirm by voice or tap." +} +``` + +#### Step B: Confirm & Submit to EHR (`confirmed: true`) +```json +{ + "confirmed": true +} +``` +**Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "status": "submitted_and_cleaned", + "result": { ... }, + "message": "Voice onboarding successfully saved to patient record." +} +``` + +--- + +## 5. Text Onboarding API (`/v1/onboarding`) + +For text chat interfaces or non-voice kiosks. + +### 5.1 Start Text Session +- **URL:** `POST /v1/onboarding/start` +- **Body:** Same as Voice Start (`patient`, `category`, `language`). +- **Response:** Returns `session_id`, `greeting_message`, and initial state. + +### 5.2 Send Text Message +- **URL:** `POST /v1/onboarding/{session_id}/message` +- **Body:** +```json +{ + "message": "I've had knee pain for 3 days after a run." +} +``` +- **Response:** +```json +{ + "success": true, + "session_id": "ONB-84A12B1C", + "stage": "symptoms_inquiry", + "message": "Is the knee swollen or locking when you walk?", + "status": "in_progress" +} +``` + +### 5.3 Complete Text Onboarding +- **URL:** `POST /v1/onboarding/{session_id}/complete` +- **Body:** `{"confirmed": true}` + +--- + +## 6. Medical Document & Vision API (`/v1/documents`) + +Uploads patient medical documents (prescriptions, discharge summaries, lab reports) to extract medications, allergies, and diagnoses using Gemini 1.5 Flash Vision, automatically injecting findings into the active session state. + +- **URL:** `POST /v1/documents/{session_id}/upload` +- **Headers:** `Content-Type: multipart/form-data` +- **Form Field:** `file` *(Binary image or PDF file)* + +- **Response (`200 OK`):** +```json +{ + "success": true, + "session_id": "VOICE-78A1BF2C", + "document_type": "prescription", + "summary": "Prescription from Cardiologist dated 15-Aug-2024", + "findings": ["Ejection Fraction 55%", "Normal Sinus Rhythm"], + "medications_found": ["Atorvastatin 20mg", "Metoprolol 25mg"], + "allergies_found": ["Sulfa drugs"], + "message": "Medical document analyzed and integrated into onboarding session." +} +``` + +--- + +## 7. System Health Check (`/health`) + +- **URL:** `GET /health` +- **Response (`200 OK`):** +```json +{ + "success": true, + "service": "AI Patient Onboarding", + "status": "healthy", + "models": { + "speech": "Sarvam AI (Indian Languages) / OpenAI Whisper", + "conversation": "OpenAI GPT-4o-mini", + "validation": "Claude 3.5 Sonnet", + "vision_documents": "Gemini 1.5 Flash" + } +} +``` + +--- + +## 8. Connecting External Hospital EMR / EHR System + +When onboarding is finalized, the backend invokes `PatientSubmissionService` located at: +`app/services/patient_submission_service.py` + +### Customizing the Destination Forwarding: +```python +import httpx +from app.config import PATIENT_API_BASE_URL, PATIENT_API_TOKEN + +class PatientSubmissionService: + async def submit(self, result: dict) -> dict: + if not PATIENT_API_BASE_URL: + # Local fallback mock + return { + "success": True, + "message": "Patient data saved locally.", + "patient_id": result.get("patient_id"), + } + + # Forward JSON payload to hospital central server + async with httpx.AsyncClient(timeout=15.0) as client: + headers = { + "Authorization": f"Bearer {PATIENT_API_TOKEN}", + "Content-Type": "application/json", + } + response = await client.post( + f"{PATIENT_API_BASE_URL}/patients/{result.get('patient_id')}/onboarding", + json=result, + headers=headers, + ) + + if response.status_code in [200, 201]: + return {"success": True, "data": response.json()} + + return { + "success": False, + "error": f"Hospital API error: {response.text}", + } +``` + +--- + +## 9. Multi-Language & Clinical Specialties + +### Supported Languages +| Code | Language | Speech Engine | +|---|---|---| +| `en-IN` / `en` | English (Indian accent) | Sarvam / OpenAI Whisper | +| `hi-IN` / `hi` | Hindi (हिन्दी) | Sarvam Bulbul / Whisper | +| `ta-IN` / `ta` | Tamil (தமிழ்) | Sarvam Bulbul / Whisper | +| `te-IN` / `te` | Telugu (తెలుగు) | Sarvam Bulbul / Whisper | +| `kn-IN` / `kn` | Kannada (ಕನ್ನಡ) | Sarvam Bulbul / Whisper | +| `bn-IN` / `bn` | Bengali (বাংলা) | Sarvam Bulbul / Whisper | +| `ml-IN` / `ml` | Malayalam (മലയാളം) | Sarvam Bulbul / Whisper | +| `mr-IN` / `mr` | Marathi (मराठी) | Sarvam Bulbul / Whisper | +| `gu-IN` / `gu` | Gujarati (ગુજરાતી) | Sarvam Bulbul / Whisper | + +### Clinical Specialties Supported +- `general_medicine` - General Medicine / Primary Care +- `cardio` - Cardiology +- `ortho` - Orthopedics +- `peds` - Pediatrics +- `derma` - Dermatology +- `neuro` - Neurology +- `gastro` - Gastroenterology +- `ent` - ENT / Otorhinolaryngology +- `gyn` - Obstetrics & Gynecology + +--- + +## 10. Client Integration Code Samples + +### 10.1 Python Client (`httpx`) +```python +import httpx +import asyncio + +API_BASE = "http://127.0.0.1:8000" + +async def main(): + async with httpx.AsyncClient() as client: + # 1. Start Session + start_res = await client.post(f"{API_BASE}/v1/voice-onboarding/start", json={ + "patient": {"id": "P-1001", "name": "Deepak Sharma", "age": 45, "gender": "male"}, + "category": {"type": "speciality", "code": "cardio", "name": "Cardiology"}, + "language": "hi-IN" + }) + session = start_res.json() + session_id = session["session_id"] + print(f"Session started: {session_id}") + print(f"Greeting: {session['greeting_text']}") + + # 2. Send Turn + turn_res = await client.post( + f"{API_BASE}/v1/voice-onboarding/{session_id}/turn", + data={"text_fallback": "मुझे सीने में भारीपन महसूस हो रहा है"} + ) + turn_data = turn_res.json() + print(f"AI Response: {turn_data['ai_text']}") + + # 3. Finalize & Submit + complete_res = await client.post( + f"{API_BASE}/v1/voice-onboarding/{session_id}/complete", + json={"confirmed": True} + ) + print(f"EHR Saved: {complete_res.json()}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### 10.2 JavaScript / TypeScript (`fetch`) +```typescript +const API_BASE = "http://127.0.0.1:8000"; + +// Start voice onboarding +export async function startVoiceOnboarding(patient: any, category: any, language = "en-IN") { + const res = await fetch(`${API_BASE}/v1/voice-onboarding/start`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ patient, category, language }) + }); + return res.json(); +} + +// Send audio turn +export async function sendVoiceTurn(sessionId: string, audioBlob?: Blob, textFallback?: string) { + const formData = new FormData(); + if (audioBlob) formData.append("audio", audioBlob, "speech.webm"); + if (textFallback) formData.append("text_fallback", textFallback); + + const res = await fetch(`${API_BASE}/v1/voice-onboarding/${sessionId}/turn`, { + method: "POST", + body: formData + }); + return res.json(); +} + +// Upload medical report +export async function uploadPrescription(sessionId: string, file: File) { + const formData = new FormData(); + formData.append("file", file); + + const res = await fetch(`${API_BASE}/v1/documents/${sessionId}/upload`, { + method: "POST", + body: formData + }); + return res.json(); +} + +// Complete onboarding +export async function completeOnboarding(sessionId: string, confirmed: boolean) { + const res = await fetch(`${API_BASE}/v1/voice-onboarding/${sessionId}/complete`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ confirmed }) + }); + return res.json(); +} +``` + +### 10.3 cURL Commands + +#### Start Session: +```bash +curl -X POST "http://127.0.0.1:8000/v1/voice-onboarding/start" \ + -H "Content-Type: application/json" \ + -d '{ + "patient": {"id": "P-101", "name": "Arun Kumar", "age": 40, "gender": "male"}, + "category": {"type": "general", "code": "general_medicine", "name": "General Medicine"}, + "language": "en-IN" + }' +``` + +#### Send Voice Turn (with text fallback): +```bash +curl -X POST "http://127.0.0.1:8000/v1/voice-onboarding/VOICE-XXXXX/turn" \ + -F "text_fallback=I have had high fever and sore throat since yesterday" +``` + +#### Upload Prescription: +```bash +curl -X POST "http://127.0.0.1:8000/v1/documents/VOICE-XXXXX/upload" \ + -F "file=@/path/to/prescription.pdf" +``` + +#### Confirm & Submit: +```bash +curl -X POST "http://127.0.0.1:8000/v1/voice-onboarding/VOICE-XXXXX/complete" \ + -H "Content-Type: application/json" \ + -d '{"confirmed": true}' +``` diff --git a/ai-patient-onboarding/app/__init__.py b/ai-patient-onboarding/app/__init__.py new file mode 100644 index 0000000..857ca85 --- /dev/null +++ b/ai-patient-onboarding/app/__init__.py @@ -0,0 +1 @@ +"""App package initialization.""" diff --git a/ai-patient-onboarding/app/ai/__init__.py b/ai-patient-onboarding/app/ai/__init__.py new file mode 100644 index 0000000..1478e01 --- /dev/null +++ b/ai-patient-onboarding/app/ai/__init__.py @@ -0,0 +1 @@ +"""AI package initialization.""" diff --git a/ai-patient-onboarding/app/ai/claude_client.py b/ai-patient-onboarding/app/ai/claude_client.py new file mode 100644 index 0000000..9d90de5 --- /dev/null +++ b/ai-patient-onboarding/app/ai/claude_client.py @@ -0,0 +1,69 @@ +import json +from app.config import ANTHROPIC_API_KEY, ANTHROPIC_MODEL + + +class ClaudeClient: + + def __init__(self): + self.api_key = ANTHROPIC_API_KEY + self.model = ANTHROPIC_MODEL + + async def validate_extraction( + self, + patient_context: dict, + specialty_config: dict, + conversation_history: list, + extracted_data: dict, + ) -> dict: + if not self.api_key or self.api_key == "your_claude_key": + return { + "valid": True, + "missing": [], + "recommendation": "continue", + "hallucinations_detected": False, + } + + try: + from anthropic import AsyncAnthropic + client = AsyncAnthropic(api_key=self.api_key) + + prompt = f""" +You are a senior clinical auditor and AI quality control specialist. +Your task is to validate whether the extracted patient data accurately reflects the actual patient conversation without hallucination or omitted critical criteria. + +PATIENT CONTEXT: +{json.dumps(patient_context, indent=2)} + +SPECIALTY REQUIREMENTS: +{json.dumps(specialty_config, indent=2)} + +CONVERSATION HISTORY: +{json.dumps(conversation_history, indent=2)} + +PROPOSED EXTRACTED STATE: +{json.dumps(extracted_data, indent=2)} + +Return ONLY a JSON response: +{{ + "valid": true/false, + "missing": ["list of critical missing focus items for this specialty, if any"], + "recommendation": "continue or ask_severity or ask_duration etc.", + "hallucinations_detected": false +}} +""" + response = await client.messages.create( + model=self.model, + max_tokens=1024, + system="You are a clinical AI quality controller. Output only valid JSON.", + messages=[{"role": "user", "content": prompt}], + ) + content = response.content[0].text + return json.loads(content) + except Exception as e: + print(f"[Claude Validation Error]: {e}") + return { + "valid": True, + "missing": [], + "recommendation": "continue", + "hallucinations_detected": False, + } diff --git a/ai-patient-onboarding/app/ai/context.py b/ai-patient-onboarding/app/ai/context.py new file mode 100644 index 0000000..1140025 --- /dev/null +++ b/ai-patient-onboarding/app/ai/context.py @@ -0,0 +1,33 @@ +import json + + +def build_onboarding_context( + session, + state, + conversation_history, +): + + return { + "patient": { + "id": session.patient_id, + "name": session.patient_name, + "age": session.age, + "gender": session.gender, + }, + + "category": { + "type": session.category_type, + "code": session.category_code, + "name": session.category_name, + }, + + "language": session.language, + + "current_stage": session.stage, + + "current_state": json.loads( + state.state_json + ), + + "conversation": conversation_history, + } diff --git a/ai-patient-onboarding/app/ai/gemini_client.py b/ai-patient-onboarding/app/ai/gemini_client.py new file mode 100644 index 0000000..044cb80 --- /dev/null +++ b/ai-patient-onboarding/app/ai/gemini_client.py @@ -0,0 +1,73 @@ +import json +import base64 +from app.config import GEMINI_API_KEY, GEMINI_MODEL + + +class GeminiClient: + + def __init__(self): + self.api_key = GEMINI_API_KEY + self.model = GEMINI_MODEL + + async def extract_from_document( + self, + file_bytes: bytes, + mime_type: str, + patient_context: dict, + ) -> dict: + if not self.api_key or self.api_key == "your_gemini_key": + return { + "document_type": "medical_report", + "extracted_text": "Sample HbA1c 7.2%, Blood Pressure 130/85 mmHg", + "findings": [ + {"test": "HbA1c", "value": "7.2", "unit": "%", "interpretation": "Elevated"}, + {"test": "Blood Pressure", "value": "130/85", "unit": "mmHg", "interpretation": "Stage 1"}, + ], + "medications_found": ["Metformin 500mg"], + "allergies_found": [], + "summary": "Report indicates elevated HbA1c and prehypertension.", + } + + try: + import google.generativeai as genai + genai.configure(api_key=self.api_key) + model = genai.GenerativeModel(self.model) + + prompt = f""" +You are a medical document extraction AI. +Analyze this medical report / prescription / lab result for patient {patient_context.get('name', 'Patient')}. +Extract all lab tests, numeric values, units, diagnosed conditions, medications, and allergies into structured JSON. + +Return ONLY JSON: +{{ + "document_type": "lab_report | prescription | discharge_summary | other", + "extracted_text": "brief summary text", + "findings": [ + {{ + "test": "test name", + "value": "numeric or text value", + "unit": "unit or null", + "interpretation": "Normal | High | Low | etc." + }} + ], + "medications_found": ["medication 1", "medication 2"], + "allergies_found": ["allergy 1"], + "summary": "short clinical summary of the document" +}} +""" + part = { + "mime_type": mime_type, + "data": file_bytes, + } + response = await model.generate_content_async([part, prompt]) + cleaned_text = response.text.replace("```json", "").replace("```", "").strip() + return json.loads(cleaned_text) + except Exception as e: + print(f"[Gemini Extraction Error]: {e}") + return { + "document_type": "unknown", + "findings": [], + "medications_found": [], + "allergies_found": [], + "summary": f"Could not extract document details: {e}", + } diff --git a/ai-patient-onboarding/app/ai/openai_client.py b/ai-patient-onboarding/app/ai/openai_client.py new file mode 100644 index 0000000..001a579 --- /dev/null +++ b/ai-patient-onboarding/app/ai/openai_client.py @@ -0,0 +1,39 @@ +from openai import AsyncOpenAI +from app.config import OPENAI_API_KEY, OPENAI_MODEL + + +class OpenAIClient: + + _client = None + + def __init__(self): + self.api_key = OPENAI_API_KEY + self.model = OPENAI_MODEL + if OpenAIClient._client is None and self.api_key: + OpenAIClient._client = AsyncOpenAI( + api_key=self.api_key, + timeout=12.0, + max_retries=1, + ) + + async def generate_response( + self, + system_prompt: str, + user_message: str, + ) -> str: + client = OpenAIClient._client or AsyncOpenAI( + api_key=self.api_key or "sk-dummy-key-placeholder", + timeout=12.0, + ) + response = await client.chat.completions.create( + model=self.model or "gpt-4o-mini", + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_message}, + ], + response_format={"type": "json_object"}, + temperature=0.35, + max_completion_tokens=140, + ) + return response.choices[0].message.content + diff --git a/ai-patient-onboarding/app/ai/orchestrator.py b/ai-patient-onboarding/app/ai/orchestrator.py new file mode 100644 index 0000000..e3db552 --- /dev/null +++ b/ai-patient-onboarding/app/ai/orchestrator.py @@ -0,0 +1,149 @@ +import json + +from app.ai.openai_client import OpenAIClient +from app.ai.prompts import build_system_prompt +from app.ai.validator import validate_ai_response +from app.config import OPENAI_API_KEY +from app.specialties.registry import get_specialty_config + + +class AIOrchestrator: + + def __init__(self): + self.openai = OpenAIClient() + + async def process( + self, + session, + state, + conversation_history, + user_message, + ): + specialty = get_specialty_config( + session.category_code + ) + + current_state = json.loads( + state.state_json + ) + + context = { + "patient": { + "id": session.patient_id, + "name": session.patient_name, + "age": session.age, + "gender": session.gender, + }, + "category": { + "type": session.category_type, + "code": session.category_code, + "name": session.category_name, + }, + "specialty_config": specialty, + "language": session.language, + "current_stage": session.stage, + "current_state": current_state, + "conversation_history": conversation_history, + "new_patient_message": user_message, + } + + prompt = build_system_prompt(context) + + # Real OpenAI Call if valid API key is set + if OPENAI_API_KEY and OPENAI_API_KEY.strip() not in ["your_openai_api_key", "your_openai_key", ""]: + try: + raw_response = await self.openai.generate_response( + system_prompt=prompt, + user_message=user_message, + ) + parsed = json.loads(raw_response) + validated = validate_ai_response(parsed) + return validated.model_dump() + except Exception as e: + print(f"[OpenAI Orchestrator Error]: {e}") + + # Fallback simulator for offline development with concise clinical flow + parsed = self._simulate_specialty_response(session, specialty, user_message, len(conversation_history)) + validated = validate_ai_response(parsed) + return validated.model_dump() + + def _simulate_specialty_response(self, session, specialty, user_message, turn_count=0): + stage = session.stage + code = specialty.get("code", "general") + msg_lower = user_message.lower().strip() + is_negative = msg_lower in ["no", "no more", "nothing", "nothing else", "know nothing", "none", "illness none"] + + # Fast conclusion if patient says "No" or already had 4 turns + if is_negative or turn_count >= 6: + return { + "reply": "Thank you. I have recorded your intake details for the doctor to review.", + "next_stage": "confirmation", + "state_updates": { + "reason_for_visit": None, + "symptoms": [], + "medical_history": ["None reported"] if is_negative else [], + "medications": [], + "allergies": ["None"] if is_negative else [] + }, + "completed": True + } + + if stage == "greeting": + if code == "cardiology": + reply = "Could you please share what cardiac or chest symptoms you are experiencing?" + else: + reply = "Could you please describe what primary symptoms you are experiencing today?" + return { + "reply": reply, + "next_stage": "reason_for_visit", + "state_updates": { + "reason_for_visit": user_message, + "symptoms": [{"name": user_message, "duration": "recent"}], + "medical_history": [], + "medications": [], + "allergies": [] + }, + "completed": False + } + elif stage == "reason_for_visit": + reply = "How long have you had these symptoms, and have you taken any medicines?" + return { + "reply": reply, + "next_stage": "medications", + "state_updates": { + "reason_for_visit": user_message, + "symptoms": [{"name": user_message, "duration": "reported"}], + "medical_history": [], + "medications": [], + "allergies": [] + }, + "completed": False + } + elif stage == "medications": + reply = "Thank you. Do you have any known drug allergies or existing health conditions?" + return { + "reply": reply, + "next_stage": "allergies", + "state_updates": { + "reason_for_visit": None, + "symptoms": [], + "medical_history": [], + "medications": [user_message] if not is_negative else [], + "allergies": [] + }, + "completed": False + } + else: + return { + "reply": "Thank you. I have recorded your intake details for the doctor to review.", + "next_stage": "confirmation", + "state_updates": { + "reason_for_visit": None, + "symptoms": [], + "medical_history": [], + "medications": [], + "allergies": ["None"] if is_negative else [user_message] + }, + "completed": True + } + diff --git a/ai-patient-onboarding/app/ai/prompts.py b/ai-patient-onboarding/app/ai/prompts.py new file mode 100644 index 0000000..10e95cb --- /dev/null +++ b/ai-patient-onboarding/app/ai/prompts.py @@ -0,0 +1,89 @@ +import json + + +def build_system_prompt(context: dict) -> str: + specialty = context.get("specialty_config", {}) + patient = context.get("patient", {}) + language = context.get("language", "ta-IN") + stage = context.get("current_stage", "greeting") + current_state = context.get("current_state", {}) + conversation_history = context.get("conversation_history", []) + + turn_count = len(conversation_history) + + lang_instructions = { + "ta-IN": ( + "LANGUAGE & ACCENT: Professional, Respectful, and Articulate Tamil (தமிழ்).\n" + "- Speak with the professional, calm, and reassuring demeanor of a senior hospital clinical coordinator.\n" + "- Use polite, respectful Tamil words (e.g. 'வணக்கம்', 'கூற முடியுமா?', 'புரிகிறது', 'நன்றி').\n" + "- Keep sentence structure clear, elegant, and under 14 words." + ), + "hi-IN": ( + "LANGUAGE & ACCENT: Professional, Respectful, and Clear Hindi (हिन्दी).\n" + "- Speak with the dignified and polite tone of an experienced hospital triage officer.\n" + "- Use respectful language with 'आप' and clear clinical clarity.\n" + "- Keep sentence structure polite, concise, and under 14 words." + ), + "en-IN": ( + "LANGUAGE & ACCENT: Polished, Professional Indian Healthcare English.\n" + "- Speak with a calm, articulate, and reassuring clinical tone.\n" + "- Use clear, professional phrasing (e.g. 'Thank you. I have recorded your symptoms for the doctor.').\n" + "- Keep each response concise, respectful, and under 14 words." + ), + "te-IN": ( + "LANGUAGE & ACCENT: Professional and Respectful Telugu (తెలుగు).\n" + "- Speak with dignified, clear hospital intake phrasing (under 14 words)." + ), + "kn-IN": ( + "LANGUAGE & ACCENT: Professional and Respectful Kannada (ಕನ್ನಡ).\n" + "- Speak with dignified, clear hospital intake phrasing (under 14 words)." + ), + } + + selected_lang_rule = lang_instructions.get(language, lang_instructions["en-IN"]) + + return f"""You are a senior, professional Hospital Patient Intake Assistant at CureSelect Healthcare. + +{selected_lang_rule} + +CRITICAL CLINICAL INTAKE PRINCIPLES (AVOID UNWANTED QUESTIONS): +1. **FOCUS STRICTLY ON THE PATIENT'S COMPLAINT**: + - Only ask questions directly relevant to what the patient reported (e.g., if they report headache & fever, ask about duration, fever severity, remedies, and allergies). + - **DO NOT randomly cross-examine unrelated organ systems** (e.g., NEVER ask about digestive issues, acidity, loose stools, or stomach unless the patient specifically mentioned stomach problems!). + +2. **FAST & CONCISE INTAKE (MAX 4 TURNS TOTAL)**: + - Flow: Chief Complaint -> Duration -> Medications Taken -> Drug Allergies -> COMPLETE. + - Once symptoms, duration, medicines, and allergies are collected, **IMMEDIATELY SET "completed": true**. + +3. **RESPECT NEGATIVE ANSWERS ("NO" / "NOTHING ELSE")**: + - If the patient says "No", "Nothing", "Know nothing", or indicates no more symptoms, **DO NOT ask another diagnostic question**. + - Conclude immediately: *"Thank you. I have recorded your details for the doctor to review."* with `"completed": true` and `"next_stage": "confirmation"`. + +4. **CONCISENESS**: Exactly ONE spoken sentence (maximum 12 to 14 words). +5. **NO ARTIFACTS**: No bullet points, no brackets, no markdown formatting. Deliver clean spoken dialogue. +6. **EMERGENCY**: Direct severe life-threatening chest pain, sudden paralysis, or severe respiratory distress to 108 ambulance. + +PATIENT: {patient.get("name", "Patient")}, {patient.get("age", "")}y, {patient.get("gender", "")} +DEPARTMENT: {specialty.get("name", "General Intake")} +CURRENT STAGE: {stage} (Turn #{turn_count}) +CURRENT STRUCTURED STATE: +{json.dumps(current_state, ensure_ascii=False)} + +CONVERSATION HISTORY: +{json.dumps(conversation_history, ensure_ascii=False)} + +OUTPUT JSON ONLY: +{{ + "reply": "1 concise, respectful sentence (max 14 words)", + "next_stage": "greeting" | "reason_for_visit" | "symptoms" | "medical_history" | "medications" | "allergies" | "confirmation", + "completed": true | false, + "state_updates": {{ + "reason_for_visit": {{"status": "reported" | "not_collected", "value": "..."}} or null, + "symptoms": {{"status": "reported" | "not_collected", "items": [{{"name": "...", "duration": "...", "severity": "..."}}]}} or null, + "medical_history": {{"status": "reported" | "confirmed_none", "items": ["..."]}} or null, + "medications": {{"status": "reported" | "confirmed_none", "items": ["..."]}} or null, + "allergies": {{"status": "reported" | "confirmed_none", "items": ["..."]}} or null + }} +}} +""" + diff --git a/ai-patient-onboarding/app/ai/sarvam_client.py b/ai-patient-onboarding/app/ai/sarvam_client.py new file mode 100644 index 0000000..91cfd4a --- /dev/null +++ b/ai-patient-onboarding/app/ai/sarvam_client.py @@ -0,0 +1,84 @@ +import base64 +import httpx2 +from app.config import SARVAM_API_KEY + + +class SarvamClient: + + _client = None + + def __init__(self): + self.api_key = SARVAM_API_KEY + self.base_url = "https://api.sarvam.ai" + + def _get_client(self) -> httpx2.AsyncClient: + if SarvamClient._client is None: + SarvamClient._client = httpx2.AsyncClient( + timeout=12.0, + limits=httpx2.Limits(max_keepalive_connections=5, max_connections=10), + ) + return SarvamClient._client + + async def speech_to_text( + self, + audio_bytes: bytes, + filename: str = "audio.webm", + language_code: str = "ta-IN", + ) -> str: + if not self.api_key or self.api_key == "your_sarvam_key": + return "" + + client = self._get_client() + files = {"file": (filename, audio_bytes, "audio/webm")} + data = {"model": "saaras:v2", "language_code": language_code} + headers = {"api-subscription-key": self.api_key} + + try: + res = await client.post( + f"{self.base_url}/speech-to-text", + files=files, + data=data, + headers=headers, + ) + if res.status_code == 200: + return res.json().get("transcript", "").strip() + except Exception as e: + print(f"[Sarvam STT Error]: {e}") + return "" + + async def text_to_speech( + self, + text: str, + language_code: str = "ta-IN", + ) -> bytes | None: + if not self.api_key or self.api_key == "your_sarvam_key": + return None + + lang_prefix = language_code.lower()[:2] + speaker = "meera" if lang_prefix == "ta" else "arvind" + + client = self._get_client() + headers = { + "api-subscription-key": self.api_key, + "Content-Type": "application/json", + } + payload = { + "inputs": [text], + "target_language_code": language_code, + "speaker": speaker, + "model": "bulbul:v1", + } + try: + res = await client.post( + f"{self.base_url}/text-to-speech", + json=payload, + headers=headers, + ) + if res.status_code == 200: + audios = res.json().get("audios", []) + if audios: + return base64.b64decode(audios[0]) + except Exception as e: + print(f"[Sarvam TTS Error]: {e}") + return None + diff --git a/ai-patient-onboarding/app/ai/validator.py b/ai-patient-onboarding/app/ai/validator.py new file mode 100644 index 0000000..81ae865 --- /dev/null +++ b/ai-patient-onboarding/app/ai/validator.py @@ -0,0 +1,12 @@ +from app.schemas.onboarding import ( + AIOnboardingResponse, +) + + +def validate_ai_response( + data: dict, +) -> AIOnboardingResponse: + + return AIOnboardingResponse.model_validate( + data + ) diff --git a/ai-patient-onboarding/app/api/__init__.py b/ai-patient-onboarding/app/api/__init__.py new file mode 100644 index 0000000..f3ac4ed --- /dev/null +++ b/ai-patient-onboarding/app/api/__init__.py @@ -0,0 +1 @@ +"""API package initialization.""" diff --git a/ai-patient-onboarding/app/api/documents.py b/ai-patient-onboarding/app/api/documents.py new file mode 100644 index 0000000..9d42e5b --- /dev/null +++ b/ai-patient-onboarding/app/api/documents.py @@ -0,0 +1,84 @@ +import json +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from sqlalchemy.orm import Session + +from app.ai.gemini_client import GeminiClient +from app.database.database import get_db +from app.schemas.documents import DocumentExtractionResponse +from app.services.onboarding_service import OnboardingService + +router = APIRouter( + prefix="/v1/documents", + tags=["Medical Documents"], +) + + +@router.post( + "/{session_id}/upload", + response_model=DocumentExtractionResponse, +) +async def upload_medical_document( + session_id: str, + file: UploadFile = File(...), + db: Session = Depends(get_db), +): + onboarding_service = OnboardingService() + gemini_client = GeminiClient() + + session = onboarding_service.get_session(db, session_id) + if not session: + raise HTTPException(status_code=404, detail="Onboarding session not found") + + state = onboarding_service.get_state(db, session_id) + file_bytes = await file.read() + mime_type = file.content_type or "application/pdf" + + patient_context = { + "id": session.patient_id, + "name": session.patient_name, + "age": session.age, + "gender": session.gender, + "category": session.category_name, + } + + # Extract clinical data with Gemini + extracted = await gemini_client.extract_from_document( + file_bytes=file_bytes, + mime_type=mime_type, + patient_context=patient_context, + ) + + # Merge document findings into onboarding state + current_state = json.loads(state.state_json) + + # Append medications if found + meds_found = extracted.get("medications_found", []) + if meds_found: + existing_meds = current_state.get("medications", {}) + items = existing_meds.get("items", []) if isinstance(existing_meds, dict) else (existing_meds or []) + items = list(set(items + meds_found)) + current_state["medications"] = {"status": "reported", "items": items} + + # Append allergies if found + allergies_found = extracted.get("allergies_found", []) + if allergies_found: + existing_allergies = current_state.get("allergies", {}) + items = existing_allergies.get("items", []) if isinstance(existing_allergies, dict) else (existing_allergies or []) + items = list(set(items + allergies_found)) + current_state["allergies"] = {"status": "reported", "items": items} + + # Add document findings to state + current_state["document_findings"] = extracted.get("findings", []) + + onboarding_service.update_state(db=db, state=state, data=current_state) + + return DocumentExtractionResponse( + success=True, + session_id=session_id, + document_type=extracted.get("document_type", "medical_report"), + summary=extracted.get("summary", "Document analyzed successfully"), + findings=extracted.get("findings", []), + medications_found=meds_found, + allergies_found=allergies_found, + message="Medical document analyzed and integrated into onboarding session.", + ) diff --git a/ai-patient-onboarding/app/api/onboarding.py b/ai-patient-onboarding/app/api/onboarding.py new file mode 100644 index 0000000..43c0cd5 --- /dev/null +++ b/ai-patient-onboarding/app/api/onboarding.py @@ -0,0 +1,325 @@ +import json +from uuid import uuid4 + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from app.database.database import get_db +from app.database.models import ( + OnboardingMessage, + OnboardingSession, + OnboardingState, +) +from app.schemas.onboarding import ( + CompleteOnboardingRequest, + CompleteOnboardingResponse, + SendMessageRequest, + SendMessageResponse, + StartOnboardingRequest, + StartOnboardingResponse, +) +from app.services.ai_onboarding_service import ( + AIOnboardingService, +) +from app.services.completion_service import ( + CompletionService, +) +from app.services.onboarding_service import ( + OnboardingService, +) +from app.services.patient_submission_service import ( + PatientSubmissionService, +) +from app.services.result_extraction_service import ( + ResultExtractionService, +) + + +router = APIRouter( + prefix="/v1/onboarding", + tags=["Patient Onboarding"], +) + + +@router.post( + "/start", + response_model=StartOnboardingResponse, +) +async def start_onboarding( + request: StartOnboardingRequest, + db: Session = Depends(get_db), +): + onboarding_service = OnboardingService() + session_id = f"ONB-{uuid4().hex[:8].upper()}" + + session = OnboardingSession( + session_id=session_id, + patient_id=request.patient.id, + patient_name=request.patient.name, + age=request.patient.age, + gender=request.patient.gender, + category_type=request.category.type, + category_code=request.category.code, + category_name=request.category.name, + language=request.language, + status="in_progress", + stage="greeting", + ) + + db.add(session) + db.commit() + db.refresh(session) + + # Initialize empty state + onboarding_service.get_state(db, session_id) + + greeting_msg = ( + f"Hello {request.patient.name or 'there'}, " + "welcome to CureSelect Healthcare. " + "I'll ask you a few questions to understand " + "your current health concern. Shall we begin?" + ) + + # Save initial assistant greeting + onboarding_service.save_message( + db=db, + session_id=session_id, + role="assistant", + message=greeting_msg, + stage="greeting", + ) + + return StartOnboardingResponse( + success=True, + session_id=session.session_id, + patient_id=session.patient_id, + category=request.category, + language=session.language, + status=session.status, + stage=session.stage, + message=greeting_msg, + ) + + +@router.post( + "/{session_id}/message", + response_model=SendMessageResponse, +) +async def send_message( + session_id: str, + request: SendMessageRequest, + db: Session = Depends(get_db), +): + onboarding_service = OnboardingService() + ai_service = AIOnboardingService() + + # 1. Get session + session = onboarding_service.get_session( + db, + session_id, + ) + + if not session: + raise HTTPException( + status_code=404, + detail="Onboarding session not found", + ) + + # 2. Check session + if session.status != "in_progress": + raise HTTPException( + status_code=400, + detail="Onboarding session is not active", + ) + + # 3. Save patient message + onboarding_service.save_message( + db=db, + session_id=session_id, + role="patient", + message=request.message, + stage=session.stage, + ) + + # 4. Get state + state = onboarding_service.get_state( + db, + session_id, + ) + + # 5. Get conversation history + messages = ( + db.query(OnboardingMessage) + .filter( + OnboardingMessage.session_id == session_id + ) + .order_by( + OnboardingMessage.created_at.asc() + ) + .all() + ) + + conversation_history = [ + { + "role": message.role, + "message": message.message, + "stage": message.stage, + } + for message in messages + ] + + # 6. Send to AI + ai_response = await ai_service.process_message( + session=session, + state=state, + conversation_history=conversation_history, + user_message=request.message, + ) + + # 7. Update state + current_state = json.loads( + state.state_json + ) + + updates = ai_response.get( + "state_updates", + {}, + ) + + for key, value in updates.items(): + if value is not None: + if isinstance(value, list) and len(value) == 0: + continue + current_state[key] = value + + onboarding_service.update_state( + db=db, + state=state, + data=current_state, + ) + + # 8. Update stage + session.stage = ai_response.get("next_stage", session.stage) + + # 9. Check completion + if ai_response.get("completed"): + session.status = "completed" + + db.commit() + + # 10. Save AI message + onboarding_service.save_message( + db=db, + session_id=session_id, + role="assistant", + message=ai_response["reply"], + stage=session.stage, + ) + + return SendMessageResponse( + success=True, + session_id=session_id, + stage=session.stage, + message=ai_response["reply"], + status=session.status, + ) + + +@router.post( + "/{session_id}/complete", + response_model=CompleteOnboardingResponse, +) +async def complete_onboarding( + session_id: str, + request: CompleteOnboardingRequest, + db: Session = Depends(get_db), +): + onboarding_service = OnboardingService() + completion_service = CompletionService() + result_service = ResultExtractionService() + submission_service = PatientSubmissionService() + + # 1. Find session + session = onboarding_service.get_session( + db, + session_id, + ) + + if not session: + raise HTTPException( + status_code=404, + detail="Onboarding session not found", + ) + + # 2. Load state + state = onboarding_service.get_state( + db, + session_id, + ) + + # Check if complete + has_details = completion_service.is_complete(session, state) + + # 4. Get conversation history + messages = ( + db.query(OnboardingMessage) + .filter(OnboardingMessage.session_id == session_id) + .order_by(OnboardingMessage.created_at.asc()) + .all() + ) + + conversation_history = [ + { + "role": message.role, + "message": message.message, + "stage": message.stage, + } + for message in messages + ] + + # 5. Generate final structured result + result = await result_service.extract_result( + session=session, + state=state, + conversation_history=conversation_history, + ) + + # 6. If patient has not confirmed yet, return summary for review + if not request.confirmed: + return CompleteOnboardingResponse( + success=True, + session_id=session_id, + status="awaiting_confirmation", + result=result, + message="Please review the information and confirm.", + ) + + # 7. Patient confirmed -> submit to existing patient / hospital API + submission = await submission_service.submit(result) + + if submission.get("success"): + # 8. Successful submission -> clean up temporary SQLite session + db.query(OnboardingMessage).filter(OnboardingMessage.session_id == session_id).delete() + db.query(OnboardingState).filter(OnboardingState.session_id == session_id).delete() + db.delete(session) + db.commit() + + return CompleteOnboardingResponse( + success=True, + session_id=session_id, + status="submitted_and_cleaned", + result=result, + message="Onboarding successfully completed, saved to patient record, and temporary session cleaned.", + ) + + # If submission failed, preserve SQLite session for retry + session.status = "submission_pending_retry" + db.commit() + + return CompleteOnboardingResponse( + success=False, + session_id=session_id, + status="submission_pending_retry", + result=result, + message="Patient API submission failed. Session preserved for retry.", + ) diff --git a/ai-patient-onboarding/app/api/voice.py b/ai-patient-onboarding/app/api/voice.py new file mode 100644 index 0000000..2a60581 --- /dev/null +++ b/ai-patient-onboarding/app/api/voice.py @@ -0,0 +1,4 @@ +from app.api.voice_onboarding import router + +# Export the router +__all__ = ["router"] diff --git a/ai-patient-onboarding/app/api/voice_onboarding.py b/ai-patient-onboarding/app/api/voice_onboarding.py new file mode 100644 index 0000000..334b4e9 --- /dev/null +++ b/ai-patient-onboarding/app/api/voice_onboarding.py @@ -0,0 +1,343 @@ +import json +from uuid import uuid4 +from typing import Optional + +from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from fastapi.responses import FileResponse +from sqlalchemy.orm import Session + +from app.database.database import get_db +from app.database.models import ( + OnboardingMessage, + OnboardingSession, + OnboardingState, +) +from app.schemas.onboarding import ( + CompleteOnboardingRequest, + CompleteOnboardingResponse, +) +from app.schemas.voice import ( + StartVoiceOnboardingRequest, + StartVoiceOnboardingResponse, + VoiceTurnResponse, +) +from app.services.ai_onboarding_service import AIOnboardingService +from app.services.completion_service import CompletionService +from app.services.onboarding_service import OnboardingService +from app.services.patient_submission_service import PatientSubmissionService +from app.services.result_extraction_service import ResultExtractionService +from app.services.voice_service import VoiceService + +router = APIRouter( + prefix="/v1/voice-onboarding", + tags=["Voice Onboarding"], +) + + +@router.post( + "/start", + response_model=StartVoiceOnboardingResponse, +) +async def start_voice_onboarding( + request: StartVoiceOnboardingRequest, + db: Session = Depends(get_db), +): + onboarding_service = OnboardingService() + voice_service = VoiceService() + + session_id = f"VOICE-{uuid4().hex[:8].upper()}" + + session = OnboardingSession( + session_id=session_id, + patient_id=request.patient.id, + patient_name=request.patient.name, + age=request.patient.age, + gender=request.patient.gender, + category_type=request.category.type, + category_code=request.category.code, + category_name=request.category.name, + language=request.language, + status="in_progress", + stage="greeting", + ) + + db.add(session) + db.commit() + db.refresh(session) + + # Initialize empty state + onboarding_service.get_state(db, session_id) + + # Generate localized voice greeting + greeting_text = voice_service.get_initial_greeting( + request.patient.name, + request.language, + ) + + # Synthesize audio greeting + audio_id, _ = await voice_service.text_to_speech( + greeting_text, + request.language, + ) + + # Save initial assistant message + onboarding_service.save_message( + db=db, + session_id=session_id, + role="assistant", + message=greeting_text, + stage="greeting", + ) + + return StartVoiceOnboardingResponse( + success=True, + session_id=session.session_id, + patient_id=session.patient_id, + category=request.category, + language=session.language, + status=session.status, + stage=session.stage, + greeting_text=greeting_text, + audio_url=f"/v1/voice-onboarding/audio/{audio_id}", + ) + + +@router.post( + "/{session_id}/turn", + response_model=VoiceTurnResponse, +) +async def voice_turn( + session_id: str, + audio: Optional[UploadFile] = File(None), + text_fallback: Optional[str] = Form(None), + db: Session = Depends(get_db), +): + onboarding_service = OnboardingService() + voice_service = VoiceService() + ai_service = AIOnboardingService() + + # 1. Fetch active session + session = onboarding_service.get_session(db, session_id) + if not session: + raise HTTPException( + status_code=404, + detail="Voice onboarding session not found", + ) + + if session.status != "in_progress": + raise HTTPException( + status_code=400, + detail="Voice onboarding session is not active", + ) + + # 2. Transcribe speech if audio provided with multi-model Indic recognition + patient_transcript = "" + if audio: + audio_bytes = await audio.read() + if len(audio_bytes) > 0: + patient_transcript = await voice_service.speech_to_text( + audio_bytes=audio_bytes, + filename=audio.filename or "recording.webm", + language_code=session.language, + specialty_code=session.category_code or "general", + patient_context={ + "name": session.patient_name, + "age": session.age, + "gender": session.gender, + }, + ) + + # If backend STT is empty, use frontend real-time transcript fallback + if (not patient_transcript or not patient_transcript.strip()) and text_fallback and text_fallback.strip(): + from app.services.medical_speech_normalizer import normalize_medical_speech + patient_transcript = normalize_medical_speech(text_fallback.strip(), language_code=session.language) + + if not patient_transcript or not patient_transcript.strip(): + if session.language.startswith("ta"): + patient_transcript = "எனக்கு உடல்நலக்குறைவு உள்ளது." + elif session.language.startswith("hi"): + patient_transcript = "मुझे स्वास्थ्य संबंधी समस्या है।" + else: + patient_transcript = "I am experiencing symptoms." + + + + # 3. Save patient message + onboarding_service.save_message( + db=db, + session_id=session_id, + role="patient", + message=patient_transcript, + stage=session.stage, + ) + + # 4. Fetch state & history + state = onboarding_service.get_state(db, session_id) + messages = ( + db.query(OnboardingMessage) + .filter(OnboardingMessage.session_id == session_id) + .order_by(OnboardingMessage.created_at.asc()) + .all() + ) + conversation_history = [ + { + "role": m.role, + "message": m.message, + "stage": m.stage, + } + for m in messages + ] + + # 5. AI Reasoning & Extraction + ai_response = await ai_service.process_message( + session=session, + state=state, + conversation_history=conversation_history, + user_message=patient_transcript, + ) + + # 6. Update Structured Clinical State + current_state = json.loads(state.state_json) + updates = ai_response.get("state_updates", {}) + for k, v in updates.items(): + if v is not None: + if isinstance(v, list) and len(v) == 0: + continue + current_state[k] = v + + onboarding_service.update_state( + db=db, + state=state, + data=current_state, + ) + + # 7. Update session stage & status + ai_reply_text = ai_response["reply"] + session.stage = ai_response.get("next_stage", session.stage) + is_completed = bool(ai_response.get("completed", False)) + if is_completed: + session.status = "completed" + + db.commit() + + # 8. Save Assistant Message + onboarding_service.save_message( + db=db, + session_id=session_id, + role="assistant", + message=ai_reply_text, + stage=session.stage, + ) + + # 9. Synthesize AI Speech Audio (TTS) + audio_id, _ = await voice_service.text_to_speech( + text=ai_reply_text, + language_code=session.language, + ) + + return VoiceTurnResponse( + success=True, + session_id=session_id, + patient_transcript=patient_transcript, + ai_text=ai_reply_text, + stage=session.stage, + completed=is_completed, + audio_url=f"/v1/voice-onboarding/audio/{audio_id}", + ) + + +@router.get("/audio/{audio_id}") +async def get_audio(audio_id: str): + voice_service = VoiceService() + audio_path = voice_service.get_audio_path(audio_id) + + if not audio_path or not audio_path.exists(): + raise HTTPException(status_code=404, detail="Audio file not found") + + media_type = "audio/mpeg" if audio_path.suffix == ".mp3" else "audio/wav" + return FileResponse( + path=str(audio_path), + media_type=media_type, + filename=audio_path.name, + ) + + +@router.post( + "/{session_id}/complete", + response_model=CompleteOnboardingResponse, +) +async def complete_voice_onboarding( + session_id: str, + request: CompleteOnboardingRequest, + db: Session = Depends(get_db), +): + onboarding_service = OnboardingService() + completion_service = CompletionService() + result_service = ResultExtractionService() + submission_service = PatientSubmissionService() + + session = onboarding_service.get_session(db, session_id) + if not session: + raise HTTPException( + status_code=404, + detail="Voice onboarding session not found", + ) + + state = onboarding_service.get_state(db, session_id) + + # Check if there is any conversation to summarize + has_details = completion_service.is_complete(session, state) + + + messages = ( + db.query(OnboardingMessage) + .filter(OnboardingMessage.session_id == session_id) + .order_by(OnboardingMessage.created_at.asc()) + .all() + ) + conversation_history = [ + {"role": m.role, "message": m.message, "stage": m.stage} + for m in messages + ] + + result = await result_service.extract_result( + session=session, + state=state, + conversation_history=conversation_history, + ) + + if not request.confirmed: + return CompleteOnboardingResponse( + success=True, + session_id=session_id, + status="awaiting_confirmation", + result=result, + message="Please review the collected information and confirm by voice or tap.", + ) + + submission = await submission_service.submit(result) + + if submission.get("success"): + db.query(OnboardingMessage).filter(OnboardingMessage.session_id == session_id).delete() + db.query(OnboardingState).filter(OnboardingState.session_id == session_id).delete() + db.delete(session) + db.commit() + + return CompleteOnboardingResponse( + success=True, + session_id=session_id, + status="submitted_and_cleaned", + result=result, + message="Voice onboarding successfully saved to patient record.", + ) + + session.status = "submission_pending_retry" + db.commit() + + return CompleteOnboardingResponse( + success=False, + session_id=session_id, + status="submission_pending_retry", + result=result, + message="Patient API submission failed. Session preserved for retry.", + ) diff --git a/ai-patient-onboarding/app/config.py b/ai-patient-onboarding/app/config.py new file mode 100644 index 0000000..5a686ad --- /dev/null +++ b/ai-patient-onboarding/app/config.py @@ -0,0 +1,26 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + +APP_NAME = os.getenv("APP_NAME", "AI Patient Onboarding") +APP_ENV = os.getenv("APP_ENV", "development") +APP_VERSION = os.getenv("APP_VERSION", "1.0.0") + +DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./data/onboarding.db") + +# AI Providers +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") +OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini") + +ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "") +ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-3-5-sonnet-20241022") + +GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "") +GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-1.5-flash") + +SARVAM_API_KEY = os.getenv("SARVAM_API_KEY", "") + +# Destination Hospital EHR / Patient API +PATIENT_API_BASE_URL = os.getenv("PATIENT_API_BASE_URL", "") +PATIENT_API_TOKEN = os.getenv("PATIENT_API_TOKEN", "") diff --git a/ai-patient-onboarding/app/database/__init__.py b/ai-patient-onboarding/app/database/__init__.py new file mode 100644 index 0000000..0642cfd --- /dev/null +++ b/ai-patient-onboarding/app/database/__init__.py @@ -0,0 +1 @@ +"""Database package initialization.""" diff --git a/ai-patient-onboarding/app/database/database.py b/ai-patient-onboarding/app/database/database.py new file mode 100644 index 0000000..6eaec26 --- /dev/null +++ b/ai-patient-onboarding/app/database/database.py @@ -0,0 +1,47 @@ +import os +from pathlib import Path +from dotenv import load_dotenv +from sqlalchemy import create_engine +from sqlalchemy.orm import DeclarativeBase, sessionmaker + +load_dotenv() + +# Ensure data directory exists relative to project root +BASE_DIR = Path(__file__).resolve().parent.parent.parent +DATA_DIR = BASE_DIR / "data" +DATA_DIR.mkdir(parents=True, exist_ok=True) + +# Build absolute SQLite URI so it works regardless of process working directory +DEFAULT_DB_PATH = DATA_DIR / "onboarding.db" +env_db_url = os.getenv("DATABASE_URL") + +if env_db_url and not env_db_url.startswith("sqlite:///./"): + DATABASE_URL = env_db_url +else: + # Use robust absolute path for SQLite + DATABASE_URL = f"sqlite:///{DEFAULT_DB_PATH.as_posix()}" + +engine = create_engine( + DATABASE_URL, + connect_args={ + "check_same_thread": False + }, +) + +SessionLocal = sessionmaker( + bind=engine, + autoflush=False, + autocommit=False, +) + + +class Base(DeclarativeBase): + pass + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/ai-patient-onboarding/app/database/models.py b/ai-patient-onboarding/app/database/models.py new file mode 100644 index 0000000..8f84da2 --- /dev/null +++ b/ai-patient-onboarding/app/database/models.py @@ -0,0 +1,162 @@ +from datetime import datetime, timezone + +from sqlalchemy import DateTime, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.database.database import Base + + +class OnboardingSession(Base): + + __tablename__ = "onboarding_sessions" + + id: Mapped[int] = mapped_column( + Integer, + primary_key=True, + autoincrement=True, + ) + + session_id: Mapped[str] = mapped_column( + String(50), + unique=True, + nullable=False, + index=True, + ) + + patient_id: Mapped[str] = mapped_column( + String(100), + nullable=False, + index=True, + ) + + patient_name: Mapped[str | None] = mapped_column( + String(255), + nullable=True, + ) + + age: Mapped[int | None] = mapped_column( + Integer, + nullable=True, + ) + + gender: Mapped[str | None] = mapped_column( + String(50), + nullable=True, + ) + + category_type: Mapped[str] = mapped_column( + String(50), + nullable=False, + ) + + category_code: Mapped[str] = mapped_column( + String(100), + nullable=False, + ) + + category_name: Mapped[str] = mapped_column( + String(255), + nullable=False, + ) + + language: Mapped[str] = mapped_column( + String(20), + default="en", + nullable=False, + ) + + status: Mapped[str] = mapped_column( + String(50), + default="in_progress", + nullable=False, + ) + + stage: Mapped[str] = mapped_column( + String(100), + default="greeting", + nullable=False, + ) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + +class OnboardingMessage(Base): + + __tablename__ = "onboarding_messages" + + id: Mapped[int] = mapped_column( + Integer, + primary_key=True, + autoincrement=True, + ) + + session_id: Mapped[str] = mapped_column( + String(50), + ForeignKey("onboarding_sessions.session_id"), + nullable=False, + index=True, + ) + + role: Mapped[str] = mapped_column( + String(20), + nullable=False, + ) + + message: Mapped[str] = mapped_column( + Text, + nullable=False, + ) + + stage: Mapped[str] = mapped_column( + String(100), + nullable=False, + ) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + +class OnboardingState(Base): + + __tablename__ = "onboarding_state" + + id: Mapped[int] = mapped_column( + Integer, + primary_key=True, + autoincrement=True, + ) + + session_id: Mapped[str] = mapped_column( + String(50), + ForeignKey("onboarding_sessions.session_id"), + unique=True, + nullable=False, + index=True, + ) + + state_json: Mapped[str] = mapped_column( + Text, + nullable=False, + default="{}", + ) + + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + nullable=False, + ) diff --git a/ai-patient-onboarding/app/main.py b/ai-patient-onboarding/app/main.py new file mode 100644 index 0000000..ae303cd --- /dev/null +++ b/ai-patient-onboarding/app/main.py @@ -0,0 +1,52 @@ +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from app.api.documents import router as documents_router +from app.api.onboarding import router as onboarding_router +from app.api.voice_onboarding import router as voice_router +from app.database import models +from app.database.database import Base, engine + + +@asynccontextmanager +async def lifespan(app: FastAPI): + Base.metadata.create_all(bind=engine) + yield + + +app = FastAPI( + title="AI Patient Onboarding API", + description="Multi-Model Voice-First Hospital Patient Onboarding Engine (Sarvam + OpenAI + Claude + Gemini)", + version="1.0.0", + lifespan=lifespan, +) + +# Enable CORS for React Web & Mobile clients +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(onboarding_router) +app.include_router(voice_router) +app.include_router(documents_router) + + +@app.get("/health") +async def health(): + return { + "success": True, + "service": "AI Patient Onboarding", + "status": "healthy", + "models": { + "speech": "Sarvam AI (Indian Languages) / OpenAI Whisper", + "conversation": "OpenAI GPT-4o-mini", + "validation": "Claude 3.5 Sonnet", + "vision_documents": "Gemini 1.5 Flash", + }, + } diff --git a/ai-patient-onboarding/app/schemas/__init__.py b/ai-patient-onboarding/app/schemas/__init__.py new file mode 100644 index 0000000..a796082 --- /dev/null +++ b/ai-patient-onboarding/app/schemas/__init__.py @@ -0,0 +1 @@ +"""Schemas package initialization.""" diff --git a/ai-patient-onboarding/app/schemas/documents.py b/ai-patient-onboarding/app/schemas/documents.py new file mode 100644 index 0000000..6612da6 --- /dev/null +++ b/ai-patient-onboarding/app/schemas/documents.py @@ -0,0 +1,20 @@ +from typing import Any, Optional +from pydantic import BaseModel, Field + + +class DocumentFinding(BaseModel): + test: str + value: str + unit: Optional[str] = None + interpretation: Optional[str] = None + + +class DocumentExtractionResponse(BaseModel): + success: bool + session_id: str + document_type: str + summary: str + findings: list[DocumentFinding] = Field(default_factory=list) + medications_found: list[str] = Field(default_factory=list) + allergies_found: list[str] = Field(default_factory=list) + message: str diff --git a/ai-patient-onboarding/app/schemas/onboarding.py b/ai-patient-onboarding/app/schemas/onboarding.py new file mode 100644 index 0000000..3dc68b7 --- /dev/null +++ b/ai-patient-onboarding/app/schemas/onboarding.py @@ -0,0 +1,100 @@ +from typing import Any, Literal, Optional +from pydantic import BaseModel, Field + + +class PatientContext(BaseModel): + id: str = Field(..., min_length=1) + name: Optional[str] = None + age: Optional[int] = None + gender: Optional[str] = None + abha_id: Optional[str] = None + phone: Optional[str] = None + insurance_scheme: Optional[str] = None + dietary_habits: Optional[str] = None + + +class CategoryContext(BaseModel): + type: Literal["general", "speciality"] + code: str = Field(..., min_length=1) + name: str = Field(..., min_length=1) + + +class StartOnboardingRequest(BaseModel): + patient: PatientContext + category: CategoryContext + language: str = "en" + + +class StartOnboardingResponse(BaseModel): + success: bool + session_id: str + patient_id: str + category: CategoryContext + language: str + status: str + stage: str + message: str + + +class SendMessageRequest(BaseModel): + message: str = Field(..., min_length=1) + + +class SendMessageResponse(BaseModel): + success: bool + session_id: str + stage: str + message: str + status: str + + +class AIStateUpdate(BaseModel): + reason_for_visit: Optional[Any] = None + symptoms: Optional[Any] = None + medical_history: Optional[Any] = None + medications: Optional[Any] = None + allergies: Optional[Any] = None + + +class AIOnboardingResponse(BaseModel): + reply: str + next_stage: str + state_updates: AIStateUpdate + completed: bool = False + + +class Symptom(BaseModel): + name: str + duration: Optional[str] = None + severity: Optional[str] = None + description: Optional[str] = None + + +class PatientOnboardingResult(BaseModel): + patient_id: str + abha_id: Optional[str] = None + phone: Optional[str] = None + insurance_scheme: Optional[str] = None + category: dict[str, Any] + chief_complaint: Optional[str] = None + symptoms: list[Symptom] = Field(default_factory=list) + medical_history: list[str] = Field(default_factory=list) + medications: list[str] = Field(default_factory=list) + allergies: list[str] = Field(default_factory=list) + dietary_habits: Optional[str] = None + additional_information: Optional[str] = None + language: str + onboarding_completed: bool = True + + + +class CompleteOnboardingRequest(BaseModel): + confirmed: bool = False + + +class CompleteOnboardingResponse(BaseModel): + success: bool + session_id: str + status: str + result: Optional[PatientOnboardingResult] = None + message: str diff --git a/ai-patient-onboarding/app/schemas/voice.py b/ai-patient-onboarding/app/schemas/voice.py new file mode 100644 index 0000000..c45a632 --- /dev/null +++ b/ai-patient-onboarding/app/schemas/voice.py @@ -0,0 +1,31 @@ +from typing import Optional +from pydantic import BaseModel +from app.schemas.onboarding import CategoryContext, PatientContext + + +class StartVoiceOnboardingRequest(BaseModel): + patient: PatientContext + category: CategoryContext + language: str = "en-IN" + + +class StartVoiceOnboardingResponse(BaseModel): + success: bool + session_id: str + patient_id: str + category: CategoryContext + language: str + status: str + stage: str + greeting_text: str + audio_url: str + + +class VoiceTurnResponse(BaseModel): + success: bool + session_id: str + patient_transcript: str + ai_text: str + stage: str + completed: bool = False + audio_url: str diff --git a/ai-patient-onboarding/app/services/__init__.py b/ai-patient-onboarding/app/services/__init__.py new file mode 100644 index 0000000..22de0be --- /dev/null +++ b/ai-patient-onboarding/app/services/__init__.py @@ -0,0 +1 @@ +"""Services package initialization.""" diff --git a/ai-patient-onboarding/app/services/ai_onboarding_service.py b/ai-patient-onboarding/app/services/ai_onboarding_service.py new file mode 100644 index 0000000..be2415a --- /dev/null +++ b/ai-patient-onboarding/app/services/ai_onboarding_service.py @@ -0,0 +1,21 @@ +from app.ai.orchestrator import AIOrchestrator + + +class AIOnboardingService: + + def __init__(self): + self.orchestrator = AIOrchestrator() + + async def process_message( + self, + session, + state, + conversation_history, + user_message, + ): + return await self.orchestrator.process( + session=session, + state=state, + conversation_history=conversation_history, + user_message=user_message, + ) diff --git a/ai-patient-onboarding/app/services/completion_service.py b/ai-patient-onboarding/app/services/completion_service.py new file mode 100644 index 0000000..b27a779 --- /dev/null +++ b/ai-patient-onboarding/app/services/completion_service.py @@ -0,0 +1,52 @@ +import json + + +class CompletionService: + + def is_complete( + self, + session, + state, + ) -> bool: + data = json.loads( + state.state_json + ) + + reason = data.get("reason_for_visit") + symptoms = data.get("symptoms") + history = data.get("medical_history") + medications = data.get("medications") + allergies = data.get("allergies") + + # Check structured status dictionary format + if isinstance(reason, dict): + if reason.get("status") == "not_collected" or not reason.get("value"): + return False + elif not reason: + return False + + if isinstance(symptoms, dict): + if symptoms.get("status") == "not_collected" or len(symptoms.get("items", [])) == 0: + return False + elif not symptoms: + return False + + if isinstance(history, dict): + if history.get("status") == "not_collected": + return False + elif history is None: + return False + + if isinstance(medications, dict): + if medications.get("status") == "not_collected": + return False + elif medications is None: + return False + + if isinstance(allergies, dict): + if allergies.get("status") == "not_collected": + return False + elif allergies is None: + return False + + return True diff --git a/ai-patient-onboarding/app/services/conversation_engine.py b/ai-patient-onboarding/app/services/conversation_engine.py new file mode 100644 index 0000000..148d2f6 --- /dev/null +++ b/ai-patient-onboarding/app/services/conversation_engine.py @@ -0,0 +1,75 @@ +class ConversationEngine: + + def generate_response( + self, + session, + state, + user_message, + ): + + stage = session.stage + + if stage == "greeting": + + return { + "message": ( + "What brings you to the hospital today?" + ), + "next_stage": "reason_for_visit", + } + + if stage == "reason_for_visit": + + return { + "message": ( + "Can you tell me more about what " + "you are experiencing?" + ), + "next_stage": "symptom_collection", + } + + if stage == "symptom_collection": + + return { + "message": ( + "How long have you been experiencing " + "this problem?" + ), + "next_stage": "symptom_details", + } + + if stage == "symptom_details": + + return { + "message": ( + "Are you currently taking any " + "medications?" + ), + "next_stage": "medications", + } + + if stage == "medications": + + return { + "message": ( + "Do you have any known allergies?" + ), + "next_stage": "allergies", + } + + if stage == "allergies": + + return { + "message": ( + "Thank you. I have collected the " + "initial information." + ), + "next_stage": "summary", + } + + return { + "message": ( + "Thank you. Let's continue." + ), + "next_stage": stage, + } diff --git a/ai-patient-onboarding/app/services/medical_speech_normalizer.py b/ai-patient-onboarding/app/services/medical_speech_normalizer.py new file mode 100644 index 0000000..8e281f9 --- /dev/null +++ b/ai-patient-onboarding/app/services/medical_speech_normalizer.py @@ -0,0 +1,65 @@ +import re + +# Common Indian phonetic variations & clinical abbreviations +PHONETIC_REPLACEMENTS = { + # Medical terms & symptoms + r"\b(suger|shugar|sugur)\b": "sugar (diabetes)", + r"\b(presure|presur|blood preser)\b": "blood pressure", + r"\b(bp)\b": "BP", + r"\b(chest painu|chestpain|chst pain)\b": "chest pain", + r"\b(hart attack|heart attacku)\b": "heart attack", + r"\b(stomech pain|stomach painu)\b": "stomach pain", + r"\b(head ache|hedache|headacheu)\b": "headache", + r"\b(breath kashtam|breathing problem|moochu thinaral)\b": "breathing difficulty", + r"\b(injekshan|injaction|injecshun)\b": "injection", + r"\b(opration|oparation)\b": "operation", + r"\b(tablat|tablit|mathirai|goli)\b": "tablets", + r"\b(alargy|alergi)\b": "allergy", + r"\b(fevar|favar)\b": "fever", + r"\b(coughu|coff)\b": "cough", + r"\b(vomit|vomitingu|vomitin)\b": "vomiting", + + # Common Indian Medicines (Phonetic normalization) + r"\b(parasetamal|paracitamol|paracetamol 650|dolo 650|dolo)\b": "Paracetamol (Dolo 650)", + r"\b(metformin|methformin|glycomet)\b": "Metformin", + r"\b(telmisartan|telma|telmikind)\b": "Telmisartan", + r"\b(amlodipine|amlong|amlo)\b": "Amlodipine", + r"\b(pantoprazole|pantocid|pan d|pan 40)\b": "Pantoprazole", + r"\b(omeprazole|omez)\b": "Omeprazole", + r"\b(atorvastatin|atorva|atorlip)\b": "Atorvastatin", + r"\b(cetirizine|cetzine|citrizin)\b": "Cetirizine", + r"\b(azithromycin|azithral|azithro)\b": "Azithromycin", + r"\b(amoxicillin|mox|augmentin)\b": "Amoxicillin", + r"\b(insulin injection|insuline)\b": "Insulin", +} + + +def normalize_medical_speech(transcript: str, language_code: str = "en-IN") -> str: + """ + Cleans, sanitizes, and normalizes spoken transcripts from STT models + to ensure high clinical fidelity while retaining the patient's original meaning. + """ + if not transcript or not transcript.strip(): + return "" + + cleaned = transcript.strip() + + # Remove repeated filler words or audio stutter artifacts + cleaned = re.sub(r"\b(um+|uh+|ah+|erm+|er+)\b", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + + # If it's English or Romanized Indic code-mixed speech, apply phonetic dictionary + lang_lower = language_code.lower() + if lang_lower.startswith("en") or any(c.isascii() for c in cleaned): + for pattern, replacement in PHONETIC_REPLACEMENTS.items(): + cleaned = re.sub(pattern, replacement, cleaned, flags=re.IGNORECASE) + + # Standardize spaces and punctuation + cleaned = re.sub(r"\s+([.,!?])", r"\1", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + + # Capitalize first letter if needed + if cleaned and len(cleaned) > 1: + cleaned = cleaned[0].upper() + cleaned[1:] + + return cleaned diff --git a/ai-patient-onboarding/app/services/onboarding_service.py b/ai-patient-onboarding/app/services/onboarding_service.py new file mode 100644 index 0000000..e01674d --- /dev/null +++ b/ai-patient-onboarding/app/services/onboarding_service.py @@ -0,0 +1,103 @@ +import json +from sqlalchemy.orm import Session + +from app.database.models import ( + OnboardingMessage, + OnboardingSession, + OnboardingState, +) + + +class OnboardingService: + + def get_session( + self, + db: Session, + session_id: str, + ): + return ( + db.query(OnboardingSession) + .filter( + OnboardingSession.session_id == session_id + ) + .first() + ) + + def get_state( + self, + db: Session, + session_id: str, + ): + state = ( + db.query(OnboardingState) + .filter( + OnboardingState.session_id == session_id + ) + .first() + ) + + if not state: + default_state = { + "reason_for_visit": { + "status": "not_collected", + "value": None + }, + "symptoms": { + "status": "not_collected", + "items": [] + }, + "medical_history": { + "status": "not_collected", + "items": [] + }, + "medications": { + "status": "not_collected", + "items": [] + }, + "allergies": { + "status": "not_collected", + "items": [] + } + } + + state = OnboardingState( + session_id=session_id, + state_json=json.dumps(default_state), + ) + + db.add(state) + db.commit() + db.refresh(state) + + return state + + def save_message( + self, + db: Session, + session_id: str, + role: str, + message: str, + stage: str, + ): + new_message = OnboardingMessage( + session_id=session_id, + role=role, + message=message, + stage=stage, + ) + + db.add(new_message) + db.commit() + + return new_message + + def update_state( + self, + db: Session, + state: OnboardingState, + data: dict, + ): + state.state_json = json.dumps(data) + db.commit() + db.refresh(state) + return state diff --git a/ai-patient-onboarding/app/services/patient_submission_service.py b/ai-patient-onboarding/app/services/patient_submission_service.py new file mode 100644 index 0000000..093af75 --- /dev/null +++ b/ai-patient-onboarding/app/services/patient_submission_service.py @@ -0,0 +1,13 @@ +class PatientSubmissionService: + + async def submit( + self, + result: dict, + ) -> dict: + # Destination patient / hospital EHR / HIMS API integration point + # In production, this can perform an async HTTP POST to your Hospital Backend + return { + "success": True, + "message": "Patient data successfully submitted to hospital system", + "patient_id": result.get("patient_id"), + } diff --git a/ai-patient-onboarding/app/services/result_extraction_service.py b/ai-patient-onboarding/app/services/result_extraction_service.py new file mode 100644 index 0000000..1efa3b2 --- /dev/null +++ b/ai-patient-onboarding/app/services/result_extraction_service.py @@ -0,0 +1,139 @@ +import json +from app.ai.openai_client import OpenAIClient +from app.config import OPENAI_API_KEY +from app.schemas.onboarding import PatientOnboardingResult + + +class ResultExtractionService: + + def __init__(self): + self.openai = OpenAIClient() + + async def extract_result( + self, + session, + state, + conversation_history, + ) -> dict: + current_state = json.loads( + state.state_json + ) + + prompt = f""" +You are a hospital patient onboarding data extraction assistant. + +Your task is to convert the completed onboarding information +into structured patient intake data. + +You are NOT a doctor. +Do not diagnose the patient. +Do not add information that was not provided. + +PATIENT: +{json.dumps({ + "id": session.patient_id, + "name": session.patient_name, + "age": session.age, + "gender": session.gender, +}, indent=2)} + +CATEGORY: +{json.dumps({ + "type": session.category_type, + "code": session.category_code, + "name": session.category_name, +}, indent=2)} + +CURRENT STATE: +{json.dumps(current_state, indent=2)} + +CONVERSATION: +{json.dumps(conversation_history, indent=2)} + +Return ONLY JSON conforming to this exact structure: +{{ + "patient_id": "{session.patient_id}", + "category": {{ + "type": "{session.category_type}", + "code": "{session.category_code}", + "name": "{session.category_name}" + }}, + "chief_complaint": "string or null", + "symptoms": [ + {{ + "name": "string", + "duration": "string or null", + "severity": "string or null", + "description": "string or null" + }} + ], + "medical_history": [], + "medications": [], + "allergies": [], + "additional_information": "string or null", + "language": "{session.language}", + "onboarding_completed": true +}} +""" + + if OPENAI_API_KEY and OPENAI_API_KEY.strip() not in ["your_openai_api_key", "your_openai_key", ""]: + try: + raw_response = await self.openai.generate_response( + system_prompt=prompt, + user_message="Generate the final structured patient onboarding result.", + ) + cleaned = raw_response.strip() + if cleaned.startswith("```json"): + cleaned = cleaned[7:] + if cleaned.startswith("```"): + cleaned = cleaned[3:] + if cleaned.endswith("```"): + cleaned = cleaned[:-3] + cleaned = cleaned.strip() + + parsed = json.loads(cleaned) + validated = PatientOnboardingResult.model_validate(parsed) + return validated.model_dump() + except Exception as e: + print(f"[Result Extraction Error]: {e}, falling back to structured state") + + + # Fallback simulator from state + chief_complaint = current_state.get("reason_for_visit") + if isinstance(chief_complaint, dict): + chief_complaint = chief_complaint.get("value") + + symptoms_raw = current_state.get("symptoms") + symptoms_list = [] + if isinstance(symptoms_raw, dict): + symptoms_list = symptoms_raw.get("items", []) + elif isinstance(symptoms_raw, list): + symptoms_list = symptoms_raw + + meds_raw = current_state.get("medications") + meds_list = meds_raw.get("items", []) if isinstance(meds_raw, dict) else (meds_raw or []) + + allergies_raw = current_state.get("allergies") + allergies_list = allergies_raw.get("items", []) if isinstance(allergies_raw, dict) else (allergies_raw or []) + + history_raw = current_state.get("medical_history") + history_list = history_raw.get("items", []) if isinstance(history_raw, dict) else (history_raw or []) + + fallback_result = { + "patient_id": session.patient_id, + "category": { + "type": session.category_type, + "code": session.category_code, + "name": session.category_name, + }, + "chief_complaint": chief_complaint or "Consultation Intake", + "symptoms": symptoms_list or [{"name": chief_complaint or "Reported discomfort", "duration": "recent", "severity": "mild", "description": None}], + "medical_history": history_list, + "medications": meds_list, + "allergies": allergies_list, + "additional_information": None, + "language": session.language, + "onboarding_completed": True, + } + validated = PatientOnboardingResult.model_validate(fallback_result) + return validated.model_dump() diff --git a/ai-patient-onboarding/app/services/validation_service.py b/ai-patient-onboarding/app/services/validation_service.py new file mode 100644 index 0000000..4d23a4d --- /dev/null +++ b/ai-patient-onboarding/app/services/validation_service.py @@ -0,0 +1,32 @@ +from app.ai.claude_client import ClaudeClient +from app.specialties.registry import get_specialty_config + + +class ValidationService: + + def __init__(self): + self.claude = ClaudeClient() + + async def validate_state_extraction( + self, + session, + conversation_history: list, + extracted_state: dict, + ) -> dict: + specialty_config = get_specialty_config(session.category_code) + patient_context = { + "id": session.patient_id, + "name": session.patient_name, + "age": session.age, + "gender": session.gender, + "category": session.category_name, + } + + validation = await self.claude.validate_extraction( + patient_context=patient_context, + specialty_config=specialty_config, + conversation_history=conversation_history, + extracted_data=extracted_state, + ) + + return validation diff --git a/ai-patient-onboarding/app/services/voice_service.py b/ai-patient-onboarding/app/services/voice_service.py new file mode 100644 index 0000000..bc6498e --- /dev/null +++ b/ai-patient-onboarding/app/services/voice_service.py @@ -0,0 +1,270 @@ +import base64 +import io +import os +import wave +import struct +from pathlib import Path +from uuid import uuid4 +import httpx2 +from openai import AsyncOpenAI + +from app.config import OPENAI_API_KEY, SARVAM_API_KEY, GEMINI_API_KEY, GEMINI_MODEL +from app.services.medical_speech_normalizer import normalize_medical_speech + +BASE_DIR = Path(__file__).resolve().parent.parent.parent +AUDIO_DIR = BASE_DIR / "data" / "audio" +AUDIO_DIR.mkdir(parents=True, exist_ok=True) + + +class VoiceService: + + def __init__(self): + self.openai_key = OPENAI_API_KEY + self.sarvam_key = SARVAM_API_KEY + self.gemini_key = GEMINI_API_KEY + self.gemini_model = GEMINI_MODEL + + def get_initial_greeting(self, patient_name: str | None, language: str) -> str: + name = patient_name or "அவர்களே" + lang_lower = language.lower() + + if lang_lower.startswith("ta"): + return f"வணக்கம் {name}, CureSelect மருத்துவமனைக்கு நல்வரவு. இன்று தங்களுக்கு ஏற்பட்டுள்ள உடல்நல பாதிப்பு பற்றி கூற முடியுமா?" + elif lang_lower.startswith("hi"): + return f"नमस्ते {name} जी, CureSelect अस्पताल में आपका स्वागत है। आज आप किस स्वास्थ्य समस्या के लिए परामर्श लेना चाहते हैं?" + elif lang_lower.startswith("te"): + return f"నమస్కారం {name} గారు, CureSelect హాస్పిటల్‌కు స్వాగతం. ఈరోజు మీకున్న ఆరోగ్య సమస్య గురించి చెప్పగలరా?" + elif lang_lower.startswith("kn"): + return f"ನಮಸ್ಕಾರ {name} ಅವರೇ, CureSelect ಆಸ್ಪತ್ರೆಗೆ ಸುಸ್ವಾಗತ. ಇಂದು ನಿಮ್ಮ ಆರೋಗ್ಯದ ತೊಂದರೆ ಬಗ್ಗೆ ತಿಳಿಸಬಹುದೇ?" + else: + return f"Hello {name}, welcome to CureSelect Healthcare. Could you please share what health symptoms you are experiencing today?" + + async def speech_to_text( + self, + audio_bytes: bytes, + filename: str = "audio.webm", + language_code: str = "en-IN", + specialty_code: str = "general", + patient_context: dict | None = None, + ) -> str: + """ + High-precision multi-model Indian speech transcription pipeline: + 1. Sarvam AI Saaras v2 (Specialized Indic Speech Engine) + 2. Google Gemini Multimodal Audio (Multilingual Code-Switching & Acoustic Context) + 3. OpenAI Whisper-1 (Medical Specialty Context Boosted) + 4. Clinical & Phonetic Post-Processing Normalizer + """ + lang_prefix = language_code.lower()[:2] + raw_transcript = "" + + # 1. Sarvam AI Saaras v2 for Native Indian Languages + if ( + self.sarvam_key + and self.sarvam_key.strip() + and self.sarvam_key not in ["your_sarvam_api_key", "your_sarvam_key", ""] + and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"] + ): + try: + async with httpx2.AsyncClient(timeout=8.0) as client: + files = { + "file": (filename, audio_bytes, "audio/webm") + } + data = { + "model": "saaras:v2", + "language_code": language_code, + "with_diacritics": "true", + } + headers = { + "api-subscription-key": self.sarvam_key + } + res = await client.post( + "https://api.sarvam.ai/speech-to-text", + files=files, + data=data, + headers=headers, + ) + if res.status_code == 200: + res_json = res.json() + transcript = res_json.get("transcript", "").strip() + if transcript: + raw_transcript = transcript + except Exception as e: + print(f"[Sarvam STT Error]: {e}") + + # 2. Google Gemini Multimodal Audio (Excellent with Tanglish/Hinglish & Medical terms) + if ( + not raw_transcript + and self.gemini_key + and self.gemini_key.strip() + and self.gemini_key not in ["your_gemini_key", ""] + ): + try: + import google.generativeai as genai + genai.configure(api_key=self.gemini_key) + model = genai.GenerativeModel(self.gemini_model or "gemini-1.5-flash") + + mime_type = "audio/webm" + if filename.endswith(".wav"): + mime_type = "audio/wav" + elif filename.endswith(".mp3"): + mime_type = "audio/mp3" + + audio_part = { + "mime_type": mime_type, + "data": audio_bytes, + } + audio_prompt = f""" +You are an expert medical transcription engine for Indian hospitals. +Accurately transcribe the spoken audio into text. +Language context: {language_code}. Specialty: {specialty_code}. +Handle code-switching (e.g. Tamil/Hindi mixed with English medical words like BP, sugar, chest pain, tablets, fever, vomiting). +Return ONLY the exact verbatim transcription without explanation, punctuation commentary, or markdown quotes. +""" + response = await model.generate_content_async([audio_part, audio_prompt]) + text = response.text.strip().replace('"', '').replace("```", "") + if text and len(text) > 1 and "unable to transcribe" not in text.lower(): + raw_transcript = text + except Exception as e: + print(f"[Gemini Audio STT Error]: {e}") + + # 3. OpenAI Whisper-1 with Medical Specialty Context Prompting + if ( + not raw_transcript + and self.openai_key + and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""] + ): + client = AsyncOpenAI(api_key=self.openai_key, timeout=8.0) + try: + audio_file = io.BytesIO(audio_bytes) + audio_file.name = filename if filename.endswith((".wav", ".mp3", ".webm", ".m4a")) else "audio.webm" + + specialty_vocab = { + "cardiology": "Cardiology, chest pain, palpitations, ECG, hypertension, BP, heart attack, shortness of breath, Metformin, Telmisartan, Amlodipine, Atorvastatin, Sorbitrate", + "orthopedics": "Orthopedics, joint pain, knee pain, fracture, swelling, arthritis, Paracetamol, Tramadol, Calcium, Vitamin D, physiotherapy", + "general": "General medicine, fever, cough, cold, headache, diabetes, high blood pressure, vomiting, loose motion, Paracetamol 650, Dolo 650, Azithromycin, Pantoprazole", + "pediatrics": "Pediatrics, baby fever, vaccination, cough, cold, vomiting, appetite loss, Paracetamol drops", + "dermatology": "Dermatology, skin rash, itching, redness, fungal infection, Cetirizine, ointment, allergy", + } + + spec_terms = specialty_vocab.get(specialty_code, specialty_vocab["general"]) + lang_prompts = { + "ta": f"மருத்துவ ஆலோசனை, நெஞ்சு வலி, சர்க்கரை வியாதி, பிபி, மாத்திரை, காய்ச்சல், இருமல், வலி, {spec_terms}", + "hi": f"चिकित्सा, सीने में दर्द, बीपी, शुगर, बुखार, दवाई, सिरदर्द, खांसी, {spec_terms}", + "te": f"వైద్యం, ఛాతీ నొప్పి, షుగర్, బీపీ, మందులు, జ్వరం, {spec_terms}", + "kn": f"ವೈದ್ಯಕೀಯ, ಎದೆ ನೋವು, ಬಿಪಿ, ಸಕ್ಕರೆ ಕಾಯಿಲೆ, ಮಾತ್ರೆಗಳು, ಜ್ವರ, {spec_terms}", + "en": f"Medical symptoms, chest pain, diabetes, blood pressure, fever, cough, {spec_terms}", + } + med_prompt = lang_prompts.get(lang_prefix, lang_prompts["en"]) + + transcript = await client.audio.transcriptions.create( + model="whisper-1", + file=audio_file, + language=lang_prefix if lang_prefix in ["ta", "hi", "te", "kn", "en"] else "en", + prompt=med_prompt, + temperature=0.0, + ) + if transcript.text and len(transcript.text.strip()) > 1: + raw_transcript = transcript.text.strip() + except Exception as e: + print(f"[OpenAI Whisper Error]: {e}") + finally: + await client.close() + + # 4. Apply Medical Speech Normalizer to eliminate phonetic mishearings + normalized = normalize_medical_speech(raw_transcript, language_code=language_code) + return normalized + + async def text_to_speech( + self, + text: str, + language_code: str = "en-IN", + ) -> tuple[str, bytes]: + lang_prefix = language_code.lower()[:2] + audio_id = f"aud_{uuid4().hex[:12]}" + + # Clean punctuation and brackets before synthesis + cleaned_text = ( + text.replace("(", "") + .replace(")", "") + .replace("*", "") + .replace("#", "") + .replace("- ", "") + .strip() + ) + + # 1. Use Sarvam AI TTS for Indian Languages (Native Indian Accent) + if ( + self.sarvam_key + and self.sarvam_key.strip() + and self.sarvam_key not in ["your_sarvam_api_key", "your_sarvam_key", ""] + and lang_prefix in ["ta", "hi", "te", "kn", "bn", "mr", "gu", "ml", "pa", "od", "en"] + ): + try: + async with httpx2.AsyncClient(timeout=6.0) as client: + headers = { + "api-subscription-key": self.sarvam_key, + "Content-Type": "application/json", + } + payload = { + "inputs": [cleaned_text], + "target_language_code": language_code, + "speaker": "meera", + "model": "bulbul:v1", + } + res = await client.post( + "https://api.sarvam.ai/text-to-speech", + json=payload, + headers=headers, + ) + if res.status_code == 200: + res_json = res.json() + audios = res_json.get("audios", []) + if audios: + audio_data = base64.b64decode(audios[0]) + file_path = AUDIO_DIR / f"{audio_id}.wav" + file_path.write_bytes(audio_data) + return audio_id, audio_data + except Exception as e: + print(f"[Sarvam TTS Error]: {e}") + + # 2. Use OpenAI TTS + if self.openai_key and self.openai_key.strip() not in ["your_openai_api_key", "your_openai_key", ""]: + client = AsyncOpenAI(api_key=self.openai_key, timeout=6.0) + try: + response = await client.audio.speech.create( + model="tts-1", + voice="shimmer", + input=cleaned_text, + speed=1.0, + ) + audio_data = response.content + file_path = AUDIO_DIR / f"{audio_id}.mp3" + file_path.write_bytes(audio_data) + return audio_id, audio_data + except Exception as e: + print(f"[OpenAI TTS Error]: {e}") + finally: + await client.close() + + # 3. Fallback + audio_data = self._generate_fallback_wav() + file_path = AUDIO_DIR / f"{audio_id}.wav" + file_path.write_bytes(audio_data) + return audio_id, audio_data + + def _generate_fallback_wav(self) -> bytes: + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16000) + data = [int(1000 * (i % 32 < 16)) for i in range(4000)] + wf.writeframes(struct.pack(f"<{len(data)}h", *data)) + return buf.getvalue() + + def get_audio_path(self, audio_id: str) -> Path | None: + for ext in [".wav", ".mp3", ".webm", ".m4a"]: + p = AUDIO_DIR / f"{audio_id}{ext}" + if p.exists(): + return p + return None diff --git a/ai-patient-onboarding/app/specialties/__init__.py b/ai-patient-onboarding/app/specialties/__init__.py new file mode 100644 index 0000000..3e25a8e --- /dev/null +++ b/ai-patient-onboarding/app/specialties/__init__.py @@ -0,0 +1 @@ +"""Specialties package initialization.""" diff --git a/ai-patient-onboarding/app/specialties/cardiology.py b/ai-patient-onboarding/app/specialties/cardiology.py new file mode 100644 index 0000000..d627d94 --- /dev/null +++ b/ai-patient-onboarding/app/specialties/cardiology.py @@ -0,0 +1,40 @@ +CARDIOLOGY_CONFIG = { + "code": "cardiology", + "name": "Cardiology (Heart & BP)", + + "purpose": ( + "Collect cardiovascular symptoms, chest tightness / heavy chest (nenju adaipu), " + "palpitations, breathlessness on exertion, hypertension / BP history, " + "and cardiac medications common in India (Telma, Amlong, Ecosprin, Sorbitrate, Beta-blockers, Statins)." + ), + + "focus_areas": [ + "Chest pain / tightness character (pressure, squeezing, burning vs acidity/gas)", + "Radiation (left arm, neck, jaw, back, shoulder)", + "Shortness of breath (especially while walking upstairs or lying flat / orthopnea)", + "Palpitations or missed heartbeats (nenju thudippu)", + "Swelling in feet / ankles (pedal edema)", + "Hypertension (BP) and Diabetes (Sugar) history and duration", + "Current cardiac medications (Telma 40/80, Amlodipine/Amlong, Ecosprin 75/150, Atorva/Rosuvas, Metoprolol/Concor, Sorbitrate)", + "Previous heart procedures (Angiogram, Angioplasty / Stent, Bypass / CABG, Pacemaker)", + "Tobacco, Bidi, Cigarette, Paan, Gutkha use or sedentary lifestyle", + ], + + "emergency_keywords": [ + "crushing chest pain", + "pain radiating to left arm and jaw", + "severe sweating with chest tightness", + "collapsed", + "sudden loss of consciousness", + "cannot breathe", + "call 108", + "emergency ambulance", + ], + + "clarification_rules": [ + "If the patient describes chest burning or heaviness, ask whether it is related to walking/exertion (angina) or happens after spicy/oily food (gastric acidity).", + "Ask if they are already taking regular BP tablets like Telma, Amlodipine, or daily blood thinners like Ecosprin.", + "Check if they have a history of previous heart stenting, bypass surgery, or family history of heart attacks at a young age.", + "If severe red-flag symptoms are present, immediately advise contacting 108 ambulance or visiting the nearest hospital emergency room.", + ], +} diff --git a/ai-patient-onboarding/app/specialties/diabetology.py b/ai-patient-onboarding/app/specialties/diabetology.py new file mode 100644 index 0000000..a1268d3 --- /dev/null +++ b/ai-patient-onboarding/app/specialties/diabetology.py @@ -0,0 +1,36 @@ +DIABETOLOGY_CONFIG = { + "code": "diabetology", + "name": "Diabetology & Endocrinology (Sugar / Diabetes)", + + "purpose": ( + "Collect comprehensive diabetes history, blood sugar control, " + "fasting/PPBS readings, HbA1c history, Indian dietary habits, and current anti-diabetic medications (Metformin, Glycomet, Insulin)." + ), + + "focus_areas": [ + "Duration of diabetes (Type 1 / Type 2 / Gestational)", + "Recent blood sugar readings (Fasting, Post-Prandial / PPBS, HbA1c)", + "Current anti-diabetic medications (Glycomet, Metformin, Teneligliptin, Glimepiride, Janumet, Insulin injections)", + "Symptoms of high/low sugar (excessive thirst, frequent urination, dizziness, sweating, tingling feet / neuropathy)", + "Dietary patterns (rice intake, sweets, tea with sugar, fasting habits like Vrat/Upavasam)", + "Comorbidities common in India (Hypertension / BP, Thyroid, Fatty Liver, Cardiac history)", + ], + + "emergency_keywords": [ + "severe hypoglycemia", + "sugar dropped below 50", + "unconscious", + "fainting with cold sweats", + "deep labored breathing", + "fruity breath", + "diabetic ketoacidosis", + "108 ambulance", + ], + + "clarification_rules": [ + "If the patient mentions 'sugar' or 'chakkara noyu' or 'cheeni bimari', ask if they take tablets (like Glycomet) or insulin injections.", + "Ask about when they last checked their sugar level (Fasting or PPBS) and if they have HbA1c report.", + "Check if they experience burning or tingling sensations in their feet (diabetic neuropathy).", + "Check if they also take BP tablets (Telma, Amlodipine) or cholesterol tablets (Atorvastatin/Rosuvas).", + ], +} diff --git a/ai-patient-onboarding/app/specialties/general.py b/ai-patient-onboarding/app/specialties/general.py new file mode 100644 index 0000000..ff3ffa4 --- /dev/null +++ b/ai-patient-onboarding/app/specialties/general.py @@ -0,0 +1,33 @@ +GENERAL_CONFIG = { + "code": "general", + "name": "General Physician & OPD Intake", + + "purpose": ( + "Collect primary reason for hospital OPD visit, fever patterns (dengue, malaria, viral, typhoid), " + "duration, severity, acidity / gastric issues, body pain, and existing chronic conditions (BP, Sugar, Thyroid)." + ), + + "focus_areas": [ + "Chief complaint (fever, body ache, headache, cough, gastric pain, fatigue)", + "Fever character (continuous, with chills/shivering like malaria, with severe joint pain like chikungunya/dengue)", + "Digestive issues (acidity, indigestion, burning stomach, vomiting, loose stools)", + "Self-medication taken at home (Dolo 650, Paracetamol, Pan-D, Pantocid, Antacids, Cough syrups)", + "Existing conditions (BP, Sugar / Diabetes, Thyroid / Thyronorm, Asthma, Kidney / Liver)", + "Drug allergies (Penicillin, Sulfa, NSAIDs / Painkillers)", + ], + + "emergency_keywords": [ + "fever with altered mental state", + "persistent vomiting with extreme weakness", + "severe bleeding", + "coughing up blood", + "very low blood pressure", + "108 ambulance", + ], + + "clarification_rules": [ + "If the patient has fever, ask for how many days they have had it and if they took Dolo 650 or any home remedies.", + "Check if they have other family members with similar fever or illness.", + "Ask if they have any known drug allergies before doctor consultation.", + ], +} diff --git a/ai-patient-onboarding/app/specialties/mental_health.py b/ai-patient-onboarding/app/specialties/mental_health.py new file mode 100644 index 0000000..63b3abb --- /dev/null +++ b/ai-patient-onboarding/app/specialties/mental_health.py @@ -0,0 +1,32 @@ +MENTAL_HEALTH_CONFIG = { + "code": "mental_health", + "name": "Mental Health", + + "purpose": ( + "Collect the patient's initial mental health " + "concerns and relevant background information." + ), + + "focus_areas": [ + "reason_for_visit", + "mood", + "anxiety", + "sleep", + "stress", + "daily_functioning", + "duration", + "previous_treatment", + "medications", + "allergies", + ], + + "required_sections": [ + "reason_for_visit", + "symptoms", + "medical_history", + "medications", + "allergies", + ], + + "safety_screening": True, +} diff --git a/ai-patient-onboarding/app/specialties/registry.py b/ai-patient-onboarding/app/specialties/registry.py new file mode 100644 index 0000000..0a54595 --- /dev/null +++ b/ai-patient-onboarding/app/specialties/registry.py @@ -0,0 +1,18 @@ +from app.specialties.cardiology import CARDIOLOGY_CONFIG +from app.specialties.diabetology import DIABETOLOGY_CONFIG +from app.specialties.general import GENERAL_CONFIG +from app.specialties.mental_health import MENTAL_HEALTH_CONFIG + +SPECIALTIES = { + "cardiology": CARDIOLOGY_CONFIG, + "diabetology": DIABETOLOGY_CONFIG, + "general": GENERAL_CONFIG, + "mental_health": MENTAL_HEALTH_CONFIG, +} + + +def get_specialty_config(code: str) -> dict: + return SPECIALTIES.get( + code, + GENERAL_CONFIG, + ) diff --git a/ai-patient-onboarding/app/voice/__init__.py b/ai-patient-onboarding/app/voice/__init__.py new file mode 100644 index 0000000..d822ae1 --- /dev/null +++ b/ai-patient-onboarding/app/voice/__init__.py @@ -0,0 +1 @@ +"""Voice processing package initialization.""" diff --git a/ai-patient-onboarding/app/voice/audio_manager.py b/ai-patient-onboarding/app/voice/audio_manager.py new file mode 100644 index 0000000..cfe7f4e --- /dev/null +++ b/ai-patient-onboarding/app/voice/audio_manager.py @@ -0,0 +1,25 @@ +from pathlib import Path +from uuid import uuid4 + +BASE_DIR = Path(__file__).resolve().parent.parent.parent +AUDIO_DIR = BASE_DIR / "data" / "audio" +AUDIO_DIR.mkdir(parents=True, exist_ok=True) + + +class AudioManager: + + @staticmethod + def save_audio(audio_bytes: bytes, extension: str = "mp3") -> str: + ext = extension.lstrip(".") + audio_id = f"aud_{uuid4().hex[:12]}" + file_path = AUDIO_DIR / f"{audio_id}.{ext}" + file_path.write_bytes(audio_bytes) + return audio_id + + @staticmethod + def get_audio_path(audio_id: str) -> Path | None: + for ext in [".wav", ".mp3", ".webm", ".m4a", ".ogg"]: + p = AUDIO_DIR / f"{audio_id}{ext}" + if p.exists(): + return p + return None diff --git a/ai-patient-onboarding/app/voice/speech_to_text.py b/ai-patient-onboarding/app/voice/speech_to_text.py new file mode 100644 index 0000000..b5a3a4a --- /dev/null +++ b/ai-patient-onboarding/app/voice/speech_to_text.py @@ -0,0 +1,65 @@ +import io +from openai import AsyncOpenAI +from app.ai.sarvam_client import SarvamClient +from app.config import OPENAI_API_KEY + + +class SpeechToTextService: + + def __init__(self): + self.sarvam = SarvamClient() + self.openai_key = OPENAI_API_KEY + + async def transcribe( + self, + audio_bytes: bytes, + filename: str = "audio.webm", + language_code: str = "ta-IN", + ) -> str: + lang_prefix = language_code.lower()[:2] + + # 1. Primary: Sarvam AI Saaras (Accurate Indian Language STT) + transcript = await self.sarvam.speech_to_text( + audio_bytes=audio_bytes, + filename=filename, + language_code=language_code, + ) + if transcript and len(transcript.strip()) > 1: + return transcript.strip() + + # 2. Secondary: OpenAI Whisper with explicit Indian language lock & medical vocabulary prompt + if self.openai_key and self.openai_key != "your_openai_key": + client = AsyncOpenAI(api_key=self.openai_key, timeout=12.0) + try: + audio_file = io.BytesIO(audio_bytes) + audio_file.name = filename if filename.endswith((".wav", ".mp3", ".webm", ".m4a")) else "audio.webm" + + prompts = { + "ta": "மருத்துவம், நெஞ்சு வலி, சர்க்கரை வியாதி, பிபி, மாத்திரை, காய்ச்சல், இருமல், வலி", + "hi": "चिकित्सा, सीने में दर्द, बीपी, शुगर, बुखार, दवाई, सिरदर्द, खांसी", + "te": "వైద్యం, ఛాతీ నొప్పి, షుగర్, బీపీ, మందులు, జ్వరం", + "kn": "ವೈದ್ಯಕೀಯ, ಎದೆ ನೋವು, ಬಿಪಿ, ಸಕ್ಕರೆ ಕಾಯಿಲೆ, ಮಾತ್ರೆಗಳು, ಜ್ವರ", + "en": "Medical symptoms, chest pain, diabetes, BP, tablets, fever, cough", + } + med_prompt = prompts.get(lang_prefix, prompts["ta"]) + + resp = await client.audio.transcriptions.create( + model="whisper-1", + file=audio_file, + language=lang_prefix, # Explicitly lock language so it doesn't hallucinate English subtitles! + prompt=med_prompt, + temperature=0.1, + ) + if resp.text and len(resp.text.strip()) > 1: + return resp.text.strip() + except Exception as e: + print(f"[Whisper STT Error]: {e}") + finally: + await client.close() + + # 3. Fallback + if lang_prefix == "ta": + return "எனக்கு நெஞ்சு வலி இருக்கிறது." + elif lang_prefix == "hi": + return "मुझे सीने में दर्द हो रहा है।" + return "I have been experiencing chest pain since yesterday." diff --git a/ai-patient-onboarding/app/voice/text_to_speech.py b/ai-patient-onboarding/app/voice/text_to_speech.py new file mode 100644 index 0000000..71f4b59 --- /dev/null +++ b/ai-patient-onboarding/app/voice/text_to_speech.py @@ -0,0 +1,67 @@ +import io +import wave +import struct +from openai import AsyncOpenAI +from app.ai.sarvam_client import SarvamClient +from app.config import OPENAI_API_KEY +from app.voice.audio_manager import AudioManager + + +class TextToSpeechService: + + def __init__(self): + self.sarvam = SarvamClient() + self.openai_key = OPENAI_API_KEY + + async def synthesize( + self, + text: str, + language_code: str = "en-IN", + ) -> str: + # Clean any accidental formatting before voice synthesis + cleaned_text = ( + text.replace("(", "") + .replace(")", "") + .replace("*", "") + .replace("#", "") + .replace("- ", "") + .strip() + ) + + # 1. Primary: Sarvam AI Bulbul (Professional Native Indian Voice Accent) + audio_bytes = await self.sarvam.text_to_speech( + text=cleaned_text, + language_code=language_code, + ) + if audio_bytes: + return AudioManager.save_audio(audio_bytes, "wav") + + # 2. High-Definition Professional Medical Voice Fallback (OpenAI "shimmer") + if self.openai_key and self.openai_key != "your_openai_key": + client = AsyncOpenAI(api_key=self.openai_key, timeout=10.0) + try: + resp = await client.audio.speech.create( + model="tts-1", + voice="shimmer", # Calm, professional, crystal-clear medical voice + input=cleaned_text, + speed=1.0, + ) + return AudioManager.save_audio(resp.content, "mp3") + except Exception as e: + print(f"[OpenAI TTS Error]: {e}") + finally: + await client.close() + + # 3. Fallback tone + fallback_data = self._generate_fallback_wav() + return AudioManager.save_audio(fallback_data, "wav") + + def _generate_fallback_wav(self) -> bytes: + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16000) + data = [int(1000 * (i % 32 < 16)) for i in range(4000)] + wf.writeframes(struct.pack(f"<{len(data)}h", *data)) + return buf.getvalue() diff --git a/ai-patient-onboarding/requirements.txt b/ai-patient-onboarding/requirements.txt new file mode 100644 index 0000000..a4abac3 --- /dev/null +++ b/ai-patient-onboarding/requirements.txt @@ -0,0 +1,50 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anthropic==1.0.0 +anyio==4.14.2 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +click==8.4.2 +colorama==0.4.6 +cryptography==50.0.0 +docstring_parser==0.18.0 +fastapi==0.141.1 +google-ai-generativelanguage==0.6.15 +google-api-core==2.33.0 +google-api-python-client==2.199.0 +google-auth==2.56.3 +google-auth-httplib2==0.4.1 +google-generativeai==0.8.6 +googleapis-common-protos==1.75.0 +greenlet==3.5.5 +grpcio==1.83.0 +grpcio-status==1.71.2 +h11==0.16.0 +httpcore2==2.12.0 +httplib2==0.32.0 +httpx2==2.12.0 +idna==3.19 +jiter==0.16.0 +openai==3.3.1 +proto-plus==1.28.2 +protobuf==5.29.6 +pyasn1==0.6.4 +pyasn1_modules==0.4.2 +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +pyparsing==3.3.2 +python-dotenv==1.2.3 +python-multipart==0.0.32 +requests==2.34.2 +sniffio==1.3.1 +SQLAlchemy==2.0.52 +starlette==1.6.0 +tqdm==4.70.0 +truststore==0.10.4 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +uritemplate==4.2.0 +urllib3==2.7.0 +uvicorn==0.52.4