feat: complete real-time voice-ai engine upgrades and frontend integration guide

2026-09-03 12:34:55 +05:30
parent c23d979338
commit 37440993f7
56 changed files with 2521 additions and 3040 deletions
@@ -0,0 +1,256 @@
+# CureSelect AI — Real-Time Voice Intake & EHR Frontend Integration Guide
+
+**Version**: `2.0.0 (Production Release)`  
+**Target Audience**: Frontend Engineers (React, Next.js, Vue, React Native, Flutter, Vanilla TypeScript)  
+**Backend Protocol**: Full-Duplex WebSockets (`ws://`) & HTTP/1.1 REST  
+**Authentication Header**: `X-API-Key: skyheal_secure_api_key_2026` or `Authorization: Bearer <TOKEN>`  
+**Default Dashboard**: `http://localhost:8000/` or `http://localhost:8000/dashboard`  
+
+---
+
+## 1. Architectural Overview
+
+The **CureSelect AI Voice Engine** is a high-speed, full-duplex conversational health intake system. It processes real-time microphone audio or text turns, classifies speaker perspectives, detects emergency distress, checks RxNorm drug safety, redacts HIPAA/ABDM PII, and generates real-time HL7 FHIR R4 Bundles and Doctor SOAP Notes.
+
+```mermaid
+sequenceDiagram
+    participant Client as Frontend (React / Web / Mobile)
+    participant WS as WebSocket Manager (ws/auto)
+    participant Engine as AI Orchestrator & Multi-Models
+    participant EHR as EHR Export (FHIR / SOAP Note)
+
+    Client->>WS: Connect (ws://localhost:8000/v1/voice-onboarding/ws/auto)
+    WS-->>Client: {"type": "connected", "session_id": "VOICE-AUTO-8F12A9C4"}
+    WS-->>Client: {"type": "ai_response", "reply": "Hello Patient, welcome to CureSelect..."}
+    
+    Client->>WS: Stream Binary Mic Bytes (MediaRecorder WebM/PCM)
+    WS-->>Client: {"type": "backchannel", "reply": "Mm-hmm..."} (< 15ms)
+    WS-->>Client: {"type": "speculative_audio_chunk", "ttfa_ms": 180} (< 200ms)
+    WS-->>Client: {"type": "transcript", "speaker_type": "caregiver", "speaker_label": "Caregiver (Mother)"}
+    WS-->>Client: {"type": "ai_response", "reply": "I'm sorry to hear about your mother's headache...", "audio_base64": "..."}
+
+    Client->>EHR: GET /v1/voice-onboarding/soap-note/{session_id}?format=html
+    EHR-->>Client: Render Printable Doctor SOAP Note Report
+```
+
+---
+
+## 2. WebSocket Connection Lifecycle (`ws://localhost:8000/v1/voice-onboarding/ws/auto`)
+
+### 2.1 Connection Setup
+Connect to the zero-effort auto-session WebSocket endpoint:
+```typescript
+const ws = new WebSocket("ws://localhost:8000/v1/voice-onboarding/ws/auto");
+```
+
+### 2.2 Inbound WebSocket Event Schema
+
+#### 1. Connection Initialized (`connected`)
+Sent immediately upon socket handshake:
+```json
+{
+  "type": "connected",
+  "session_id": "VOICE-AUTO-9DEC79EC",
+  "message": "Connected to CureSelect Real-Time Voice Engine"
+}
+```
+
+#### 2. Speaker Diarized Transcript (`transcript`)
+Sent as soon as patient or caregiver speech is recognized:
+```json
+{
+  "type": "transcript",
+  "role": "patient",
+  "speaker_type": "caregiver",
+  "speaker_label": "Caregiver (Mother)",
+  "text": "My mother has severe headache for 10 days and she is taking Metformin 500mg."
+}
+```
+*Note*: `speaker_type` is either `"patient"` or `"caregiver"`.
+
+#### 3. Low-Latency Acoustic Backchannel (`backchannel`)
+Emitted in **< 15ms** during patient pauses to signal active listening:
+```json
+{
+  "type": "backchannel",
+  "reply": "Mm-hmm...",
+  "audio_base64": "..."
+}
+```
+
+#### 4. Speculative Audio Chunk (`speculative_audio_chunk`)
+Emitted in **< 200ms** on the first 3-4 words generated by the LLM:
+```json
+{
+  "type": "speculative_audio_chunk",
+  "clause_text": "I am sorry to hear that.",
+  "audio_base64": "...",
+  "ttfa_ms": 180
+}
+```
+
+#### 5. Complete AI Response & State Update (`ai_response`)
+Contains full reply, TTS audio, next stage, and updated clinical state:
+```json
+{
+  "type": "ai_response",
+  "reply": "Any known drug allergies?",
+  "audio_id": "aud_771829",
+  "audio_base64": "...",
+  "next_stage": "symptom_collection",
+  "completed": false,
+  "state_updates": {
+    "symptoms": [{"name": "headache", "duration": "10 days", "severity": "severe"}],
+    "medications": [{"name": "Metformin", "dosage": "500mg daily"}]
+  }
+}
+```
+
+#### 6. Emergency Nurse Escalation Alert (`nurse_escalation_alert`)
+Emitted when Pain Score >= 8/10 or Distress Score > 0.75:
+```json
+{
+  "type": "nurse_escalation_alert",
+  "session_id": "VOICE-AUTO-9DEC79EC",
+  "distress_score": 0.85,
+  "pain_score": 9,
+  "message": "CRITICAL: Patient reported severe pain (9/10). Nurse escalation triggered!"
+}
+```
+
+#### 7. RxNorm / FDA Drug Safety Warning (`drug_safety_warning`)
+Emitted when drug interactions or drug-allergy contraindications are flagged:
+```json
+{
+  "type": "drug_safety_warning",
+  "warnings": [
+    {
+      "severity": "CRITICAL",
+      "category": "DRUG_INTERACTION",
+      "description": "High risk of bleeding: Combined use of Warfarin and Aspirin."
+    }
+  ]
+}
+```
+
+---
+
+## 3. Outbound Client Messages
+
+### 3.1 Streaming Audio Bytes
+Stream raw WebM / PCM audio blobs from `MediaRecorder`:
+```javascript
+mediaRecorder.ondataavailable = (e) => {
+  if (e.data.size > 0 && ws.readyState === WebSocket.OPEN) {
+    ws.send(e.data); // Send raw Blob
+  }
+};
+```
+
+### 3.2 Text Turn Fallback
+```json
+{
+  "type": "text_turn",
+  "text": "10 days-ah severe headache iruku, Metformin 500mg sapudren."
+}
+```
+
+### 3.3 Full-Duplex Barge-In Interruption
+Send when the user speaks or clicks "Interrupt AI" while AI audio is playing:
+```json
+{
+  "type": "interrupt"
+}
+```
+
+---
+
+## 4. REST API Integration Endpoints
+
+All REST requests require the authentication header:  
+`X-API-Key: skyheal_secure_api_key_2026`
+
+### 4.1 HL7 FHIR R4 Bundle Export
+- **Endpoint**: `GET /v1/voice-onboarding/fhir/{session_id}`
+- **Response**: Production-grade HL7 FHIR R4 Bundle (`Patient`, `Encounter`, `Condition`, `MedicationStatement`, `AllergyIntolerance`).
+
+```bash
+curl -X GET "http://localhost:8000/v1/voice-onboarding/fhir/VOICE-AUTO-9DEC79EC" \
+     -H "X-API-Key: skyheal_secure_api_key_2026"
+```
+
+### 4.2 Doctor SOAP Note & Printable PDF/HTML Report
+- **Endpoint**: `GET /v1/voice-onboarding/soap-note/{session_id}?format=json` (or `format=html`)
+- **Query Params**:
+  - `format=json`: Returns structured Subjective, Objective, Assessment (ICD-10-CM & SNOMED-CT codes), and Plan JSON.
+  - `format=html`: Renders a clean printable HTML document for 1-click doctor chart printing.
+
+```bash
+# Get JSON SOAP Note with ICD-10 & SNOMED CT
+curl -X GET "http://localhost:8000/v1/voice-onboarding/soap-note/VOICE-AUTO-9DEC79EC?format=json" \
+     -H "X-API-Key: skyheal_secure_api_key_2026"
+
+# Get Printable HTML Report
+curl -X GET "http://localhost:8000/v1/voice-onboarding/soap-note/VOICE-AUTO-9DEC79EC?format=html" \
+     -H "X-API-Key: skyheal_secure_api_key_2026"
+```
+
+---
+
+## 5. React / Next.js Custom Hook Example
+
+```typescript
+import { useState, useEffect, useRef } from "react";
+
+export function useVoiceIntakeWebSocket(url: string = "ws://localhost:8000/v1/voice-onboarding/ws/auto") {
+  const [sessionId, setSessionId] = useState<string>("");
+  const [messages, setMessages] = useState<Array<any>>([]);
+  const [clinicalState, setClinicalState] = useState<any>({});
+  const [escalationAlert, setEscalationAlert] = useState<string | null>(null);
+  const [drugWarnings, setDrugWarnings] = useState<Array<any>>([]);
+  const wsRef = useRef<WebSocket | null>(null);
+
+  useEffect(() => {
+    const ws = new WebSocket(url);
+    wsRef.current = ws;
+
+    ws.onmessage = (event) => {
+      const data = JSON.parse(event.data);
+      if (data.type === "connected") {
+        setSessionId(data.session_id);
+      } else if (data.type === "transcript") {
+        setMessages(prev => [...prev, { role: data.role, text: data.text, label: data.speaker_label }]);
+      } else if (data.type === "ai_response") {
+        setMessages(prev => [...prev, { role: "assistant", text: data.reply, label: "AI Assistant" }]);
+        if (data.audio_base64) {
+          const audio = new Audio("data:audio/wav;base64," + data.audio_base64);
+          audio.play().catch(console.error);
+        }
+        if (data.state_updates) {
+          setClinicalState(data.state_updates);
+        }
+      } else if (data.type === "nurse_escalation_alert") {
+        setEscalationAlert(data.message);
+      } else if (data.type === "drug_safety_warning") {
+        setDrugWarnings(data.warnings);
+      }
+    };
+
+    return () => ws.close();
+  }, [url]);
+
+  const sendTextTurn = (text: string) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify({ type: "text_turn", text }));
+    }
+  };
+
+  const interruptAI = () => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify({ type: "interrupt" }));
+    }
+  };
+
+  return { sessionId, messages, clinicalState, escalationAlert, drugWarnings, sendTextTurn, interruptAI };
+}
+```