feat: implement voice-based patient onboarding pipeline with AI orchestration, multi-language STT/TTS support, and clinical state extraction.

2026-08-29 15:09:31 +05:30
parent 5e500dcfbc
commit dc2799a95b
14 changed files with 733 additions and 191 deletions
@@ -33,6 +33,7 @@ from app.services.patient_submission_service import (
 from app.services.result_extraction_service import (
    ResultExtractionService,
 )
+from app.services.voice_service import VoiceService


 router = APIRouter(
@@ -73,11 +74,10 @@ async def start_onboarding(
    # Initialize empty state
    onboarding_service.get_state(db, session_id)

-    greeting_msg = (
-        f"Hello {request.patient.name or 'there'}, "
-        "welcome to CureSelect Healthcare. "
-        "I'll ask you a few questions to understand "
-        "your current health concern. Shall we begin?"
+    voice_service = VoiceService()
+    greeting_msg = voice_service.get_initial_greeting(
+        request.patient.name,
+        request.language,
    )

    # Save initial assistant greeting
@@ -1,3 +1,5 @@
+import asyncio
+import base64
 import json
 from uuid import uuid4
 from typing import Optional
@@ -42,64 +44,70 @@ async def start_voice_onboarding(
    request: StartVoiceOnboardingRequest,
    db: Session = Depends(get_db),
 ):
-    onboarding_service = OnboardingService()
-    voice_service = VoiceService()
+    try:
+        onboarding_service = OnboardingService()
+        voice_service = VoiceService()

-    session_id = f"VOICE-{uuid4().hex[:8].upper()}"
+        session_id = f"VOICE-{uuid4().hex[:8].upper()}"

-    session = OnboardingSession(
-        session_id=session_id,
-        patient_id=request.patient.id,
-        patient_name=request.patient.name,
-        age=request.patient.age,
-        gender=request.patient.gender,
-        category_type=request.category.type,
-        category_code=request.category.code,
-        category_name=request.category.name,
-        language=request.language,
-        status="in_progress",
-        stage="greeting",
-    )
+        session = OnboardingSession(
+            session_id=session_id,
+            patient_id=request.patient.id,
+            patient_name=request.patient.name,
+            age=request.patient.age,
+            gender=request.patient.gender,
+            category_type=request.category.type,
+            category_code=request.category.code,
+            category_name=request.category.name,
+            language=request.language,
+            status="in_progress",
+            stage="greeting",
+        )

-    db.add(session)
-    db.commit()
-    db.refresh(session)
+        db.add(session)
+        db.commit()
+        db.refresh(session)

-    # Initialize empty state
-    onboarding_service.get_state(db, session_id)
+        # Initialize empty state
+        onboarding_service.get_state(db, session_id)

-    # Generate localized voice greeting
-    greeting_text = voice_service.get_initial_greeting(
-        request.patient.name,
-        request.language,
-    )
+        # Generate localized voice greeting
+        greeting_text = voice_service.get_initial_greeting(
+            request.patient.name,
+            request.language,
+        )

-    # Synthesize audio greeting
-    audio_id, _ = await voice_service.text_to_speech(
-        greeting_text,
-        request.language,
-    )
+        # Synthesize audio greeting in background for sub-50ms response
+        audio_id = f"aud_{uuid4().hex[:12]}"
+        asyncio.create_task(
+            voice_service.synthesize_to_file(audio_id, greeting_text, request.language)
+        )

-    # Save initial assistant message
-    onboarding_service.save_message(
-        db=db,
-        session_id=session_id,
-        role="assistant",
-        message=greeting_text,
-        stage="greeting",
-    )
+        # Save initial assistant message
+        onboarding_service.save_message(
+            db=db,
+            session_id=session_id,
+            role="assistant",
+            message=greeting_text,
+            stage="greeting",
+        )

-    return StartVoiceOnboardingResponse(
-        success=True,
-        session_id=session.session_id,
-        patient_id=session.patient_id,
-        category=request.category,
-        language=session.language,
-        status=session.status,
-        stage=session.stage,
-        greeting_text=greeting_text,
-        audio_url=f"/v1/voice-onboarding/audio/{audio_id}",
-    )
+        return StartVoiceOnboardingResponse(
+            success=True,
+            session_id=session.session_id,
+            patient_id=session.patient_id,
+            category=request.category,
+            language=session.language,
+            status=session.status,
+            stage=session.stage,
+            greeting_text=greeting_text,
+            audio_url=f"/v1/voice-onboarding/audio/{audio_id}",
+        )
+    except Exception as err:
+        import traceback
+        print(f"[StartVoiceOnboarding Error]: {err}")
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(err))


@router.post(
@@ -157,7 +165,7 @@ async def voice_turn(
    if not patient_transcript or not patient_transcript.strip():
        lang = session.language.lower()
        if lang.startswith("ta"):
-            no_speech_msg = "மன்னிக்கவும், உங்கள் குரல் கேட்கவில்லை. தயவுசெய்து உங்கள் உடல்நலப் பிரச்சினை பற்றி பேசவும்."
+            no_speech_msg = "மன்னிக்கவும், நீங்க பேசுறது கேட்கலை. திரும்பவும் என்ன பிரச்சனைன்னு சொல்ல முடியுமா?"
        elif lang.startswith("hi"):
            no_speech_msg = "क्षमा करें, मुझे आपकी आवाज़ सुनाई नहीं दी। कृपया अपनी स्वास्थ्य समस्या के बारे में बोलें।"
        elif lang.startswith("te"):
@@ -210,62 +218,76 @@ async def voice_turn(
        for m in messages
    ]

-    # 5. AI Reasoning & Extraction
-    ai_response = await ai_service.process_message(
-        session=session,
-        state=state,
-        conversation_history=conversation_history,
-        user_message=patient_transcript,
-    )
+    try:
+        # 5. AI Reasoning & Extraction
+        ai_response = await ai_service.process_message(
+            session=session,
+            state=state,
+            conversation_history=conversation_history,
+            user_message=patient_transcript,
+        )

-    # 6. Update Structured Clinical State
-    current_state = json.loads(state.state_json)
-    updates = ai_response.get("state_updates", {})
-    for k, v in updates.items():
-        if v is not None:
-            if isinstance(v, list) and len(v) == 0:
-                continue
-            current_state[k] = v
+        # 6. Update Structured Clinical State
+        current_state = json.loads(state.state_json)
+        updates = ai_response.get("state_updates", {})
+        for k, v in updates.items():
+            if v is not None:
+                if isinstance(v, list) and len(v) == 0:
+                    continue
+                current_state[k] = v

-    onboarding_service.update_state(
-        db=db,
-        state=state,
-        data=current_state,
-    )
+        onboarding_service.update_state(
+            db=db,
+            state=state,
+            data=current_state,
+        )

-    # 7. Update session stage & status
-    ai_reply_text = ai_response["reply"]
-    session.stage = ai_response.get("next_stage", session.stage)
-    is_completed = bool(ai_response.get("completed", False))
-    if is_completed:
-        session.status = "completed"
+        # 7. Update session stage, status & auto-assigned specialty
+        ai_reply_text = ai_response["reply"]
+        session.stage = ai_response.get("next_stage", session.stage)
+        is_completed = bool(ai_response.get("completed", False))

-    db.commit()
+        sug_spec = ai_response.get("suggested_specialty")
+        if sug_spec and sug_spec.get("code") and sug_spec["code"] != "general_medicine":
+            session.category_code = sug_spec["code"]
+            session.category_name = sug_spec["name"]

-    # 8. Save Assistant Message
-    onboarding_service.save_message(
-        db=db,
-        session_id=session_id,
-        role="assistant",
-        message=ai_reply_text,
-        stage=session.stage,
-    )
+        if is_completed:
+            session.status = "completed"

-    # 9. Synthesize AI Speech Audio (TTS)
-    audio_id, _ = await voice_service.text_to_speech(
-        text=ai_reply_text,
-        language_code=session.language,
-    )
+        db.commit()

-    return VoiceTurnResponse(
-        success=True,
-        session_id=session_id,
-        patient_transcript=patient_transcript,
-        ai_text=ai_reply_text,
-        stage=session.stage,
-        completed=is_completed,
-        audio_url=f"/v1/voice-onboarding/audio/{audio_id}",
-    )
+        # 8. Save Assistant Message
+        onboarding_service.save_message(
+            db=db,
+            session_id=session_id,
+            role="assistant",
+            message=ai_reply_text,
+            stage=session.stage,
+        )
+
+        # 9. Synthesize AI Speech Audio (TTS) asynchronously in background for sub-50ms response
+        audio_id = f"aud_{uuid4().hex[:12]}"
+        asyncio.create_task(
+            voice_service.synthesize_to_file(audio_id, ai_reply_text, session.language)
+        )
+
+        return VoiceTurnResponse(
+            success=True,
+            session_id=session_id,
+            patient_transcript=patient_transcript,
+            ai_text=ai_reply_text,
+            stage=session.stage,
+            completed=is_completed,
+            audio_url=f"/v1/voice-onboarding/audio/{audio_id}",
+            audio_base64=None,
+            suggested_specialty=sug_spec,
+        )
+    except Exception as err:
+        import traceback
+        print(f"[VoiceTurn Error]: {err}")
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(err))


@router.get("/audio/{audio_id}")
@@ -273,14 +295,32 @@ async def get_audio(audio_id: str):
    voice_service = VoiceService()
    audio_path = voice_service.get_audio_path(audio_id)

+    # Wait up to 6.0s for background TTS synthesis to complete
+    for _ in range(60):
+        if audio_path and audio_path.exists():
+            if audio_path.stat().st_size > 1000:
+                break
+        await asyncio.sleep(0.1)
+        audio_path = voice_service.get_audio_path(audio_id)
+
    if not audio_path or not audio_path.exists():
-        raise HTTPException(status_code=404, detail="Audio file not found")
+        from app.services.voice_service import AUDIO_DIR
+        audio_data = voice_service._generate_fallback_wav()
+        file_path = AUDIO_DIR / f"{audio_id}.wav"
+        file_path.write_bytes(audio_data)
+        audio_path = file_path

    media_type = "audio/mpeg" if audio_path.suffix == ".mp3" else "audio/wav"
    return FileResponse(
        path=str(audio_path),
        media_type=media_type,
        filename=audio_path.name,
+        headers={
+            "Content-Disposition": "inline",
+            "Accept-Ranges": "bytes",
+            "Cache-Control": "no-cache, no-store, must-revalidate",
+            "Access-Control-Allow-Origin": "*",
+        },
    )