Update 2026-19-03 11:13

avatar-frontend/app/app/page.tsx

@@ -23,6 +23,23 @@ interface AssistantAudioPayload {
   audio_url: string;
 }
 
+interface AssistantStreamPayload {
+  stream_id: string;
+  audio_stream?: boolean;
+  text_only?: boolean;
+  audio_url?: string;
+}
+
+interface TtsStreamStartPayload {
+  stream_id: string;
+  sample_rate: number;
+}
+
+interface TtsStreamChunkPayload {
+  stream_id: string;
+  audio: string;
+}
+
 interface SpeechRecognitionAlternativeLike {
   transcript: string;
 }
@@ -533,6 +550,13 @@ export default function AppPage() {
 
   const audioContextRef = useRef<AudioContext | null>(null);
   const audioProcessorRef = useRef<ScriptProcessorNode | null>(null);
+  const ttsPlaybackContextRef = useRef<AudioContext | null>(null);
+  const ttsPlaybackTimeRef = useRef(0);
+  const ttsPendingSourcesRef = useRef(0);
+  const ttsPlaybackEndedRef = useRef(false);
+  const ttsStreamIdRef = useRef<string | null>(null);
+  const ttsStreamSampleRateRef = useRef<number>(24000);
+  const ttsSourceNodesRef = useRef<AudioBufferSourceNode[]>([]);
 
   const selectedVoiceRef = useRef(selectedVoice);
   const selectedPersonalityRef = useRef(selectedPersonality);
@@ -603,6 +627,29 @@ export default function AppPage() {
       }
       audioProcessorRef.current = null;
     }
+    if (ttsPlaybackContextRef.current) {
+      try {
+        ttsPlaybackContextRef.current.close();
+      } catch {
+      }
+      ttsPlaybackContextRef.current = null;
+    }
+    ttsSourceNodesRef.current.forEach((source) => {
+      try {
+        source.stop();
+      } catch {
+      }
+      try {
+        source.disconnect();
+      } catch {
+      }
+    });
+    ttsSourceNodesRef.current = [];
+    ttsPlaybackTimeRef.current = 0;
+    ttsPendingSourcesRef.current = 0;
+    ttsPlaybackEndedRef.current = false;
+    ttsStreamIdRef.current = null;
+    ttsStreamSampleRateRef.current = 24000;
     if (wsRef.current) {
       try {
         wsRef.current.close(1000, "Session ended");
@@ -696,6 +743,104 @@ export default function AppPage() {
     }
   };
 
+  const finishAssistantPlayback = () => {
+    setIsResponsePlaying(false);
+    setStatus(isPausedRef.current ? "Paused" : "Listening...");
+    shouldAutoRestartRef.current = true;
+    if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify({ type: "tts_end" }));
+    }
+    if (recognitionRef.current && !isPausedRef.current) {
+      try {
+        recognitionRef.current.start();
+      } catch {
+      }
+    }
+  };
+
+  const resetStreamingPlayback = () => {
+    if (audioRef.current) {
+      audioRef.current.pause();
+      audioRef.current.removeAttribute("src");
+      audioRef.current.load();
+    }
+    ttsSourceNodesRef.current.forEach((source) => {
+      try {
+        source.stop();
+      } catch {
+      }
+      try {
+        source.disconnect();
+      } catch {
+      }
+    });
+    ttsSourceNodesRef.current = [];
+    ttsPlaybackTimeRef.current = 0;
+    ttsPendingSourcesRef.current = 0;
+    ttsPlaybackEndedRef.current = false;
+    ttsStreamSampleRateRef.current = 24000;
+  };
+
+  const ensureTtsPlaybackContext = async () => {
+    if (!ttsPlaybackContextRef.current) {
+      ttsPlaybackContextRef.current = new AudioContext();
+    }
+    if (ttsPlaybackContextRef.current.state === "suspended") {
+      await ttsPlaybackContextRef.current.resume();
+    }
+    return ttsPlaybackContextRef.current;
+  };
+
+  const finalizeStreamIfComplete = () => {
+    if (!ttsPlaybackEndedRef.current || ttsPendingSourcesRef.current > 0) return;
+    ttsStreamIdRef.current = null;
+    finishAssistantPlayback();
+  };
+
+  const startStreamingPlayback = async (data: TtsStreamStartPayload) => {
+    if (ttsStreamIdRef.current !== data.stream_id) return;
+    const playbackContext = await ensureTtsPlaybackContext();
+    ttsStreamSampleRateRef.current = data.sample_rate;
+    ttsPlaybackTimeRef.current = Math.max(ttsPlaybackTimeRef.current, playbackContext.currentTime + 0.08);
+    setStatus("Speaking...");
+  };
+
+  const enqueueStreamingAudioChunk = async (data: TtsStreamChunkPayload) => {
+    if (ttsStreamIdRef.current !== data.stream_id) return;
+    const playbackContext = await ensureTtsPlaybackContext();
+    const binary = atob(data.audio);
+    const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0));
+    const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 2));
+    const floatSamples = new Float32Array(pcm.length);
+    for (let i = 0; i < pcm.length; i++) {
+      floatSamples[i] = pcm[i] / 32768;
+    }
+
+    const audioBuffer = playbackContext.createBuffer(1, floatSamples.length, ttsStreamSampleRateRef.current);
+    audioBuffer.copyToChannel(floatSamples, 0);
+
+    const source = playbackContext.createBufferSource();
+    source.buffer = audioBuffer;
+    source.connect(playbackContext.destination);
+
+    const startAt = Math.max(ttsPlaybackTimeRef.current, playbackContext.currentTime + 0.05);
+    ttsPlaybackTimeRef.current = startAt + audioBuffer.duration;
+    ttsPendingSourcesRef.current += 1;
+    ttsSourceNodesRef.current.push(source);
+
+    source.onended = () => {
+      ttsPendingSourcesRef.current = Math.max(0, ttsPendingSourcesRef.current - 1);
+      ttsSourceNodesRef.current = ttsSourceNodesRef.current.filter((node) => node !== source);
+      try {
+        source.disconnect();
+      } catch {
+      }
+      finalizeStreamIfComplete();
+    };
+
+    source.start(startAt);
+  };
+
   function connectWebSocket() {
     const token = localStorage.getItem("mrrrme_token");
     if (!token) {
@@ -745,6 +890,24 @@ export default function AppPage() {
           },
         ]);
 
+        if (data.audio_stream && data.stream_id) {
+          resetStreamingPlayback();
+          ttsStreamIdRef.current = (data as AssistantStreamPayload).stream_id;
+          setStatus("Preparing audio...");
+          setIsResponsePlaying(true);
+          shouldAutoRestartRef.current = false;
+          if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+            wsRef.current.send(JSON.stringify({ type: "tts_start" }));
+          }
+          if (recognitionRef.current) {
+            try {
+              recognitionRef.current.stop();
+            } catch {
+            }
+          }
+          return;
+        }
+
         if (data.text_only || !data.audio_url) {
           setStatus("Response ready");
           setTimeout(() => {
@@ -770,6 +933,23 @@ export default function AppPage() {
           }
         }
         await playAssistantResponse(data);
+      } else if (data.type === "tts_stream_start") {
+        await startStreamingPlayback(data as TtsStreamStartPayload);
+      } else if (data.type === "tts_stream_chunk") {
+        await enqueueStreamingAudioChunk(data as TtsStreamChunkPayload);
+      } else if (data.type === "tts_stream_end") {
+        if (ttsStreamIdRef.current === data.stream_id) {
+          ttsPlaybackEndedRef.current = true;
+          finalizeStreamIfComplete();
+        }
+      } else if (data.type === "tts_stream_error") {
+        if (!ttsStreamIdRef.current || ttsStreamIdRef.current === data.stream_id) {
+          console.error("[Audio] Stream error:", data.message);
+          setStatus("Audio error");
+          ttsStreamIdRef.current = null;
+          resetStreamingPlayback();
+          finishAssistantPlayback();
+        }
       }
     };
     ws.onerror = () => setStatus("Connection error");
@@ -948,6 +1128,9 @@ export default function AppPage() {
   async function playAssistantResponse(data: AssistantAudioPayload) {
     if (!audioRef.current) return;
 
+    resetStreamingPlayback();
+    ttsStreamIdRef.current = null;
+
     if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
       wsRef.current.send(JSON.stringify({ type: "tts_start" }));
     }
@@ -981,18 +1164,7 @@ export default function AppPage() {
       setStatus("Audio error");
     }
 
-    setIsResponsePlaying(false);
-    setStatus(isPausedRef.current ? "Paused" : "Listening...");
-    shouldAutoRestartRef.current = true;
-    if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
-      wsRef.current.send(JSON.stringify({ type: "tts_end" }));
-    }
-    if (recognitionRef.current && !isPausedRef.current) {
-      try {
-        recognitionRef.current.start();
-      } catch {
-      }
-    }
+    finishAssistantPlayback();
   }
 
   return (

avatar/speak_server.py

@@ -1,21 +1,19 @@
-"""Avatar Backend - Qwen 3 TTS with STREAMING (Ultra-Fast 97ms latency!)"""
-import os
-import uuid
-import time
-import wave
-import subprocess
-import json as json_lib
-import asyncio
-from fastapi import FastAPI, Form, WebSocket
+"""Avatar Backend - Qwen 3 TTS with chunked PCM streaming."""
+import base64
+import os
+import uuid
+import time
+import subprocess
+import json as json_lib
+import asyncio
+from fastapi import FastAPI, Form, WebSocket
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
-from fastapi.responses import JSONResponse, StreamingResponse
-from pydub import AudioSegment
-from typing import List, Optional
-from dotenv import load_dotenv
-import torch
-import numpy as np
-import io
+from fastapi.responses import JSONResponse, StreamingResponse
+from typing import List, Optional
+from dotenv import load_dotenv
+import torch
+import numpy as np
 
 load_dotenv()
 
@@ -133,20 +131,121 @@ async def websocket_endpoint(websocket: WebSocket):
             active_connections.remove(websocket)
 
 
-async def broadcast_to_avatars(data: dict):
-    for connection in active_connections[:]:
-        try:
-            await connection.send_json(data)
-        except:
-            if connection in active_connections:
-                active_connections.remove(connection)
-
-
-# ============ FAST TTS ENDPOINT ============
-
-@app.post("/speak")
-async def speak(text: str = Form(...), voice: str = Form("female"), language: str = Form("en")):
-    """Generate TTS quickly with optimized settings"""
+async def broadcast_to_avatars(data: dict):
+    for connection in active_connections[:]:
+        try:
+            await connection.send_json(data)
+        except:
+            if connection in active_connections:
+                active_connections.remove(connection)
+
+
+def synthesize_speech(
+    text: str,
+    voice: str,
+    language: str,
+    request_id: Optional[str] = None,
+) -> dict:
+    """Run TTS once and return reusable response metadata."""
+    t_start = time.time()
+    uid = request_id or uuid.uuid4().hex[:8]
+    wav_path = os.path.join(OUT_DIR, f"{uid}.wav")
+
+    language_map = {
+        "en": "English", "nl": "Auto", "zh": "Chinese", "ja": "Japanese",
+        "ko": "Korean", "de": "German", "fr": "French", "ru": "Russian",
+        "pt": "Portuguese", "es": "Spanish", "it": "Italian"
+    }
+
+    qwen_language = language_map.get(language, "English")
+    speaker_name = VOICE_MAP.get(voice, "Serena")
+
+    print(f"[TTS] 🎤 {speaker_name}: '{text[:40]}...'")
+
+    with torch.no_grad():
+        wavs, sample_rate = qwen_tts_model.generate_custom_voice(
+            text=text,
+            language=qwen_language,
+            speaker=speaker_name,
+            max_new_tokens=1024,
+        )
+
+    audio_array = wavs[0] if isinstance(wavs, list) else wavs
+    if isinstance(audio_array, torch.Tensor):
+        audio_array = audio_array.cpu().numpy()
+    audio_array = audio_array.squeeze()
+
+    if audio_array.max() <= 1.0:
+        audio_array = (audio_array * 32767).astype(np.int16)
+    else:
+        audio_array = audio_array.astype(np.int16)
+
+    import scipy.io.wavfile
+    scipy.io.wavfile.write(wav_path, rate=sample_rate, data=audio_array)
+
+    duration_sec = len(audio_array) / sample_rate
+    generation_time = time.time() - t_start
+    print(f"[TTS] ⚡ Generated in {generation_time:.2f}s ({duration_sec:.1f}s audio)")
+
+    visemes = generate_visemes_fast(text, duration_sec)
+
+    return {
+        "stream_id": uid,
+        "audio_array": audio_array,
+        "sample_rate": sample_rate,
+        "wav_path": wav_path,
+        "visemes": visemes,
+        "duration": duration_sec,
+        "text": text,
+        "method": "fast_phoneme",
+        "generation_time": round(generation_time, 2),
+    }
+
+
+def iter_ndjson_stream(payload: dict, chunk_duration_ms: int = 120):
+    """Yield newline-delimited JSON stream frames with PCM16 audio chunks."""
+    audio_bytes = payload["audio_array"].tobytes()
+    sample_rate = int(payload["sample_rate"])
+    samples_per_chunk = max(1, int(sample_rate * (chunk_duration_ms / 1000.0)))
+    chunk_size_bytes = samples_per_chunk * 2  # mono PCM16
+
+    start_event = {
+        "type": "start",
+        "stream_id": payload["stream_id"],
+        "sample_rate": sample_rate,
+        "channels": 1,
+        "duration": payload["duration"],
+        "generation_time": payload["generation_time"],
+    }
+    yield f"{json_lib.dumps(start_event)}\n"
+
+    for offset in range(0, len(audio_bytes), chunk_size_bytes):
+        chunk = audio_bytes[offset: offset + chunk_size_bytes]
+        chunk_event = {
+            "type": "chunk",
+            "stream_id": payload["stream_id"],
+            "audio": base64.b64encode(chunk).decode("ascii"),
+        }
+        yield f"{json_lib.dumps(chunk_event)}\n"
+
+    end_event = {
+        "type": "end",
+        "stream_id": payload["stream_id"],
+        "duration": payload["duration"],
+    }
+    yield f"{json_lib.dumps(end_event)}\n"
+
+
+# ============ FAST TTS ENDPOINTS ============
+
+@app.post("/speak")
+async def speak(
+    text: str = Form(...),
+    voice: str = Form("female"),
+    language: str = Form("en"),
+    request_id: Optional[str] = Form(None),
+):
+    """Generate TTS quickly with optimized settings"""
     
     # Quick model check
     if qwen_tts_model is None or not MODEL_READY:
@@ -156,78 +255,68 @@ async def speak(text: str = Form(...), voice: str = Form("female"), language: st
             content={"error": "TTS model still loading, please wait..."}
         )
     
-    t_start = time.time()
-    uid = uuid.uuid4().hex[:8]
-    wav_path = os.path.join(OUT_DIR, f"{uid}.wav")
-    mp3_path = os.path.join(OUT_DIR, f"{uid}.mp3")
-    
-    language_map = {
-        "en": "English", "nl": "Auto", "zh": "Chinese", "ja": "Japanese",
-        "ko": "Korean", "de": "German", "fr": "French", "ru": "Russian",
-        "pt": "Portuguese", "es": "Spanish", "it": "Italian"
-    }
-    
-    qwen_language = language_map.get(language, "English")
-    speaker_name = VOICE_MAP.get(voice, "Serena")
-    
-    print(f"[TTS] 🎤 {speaker_name}: '{text[:40]}...'")
-    
-    try:
-        # ✅ OPTIMIZED: Use faster generation with minimal settings
-        with torch.no_grad():  # Disable gradients for speed
-            wavs, sample_rate = qwen_tts_model.generate_custom_voice(
-                text=text,
-                language=qwen_language,
-                speaker=speaker_name,
-                # Performance optimizations:
-                max_new_tokens=1024,  # Limit length for speed
-                # Note: Streaming mode would be even faster but requires different API
-            )
-        
-        # Quick extraction
-        audio_array = wavs[0] if isinstance(wavs, list) else wavs
-        if isinstance(audio_array, torch.Tensor):
-            audio_array = audio_array.cpu().numpy()
-        audio_array = audio_array.squeeze()
-        
-        # Convert to int16
-        if audio_array.max() <= 1.0:
-            audio_array = (audio_array * 32767).astype(np.int16)
-        else:
-            audio_array = audio_array.astype(np.int16)
-        
-        # Save WAV (skip MP3 conversion for speed)
-        import scipy.io.wavfile
-        scipy.io.wavfile.write(wav_path, rate=sample_rate, data=audio_array)
-        
-        duration_sec = len(audio_array) / sample_rate
-        
-        t_gen = time.time() - t_start
-        print(f"[TTS] ⚡ Generated in {t_gen:.2f}s ({duration_sec:.1f}s audio)")
-        
-        # Fast viseme generation (skip Rhubarb for speed)
-        visemes = generate_visemes_fast(text, duration_sec)
-        
-        response_data = {
-            "audio_url": f"/static/{os.path.basename(wav_path)}",
-            "visemes": visemes,
-            "duration": duration_sec,
-            "text": text,
-            "method": "fast_phoneme",
-            "generation_time": round(t_gen, 2)
-        }
-        
-        await broadcast_to_avatars(response_data)
-        
-        print(f"[TTS] ✅ Total: {time.time() - t_start:.2f}s\n")
-        
-        return response_data
-        
-    except Exception as e:
-        print(f"[TTS] ❌ Error: {e}")
-        import traceback
-        traceback.print_exc()
-        return JSONResponse(status_code=500, content={"error": str(e)})
+    try:
+        payload = synthesize_speech(text=text, voice=voice, language=language, request_id=request_id)
+        response_data = {
+            "stream_id": payload["stream_id"],
+            "audio_url": f"/static/{os.path.basename(payload['wav_path'])}",
+            "visemes": payload["visemes"],
+            "duration": payload["duration"],
+            "text": payload["text"],
+            "method": payload["method"],
+            "generation_time": payload["generation_time"],
+        }
+        
+        await broadcast_to_avatars(response_data)
+        
+        print(f"[TTS] ✅ Total: {payload['generation_time']:.2f}s\n")
+        
+        return response_data
+        
+    except Exception as e:
+        print(f"[TTS] ❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+@app.post("/speak_stream")
+async def speak_stream(
+    text: str = Form(...),
+    voice: str = Form("female"),
+    language: str = Form("en"),
+    request_id: Optional[str] = Form(None),
+):
+    """Stream synthesized PCM chunks as newline-delimited JSON."""
+    if qwen_tts_model is None or not MODEL_READY:
+        return JSONResponse(
+            status_code=503,
+            content={"error": "TTS model still loading, please wait..."},
+        )
+
+    try:
+        payload = synthesize_speech(text=text, voice=voice, language=language, request_id=request_id)
+        response_data = {
+            "stream_id": payload["stream_id"],
+            "audio_url": f"/static/{os.path.basename(payload['wav_path'])}",
+            "visemes": payload["visemes"],
+            "duration": payload["duration"],
+            "text": payload["text"],
+            "method": payload["method"],
+            "generation_time": payload["generation_time"],
+        }
+        await broadcast_to_avatars(response_data)
+
+        return StreamingResponse(
+            iter_ndjson_stream(payload),
+            media_type="application/x-ndjson",
+            headers={"Cache-Control": "no-cache"},
+        )
+    except Exception as e:
+        print(f"[TTS] ❌ Stream error: {e}")
+        import traceback
+        traceback.print_exc()
+        return JSONResponse(status_code=500, content={"error": str(e)})
 
 
 @app.get("/health")
@@ -261,4 +350,4 @@ if __name__ == "__main__":
     print("Optimization: torch.no_grad() + fast visemes")
     print("Expected latency: 1-3 seconds (vs 5-10s before)")
     print("="*60 + "\n")
-    uvicorn.run(app, host="0.0.0.0", port=8765)
+    uvicorn.run(app, host="0.0.0.0", port=8765)

mrrrme/backend/processing/speech.py

@@ -20,6 +20,7 @@ import requests
 import numpy as np
 import secrets
 import json
+import asyncio
 from datetime import datetime
 from typing import Optional, Dict
 from ..models.loader import get_models
@@ -71,6 +72,86 @@ def filter_transcription(transcription: str) -> tuple:
     return True, None
 
 
+async def stream_tts_audio_to_websocket(
+    websocket,
+    text: str,
+    voice: str,
+    language: str,
+    stream_id: str,
+) -> None:
+    """Proxy chunked PCM audio from the TTS service to the browser websocket."""
+    loop = asyncio.get_running_loop()
+    event_queue: asyncio.Queue[tuple[str, Optional[dict], Optional[str]]] = asyncio.Queue()
+
+    def read_stream() -> None:
+        try:
+            with requests.post(
+                f"{AVATAR_API}/speak_stream",
+                data={
+                    "text": text,
+                    "voice": voice,
+                    "language": language,
+                    "request_id": stream_id,
+                },
+                stream=True,
+                timeout=(10, 120),
+            ) as response:
+                response.raise_for_status()
+                for raw_line in response.iter_lines(decode_unicode=True):
+                    if not raw_line:
+                        continue
+                    event = json.loads(raw_line)
+                    asyncio.run_coroutine_threadsafe(
+                        event_queue.put(("event", event, None)),
+                        loop,
+                    ).result()
+        except Exception as exc:
+            asyncio.run_coroutine_threadsafe(
+                event_queue.put(("error", None, str(exc))),
+                loop,
+            ).result()
+        finally:
+            asyncio.run_coroutine_threadsafe(
+                event_queue.put(("done", None, None)),
+                loop,
+            ).result()
+
+    reader_task = asyncio.create_task(asyncio.to_thread(read_stream))
+
+    try:
+        while True:
+            event_type, event, error_message = await event_queue.get()
+            if event_type == "event" and event is not None:
+                upstream_type = event.get("type")
+                if upstream_type == "start":
+                    await websocket.send_json({
+                        "type": "tts_stream_start",
+                        "stream_id": stream_id,
+                        "sample_rate": event.get("sample_rate"),
+                        "channels": event.get("channels", 1),
+                        "duration": event.get("duration"),
+                        "generation_time": event.get("generation_time"),
+                    })
+                elif upstream_type == "chunk":
+                    await websocket.send_json({
+                        "type": "tts_stream_chunk",
+                        "stream_id": stream_id,
+                        "audio": event.get("audio"),
+                    })
+                elif upstream_type == "end":
+                    await websocket.send_json({
+                        "type": "tts_stream_end",
+                        "stream_id": stream_id,
+                        "duration": event.get("duration"),
+                    })
+            elif event_type == "error":
+                raise RuntimeError(error_message or "Unknown TTS stream error")
+            elif event_type == "done":
+                break
+    finally:
+        await reader_task
+
+
 def save_emotion_prediction(
     message_id: str,
     session_id: str,
@@ -533,40 +614,34 @@ async def process_speech_end(
         
         save_message(session_id, "assistant", response_text, fused_emotion, user_id)
         
-        # ========== SEND TO AVATAR ==========
+        # ========== SEND TEXT IMMEDIATELY, THEN STREAM TTS ==========
+        stream_id = secrets.token_urlsafe(8)
+        await websocket.send_json({
+            "type": "llm_response",
+            "text": response_text,
+            "emotion": fused_emotion,
+            "intensity": intensity,
+            "is_masking": is_masking,
+            "masking_type": masking_type,
+            "frames_captured": frame_count,
+            "audio_stream": True,
+            "stream_id": stream_id,
+        })
+
         try:
-            avatar_response = requests.post(
-                f"{AVATAR_API}/speak",
-                data={
-                    "text": response_text,
-                    "voice": user_preferences.get("voice", "female"),
-                    "language": user_preferences.get("language", "en")
-                },
-                timeout=45
+            await stream_tts_audio_to_websocket(
+                websocket=websocket,
+                text=response_text,
+                voice=user_preferences.get("voice", "female"),
+                language=user_preferences.get("language", "en"),
+                stream_id=stream_id,
             )
-            avatar_response.raise_for_status()
-            avatar_data = avatar_response.json()
-            
-            await websocket.send_json({
-                "type": "llm_response",
-                "text": response_text,
-                "emotion": fused_emotion,
-                "intensity": intensity,
-                "is_masking": is_masking,
-                "masking_type": masking_type,
-                "audio_url": avatar_data.get("audio_url"),
-                "visemes": avatar_data.get("visemes"),
-                "frames_captured": frame_count  #  NEW: Include frame count
-            })
         except Exception as avatar_err:
             safe_print(f"[TTS] Error: {avatar_err}")
             await websocket.send_json({
-                "type": "llm_response",
-                "text": response_text,
-                "emotion": fused_emotion,
-                "intensity": intensity,
-                "is_masking": is_masking,
-                "text_only": True
+                "type": "tts_stream_error",
+                "stream_id": stream_id,
+                "message": str(avatar_err),
             })
         
         safe_print(f"[Pipeline]  Complete for {username}")
@@ -575,4 +650,4 @@ async def process_speech_end(
     except Exception as e:
         safe_print(f"[Pipeline] Error: {e}")
         import traceback
-        traceback.print_exc()
+        traceback.print_exc()

mrrrme/backend/websocket.py

@@ -16,19 +16,18 @@ from fastapi import WebSocket, WebSocketDisconnect
 from starlette.websockets import WebSocketState
 import asyncio
 import base64
+import secrets
 import numpy as np
 import cv2
 import io
 from PIL import Image
-import requests
 
 from . import models as models_module
 from .session.manager import validate_token, save_message, load_user_history
 from .session.summary import generate_session_summary
 from .auth.database import get_db_connection
-from .utils.helpers import get_avatar_api_url
 from .config import GREETINGS
-from .processing.speech import process_speech_end
+from .processing.speech import process_speech_end, stream_tts_audio_to_websocket
 from .processing.face_emotion_aggregator import FaceEmotionAggregator
 
 # Import multi-frame capture
@@ -39,8 +38,6 @@ except ImportError:
     HAS_FRAME_BUFFER = False
     print("[WebSocket] FrameBuffer not available - using single frame capture")
 
-AVATAR_API = get_avatar_api_url()
-
 async def websocket_endpoint(websocket: WebSocket):
     """Main WebSocket endpoint handler with multi-frame capture"""
     
@@ -200,36 +197,34 @@ async def websocket_endpoint(websocket: WebSocket):
                     try:
                         lang = user_preferences.get("language", "en")
                         greeting_text = GREETINGS[lang]["returning" if user_summary else "new"].format(username=username)
-                        
-                        audio_url, visemes = None, None
-                        try:
-                            resp = requests.post(f"{AVATAR_API}/speak", data={
-                                "text": greeting_text,
-                                "voice": user_preferences.get("voice", "female"),
-                                "language": lang
-                            }, timeout=10)
-                            if resp.status_code == 200:
-                                avatar_data = resp.json()
-                                audio_url = avatar_data.get("audio_url")
-                                visemes = avatar_data.get("visemes")
-                        except:
-                            pass
-                        
+                        stream_id = secrets.token_urlsafe(8)
                         response_data = {
                             "type": "llm_response",
                             "text": greeting_text,
                             "emotion": "Neutral",
                             "intensity": 0.5,
-                            "is_greeting": True
+                            "is_greeting": True,
+                            "audio_stream": True,
+                            "stream_id": stream_id,
                         }
-                        if audio_url and visemes:
-                            response_data["audio_url"] = audio_url
-                            response_data["visemes"] = visemes
-                        else:
-                            response_data["text_only"] = True
-                        
                         await websocket.send_json(response_data)
                         save_message(session_id, "assistant", greeting_text, "Neutral")
+
+                        try:
+                            await stream_tts_audio_to_websocket(
+                                websocket=websocket,
+                                text=greeting_text,
+                                voice=user_preferences.get("voice", "female"),
+                                language=lang,
+                                stream_id=stream_id,
+                            )
+                        except Exception as tts_err:
+                            print(f"[Greeting] TTS stream error: {tts_err}", flush=True)
+                            await websocket.send_json({
+                                "type": "tts_stream_error",
+                                "stream_id": stream_id,
+                                "message": str(tts_err),
+                            })
                     except Exception as err:
                         print(f"[Greeting] Error: {err}", flush=True)
                 
@@ -392,4 +387,4 @@ async def websocket_endpoint(websocket: WebSocket):
                 print(f"[Summary]  Error: {e}", flush=True)
         
         print(f"[WebSocket] Closed for {username or 'Unknown'}", flush=True)
-        print("="*80 + "\n", flush=True)
+        print("="*80 + "\n", flush=True)