diff --git a/openai/docker-compose.yml b/openai/docker-compose.yml index 9175e59..c520fb3 100644 --- a/openai/docker-compose.yml +++ b/openai/docker-compose.yml @@ -13,13 +13,10 @@ services: - OLLAMA_MAX_LOADED_MODELS=2 # One GPU inference at a time — prevents compute contention between models - OLLAMA_NUM_PARALLEL=1 - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] + # Force all layers to GPU — fail instead of falling back to CPU + - OLLAMA_NUM_GPU=999 + runtime: nvidia + mem_limit: 4g ollama-cpu: image: ollama/ollama @@ -37,6 +34,8 @@ services: - "3125:8080" volumes: - /mnt/ssd/ai/open-webui:/app/backend/data + extra_hosts: + - "host.docker.internal:host-gateway" restart: always deploy: resources: @@ -47,6 +46,78 @@ services: capabilities: [gpu] environment: - ANTHROPIC_API_KEY=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 + - OPENAI_API_BASE_URL=http://host.docker.internal:4000/v1 + - OPENAI_API_KEY=dummy + # STT — Faster-Whisper large-v3-turbo + - AUDIO_STT_ENGINE=openai + - AUDIO_STT_OPENAI_API_BASE_URL=http://host.docker.internal:8880/v1 + - AUDIO_STT_OPENAI_API_KEY=dummy + - AUDIO_STT_MODEL=deepdml/faster-whisper-large-v3-turbo-ct2 + # TTS — Silero v4 + - AUDIO_TTS_ENGINE=openai + - AUDIO_TTS_OPENAI_API_BASE_URL=http://host.docker.internal:8881/v1 + - AUDIO_TTS_OPENAI_API_KEY=dummy + - AUDIO_TTS_MODEL=silero + - AUDIO_TTS_VOICE=onyx + + litellm-db: + image: postgres:16-alpine + container_name: litellm-db + environment: + - POSTGRES_DB=litellm + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=litellm + volumes: + - /mnt/ssd/dbs/litellm/postgres:/var/lib/postgresql/data + restart: always + + litellm: + image: ghcr.io/berriai/litellm:main-latest + container_name: litellm + ports: + - "4000:4000" + volumes: + - ./litellm-config.yaml:/app/config.yaml + environment: + - DATABASE_URL=postgresql://litellm:litellm@litellm-db:5432/litellm + - LITELLM_MASTER_KEY=sk-fjQC1BxAiGFSMs + - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-changeme} + - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-changeme} + - LANGFUSE_HOST=http://langfuse:3000 + - OPENROUTER_API_KEY=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 + command: ["--config", "/app/config.yaml", "--port", "4000"] + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + - litellm-db + - langfuse + restart: always + + langfuse-db: + image: postgres:16-alpine + container_name: langfuse-db + environment: + - POSTGRES_DB=langfuse + - POSTGRES_USER=langfuse + - POSTGRES_PASSWORD=langfuse + volumes: + - /mnt/ssd/dbs/langfuse/postgres:/var/lib/postgresql/data + restart: always + + langfuse: + image: ghcr.io/langfuse/langfuse:2 + container_name: langfuse + ports: + - "3200:3000" + environment: + - DATABASE_URL=postgresql://langfuse:langfuse@langfuse-db:5432/langfuse + - NEXTAUTH_URL=https://lf.alogins.net + - NEXTAUTH_SECRET=532a746b24ac40afa39f9d317031cab94d4d6881107ea3b1209b28020f1a9761 + - SALT=7927b3b0092afe4542274940b557becea6418a5fed79f7acd25c3a789349fdc9 + - AUTH_DISABLE_SIGNUP=true + depends_on: + - langfuse-db + restart: always searxng: image: docker.io/searxng/searxng:latest @@ -67,3 +138,48 @@ services: restart: always volumes: - /mnt/ssd/dbs/qdrant:/qdrant/storage:z + + faster-whisper: + image: fedirz/faster-whisper-server:latest-cuda + container_name: faster-whisper + runtime: nvidia + ports: + - "8880:8000" + environment: + - WHISPER__MODEL=deepdml/faster-whisper-large-v3-turbo-ct2 + - WHISPER__INFERENCE_DEVICE=cuda + - WHISPER__COMPUTE_TYPE=int8 + - WHISPER__LANGUAGE=ru + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + volumes: + - /mnt/ssd/ai/faster-whisper:/root/.cache/huggingface + restart: always + + silero-tts: + build: ./silero-tts + container_name: silero-tts + ports: + - "8881:8881" + volumes: + - /mnt/ssd/ai/silero-tts:/cache/torch + restart: always + + pipecat: + build: ./pipecat + container_name: pipecat + ports: + - "8882:8882" + environment: + - LIVEKIT_URL=ws://host.docker.internal:7880 + - LIVEKIT_PUBLIC_URL=wss://lk.alogins.net + - LIVEKIT_API_KEY=devkey + - LIVEKIT_SECRET=ef3ef4b903ca8469b09b2dd7ab6af529c4d2f3c95668f53832fc351cf67777a9 + - ADOLF_URL=http://host.docker.internal:8000/v1 + - STT_URL=http://host.docker.internal:8880/v1 + - TTS_URL=http://host.docker.internal:8881/v1 + - STT_MODEL=deepdml/faster-whisper-large-v3-turbo-ct2 + - TTS_VOICE=onyx + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped diff --git a/openai/litellm-config.yaml b/openai/litellm-config.yaml new file mode 100644 index 0000000..d945576 --- /dev/null +++ b/openai/litellm-config.yaml @@ -0,0 +1,120 @@ +model_list: + # ── oO aliases (used by ml/serving; see oO/CLAUDE.md AI stack) ────────── + - model_name: tip-generator + litellm_params: + model: ollama/qwen2.5:1.5b + api_base: http://host.docker.internal:11434 + + - model_name: embedder + litellm_params: + model: ollama/nomic-embed-text + api_base: http://host.docker.internal:11434 + + - model_name: judge + litellm_params: + model: anthropic/claude-haiku-4-5-20251001 + api_key: os.environ/ANTHROPIC_API_KEY + + # ── raw model exposure ───────────────────────────────────────────────── + - model_name: ollama/qwen3.5:4b + litellm_params: + model: ollama/qwen3.5:4b + api_base: http://host.docker.internal:11436 + + - model_name: ollama/qwen3:8b + litellm_params: + model: ollama/qwen3:8b + api_base: http://host.docker.internal:11436 + + - model_name: ollama/qwen2.5:1.5b + litellm_params: + model: ollama/qwen2.5:1.5b + api_base: http://host.docker.internal:11436 + + - model_name: ollama/qwen2.5:0.5b + litellm_params: + model: ollama/qwen2.5:0.5b + api_base: http://host.docker.internal:11436 + + - model_name: ollama/gemma3:4b + litellm_params: + model: ollama/gemma3:4b + api_base: http://host.docker.internal:11436 + + - model_name: ollama/gemma3:1b + litellm_params: + model: ollama/gemma3:1b + api_base: http://host.docker.internal:11435 + + - model_name: ollama/nomic-embed-text + litellm_params: + model: ollama/nomic-embed-text + api_base: http://host.docker.internal:11435 + + # OpenRouter free-tier models + - model_name: meta-llama/llama-3.3-70b-instruct:free + litellm_params: + model: openrouter/meta-llama/llama-3.3-70b-instruct:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: meta-llama/llama-3.2-3b-instruct:free + litellm_params: + model: openrouter/meta-llama/llama-3.2-3b-instruct:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: deepseek/deepseek-r1:free + litellm_params: + model: openrouter/deepseek/deepseek-r1:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: qwen/qwen3-4b:free + litellm_params: + model: openrouter/qwen/qwen3-4b:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: qwen/qwen3-coder:free + litellm_params: + model: openrouter/qwen/qwen3-coder:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: google/gemma-3-27b-it:free + litellm_params: + model: openrouter/google/gemma-3-27b-it:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: google/gemma-3-12b-it:free + litellm_params: + model: openrouter/google/gemma-3-12b-it:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: mistralai/mistral-small-3.1-24b-instruct:free + litellm_params: + model: openrouter/mistralai/mistral-small-3.1-24b-instruct:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: nvidia/nemotron-3-super-120b-a12b:free + litellm_params: + model: openrouter/nvidia/nemotron-3-super-120b-a12b:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: openai/gpt-oss-120b:free + litellm_params: + model: openrouter/openai/gpt-oss-120b:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: minimax/minimax-m2.5:free + litellm_params: + model: openrouter/minimax/minimax-m2.5:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: nousresearch/hermes-3-llama-3.1-405b:free + litellm_params: + model: openrouter/nousresearch/hermes-3-llama-3.1-405b:free + api_key: os.environ/OPENROUTER_API_KEY + +litellm_settings: + success_callback: ["langfuse"] + failure_callback: ["langfuse"] + drop_params: true + fallbacks: + - deepseek/deepseek-r1:free: ["ollama/qwen3.5:4b"] diff --git a/openai/pipecat/Dockerfile b/openai/pipecat/Dockerfile new file mode 100644 index 0000000..8446296 --- /dev/null +++ b/openai/pipecat/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y --no-install-recommends gcc g++ && rm -rf /var/lib/apt/lists/* + +# CPU torch first — prevents silero-vad from pulling in the CUDA variant +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu + +RUN pip install --no-cache-dir \ + "pipecat-ai[openai,livekit,silero]" \ + "livekit-api" \ + fastapi \ + "uvicorn[standard]" + +WORKDIR /app +COPY . . + +EXPOSE 8882 +CMD ["uvicorn", "bot:app", "--host", "0.0.0.0", "--port", "8882"] diff --git a/openai/pipecat/bot.py b/openai/pipecat/bot.py new file mode 100644 index 0000000..ad2fc6d --- /dev/null +++ b/openai/pipecat/bot.py @@ -0,0 +1,228 @@ +import asyncio +import os +import re +import uuid +import logging + +from fastapi import FastAPI +from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel + +from livekit import api as lkapi + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import TextFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameProcessor, FrameDirection +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.stt import OpenAISTTService +from pipecat.services.openai.tts import OpenAITTSService +from pipecat.transports.livekit.transport import LiveKitTransport, LiveKitParams + + +# ── TTS text normalizer ────────────────────────────────────────────────────── +# Replaces symbols and abbreviations with spoken Russian words so Silero TTS +# doesn't truncate on unknown characters. + +_NORM_RULES: list[tuple[re.Pattern, str]] = [ + # Temperature: +12°C / -5°С / 12 °C → плюс двенадцать градусов цельсия + (re.compile(r"([+-]?\d+)\s*°\s*[CСcс]", re.IGNORECASE), r"\1 градусов цельсия"), + # Bare degree sign: 90° → 90 градусов + (re.compile(r"(\d+)\s*°"), r"\1 градусов"), + # Percent + (re.compile(r"(\d+)\s*%"), r"\1 процентов"), + # Speed: m/s, м/с, km/h, км/ч + (re.compile(r"\bm/s\b", re.IGNORECASE), "метров в секунду"), + (re.compile(r"\bм/с\b"), "метров в секунду"), + (re.compile(r"\bkm/h\b", re.IGNORECASE), "километров в час"), + (re.compile(r"\bкм/ч\b"), "километров в час"), + # Currency + (re.compile(r"\$\s*(\d+)"), r"\1 долларов"), + (re.compile(r"(\d+)\s*\$"), r"\1 долларов"), + (re.compile(r"€\s*(\d+)"), r"\1 евро"), + (re.compile(r"(\d+)\s*€"), r"\1 евро"), + (re.compile(r"(\d+)\s*₽"), r"\1 рублей"), + # Plus/minus signs before numbers + (re.compile(r"\+(\d)"), r"плюс \1"), + (re.compile(r"-(\d)"), r"минус \1"), + # Common abbreviations + (re.compile(r"\bкг\b"), "килограмм"), + (re.compile(r"\bг\b(?=\s|$)"), "грамм"), + (re.compile(r"\bмм\b"), "миллиметров"), + (re.compile(r"\bсм\b"), "сантиметров"), + (re.compile(r"\bкм\b"), "километров"), + # Strip remaining special chars that TTS can't handle + (re.compile(r"[°•·†‡§¶©®™«»<>{}[\]|\\~^`]"), ""), +] + + +def normalize_for_tts(text: str) -> str: + """Replace symbols with spoken Russian equivalents.""" + for pattern, replacement in _NORM_RULES: + text = pattern.sub(replacement, text) + return text + + +class TTSTextNormalizer(FrameProcessor): + """Intercepts TextFrames between LLM and TTS, normalizing symbols to words.""" + + async def process_frame(self, frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): + original = frame.text + normalized = normalize_for_tts(original) + if normalized != original: + logger.debug(f"TTSTextNormalizer: {original!r} → {normalized!r}") + await self.push_frame(TextFrame(text=normalized), direction) + else: + await self.push_frame(frame, direction) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# ── Config ──────────────────────────────────────────────────────────────────── +LIVEKIT_URL = os.getenv("LIVEKIT_URL", "ws://host.docker.internal:7880") # bot connects here +LIVEKIT_PUBLIC_URL = os.getenv("LIVEKIT_PUBLIC_URL", "wss://lk.alogins.net") # browser connects here +LIVEKIT_API_KEY = os.getenv("LIVEKIT_API_KEY", "devkey") +LIVEKIT_SECRET = os.getenv("LIVEKIT_SECRET", "") +ADOLF_URL = os.getenv("ADOLF_URL", "http://host.docker.internal:8000/v1") +STT_URL = os.getenv("STT_URL", "http://host.docker.internal:8880/v1") +TTS_URL = os.getenv("TTS_URL", "http://host.docker.internal:8881/v1") +STT_MODEL = os.getenv("STT_MODEL", "deepdml/faster-whisper-large-v3-turbo-ct2") +TTS_VOICE = os.getenv("TTS_VOICE", "onyx") + +SYSTEM_PROMPT = "You are Adolf, a helpful voice assistant. Keep replies concise — 1-3 sentences. No markdown." + +app = FastAPI(title="Pipecat Voice Bot") +app.mount("/static", StaticFiles(directory="static"), name="static") + + +# ── LiveKit helpers ─────────────────────────────────────────────────────────── +def _lk_token(room: str, identity: str, is_bot: bool = False) -> str: + grants = lkapi.VideoGrants( + room_join=True, + room=room, + can_publish=True, + can_subscribe=True, + can_publish_data=True, + ) + token = ( + lkapi.AccessToken(LIVEKIT_API_KEY, LIVEKIT_SECRET) + .with_identity(identity) + .with_name("Adolf Bot" if is_bot else identity) + .with_grants(grants) + ) + return token.to_jwt() + + +async def _create_room(room_name: str) -> None: + lk = lkapi.LiveKitAPI(LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_SECRET) + try: + await lk.room.create_room( + lkapi.CreateRoomRequest(name=room_name, empty_timeout=300, max_participants=5) + ) + finally: + await lk.aclose() + + +# ── Pipecat pipeline ────────────────────────────────────────────────────────── +async def _run_bot(room_name: str) -> None: + bot_token = _lk_token(room_name, "pipecat-bot", is_bot=True) + + transport = LiveKitTransport( + url=LIVEKIT_URL, + token=bot_token, + room_name=room_name, + params=LiveKitParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams( + stop_secs=0.8, # wait 0.8s of silence before end-of-speech + start_secs=0.2, # start speech detection after 0.2s + confidence=0.85, # high confidence to avoid triggering on ambient noise + )), + ), + ) + + stt = OpenAISTTService( + api_key="dummy", + base_url=STT_URL, + model=STT_MODEL, + language="ru", + ) + + llm = OpenAILLMService( + api_key="dummy", + base_url=ADOLF_URL, + model="adolf-light", + ) + + tts = OpenAITTSService( + api_key="dummy", + base_url=TTS_URL, + model="silero", + voice=TTS_VOICE, + ) + + messages = [{"role": "system", "content": SYSTEM_PROMPT}] + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + normalizer = TTSTextNormalizer() + + pipeline = Pipeline([ + transport.input(), + stt, + context_aggregator.user(), + llm, + normalizer, + tts, + transport.output(), + context_aggregator.assistant(), + ]) + + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=False)) + + @transport.event_handler("on_participant_disconnected") + async def on_disconnect(transport, participant): + identity = participant if isinstance(participant, str) else getattr(participant, "identity", str(participant)) + logger.info(f"Participant {identity} left — stopping bot") + await task.cancel() + + runner = PipelineRunner() + logger.info(f"Bot starting in room={room_name}") + await runner.run(task) + logger.info(f"Bot done in room={room_name}") + + +# ── API ─────────────────────────────────────────────────────────────────────── +class ConnectResponse(BaseModel): + room: str + token: str + url: str + + +@app.post("/connect", response_model=ConnectResponse) +async def connect(): + room_name = f"voice-{uuid.uuid4().hex[:6]}" + await _create_room(room_name) + user_token = _lk_token(room_name, "user") + asyncio.create_task(_run_bot(room_name)) + return ConnectResponse(room=room_name, token=user_token, url=LIVEKIT_PUBLIC_URL) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.get("/", response_class=HTMLResponse) +async def index(): + with open("static/index.html") as f: + return f.read() diff --git a/openai/pipecat/static/index.html b/openai/pipecat/static/index.html new file mode 100644 index 0000000..7017aa6 --- /dev/null +++ b/openai/pipecat/static/index.html @@ -0,0 +1,241 @@ + + + + + +Adolf Voice + + + +
+

Adolf

+

Voice assistant

+
🎙️
+
Press to connect
+
+ +
+ + + + + diff --git a/openai/pipecat/test_pipeline.py b/openai/pipecat/test_pipeline.py new file mode 100644 index 0000000..1b8cfb8 --- /dev/null +++ b/openai/pipecat/test_pipeline.py @@ -0,0 +1,122 @@ +""" +End-to-end pipeline test: +1. Call /connect to get a room + token +2. Join the LiveKit room as a Python client +3. Publish TTS audio (pre-generated) as microphone input +4. Capture bot's audio response and save to file +""" +import asyncio +import wave +import struct +import httpx +import numpy as np +from livekit import rtc + +PIPECAT_URL = "http://localhost:8882" +TTS_URL = "http://host.docker.internal:8881" +OUTPUT_FILE = "/tmp/bot_response.wav" +SAMPLE_RATE = 48000 +NUM_CHANNELS = 1 + + +async def generate_tts_pcm(text: str) -> bytes: + """Get WAV audio from Silero TTS, return raw PCM int16 bytes.""" + async with httpx.AsyncClient(timeout=30) as c: + r = await c.post(f"{TTS_URL}/v1/audio/speech", json={ + "input": text, "voice": "onyx", "response_format": "wav" + }) + r.raise_for_status() + # Skip WAV header (44 bytes) to get raw PCM + return r.content[44:] + + +async def main(): + # Step 1 — create room + print("[test] Creating room...") + async with httpx.AsyncClient() as c: + r = await c.post(f"{PIPECAT_URL}/connect") + r.raise_for_status() + creds = r.json() + print(f"[test] Room: {creds['room']} URL: {creds['url']}") + + # Step 2 — generate test audio + test_phrase = "Привет! Как тебя зовут?" + print(f"[test] Generating TTS for: {test_phrase!r}") + pcm_bytes = await generate_tts_pcm(test_phrase) + print(f"[test] TTS PCM: {len(pcm_bytes)} bytes (~{len(pcm_bytes)//(SAMPLE_RATE*2):.1f}s)") + + # Step 3 — join room + room = rtc.Room() + received_frames: list[bytes] = [] + + @room.on("track_subscribed") + def on_track(track, pub, participant): + if track.kind == rtc.TrackKind.KIND_AUDIO and participant.identity == "pipecat-bot": + print(f"[test] Subscribed to bot audio track") + audio_stream = rtc.AudioStream(track, sample_rate=SAMPLE_RATE, num_channels=NUM_CHANNELS) + asyncio.ensure_future(_collect_audio(audio_stream, received_frames)) + + ws_url = creds["url"].replace("https://", "wss://").replace("http://", "ws://") + # Connect internally via host.docker.internal + internal_url = "ws://host.docker.internal:7880" + print(f"[test] Connecting to LiveKit at {internal_url}...") + await room.connect(internal_url, creds["token"]) + print(f"[test] Connected. Waiting for bot to join...") + + # Wait for bot participant + for _ in range(20): + if any(p.identity == "pipecat-bot" for p in room.remote_participants.values()): + break + await asyncio.sleep(0.5) + print(f"[test] Participants: {[p.identity for p in room.remote_participants.values()]}") + + # Step 4 — publish audio as microphone + print("[test] Publishing audio track...") + source = rtc.AudioSource(SAMPLE_RATE, NUM_CHANNELS) + local_track = rtc.LocalAudioTrack.create_audio_track("microphone", source) + opts = rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_MICROPHONE) + await room.local_participant.publish_track(local_track, opts) + + # Send PCM in 20ms chunks + chunk_samples = SAMPLE_RATE * 20 // 1000 # 960 samples per chunk + chunk_bytes = chunk_samples * 2 # int16 + print(f"[test] Sending {len(pcm_bytes) // chunk_bytes} audio chunks...") + for i in range(0, len(pcm_bytes), chunk_bytes): + chunk = pcm_bytes[i:i + chunk_bytes] + if len(chunk) < chunk_bytes: + chunk = chunk + b'\x00' * (chunk_bytes - len(chunk)) + samples = np.frombuffer(chunk, dtype=np.int16) + frame = rtc.AudioFrame( + data=samples.tobytes(), + sample_rate=SAMPLE_RATE, + num_channels=NUM_CHANNELS, + samples_per_channel=chunk_samples, + ) + await source.capture_frame(frame) + await asyncio.sleep(0.02) + + print("[test] Audio sent. Waiting for bot response (up to 30s)...") + await asyncio.sleep(30) + + await room.disconnect() + + # Step 5 — save response + if received_frames: + total = b"".join(received_frames) + print(f"[test] Received {len(total)} bytes of bot audio ({len(total)//(SAMPLE_RATE*2):.1f}s)") + with wave.open(OUTPUT_FILE, "wb") as wf: + wf.setnchannels(NUM_CHANNELS) + wf.setsampwidth(2) + wf.setframerate(SAMPLE_RATE) + wf.writeframes(total) + print(f"[test] Saved to {OUTPUT_FILE}") + else: + print("[test] No audio received from bot!") + + +async def _collect_audio(stream: rtc.AudioStream, buf: list): + async for event in stream: + buf.append(bytes(event.frame.data)) + + +asyncio.run(main()) diff --git a/openai/silero-tts/Dockerfile b/openai/silero-tts/Dockerfile new file mode 100644 index 0000000..1af5194 --- /dev/null +++ b/openai/silero-tts/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* + +# CPU-only torch keeps image ~500MB vs ~2GB for CUDA +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu +RUN pip install --no-cache-dir fastapi uvicorn scipy numpy pydub omegaconf + +WORKDIR /app +COPY server.py . + +ENV TORCH_HOME=/cache/torch + +EXPOSE 8881 +CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8881"] diff --git a/openai/silero-tts/server.py b/openai/silero-tts/server.py new file mode 100644 index 0000000..9d44c90 --- /dev/null +++ b/openai/silero-tts/server.py @@ -0,0 +1,153 @@ +import io +import re +import logging +import numpy as np +import torch +from fastapi import FastAPI, HTTPException +from fastapi.responses import Response +from pydantic import BaseModel +import scipy.io.wavfile as wavfile +from pydub import AudioSegment + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +app = FastAPI(title="Silero TTS") + +# ── Config ──────────────────────────────────────────────────────────────────── +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" +SAMPLE_RATE = 24000 +MAX_CHUNK = 800 # chars per Silero call + +# Model identifiers (passed as `speaker` to torch.hub.load — selects model file) +MODEL_ID = {"ru": "v3_1_ru", "en": "v3_en"} + +# Silero speakers (passed to apply_tts) +RU_SPEAKERS = ["aidar", "baya", "kseniya", "xenia", "eugene"] +EN_SPEAKERS = [f"en_{i}" for i in range(10)] + +# OpenAI voice → Silero speaker +VOICE_MAP = { + "ru": {"alloy": "eugene", "echo": "aidar", "fable": "baya", + "onyx": "eugene", "nova": "kseniya", "shimmer": "xenia"}, + "en": {"alloy": "en_3", "echo": "en_1", "fable": "en_2", + "onyx": "en_3", "nova": "en_4", "shimmer": "en_5"}, +} + +# ── Model cache ─────────────────────────────────────────────────────────────── +_models: dict[str, object] = {} + + +def _get_model(language: str): + if language not in _models: + logger.info(f"Loading Silero model {MODEL_ID[language]} lang={language} device={DEVICE}") + model, _ = torch.hub.load( + repo_or_dir="snakers4/silero-models", + model="silero_tts", + language=language, + speaker=MODEL_ID[language], + trust_repo=True, + ) + model.to(DEVICE) + _models[language] = model + logger.info(f"Model ready: lang={language}") + return _models[language] + + +@app.on_event("startup") +async def preload(): + """Preload both language models to avoid cold-start on first request.""" + for lang in ("ru", "en"): + try: + _get_model(lang) + except Exception as e: + logger.warning(f"Preload failed for lang={lang}: {e}") + + +# ── Helpers ─────────────────────────────────────────────────────────────────── +def _is_russian(text: str) -> bool: + return bool(re.search(r"[а-яёА-ЯЁ]", text)) + + +def _split_sentences(text: str) -> list[str]: + """Split on sentence boundaries, keeping chunks under MAX_CHUNK chars.""" + if len(text) <= MAX_CHUNK: + return [text] + parts = re.split(r"(?<=[.!?;])\s+", text.strip()) + chunks, cur = [], "" + for part in parts: + if len(cur) + len(part) + 1 <= MAX_CHUNK: + cur = f"{cur} {part}" if cur else part + else: + if cur: + chunks.append(cur) + # If single part is too long, split mid-word as last resort + cur = part[:MAX_CHUNK] if len(part) > MAX_CHUNK else part + if cur: + chunks.append(cur) + return chunks or [text[:MAX_CHUNK]] + + +def _to_bytes(audio: torch.Tensor, fmt: str) -> bytes: + pcm = (audio.cpu().numpy() * 32767).astype(np.int16) + if fmt == "pcm": + return pcm.tobytes() + buf = io.BytesIO() + wavfile.write(buf, SAMPLE_RATE, pcm) + if fmt == "wav": + return buf.getvalue() + seg = AudioSegment.from_wav(io.BytesIO(buf.getvalue())) + out = io.BytesIO() + seg.export(out, format="mp3") + return out.getvalue() + + +# ── API ─────────────────────────────────────────────────────────────────────── +class SpeechRequest(BaseModel): + model: str = "silero" + input: str + voice: str = "alloy" + response_format: str = "mp3" + speed: float = 1.0 + + +@app.get("/health") +async def health(): + return {"status": "ok", "device": DEVICE} + + +@app.get("/v1/models") +async def list_models(): + return { + "object": "list", + "data": [{"id": "silero", "object": "model", "owned_by": "silero"}], + } + + +@app.post("/v1/audio/speech") +async def speech(req: SpeechRequest): + text = req.input.strip() + if not text: + raise HTTPException(status_code=400, detail="input is empty") + + language = "ru" if _is_russian(text) else "en" + vm = VOICE_MAP[language] + speaker = vm.get(req.voice, vm["alloy"]) + + try: + model = _get_model(language) + chunks = _split_sentences(text) + parts = [ + model.apply_tts(text=chunk, speaker=speaker, sample_rate=SAMPLE_RATE) + for chunk in chunks + ] + audio = parts[0] if len(parts) == 1 else torch.cat(parts) + except Exception as e: + logger.error(f"TTS error: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + fmt = req.response_format.lower() + audio_bytes = _to_bytes(audio, fmt) + media_types = {"wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg"} + media_type = media_types.get(fmt, "audio/mpeg") + return Response(content=audio_bytes, media_type=media_type)