Gradio и FastAPI на GPU: API и UI для моделей

				
					from fastapi import FastAPI, Body, WebSocket
from sse_starlette.sse import EventSourceResponse
import asyncio, time, json

app = FastAPI()

@app.get("/healthz")
def health(): return {"ok": True}
@app.get("/livez")
def live():   return {"ok": True}

# REST: синхронный инференс (условный вызов локальной модели)
@app.post("/v1/complete")
def complete(payload: dict = Body(...)):
    prompt = payload["prompt"]; t0 = time.time()
    text = run_llm_sync(prompt, max_tokens=payload.get("max_tokens", 256))
    return {"ok": True, "latency_ms": int((time.time()-t0)*1000), "text": text}

# SSE: поток частичных токенов
@app.post("/v1/stream")
def stream(payload: dict = Body(...)):
    prompt = payload["prompt"]
    async def gen():
        async for token in run_llm_stream(prompt):
            yield {"event": "token", "data": token}
        yield {"event": "done", "data": "{}"}
    return EventSourceResponse(gen())

# WebSocket: поток аудио (ASR chunking 200–500 мс) → частичные гипотезы
@app.websocket("/ws/asr")
async def ws_asr(ws: WebSocket):
    await ws.accept()
    buffer = b""
    try:
        while True:
            chunk = await ws.receive_bytes()
            buffer += chunk
            if len(buffer) > 16000*2*0.3:  # ~300ms @16kHz, 16-bit
                text_partial = asr_score(buffer); buffer = b""
                await ws.send_json({"partial": text_partial})
    except Exception:
        await ws.close()