NVIDIA Triton: мультифреймворк‑сервинг на GPU

				
					from fastapi import FastAPI
from fastapi.responses import StreamingResponse
import tritonclient.http as http
import numpy as np, io

TRITON_URL = "http://localhost:8000"
cli = http.InferenceServerClient(url=TRITON_URL)
app = FastAPI()

@app.post("/detect")
def detect(image: bytes):
    inputs = [http.InferInput("image_bytes", [1, len(image)], "UINT8")]
    inputs[0].set_data_from_numpy(np.frombuffer(image, dtype=np.uint8).reshape(1, -1))
    outputs = [http.InferRequestedOutput("boxes"),
               http.InferRequestedOutput("scores"),
               http.InferRequestedOutput("labels")]
    res = cli.infer("cv_ensemble", inputs=inputs, outputs=outputs)
    return {
        "boxes":  res.as_numpy("boxes").tolist(),
        "scores": res.as_numpy("scores").tolist(),
        "labels": res.as_numpy("labels").tolist(),
    }

@app.get("/generate")
def generate(prompt: str):
    def stream():
        # пример: ранняя отдача токенов; сервер внутри батчит/декодит
        for tok in llm_tokens_from_triton(prompt, delta_ms=20):
            yield f"data: {tok}\n\n"
    return StreamingResponse(stream(), media_type="text/event-stream")