routing benchmark: 1s strict deadline per query

QUERY_TIMEOUT=1s — classification and routing must complete within
1 second or the query is recorded as 'timeout'.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-24 07:35:13 +00:00
parent fc53632c7b
commit b7d5896076

View File

@@ -30,7 +30,7 @@ import httpx
ADOLF_URL = "http://localhost:8000"
DATASET = Path(__file__).parent / "benchmark.json"
RESULTS = Path(__file__).parent / "routing_results_latest.json"
QUERY_TIMEOUT = 30 # seconds — routing is fast, no LLM wait
QUERY_TIMEOUT = 1 # 1s strict deadline — routing must decide within 1 second
# ── Log helpers ────────────────────────────────────────────────────────────────
@@ -139,9 +139,10 @@ async def run(queries: list[dict]) -> list[dict]:
except Exception:
pass # timeout or connection issue — check logs anyway
await asyncio.sleep(0.3)
logs_after = get_log_tail(300)
actual = extract_tier_from_logs(logs_before, logs_after)
if actual is None:
actual = "timeout"
elapsed = time.monotonic() - t0
match = actual == expected or (actual == "fast" and expected == "medium")
@@ -149,7 +150,7 @@ async def run(queries: list[dict]) -> list[dict]:
correct += 1
mark = "" if match else ""
actual_str = actual or "?"
actual_str = actual
print(f"{actual_str:8} {mark:3} {elapsed:5.1f}s {category:22} {query_text[:40]}")
results.append({