Merge pull request 'feat: add run_routing_benchmark.py — routing-only benchmark' (#19) from feat/routing-benchmark into main

Reviewed-on: #19

benchmarks/run_benchmark.py

@@ -120,7 +120,7 @@ def extract_tier_from_logs(logs_before: str, logs_after: str) -> str | None:
     """Find new tier= lines that appeared after we sent the query."""
     before_lines = set(logs_before.splitlines())
     new_lines = [l for l in logs_after.splitlines() if l not in before_lines]
-    for line in reversed(new_lines):
+    for line in new_lines:
         m = re.search(r"tier=(\w+(?:\s*\(dry-run\))?)", line)
         if m:
             tier_raw = m.group(1)
@@ -203,7 +203,7 @@ async def run(queries: list[dict], dry_run: bool = False) -> list[dict]:
 
             print(f"{qid:>3}  {expected:8}  ", end="", flush=True)
 
-            logs_before = get_log_tail(80)
+            logs_before = get_log_tail(300)
             t0 = time.monotonic()
 
             ok_post = await post_message(client, qid, query_text, dry_run=send_dry)
@@ -225,7 +225,7 @@ async def run(queries: list[dict], dry_run: bool = False) -> list[dict]:
 
             # Now the query is done — check logs for tier
             await asyncio.sleep(0.3)
-            logs_after = get_log_tail(80)
+            logs_after = get_log_tail(300)
             actual = extract_tier_from_logs(logs_before, logs_after)
 
             elapsed = time.monotonic() - t0

benchmarks/run_routing_benchmark.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+"""
+Adolf routing benchmark — tests routing decisions only, no LLM inference.
+
+Sends each query with no_inference=True, waits for the routing decision to
+appear in docker logs, and records whether the correct tier was selected.
+
+Usage:
+    python3 run_routing_benchmark.py [options]
+    python3 run_routing_benchmark.py --tier light|medium|complex
+    python3 run_routing_benchmark.py --category <name>
+    python3 run_routing_benchmark.py --ids 1,2,3
+    python3 run_routing_benchmark.py --list-categories
+
+No GPU check needed — inference is disabled for all queries.
+Adolf must be running at http://localhost:8000.
+"""
+
+import argparse
+import asyncio
+import json
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import httpx
+
+ADOLF_URL = "http://localhost:8000"
+DATASET = Path(__file__).parent / "benchmark.json"
+RESULTS = Path(__file__).parent / "routing_results_latest.json"
+QUERY_TIMEOUT = 30  # seconds — routing is fast, no LLM wait
+
+
+# ── Log helpers ────────────────────────────────────────────────────────────────
+
+def get_log_tail(n: int = 50) -> str:
+    result = subprocess.run(
+        ["docker", "logs", "deepagents", "--tail", str(n)],
+        capture_output=True, text=True,
+    )
+    return result.stdout + result.stderr
+
+
+def extract_tier_from_logs(logs_before: str, logs_after: str) -> str | None:
+    """Find new tier= lines that appeared after we sent the query."""
+    before_lines = set(logs_before.splitlines())
+    new_lines = [line for line in logs_after.splitlines() if line not in before_lines]
+    for line in new_lines:
+        m = re.search(r"tier=(\w+(?:\s*\(no-inference\))?)", line)
+        if m:
+            tier_raw = m.group(1)
+            return tier_raw.split()[0]
+    return None
+
+
+# ── Request helpers ────────────────────────────────────────────────────────────
+
+async def post_message(client: httpx.AsyncClient, query_id: int, query: str) -> bool:
+    payload = {
+        "text": query,
+        "session_id": f"routing-bench-{query_id}",
+        "channel": "cli",
+        "user_id": "benchmark",
+        "metadata": {"no_inference": True, "benchmark": True},
+    }
+    try:
+        r = await client.post(f"{ADOLF_URL}/message", json=payload, timeout=10)
+        r.raise_for_status()
+        return True
+    except Exception as e:
+        print(f" POST_ERROR: {e}", end="")
+        return False
+
+
+# ── Dataset ────────────────────────────────────────────────────────────────────
+
+def load_dataset() -> list[dict]:
+    with open(DATASET) as f:
+        return json.load(f)["queries"]
+
+
+def filter_queries(queries, tier, category, ids):
+    if tier:
+        queries = [q for q in queries if q["tier"] == tier]
+    if category:
+        queries = [q for q in queries if q["category"] == category]
+    if ids:
+        queries = [q for q in queries if q["id"] in ids]
+    return queries
+
+
+# ── Main run ───────────────────────────────────────────────────────────────────
+
+async def run(queries: list[dict]) -> list[dict]:
+    results = []
+
+    async with httpx.AsyncClient() as client:
+        try:
+            r = await client.get(f"{ADOLF_URL}/health", timeout=5)
+            r.raise_for_status()
+        except Exception as e:
+            print(f"ERROR: Adolf not reachable: {e}", file=sys.stderr)
+            sys.exit(1)
+
+        total = len(queries)
+        correct = 0
+
+        print(f"\nRunning {total} queries [NO-INFERENCE: routing only]\n")
+        print(f"{'ID':>3}  {'EXPECTED':8}  {'ACTUAL':8}  {'OK':3}  {'TIME':6}  {'CATEGORY':22}  QUERY")
+        print("─" * 110)
+
+        for q in queries:
+            qid = q["id"]
+            expected = q["tier"]
+            category = q["category"]
+            query_text = q["query"]
+            session_id = f"routing-bench-{qid}"
+
+            print(f"{qid:>3}  {expected:8}  ", end="", flush=True)
+
+            logs_before = get_log_tail(300)
+            t0 = time.monotonic()
+
+            ok_post = await post_message(client, qid, query_text)
+            if not ok_post:
+                print(f"{'?':8}  {'ERR':3}  {'?':6}  {category:22}  {query_text[:40]}")
+                results.append({"id": qid, "expected": expected, "actual": None, "ok": False})
+                continue
+
+            try:
+                async with client.stream(
+                    "GET", f"{ADOLF_URL}/stream/{session_id}", timeout=QUERY_TIMEOUT
+                ) as sse:
+                    async for line in sse.aiter_lines():
+                        if "data: [DONE]" in line:
+                            break
+            except Exception:
+                pass  # timeout or connection issue — check logs anyway
+
+            await asyncio.sleep(0.3)
+            logs_after = get_log_tail(300)
+            actual = extract_tier_from_logs(logs_before, logs_after)
+
+            elapsed = time.monotonic() - t0
+            match = actual == expected or (actual == "fast" and expected == "medium")
+            if match:
+                correct += 1
+
+            mark = "✓" if match else "✗"
+            actual_str = actual or "?"
+            print(f"{actual_str:8}  {mark:3}  {elapsed:5.1f}s  {category:22}  {query_text[:40]}")
+
+            results.append({
+                "id": qid,
+                "expected": expected,
+                "actual": actual_str,
+                "ok": match,
+                "elapsed": round(elapsed, 1),
+                "category": category,
+                "query": query_text,
+            })
+
+        print("─" * 110)
+        accuracy = correct / total * 100 if total else 0
+        print(f"\nAccuracy: {correct}/{total} ({accuracy:.0f}%)")
+
+        for tier_name in ["light", "medium", "complex"]:
+            tier_qs = [r for r in results if r["expected"] == tier_name]
+            if tier_qs:
+                tier_ok = sum(1 for r in tier_qs if r["ok"])
+                print(f"  {tier_name:8}: {tier_ok}/{len(tier_qs)}")
+
+        wrong = [r for r in results if not r["ok"]]
+        if wrong:
+            print(f"\nMisclassified ({len(wrong)}):")
+            for r in wrong:
+                print(f"  id={r['id']:3}  expected={r['expected']:8}  actual={r['actual']:8}  {r['query'][:60]}")
+
+    with open(RESULTS, "w") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+    print(f"\nResults saved to {RESULTS}")
+
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Adolf routing benchmark — routing decisions only, no LLM inference",
+    )
+    parser.add_argument("--tier", choices=["light", "medium", "complex"])
+    parser.add_argument("--category")
+    parser.add_argument("--ids", help="Comma-separated IDs")
+    parser.add_argument("--list-categories", action="store_true")
+    args = parser.parse_args()
+
+    queries = load_dataset()
+
+    if args.list_categories:
+        cats = sorted(set(q["category"] for q in queries))
+        tiers = {t: sum(1 for q in queries if q["tier"] == t) for t in ["light", "medium", "complex"]}
+        print(f"Total: {len(queries)} | Tiers: {tiers}")
+        print(f"Categories: {cats}")
+        return
+
+    ids = [int(i) for i in args.ids.split(",")] if args.ids else None
+    queries = filter_queries(queries, args.tier, args.category, ids)
+    if not queries:
+        print("No queries match filters.")
+        sys.exit(1)
+
+    asyncio.run(run(queries))
+
+
+if __name__ == "__main__":
+    main()

router.py

@@ -52,6 +52,13 @@ _LIGHT_PATTERNS = re.compile(
     r"|окей|хорошо|отлично|понятно|ок|ладно|договорились|спс|благодарю"
     r"|пожалуйста|не за что|всё понятно|ясно"
     r"|как дела|как ты|как жизнь|всё хорошо|всё ок"
+    # Russian tech definitions — static knowledge (no tools needed)
+    r"|что\s+такое\s+\S+"
+    r"|что\s+означает\s+\S+"
+    r"|сколько\s+(?:бит|байт|байтов|мегабайт|мегабайтов|гигабайт|гигабайтов)(?:\s+\w+)*"
+    # Compound Russian greetings
+    r"|привет[,!]?\s+как\s+дела"
+    r"|добрый\s+(?:день|вечер|утро)[,!]?\s+как\s+дела"
     r")[\s!.?]*$",
     re.IGNORECASE,
 )
@@ -314,6 +321,10 @@ _MEDIUM_PATTERNS = re.compile(
     r"|курс (?:доллара|биткоина|евро|рубл)"
     r"|(?:последние |свежие )?новости\b"
     r"|(?:погода|температура)\s+(?:на завтра|на неделю)"
+    # Smart home commands that don't use verb-first pattern
+    r"|(?:свет|лампочк|освещени)\w*\s+(?:включ|выключ|убавь|прибавь)"
+    r"|(?:дома|в доме|по всему дому)\s+(?:свет|лампочк)"
+    r"|(?:режим|сцена)\s+(?:ночной|утренний|вечерний|кинотеатр)"
     r")",
     re.IGNORECASE,
 )

tests/integration/common.py

@@ -11,7 +11,7 @@ import urllib.request
 
 # ── config ────────────────────────────────────────────────────────────────────
 DEEPAGENTS   = "http://localhost:8000"
-BIFROST      = "http://localhost:8080"
+LITELLM      = "http://localhost:4000"
 OPENMEMORY   = "http://localhost:8765"
 GRAMMY_HOST  = "localhost"
 GRAMMY_PORT  = 3001
@@ -156,19 +156,6 @@ def fetch_logs(since_s=600):
         return []
 
 
-def fetch_bifrost_logs(since_s=120):
-    """Return bifrost container log lines from the last since_s seconds."""
-    try:
-        r = subprocess.run(
-            ["docker", "compose", "-f", COMPOSE_FILE, "logs", "bifrost",
-             f"--since={int(since_s)}s", "--no-log-prefix"],
-            capture_output=True, text=True, timeout=10,
-        )
-        return r.stdout.splitlines()
-    except Exception:
-        return []
-
-
 def parse_run_block(lines, msg_prefix):
     """
     Scan log lines for the LAST '[agent] running: <msg_prefix>' block.

tests/integration/test_memory.py

@@ -6,7 +6,7 @@ Tests:
   1. Name store   — POST "remember that your name is <RandomName>"
   2. Qdrant point — verifies a new vector was written after store
   3. Name recall  — POST "what is your name?" → reply must contain <RandomName>
-  4. Bifrost      — verifies store/recall requests passed through Bifrost
+  4. LiteLLM      — verifies LiteLLM proxy is reachable (replaced Bifrost)
   5. Timing profile — breakdown of store and recall latencies
   6. Memory benchmark — store 5 personal facts, recall with 10 questions
   7. Dedup test   — same fact stored twice must not grow Qdrant by 2 points
@@ -24,11 +24,11 @@ import time
 import urllib.request
 
 from common import (
-    DEEPAGENTS, QDRANT, COMPOSE_FILE, DEFAULT_CHAT_ID,
+    DEEPAGENTS, LITELLM, QDRANT, COMPOSE_FILE, DEFAULT_CHAT_ID,
     NAMES,
     INFO, PASS, FAIL, WARN,
     report, print_summary, tf,
-    get, post_json, qdrant_count, fetch_logs, fetch_bifrost_logs,
+    get, post_json, qdrant_count, fetch_logs,
     parse_run_block, wait_for,
 )
 
@@ -155,14 +155,13 @@ if _run_name:
         report(results, "Agent replied to recall message", False, "timeout")
         report(results, f"Reply contains '{random_name}'", False, "no reply")
 
-    # ── 4. Bifrost pass-through check ─────────────────────────────────────────
-    bifrost_lines = fetch_bifrost_logs(since_s=300)
-    report(results, "Bifrost container has log output (requests forwarded)",
-           len(bifrost_lines) > 0, f"{len(bifrost_lines)} lines in bifrost logs")
-    bifrost_raw = "\n".join(bifrost_lines)
-    report(results, "  Bifrost log shows AsyncOpenAI agent requests",
-           "AsyncOpenAI" in bifrost_raw,
-           f"{'found' if 'AsyncOpenAI' in bifrost_raw else 'NOT found'} in bifrost logs")
+    # ── 4. LiteLLM proxy reachable (replaced Bifrost) ─────────────────────────
+    try:
+        status, _ = get(f"{LITELLM}/health", timeout=5)
+        litellm_ok = status == 200
+    except Exception:
+        litellm_ok = False
+    report(results, "LiteLLM proxy reachable", litellm_ok)
 
     # ── 5. Timing profile ─────────────────────────────────────────────────────
     print(f"\n[{INFO}] 5. Timing profile")