feat(ml): prompt registry + per-request variant selection
Replaces the hardcoded "v1" label with a real prompt registry:
ml/serving/prompts.py — keyed by version: v1 (baseline),
v2-mentor (calm/specific persona),
v3-few-shot (v1 persona + curated examples)
ml/serving/main.py — POST /generate accepts optional prompt_version,
422 on unknown, echoes the version actually used
back in the response
services/api/src/config.ts — TIP_PROMPT_VERSION: empty / single / comma-list
(uniform random per request)
services/api/src/routes/recommender.ts
— pickPromptVersion() drives selection; the
response's prompt_version (not a stale TS
constant) is what lands in tip_scores so the
#92 reward-analytics dashboard shows real
per-variant reaction rates
Closes #84.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,10 @@ import httpx
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from httpx import AsyncClient, ASGITransport, Response
|
||||
|
||||
from main import app, _build_prompt, PromptContext
|
||||
from main import app, PromptContext
|
||||
from prompts import PROMPTS, get_prompt
|
||||
|
||||
_build_user_v1 = PROMPTS["v1"].build_user
|
||||
|
||||
|
||||
def _litellm_response(candidates: list[dict]) -> Response:
|
||||
@@ -96,7 +99,7 @@ def test_build_prompt_includes_tasks():
|
||||
hour_of_day=9,
|
||||
day_of_week=2,
|
||||
)
|
||||
prompt = _build_prompt(ctx, n=3)
|
||||
prompt = _build_user_v1(ctx, n=3)
|
||||
assert "Write report" in prompt
|
||||
assert "09:00" in prompt
|
||||
assert "Generate 3 tips" in prompt
|
||||
@@ -105,21 +108,21 @@ def test_build_prompt_includes_tasks():
|
||||
def test_build_prompt_truncates_at_five():
|
||||
tasks = [{"content": f"Task {i}", "priority": 1, "is_overdue": False, "due_date": None} for i in range(8)]
|
||||
ctx = PromptContext(tasks=tasks, hour_of_day=12)
|
||||
prompt = _build_prompt(ctx, n=2)
|
||||
prompt = _build_user_v1(ctx, n=2)
|
||||
assert "Task 4" in prompt
|
||||
assert "Task 5" not in prompt
|
||||
|
||||
|
||||
def test_build_prompt_extra_fields():
|
||||
ctx = PromptContext(tasks=[], hour_of_day=8, extra={"mood": "focused", "energy": "high"})
|
||||
prompt = _build_prompt(ctx, n=1)
|
||||
prompt = _build_user_v1(ctx, n=1)
|
||||
assert "mood: focused" in prompt
|
||||
assert "energy: high" in prompt
|
||||
|
||||
|
||||
def test_build_prompt_empty_tasks_no_task_line():
|
||||
ctx = PromptContext(tasks=[], hour_of_day=10)
|
||||
prompt = _build_prompt(ctx, n=2)
|
||||
prompt = _build_user_v1(ctx, n=2)
|
||||
assert "Tasks:" not in prompt
|
||||
assert "Generate 2 tips" in prompt
|
||||
|
||||
@@ -223,3 +226,57 @@ def test_parse_llm_json_raises_on_invalid():
|
||||
from main import _parse_llm_json
|
||||
with pytest.raises((ValueError, Exception)):
|
||||
_parse_llm_json("this is not json")
|
||||
|
||||
|
||||
# ── Prompt registry / selection (#84) ──────────────────────────────────────
|
||||
|
||||
def test_prompt_registry_contains_expected_versions():
|
||||
assert set(PROMPTS.keys()) >= {"v1", "v2-mentor", "v3-few-shot"}
|
||||
# v2-mentor must differ from v1 in tone — easiest assertion: different system prompt.
|
||||
assert PROMPTS["v1"].system != PROMPTS["v2-mentor"].system
|
||||
# v3-few-shot must include curated example content in its system prompt.
|
||||
assert "Examples" in PROMPTS["v3-few-shot"].system
|
||||
|
||||
|
||||
def test_get_prompt_unknown_raises_keyerror():
|
||||
with pytest.raises(KeyError):
|
||||
get_prompt("does-not-exist")
|
||||
|
||||
|
||||
def test_get_prompt_default_when_none():
|
||||
p = get_prompt(None)
|
||||
assert p.version == "v1" # current DEFAULT_PROMPT_VERSION
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_generate_echoes_selected_prompt_version():
|
||||
"""Server should report back which prompt_version it actually used."""
|
||||
fake_items = [{"id": "tip-1", "content": "x", "rationale": "y"}]
|
||||
mock_resp = _litellm_response(fake_items)
|
||||
|
||||
with patch("main.httpx.AsyncClient") as MockClient:
|
||||
instance = AsyncMock()
|
||||
instance.post = AsyncMock(return_value=mock_resp)
|
||||
instance.__aenter__ = AsyncMock(return_value=instance)
|
||||
instance.__aexit__ = AsyncMock(return_value=False)
|
||||
MockClient.return_value = instance
|
||||
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
resp = await client.post(
|
||||
"/generate",
|
||||
json={"user_id": "u1", "n": 1, "prompt_version": "v2-mentor"},
|
||||
)
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["prompt_version"] == "v2-mentor"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_generate_422_on_unknown_prompt_version():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
resp = await client.post(
|
||||
"/generate",
|
||||
json={"user_id": "u1", "n": 1, "prompt_version": "nonsense"},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
assert "Unknown prompt_version" in resp.json()["detail"]
|
||||
|
||||
Reference in New Issue
Block a user