From 85a332b22b3ddc05cd8c65bd0ab6f2a5ff144fe0 Mon Sep 17 00:00:00 2001 From: alvis Date: Tue, 12 May 2026 15:21:03 +0000 Subject: [PATCH] feat(recommender): LLM schema validation + hardcoded fallback tips on AI failure (#90) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python (ml/serving): - Validate tip item after JSON parse: non-empty content, valid kind - Retry on schema failure with a targeted clarification prompt, same 2× retry budget - JSON parse failures keep the existing retry suffix TypeScript (recommender): - Add TipSource 'fallback' to shared-types - FALLBACK_TIPS: 12 general-purpose life tips (hardcoded, no DB read) - fetchOrchestratorTip returns {ok} discriminated union instead of null - On !res.ok or fetch error: serve a random fallback tip with rationale 'AI service issues' - Update tests: 204 path removed; both failure cases now expect source='fallback' Co-Authored-By: Claude Sonnet 4.6 --- ml/serving/main.py | 25 ++++++- packages/shared-types/src/http/tip.ts | 2 +- .../src/routes/__tests__/recommender.test.ts | 22 +++---- services/api/src/routes/recommender.ts | 66 ++++++++++++++----- 4 files changed, 85 insertions(+), 30 deletions(-) diff --git a/ml/serving/main.py b/ml/serving/main.py index 7799cfc..1f751f7 100644 --- a/ml/serving/main.py +++ b/ml/serving/main.py @@ -293,6 +293,25 @@ _RETRY_SUFFIX_OBJ = ( "Reply ONLY with the JSON object — no prose, no markdown fences." ) +_RETRY_SUFFIX_SCHEMA = ( + "\n\nYour previous response parsed as JSON but was missing required fields. " + 'Reply ONLY with a JSON object containing "content" (non-empty string) and "kind" ' + '(one of: advice, task, insight, reminder) — no prose, no markdown fences.' +) + +_VALID_KINDS = {"advice", "task", "insight", "reminder"} + + +def _validate_tip_item(item: dict) -> str | None: + """Return an error string if item fails schema, else None.""" + content = item.get("content", "") + if not isinstance(content, str) or not content.strip(): + return "missing or empty 'content' field" + kind = item.get("kind", "") + if kind and kind not in _VALID_KINDS: + return f"invalid kind '{kind}', must be one of {_VALID_KINDS}" + return None + @app.post("/agents/{agent_id}/compute", response_model=AgentComputeResponse) async def compute_agent(agent_id: str, req: AgentComputeRequest) -> AgentComputeResponse: @@ -504,11 +523,15 @@ async def recommend(req: RecommendRequest) -> RecommendResponse: text = text[4:] parsed = json.loads(text) item: dict = parsed[0] if isinstance(parsed, list) else parsed + schema_err = _validate_tip_item(item) + if schema_err: + raise ValueError(schema_err) break except (json.JSONDecodeError, ValueError, IndexError) as exc: last_parse_error = str(exc) messages.append({"role": "assistant", "content": last_raw}) - messages.append({"role": "user", "content": _RETRY_SUFFIX_OBJ}) + is_schema_err = not isinstance(exc, json.JSONDecodeError) + messages.append({"role": "user", "content": _RETRY_SUFFIX_SCHEMA if is_schema_err else _RETRY_SUFFIX_OBJ}) else: _end_span(llm_span, status="ERROR") _end_span(root, status="ERROR") diff --git a/packages/shared-types/src/http/tip.ts b/packages/shared-types/src/http/tip.ts index 05a254a..0014850 100644 --- a/packages/shared-types/src/http/tip.ts +++ b/packages/shared-types/src/http/tip.ts @@ -2,7 +2,7 @@ export type TipKind = 'task' | 'advice' | 'insight' | 'reminder'; /** Where the tip content originated */ -export type TipSource = 'todoist' | 'llm' | 'advice'; +export type TipSource = 'todoist' | 'llm' | 'advice' | 'fallback'; /** A single recommendation surfaced to the user */ export interface Tip { diff --git a/services/api/src/routes/__tests__/recommender.test.ts b/services/api/src/routes/__tests__/recommender.test.ts index e3923fc..0a58eba 100644 --- a/services/api/src/routes/__tests__/recommender.test.ts +++ b/services/api/src/routes/__tests__/recommender.test.ts @@ -83,16 +83,17 @@ describe('POST /recommend integration', () => { clearSignalCache?.(); }); - it('returns 204 when Todoist is empty and orchestrator fails', async () => { + it('returns fallback tip when orchestrator fails', async () => { globalThis.fetch = vi.fn().mockImplementation((url: string) => { if (String(url).includes('todoist.com')) { return Promise.resolve({ ok: true, status: 200, json: async () => ({ results: [] }) } as any); } - // /recommend fails → orchestrator returns null, random fallback also empty → 204 return Promise.resolve({ ok: false, status: 503 } as any); }); - const { status } = await post(`${baseUrl}/api/recommend`); - expect(status).toBe(204); + const { status, body } = await post(`${baseUrl}/api/recommend`); + expect(status).toBe(200); + expect(body.tip.source).toBe('fallback'); + expect(body.tip.rationale).toBe('AI service issues'); }); it('serves orchestrator tip and writes correct tip_scores columns', async () => { @@ -132,7 +133,7 @@ describe('POST /recommend integration', () => { expect(row.tipKind).toBe('advice'); }); - it('falls back to random signal tip when orchestrator fails', async () => { + it('falls back to hardcoded tip when orchestrator fails', async () => { globalThis.fetch = vi.fn().mockImplementation((url: string) => { if (String(url).includes('todoist.com')) { return Promise.resolve({ @@ -142,19 +143,14 @@ describe('POST /recommend integration', () => { }), } as any); } - // /recommend fails → falls back to random signal candidate return Promise.resolve({ ok: false, status: 502 } as any); }); const { status, body } = await post(`${baseUrl}/api/recommend`); expect(status).toBe(200); - expect(body.tip.source).toBe('todoist'); - - const rows = await testDb.select().from(tipScores); - const row = rows[rows.length - 1]; - expect(row.policy).toBe('random'); - expect(row.promptVersion).toBeNull(); - expect(row.llmModel).toBeNull(); + expect(body.tip.source).toBe('fallback'); + expect(body.tip.rationale).toBe('AI service issues'); + expect(body.tip.kind).toBe('advice'); }); it('eligibility filter: only passes consented agent outputs to ml/serving', async () => { diff --git a/services/api/src/routes/recommender.ts b/services/api/src/routes/recommender.ts index 8b1bb89..9c3ed0c 100644 --- a/services/api/src/routes/recommender.ts +++ b/services/api/src/routes/recommender.ts @@ -17,6 +17,36 @@ import { getEligibleAgentIds } from '../profile/eligibility.js'; const router: ExpressRouter = Router(); +// --------------------------------------------------------------------------- +// Fallback tips — shown when the AI service is unavailable +// --------------------------------------------------------------------------- +const FALLBACK_TIPS = [ + "Take a moment to stretch and breathe — your body and mind will thank you.", + "Write down one thing you're grateful for today.", + "Drink a glass of water. Small acts of self-care add up.", + "Reach out to someone you haven't spoken to in a while.", + "Close a tab you've been meaning to close for days.", + "Step outside for five minutes, even briefly.", + "Put your phone down for the next 30 minutes and see how it feels.", + "Do the smallest possible version of a task you've been avoiding.", + "Tidy one small area — a clear space helps a clear mind.", + "Pause and ask: what would make today feel like a win?", + "Rest is productive. Give yourself permission to recharge.", + "You don't have to do everything today. Pick one thing and do it well.", +]; + +function randomFallbackTip(): import('@oo/shared-types').Tip { + const content = FALLBACK_TIPS[Math.floor(Math.random() * FALLBACK_TIPS.length)]; + return { + id: `fallback:${nanoid()}`, + content, + source: 'fallback', + kind: 'advice', + rationale: 'AI service issues', + createdAt: new Date().toISOString(), + }; +} + // --------------------------------------------------------------------------- // Signal aggregator — register sources here as new integrations are added // --------------------------------------------------------------------------- @@ -46,6 +76,8 @@ async function loadOrchestratorPref(userId: string, key: string): Promise { +): Promise { const [allAgentRows, eligibleIds, scienceDestiny] = await Promise.all([ getActiveAgentOutputs(userId), getEligibleAgentIds(userId), @@ -77,26 +109,29 @@ async function fetchOrchestratorTip( body: JSON.stringify({ user_id: userId, agent_outputs: agentOutputs, tasks, hour_of_day: hour, day_of_week: dayOfWeek, science_destiny: scienceDestiny ?? 50, recent_tip: recentTip ?? null }), signal: AbortSignal.timeout(15_000), }); - if (!res.ok) return null; + if (!res.ok) return { ok: false }; const data = (await res.json()) as { tip: { id: string; content: string; rationale?: string }; model?: string; }; const now = new Date().toISOString(); return { - tip: { - id: `llm:${data.tip.id}`, - content: data.tip.content, - source: 'llm' as const, - kind: 'advice' as const, - rationale: data.tip.rationale, - createdAt: now, + ok: true, + result: { + tip: { + id: `llm:${data.tip.id}`, + content: data.tip.content, + source: 'llm' as const, + kind: 'advice' as const, + rationale: data.tip.rationale, + createdAt: now, + }, + model: data.model ?? null, + agentIds: agentOutputs.map((a) => a.agent_id), }, - model: data.model ?? null, - agentIds: agentOutputs.map((a) => a.agent_id), }; } catch { - return null; + return { ok: false }; } } @@ -123,14 +158,15 @@ router.post('/recommend', requireAuth, async (req: AuthenticatedRequest, res: Re const signals = await aggregator.fetchAll(req.userId!); const t0 = Date.now(); - const orchestrated = await fetchOrchestratorTip(req.userId!, signals, hour, dayOfWeek, req.traceparent, recentTip); + const outcome = await fetchOrchestratorTip(req.userId!, signals, hour, dayOfWeek, req.traceparent, recentTip); const latencyMs = Date.now() - t0; - if (!orchestrated) { - res.status(204).end(); + if (!outcome.ok) { + res.json({ tip: randomFallbackTip() }); return; } + const orchestrated = outcome.result; const tip = orchestrated.tip; const policy = 'orchestrator'; const servedAt = new Date().toISOString();