feat(recommender): LLM schema validation + hardcoded fallback tips on AI failure (#90)

Python (ml/serving):
- Validate tip item after JSON parse: non-empty content, valid kind
- Retry on schema failure with a targeted clarification prompt, same 2× retry budget
- JSON parse failures keep the existing retry suffix

TypeScript (recommender):
- Add TipSource 'fallback' to shared-types
- FALLBACK_TIPS: 12 general-purpose life tips (hardcoded, no DB read)
- fetchOrchestratorTip returns {ok} discriminated union instead of null
- On !res.ok or fetch error: serve a random fallback tip with rationale 'AI service issues'
- Update tests: 204 path removed; both failure cases now expect source='fallback'

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

ml/serving/main.py

@@ -293,6 +293,25 @@ _RETRY_SUFFIX_OBJ = (
     "Reply ONLY with the JSON object — no prose, no markdown fences."
 )
 
+_RETRY_SUFFIX_SCHEMA = (
+    "\n\nYour previous response parsed as JSON but was missing required fields. "
+    'Reply ONLY with a JSON object containing "content" (non-empty string) and "kind" '
+    '(one of: advice, task, insight, reminder) — no prose, no markdown fences.'
+)
+
+_VALID_KINDS = {"advice", "task", "insight", "reminder"}
+
+
+def _validate_tip_item(item: dict) -> str | None:
+    """Return an error string if item fails schema, else None."""
+    content = item.get("content", "")
+    if not isinstance(content, str) or not content.strip():
+        return "missing or empty 'content' field"
+    kind = item.get("kind", "")
+    if kind and kind not in _VALID_KINDS:
+        return f"invalid kind '{kind}', must be one of {_VALID_KINDS}"
+    return None
+
 
 @app.post("/agents/{agent_id}/compute", response_model=AgentComputeResponse)
 async def compute_agent(agent_id: str, req: AgentComputeRequest) -> AgentComputeResponse:
@@ -504,11 +523,15 @@ async def recommend(req: RecommendRequest) -> RecommendResponse:
                             text = text[4:]
                     parsed = json.loads(text)
                     item: dict = parsed[0] if isinstance(parsed, list) else parsed
+                    schema_err = _validate_tip_item(item)
+                    if schema_err:
+                        raise ValueError(schema_err)
                     break
                 except (json.JSONDecodeError, ValueError, IndexError) as exc:
                     last_parse_error = str(exc)
                     messages.append({"role": "assistant", "content": last_raw})
-                    messages.append({"role": "user", "content": _RETRY_SUFFIX_OBJ})
+                    is_schema_err = not isinstance(exc, json.JSONDecodeError)
+                    messages.append({"role": "user", "content": _RETRY_SUFFIX_SCHEMA if is_schema_err else _RETRY_SUFFIX_OBJ})
             else:
                 _end_span(llm_span, status="ERROR")
                 _end_span(root, status="ERROR")

packages/shared-types/src/http/tip.ts

@@ -2,7 +2,7 @@
 export type TipKind = 'task' | 'advice' | 'insight' | 'reminder';
 
 /** Where the tip content originated */
-export type TipSource = 'todoist' | 'llm' | 'advice';
+export type TipSource = 'todoist' | 'llm' | 'advice' | 'fallback';
 
 /** A single recommendation surfaced to the user */
 export interface Tip {

services/api/src/routes/__tests__/recommender.test.ts

@@ -83,16 +83,17 @@ describe('POST /recommend integration', () => {
     clearSignalCache?.();
   });
 
-  it('returns 204 when Todoist is empty and orchestrator fails', async () => {
+  it('returns fallback tip when orchestrator fails', async () => {
     globalThis.fetch = vi.fn().mockImplementation((url: string) => {
       if (String(url).includes('todoist.com')) {
         return Promise.resolve({ ok: true, status: 200, json: async () => ({ results: [] }) } as any);
       }
-      // /recommend fails → orchestrator returns null, random fallback also empty → 204
       return Promise.resolve({ ok: false, status: 503 } as any);
     });
-    const { status } = await post(`${baseUrl}/api/recommend`);
+    const { status, body } = await post(`${baseUrl}/api/recommend`);
-    expect(status).toBe(204);
+    expect(status).toBe(200);
+    expect(body.tip.source).toBe('fallback');
+    expect(body.tip.rationale).toBe('AI service issues');
   });
 
   it('serves orchestrator tip and writes correct tip_scores columns', async () => {
@@ -132,7 +133,7 @@ describe('POST /recommend integration', () => {
     expect(row.tipKind).toBe('advice');
   });
 
-  it('falls back to random signal tip when orchestrator fails', async () => {
+  it('falls back to hardcoded tip when orchestrator fails', async () => {
     globalThis.fetch = vi.fn().mockImplementation((url: string) => {
       if (String(url).includes('todoist.com')) {
         return Promise.resolve({
@@ -142,19 +143,14 @@ describe('POST /recommend integration', () => {
           }),
         } as any);
       }
-      // /recommend fails → falls back to random signal candidate
       return Promise.resolve({ ok: false, status: 502 } as any);
     });
 
     const { status, body } = await post(`${baseUrl}/api/recommend`);
     expect(status).toBe(200);
-    expect(body.tip.source).toBe('todoist');
+    expect(body.tip.source).toBe('fallback');
-
+    expect(body.tip.rationale).toBe('AI service issues');
-    const rows = await testDb.select().from(tipScores);
+    expect(body.tip.kind).toBe('advice');
-    const row = rows[rows.length - 1];
-    expect(row.policy).toBe('random');
-    expect(row.promptVersion).toBeNull();
-    expect(row.llmModel).toBeNull();
   });
 
   it('eligibility filter: only passes consented agent outputs to ml/serving', async () => {

services/api/src/routes/recommender.ts

@@ -17,6 +17,36 @@ import { getEligibleAgentIds } from '../profile/eligibility.js';
 
 const router: ExpressRouter = Router();
 
+// ---------------------------------------------------------------------------
+// Fallback tips — shown when the AI service is unavailable
+// ---------------------------------------------------------------------------
+const FALLBACK_TIPS = [
+  "Take a moment to stretch and breathe — your body and mind will thank you.",
+  "Write down one thing you're grateful for today.",
+  "Drink a glass of water. Small acts of self-care add up.",
+  "Reach out to someone you haven't spoken to in a while.",
+  "Close a tab you've been meaning to close for days.",
+  "Step outside for five minutes, even briefly.",
+  "Put your phone down for the next 30 minutes and see how it feels.",
+  "Do the smallest possible version of a task you've been avoiding.",
+  "Tidy one small area — a clear space helps a clear mind.",
+  "Pause and ask: what would make today feel like a win?",
+  "Rest is productive. Give yourself permission to recharge.",
+  "You don't have to do everything today. Pick one thing and do it well.",
+];
+
+function randomFallbackTip(): import('@oo/shared-types').Tip {
+  const content = FALLBACK_TIPS[Math.floor(Math.random() * FALLBACK_TIPS.length)];
+  return {
+    id: `fallback:${nanoid()}`,
+    content,
+    source: 'fallback',
+    kind: 'advice',
+    rationale: 'AI service issues',
+    createdAt: new Date().toISOString(),
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Signal aggregator — register sources here as new integrations are added
 // ---------------------------------------------------------------------------
@@ -46,6 +76,8 @@ async function loadOrchestratorPref<T>(userId: string, key: string): Promise<T |
   try { return JSON.parse(rows[0].valueJson) as T; } catch { return undefined; }
 }
 
+type OrchestratorOutcome = { ok: true; result: OrchestratorResult } | { ok: false };
+
 async function fetchOrchestratorTip(
   userId: string,
   signals: Signal[],
@@ -53,7 +85,7 @@ async function fetchOrchestratorTip(
   dayOfWeek: number,
   traceparent?: string,
   recentTip?: string,
-): Promise<OrchestratorResult | null> {
+): Promise<OrchestratorOutcome> {
   const [allAgentRows, eligibleIds, scienceDestiny] = await Promise.all([
     getActiveAgentOutputs(userId),
     getEligibleAgentIds(userId),
@@ -77,13 +109,15 @@ async function fetchOrchestratorTip(
       body: JSON.stringify({ user_id: userId, agent_outputs: agentOutputs, tasks, hour_of_day: hour, day_of_week: dayOfWeek, science_destiny: scienceDestiny ?? 50, recent_tip: recentTip ?? null }),
       signal: AbortSignal.timeout(15_000),
     });
-    if (!res.ok) return null;
+    if (!res.ok) return { ok: false };
     const data = (await res.json()) as {
       tip: { id: string; content: string; rationale?: string };
       model?: string;
     };
     const now = new Date().toISOString();
     return {
+      ok: true,
+      result: {
         tip: {
           id: `llm:${data.tip.id}`,
           content: data.tip.content,
@@ -94,9 +128,10 @@ async function fetchOrchestratorTip(
         },
         model: data.model ?? null,
         agentIds: agentOutputs.map((a) => a.agent_id),
+      },
     };
   } catch {
-    return null;
+    return { ok: false };
   }
 }
 
@@ -123,14 +158,15 @@ router.post('/recommend', requireAuth, async (req: AuthenticatedRequest, res: Re
   const signals = await aggregator.fetchAll(req.userId!);
 
   const t0 = Date.now();
-  const orchestrated = await fetchOrchestratorTip(req.userId!, signals, hour, dayOfWeek, req.traceparent, recentTip);
+  const outcome = await fetchOrchestratorTip(req.userId!, signals, hour, dayOfWeek, req.traceparent, recentTip);
   const latencyMs = Date.now() - t0;
 
-  if (!orchestrated) {
+  if (!outcome.ok) {
-    res.status(204).end();
+    res.json({ tip: randomFallbackTip() });
     return;
   }
 
+  const orchestrated = outcome.result;
   const tip = orchestrated.tip;
   const policy = 'orchestrator';
   const servedAt = new Date().toISOString();