alvis/oO
1
0
Fork 0
Code Issues 34 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: M1 admin console — all 10 remaining pages + signal/quality/ops infrastructure

Browse Source 
Admin console (issues #63–72):
- Event stream viewer: live-tail ring buffer (500 events) with subject/user filters
- Feature store browser: per-user feature vector history from ml/serving
- Model registry panel: MLflow embed at /admin/models
- Experiment dashboard: LinUCB per-user stats (pulls, reward, θ) + bandit reset
- Recommendation log: per-tip explainability (policy, score, features, latency)
- Reward analytics: daily reaction breakdown + per-policy compare
- Data quality widget: missing-feature rate, stale-token rate, daily completeness
- Ops actions: replay-signal, policy enable/disable; user actions link to Users page
- SQL runner: read-only SELECT runner with saved queries
- Health rollup: fan-out to api/ml/sqlite/event-bus with auto-refresh

Backend:
- tip_scores table: logs features+policy+score+latency at every scoring call (#67)
- saved_queries table: per-admin saved SQL (#71)
- Event bus: 500-event ring buffer + tail() API (#63)
- Admin routes: /events, /tips, /reward-analytics, /data-quality, /health,
  /policies, /replay-signal, /sql, /saved-queries endpoints
- /api/ml/* admin-gated proxy to ml/serving (#64, #66)
- Shadow-policy registry in recommender (#56)

ML serving:
- /reset/{user_id}: clear bandit state + feature history (#66)
- /stats/{user_id}: pulls, cumulative reward, estimated mean, θ (#66)
- /features/{user_id}: last 100 feature vectors logged at scoring time (#64)
- Meta (pulls, rewards) persisted alongside A/b matrices

Web:
- Tip action sheet adds Helpful / Not helpful buttons (#62)
- TipFeedback type extended with helpful/not_helpful actions
- Rewards mapped: helpful=+0.5, not_helpful=−0.5

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
alvis
2026-04-16 03:56:48 +00:00
parent 2402a140e9
commit e62c726ea4
37 changed files with 3386 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
services/api/src/routes/recommender.ts
View File

@@ -1,7 +1,7 @@
import { type Router as ExpressRouter, Router, Response } from 'express';
import { nanoid } from 'nanoid';
import { db } from '../db/index.js';
import { integrationTokens, tipFeedback, tipViews } from '../db/schema.js';
import { integrationTokens, tipFeedback, tipViews, tipScores } from '../db/schema.js';
import { eq, and } from 'drizzle-orm';
import { requireAuth, AuthenticatedRequest } from '../middleware/session.js';
import { config } from '../config.js';
@@ -24,7 +24,31 @@ interface CachedTask extends Tip {
const taskCache = new Map<string, { tasks: CachedTask[]; fetchedAt: number }>();
/** Parse a Todoist due date string into age in days (relative to now) */
// ---------------------------------------------------------------------------
// Shadow-policy registry
// ---------------------------------------------------------------------------
// A shadow policy runs alongside the active policy, logs its picks, but does
// NOT affect what the user sees. Promotion to A/B or live is a manual step.
// Structure: Map<policyName, { active: boolean }>
const shadowPolicies = new Map<string, { active: boolean }>([
// Example: enable random as a shadow baseline
// ('random-shadow', { active: true }),
]);
export function getShadowPolicies() {
return Array.from(shadowPolicies.entries()).map(([name, s]) => ({ name, ...s }));
}
export function setPolicyActive(name: string, active: boolean): boolean {
if (!shadowPolicies.has(name)) return false;
shadowPolicies.set(name, { active });
return true;
}
// ---------------------------------------------------------------------------
// Todoist helpers
// ---------------------------------------------------------------------------
function dueAgeDays(due: { date?: string; datetime?: string } | null | undefined): number {
if (!due) return 0;
const dateStr = due.datetime ?? due.date;
@@ -71,11 +95,17 @@ async function fetchTodoistTasks(userId: string, accessToken: string): Promise<C
});
taskCache.set(userId, { tasks, fetchedAt: Date.now() });
bus.publish('signals.task.synced', { userId, count: tasks.length, syncedAt: now.toISOString() });
return tasks;
}
/** Call ml/serving for scored selection; returns tip_id or null on failure */
async function remotePolicy(userId: string, tasks: CachedTask[]): Promise<string | null> {
/** Call ml/serving for scored selection; returns { tip_id, score } or null on failure */
async function remotePolicy(
userId: string,
tasks: CachedTask[],
): Promise<{ tipId: string; score: number } | null> {
const hour = new Date().getHours();
const dayOfWeek = new Date().getDay();
@@ -99,8 +129,8 @@ async function remotePolicy(userId: string, tasks: CachedTask[]): Promise<string
signal: AbortSignal.timeout(3000),
});
if (!res.ok) return null;
const { tip_id } = (await res.json()) as { tip_id: string };
return tip_id;
const data = (await res.json()) as { tip_id: string; score: number };
return { tipId: data.tip_id, score: data.score };
} catch {
return null;
}
@@ -111,7 +141,9 @@ function randomPolicy(candidates: CachedTask[]): CachedTask | null {
return candidates[Math.floor(Math.random() * candidates.length)];
}
/** POST /api/recommend */
// ---------------------------------------------------------------------------
// POST /api/recommend
// ---------------------------------------------------------------------------
router.post('/recommend', requireAuth, async (req: AuthenticatedRequest, res: Response) => {
const [token] = await db
.select()
@@ -130,10 +162,15 @@ router.post('/recommend', requireAuth, async (req: AuthenticatedRequest, res: Re
return;
}
const hour = new Date().getHours();
const dayOfWeek = new Date().getDay();
const t0 = Date.now();
// RemotePolicy with RandomPolicy fallback
const scoredId = await remotePolicy(req.userId!, tasks);
const tip = scoredId
? (tasks.find((t) => t.id === scoredId) ?? randomPolicy(tasks))
const scored = await remotePolicy(req.userId!, tasks);
const latencyMs = Date.now() - t0;
const tip = scored
? (tasks.find((t) => t.id === scored.tipId) ?? randomPolicy(tasks))
: randomPolicy(tasks);
if (!tip) {
@@ -141,25 +178,63 @@ router.post('/recommend', requireAuth, async (req: AuthenticatedRequest, res: Re
return;
}
const policy = scored ? 'linucb-v1' : 'random';
const servedAt = new Date().toISOString();
await db.insert(tipViews).values({ id: nanoid(), userId: req.userId!, tipId: tip.id, servedAt });
// Log recommendation explainability
await db.insert(tipScores).values({
id: nanoid(),
userId: req.userId!,
tipId: tip.id,
policy,
mlScore: scored ? Math.round(scored.score * 1000) : null,
featuresJson: JSON.stringify({
is_overdue: tip.features.is_overdue,
task_age_days: tip.features.task_age_days,
priority: tip.features.priority,
hour_of_day: hour,
day_of_week: dayOfWeek,
}),
candidateCount: tasks.length,
latencyMs,
servedAt,
});
bus.publish('signals.tip.served', {
userId: req.userId!,
tipId: tip.id,
policy: scoredId ? 'linucb-v1' : 'random',
policy,
servedAt,
});
// Run shadow policies (fire-and-forget, no effect on user)
for (const [name, s] of shadowPolicies) {
if (!s.active) continue;
if (name.startsWith('random')) {
const shadowTip = randomPolicy(tasks);
bus.publish('signals.tip.served', {
userId: req.userId!,
tipId: shadowTip?.id ?? 'none',
policy: `shadow:${name}`,
servedAt,
});
}
}
res.json({ tip });
});
/** POST /api/tip/:id/feedback */
// ---------------------------------------------------------------------------
// POST /api/tip/:id/feedback
// ---------------------------------------------------------------------------
router.post('/tip/:id/feedback', requireAuth, async (req: AuthenticatedRequest, res: Response) => {
const { action } = req.body as { action: string };
const tipId = String(req.params.id);
if (!['done', 'dismiss', 'snooze'].includes(action)) {
const validActions = ['done', 'dismiss', 'snooze', 'helpful', 'not_helpful'];
if (!validActions.includes(action)) {
res.status(400).json({ error: 'Invalid action' });
return;
}
@@ -173,18 +248,31 @@ router.post('/tip/:id/feedback', requireAuth, async (req: AuthenticatedRequest,
createdAt: new Date().toISOString(),
});
// Capture task features before clearing cache
const reward = action === 'done' ? 1.0 : action === 'dismiss' ? -1.0 : 0.0;
// Map action to reward (helpful/not_helpful supplement behavioural signals)
const rewardMap: Record<string, number> = {
done: 1.0,
helpful: 0.5,
snooze: 0.0,
not_helpful: -0.5,
dismiss: -1.0,
};
const reward = rewardMap[action] ?? 0.0;
const task = taskCache.get(req.userId!)?.tasks.find((t) => t.id === tipId);
taskCache.delete(req.userId!);
// Clear cache on behavioural actions (not on explicit helpful/not_helpful)
if (['done', 'dismiss', 'snooze'].includes(action)) {
taskCache.delete(req.userId!);
}
bus.publish('signals.tip.feedback', {
userId: req.userId!,
tipId,
action: action as 'done' | 'dismiss' | 'snooze',
action: action as 'done' | 'dismiss' | 'snooze' | 'helpful' | 'not_helpful',
reward,
createdAt: new Date().toISOString(),
});
if (task) {
fetch(`${config.ML_SERVING_URL}/reward`, {
method: 'POST',