feat: ε-greedy v1 as active policy; dwell-time reward inference; offline sim framework
- Promote egreedy-v1 to active serving policy (ADR-0007): /score/egreedy + /reward/egreedy
replaces linucb-v1 endpoints after offline sim shows +10.7% mean reward (−0.548 vs −0.606)
- Replace explicit helpful/not_helpful feedback with dwell-time inferred reward (inferReward):
dismiss=−1.0, snooze=+0.1, done<15s=−0.3, done 15s–2min=+1.0, done 2–10min=+0.6, done>10min=+0.3
- Add ml/serving ε-greedy endpoints: /score/egreedy, /reward/egreedy, /stats/egreedy/{user_id}
with d=7 feature vector (base 5 + sin/cos day-of-week encoding)
- Add offline simulation framework (ml/experiments/sim): rule/LLM/claude-code judges,
two-phase score+reward, synthetic personas, task generator; results stored in sim_runs/sim_events
- Add /admin/simulations page: start runs, live-poll status, reward curve SVG, action/persona tables
- Fix egreedy day_of_week training skew: reward endpoint now uses actual dow instead of hardcoded 0
- Fix runner.py proxy bypass: httpx.Client(trust_env=False) for localhost ML calls
- Add dwellMs to TipFeedbackEvent contract and bus.test.ts fixture
- Schema: sim_runs, sim_events tables; tip_feedback gains dwell_ms, reward_milli columns
- ADR-0006: admin console framework; ADR-0007: egreedy-v1 policy selection rationale
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
37
apps/admin/README.md
Normal file
37
apps/admin/README.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# apps/admin — oO Admin Console
|
||||
|
||||
Next.js 15 app. Deployed at `admin.o.alogins.net` (dev: `http://localhost:3080`).
|
||||
|
||||
## Contract
|
||||
|
||||
- All routes are admin-only. The Next.js middleware calls `GET /api/user/me` on every request
|
||||
and checks `role === 'admin'`. First admin is seeded via `ADMIN_SEED_EMAIL` env var at API startup.
|
||||
- Admin write actions are appended to the `admin_actions` audit log in the DB.
|
||||
|
||||
## Pages
|
||||
|
||||
| Route | Description |
|
||||
|-------|-------------|
|
||||
| `/` | Overview: DAU/WAU KPI cards, tips served, reaction breakdown, activation funnel |
|
||||
| `/users` | User list (paginated) |
|
||||
| `/users/:id` | User detail: identity, consents, integrations, tip stats, reward history; revoke-integration + reset-bandit actions |
|
||||
| `/audit` | Admin action audit log |
|
||||
| `/events` | Event stream viewer (stub — pending API history endpoint) |
|
||||
|
||||
## Dev
|
||||
|
||||
```bash
|
||||
pnpm --filter @oo/admin dev # starts on :3080
|
||||
# also run the API: pnpm --filter @oo/api dev (port 3078)
|
||||
```
|
||||
|
||||
## Extraction criteria
|
||||
|
||||
Stays as a Next.js app in the monorepo permanently — it's not a candidate for extraction.
|
||||
It gets richer (more pages, embedded MLflow/Grafana) but not split.
|
||||
|
||||
## Known issues
|
||||
|
||||
- `@tremor/react 3.x` declares a peer dep on React 18; the workspace uses React 19.
|
||||
Works in practice. Will resolve naturally when Tremor ships React 19 support or when
|
||||
we switch to Tremor v4 (which targets React 18+).
|
||||
14
apps/admin/next.config.ts
Normal file
14
apps/admin/next.config.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import type { NextConfig } from 'next';
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
async rewrites() {
|
||||
return [
|
||||
{
|
||||
source: '/api/:path*',
|
||||
destination: `${process.env.NEXT_PUBLIC_API_URL ?? 'http://localhost:3078'}/api/:path*`,
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
32
apps/admin/package.json
Normal file
32
apps/admin/package.json
Normal file
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"name": "@oo/admin",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev -p 3080",
|
||||
"build": "next build",
|
||||
"start": "next start -p 3080",
|
||||
"lint": "next lint",
|
||||
"type-check": "tsc --noEmit",
|
||||
"clean": "rm -rf .next"
|
||||
},
|
||||
"dependencies": {
|
||||
"@oo/shared-types": "workspace:*",
|
||||
"@tremor/react": "^3.18.3",
|
||||
"@tanstack/react-table": "^8.20.5",
|
||||
"next": "^15.1.6",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"recharts": "^2.15.3",
|
||||
"marked": "^14.1.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.5",
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
"tailwindcss": "^3.4.17",
|
||||
"autoprefixer": "^10.4.20",
|
||||
"postcss": "^8.5.1",
|
||||
"typescript": "^5.7.3"
|
||||
}
|
||||
}
|
||||
6
apps/admin/postcss.config.js
Normal file
6
apps/admin/postcss.config.js
Normal file
@@ -0,0 +1,6 @@
|
||||
module.exports = {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
};
|
||||
499
apps/admin/src/app/simulations/page.tsx
Normal file
499
apps/admin/src/app/simulations/page.tsx
Normal file
@@ -0,0 +1,499 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { AdminShell } from '@/components/AdminShell';
|
||||
import {
|
||||
type PolicySummary,
|
||||
type SimEvent,
|
||||
type SimRun,
|
||||
getSimRun,
|
||||
getSimRuns,
|
||||
startSimulation,
|
||||
} from '@/lib/api';
|
||||
|
||||
const KNOWN_POLICIES = ['linucb-v1', 'egreedy-v1'];
|
||||
const ACTIONS = ['done', 'snooze', 'dismiss'];
|
||||
// Shown as reference only — actual reward is dwell-time inferred for 'done'
|
||||
const ACTION_REWARDS: Record<string, number> = {
|
||||
done: 1.0, snooze: 0.1, dismiss: -1.0,
|
||||
};
|
||||
|
||||
// ── SVG reward curve ────────────────────────────────────────────────────────
|
||||
|
||||
function RewardCurve({ summary, policies }: { summary: Record<string, PolicySummary>; policies: string[] }) {
|
||||
const W = 520, H = 160, PAD = { t: 10, r: 10, b: 30, l: 40 };
|
||||
const iW = W - PAD.l - PAD.r;
|
||||
const iH = H - PAD.t - PAD.b;
|
||||
|
||||
const allVals = policies.flatMap((p) => summary[p]?.cumulative_rewards ?? []);
|
||||
const minY = Math.min(0, ...allVals);
|
||||
const maxY = Math.max(1, ...allVals);
|
||||
const n = Math.max(...policies.map((p) => (summary[p]?.cumulative_rewards ?? []).length));
|
||||
|
||||
const xScale = (i: number) => PAD.l + (i / Math.max(1, n - 1)) * iW;
|
||||
const yScale = (v: number) => PAD.t + iH - ((v - minY) / (maxY - minY)) * iH;
|
||||
|
||||
const COLORS = ['#818cf8', '#34d399', '#f87171', '#fbbf24'];
|
||||
|
||||
const path = (vals: number[]) =>
|
||||
vals
|
||||
.map((v, i) => `${i === 0 ? 'M' : 'L'}${xScale(i).toFixed(1)},${yScale(v).toFixed(1)}`)
|
||||
.join(' ');
|
||||
|
||||
// Axis labels
|
||||
const yLabels = [minY, (minY + maxY) / 2, maxY];
|
||||
|
||||
return (
|
||||
<svg width={W} height={H} className="overflow-visible">
|
||||
{/* Grid */}
|
||||
{yLabels.map((v, i) => (
|
||||
<g key={i}>
|
||||
<line
|
||||
x1={PAD.l} y1={yScale(v)} x2={W - PAD.r} y2={yScale(v)}
|
||||
stroke="#374151" strokeWidth={0.5} strokeDasharray="3,3"
|
||||
/>
|
||||
<text x={PAD.l - 4} y={yScale(v) + 4} textAnchor="end" fontSize={10} fill="#9ca3af">
|
||||
{v.toFixed(1)}
|
||||
</text>
|
||||
</g>
|
||||
))}
|
||||
{/* Zero line */}
|
||||
{minY < 0 && (
|
||||
<line x1={PAD.l} y1={yScale(0)} x2={W - PAD.r} y2={yScale(0)}
|
||||
stroke="#6b7280" strokeWidth={1} />
|
||||
)}
|
||||
{/* Curves */}
|
||||
{policies.map((p, pi) => {
|
||||
const vals = summary[p]?.cumulative_rewards ?? [];
|
||||
if (!vals.length) return null;
|
||||
return (
|
||||
<g key={p}>
|
||||
<path d={path(vals)} fill="none" stroke={COLORS[pi % COLORS.length]} strokeWidth={2} />
|
||||
<circle
|
||||
cx={xScale(vals.length - 1)} cy={yScale(vals[vals.length - 1])}
|
||||
r={3} fill={COLORS[pi % COLORS.length]}
|
||||
/>
|
||||
</g>
|
||||
);
|
||||
})}
|
||||
{/* X axis */}
|
||||
<line x1={PAD.l} y1={H - PAD.b} x2={W - PAD.r} y2={H - PAD.b} stroke="#4b5563" />
|
||||
<text x={W / 2} y={H - 2} textAnchor="middle" fontSize={10} fill="#6b7280">Round</text>
|
||||
{/* Legend */}
|
||||
{policies.map((p, pi) => (
|
||||
<g key={p} transform={`translate(${PAD.l + pi * 130},${H - PAD.b + 14})`}>
|
||||
<rect width={12} height={3} y={3} fill={COLORS[pi % COLORS.length]} />
|
||||
<text x={16} y={8} fontSize={10} fill="#d1d5db">{p}</text>
|
||||
</g>
|
||||
))}
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Action distribution table ───────────────────────────────────────────────
|
||||
|
||||
function ActionTable({
|
||||
summary,
|
||||
policies,
|
||||
}: {
|
||||
summary: Record<string, PolicySummary>;
|
||||
policies: string[];
|
||||
}) {
|
||||
return (
|
||||
<table className="text-sm w-full">
|
||||
<thead>
|
||||
<tr className="text-left text-gray-500 border-b border-gray-800">
|
||||
<th className="py-1 pr-4 font-medium">Action</th>
|
||||
{policies.map((p) => (
|
||||
<th key={p} className="py-1 pr-4 font-medium">{p}</th>
|
||||
))}
|
||||
<th className="py-1 font-medium text-gray-400">Reward</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{ACTIONS.map((action) => (
|
||||
<tr key={action} className="border-b border-gray-900">
|
||||
<td className="py-1.5 pr-4 text-gray-300">{action}</td>
|
||||
{policies.map((p) => {
|
||||
const n = summary[p]?.action_counts?.[action] ?? 0;
|
||||
const total = Object.values(summary[p]?.action_counts ?? {}).reduce(
|
||||
(a, b) => a + b, 0
|
||||
);
|
||||
const pct = total > 0 ? ((n / total) * 100).toFixed(1) : '—';
|
||||
return (
|
||||
<td key={p} className="py-1.5 pr-4 text-gray-200">
|
||||
{n} <span className="text-gray-500 text-xs">({pct}%)</span>
|
||||
</td>
|
||||
);
|
||||
})}
|
||||
<td className={`py-1.5 text-xs font-mono ${ACTION_REWARDS[action] > 0 ? 'text-green-400' : ACTION_REWARDS[action] < 0 ? 'text-red-400' : 'text-gray-500'}`}>
|
||||
{ACTION_REWARDS[action] >= 0 ? '+' : ''}{ACTION_REWARDS[action]}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Per-persona breakdown ───────────────────────────────────────────────────
|
||||
|
||||
function PersonaTable({
|
||||
breakdown,
|
||||
policies,
|
||||
}: {
|
||||
breakdown: Record<string, Record<string, { reward: number; n: number }>>;
|
||||
policies: string[];
|
||||
}) {
|
||||
const personas = Object.keys(breakdown);
|
||||
return (
|
||||
<table className="text-sm w-full">
|
||||
<thead>
|
||||
<tr className="text-left text-gray-500 border-b border-gray-800">
|
||||
<th className="py-1 pr-6 font-medium">Persona</th>
|
||||
{policies.map((p) => (
|
||||
<th key={p} className="py-1 pr-6 font-medium">{p}<br /><span className="font-normal text-xs">mean reward</span></th>
|
||||
))}
|
||||
<th className="py-1 font-medium">Winner</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{personas.map((persona) => {
|
||||
const pdata = breakdown[persona];
|
||||
const best = policies.reduce((a, b) =>
|
||||
(pdata[a]?.reward ?? -Infinity) >= (pdata[b]?.reward ?? -Infinity) ? a : b
|
||||
);
|
||||
return (
|
||||
<tr key={persona} className="border-b border-gray-900">
|
||||
<td className="py-1.5 pr-6 text-gray-300">{persona}</td>
|
||||
{policies.map((p) => {
|
||||
const d = pdata[p];
|
||||
const mean = d && d.n > 0 ? (d.reward / d.n).toFixed(3) : '—';
|
||||
return (
|
||||
<td key={p} className={`py-1.5 pr-6 font-mono text-xs ${p === best ? 'text-green-400' : 'text-gray-400'}`}>
|
||||
{mean}
|
||||
</td>
|
||||
);
|
||||
})}
|
||||
<td className="py-1.5 text-xs text-indigo-400">{best}</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Run detail panel ────────────────────────────────────────────────────────
|
||||
|
||||
function RunDetail({ runId, onClose }: { runId: string; onClose: () => void }) {
|
||||
const [data, setData] = useState<{ run: SimRun; events: SimEvent[] } | null>(null);
|
||||
const [error, setError] = useState('');
|
||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
const load = async () => {
|
||||
try {
|
||||
const d = await getSimRun(runId);
|
||||
setData(d);
|
||||
if (d.run.status !== 'running' && d.run.status !== 'pending') {
|
||||
if (pollRef.current) clearInterval(pollRef.current);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
setError(e instanceof Error ? e.message : 'Failed to load');
|
||||
if (pollRef.current) clearInterval(pollRef.current);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
pollRef.current = setInterval(load, 3000);
|
||||
return () => { if (pollRef.current) clearInterval(pollRef.current); };
|
||||
}, [runId]);
|
||||
|
||||
const run = data?.run;
|
||||
const summary: Record<string, PolicySummary> | null = run?.summaryJson
|
||||
? JSON.parse(run.summaryJson)
|
||||
: null;
|
||||
const breakdown: Record<string, Record<string, { reward: number; n: number }>> | null =
|
||||
run?.personaBreakdownJson ? JSON.parse(run.personaBreakdownJson) : null;
|
||||
const policies = run ? [run.policyA, run.policyB] : [];
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 bg-black/70 z-50 flex items-start justify-center pt-16 px-4 overflow-auto">
|
||||
<div className="bg-gray-950 border border-gray-800 rounded-lg w-full max-w-3xl p-6 space-y-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold">Simulation {runId}</h2>
|
||||
{run && (
|
||||
<p className="text-xs text-gray-500 mt-0.5">
|
||||
{run.nUsers} users × {run.nRounds} rounds × {run.tasksPerRound} tasks
|
||||
{' · '}{run.useLlm ? 'LLM judge' : 'Rule judge'}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<button onClick={onClose} className="text-gray-500 hover:text-white text-sm">✕ Close</button>
|
||||
</div>
|
||||
|
||||
{error && <p className="text-red-400 text-sm">{error}</p>}
|
||||
|
||||
{run && (
|
||||
<div className="flex items-center gap-3">
|
||||
<StatusBadge status={run.status} />
|
||||
{run.winner && run.status === 'done' && (
|
||||
<span className="px-2 py-0.5 bg-indigo-900/60 border border-indigo-700 rounded text-indigo-300 text-xs font-medium">
|
||||
Winner: {run.winner}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{summary && (
|
||||
<>
|
||||
{/* Metric cards */}
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
{policies.map((p) => (
|
||||
<div key={p} className="bg-gray-900 border border-gray-800 rounded p-4 space-y-2">
|
||||
<div className="text-xs font-medium text-gray-400 truncate">{p}</div>
|
||||
<div className="flex gap-4">
|
||||
<Metric label="Total reward" value={summary[p]?.total_reward.toFixed(2)} />
|
||||
<Metric label="Mean/pull" value={summary[p]?.mean_reward.toFixed(3)} />
|
||||
<Metric label="Pulls" value={String(summary[p]?.n_pulls)} />
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Cumulative reward chart */}
|
||||
<div className="space-y-2">
|
||||
<h3 className="text-sm font-medium text-gray-400">Cumulative reward over rounds</h3>
|
||||
<div className="bg-gray-900 border border-gray-800 rounded p-3">
|
||||
<RewardCurve summary={summary} policies={policies} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Action distribution */}
|
||||
<div className="space-y-2">
|
||||
<h3 className="text-sm font-medium text-gray-400">Action distribution</h3>
|
||||
<ActionTable summary={summary} policies={policies} />
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{breakdown && (
|
||||
<div className="space-y-2">
|
||||
<h3 className="text-sm font-medium text-gray-400">Per-persona mean reward</h3>
|
||||
<PersonaTable breakdown={breakdown} policies={policies} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{run?.status === 'running' && (
|
||||
<p className="text-yellow-400 text-xs animate-pulse">Simulation running — auto-refreshing every 3s…</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Status badge ────────────────────────────────────────────────────────────
|
||||
|
||||
function StatusBadge({ status }: { status: string }) {
|
||||
const styles: Record<string, string> = {
|
||||
pending: 'bg-gray-800 text-gray-400',
|
||||
running: 'bg-yellow-900/60 text-yellow-300 border border-yellow-700',
|
||||
done: 'bg-green-900/60 text-green-300 border border-green-700',
|
||||
failed: 'bg-red-900/60 text-red-300 border border-red-700',
|
||||
};
|
||||
return (
|
||||
<span className={`px-2 py-0.5 rounded text-xs font-medium ${styles[status] ?? styles.pending}`}>
|
||||
{status}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function Metric({ label, value }: { label: string; value: string | undefined }) {
|
||||
return (
|
||||
<div>
|
||||
<div className="text-[10px] text-gray-500 mb-0.5">{label}</div>
|
||||
<div className="text-sm font-mono text-white">{value ?? '—'}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Main page ───────────────────────────────────────────────────────────────
|
||||
|
||||
export default function SimulationsPage() {
|
||||
const [runs, setRuns] = useState<SimRun[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null);
|
||||
|
||||
// Form state
|
||||
const [nUsers, setNUsers] = useState(5);
|
||||
const [nRounds, setNRounds] = useState(20);
|
||||
const [tasksPerRound, setTasksPerRound] = useState(8);
|
||||
const [useLlm, setUseLlm] = useState(false);
|
||||
const [policyA, setPolicyA] = useState('linucb-v1');
|
||||
const [policyB, setPolicyB] = useState('egreedy-v1');
|
||||
const [launching, setLaunching] = useState(false);
|
||||
const [launchError, setLaunchError] = useState('');
|
||||
|
||||
const loadRuns = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const { runs: r } = await getSimRuns();
|
||||
setRuns(r);
|
||||
} catch (e: unknown) {
|
||||
setError(e instanceof Error ? e.message : 'Failed to load');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => { loadRuns(); }, []);
|
||||
|
||||
const handleStart = async () => {
|
||||
if (policyA === policyB) {
|
||||
setLaunchError('Policies must be different');
|
||||
return;
|
||||
}
|
||||
setLaunching(true);
|
||||
setLaunchError('');
|
||||
try {
|
||||
const { id } = await startSimulation({
|
||||
nUsers,
|
||||
nRounds,
|
||||
tasksPerRound,
|
||||
useLlm,
|
||||
policies: [policyA, policyB],
|
||||
});
|
||||
await loadRuns();
|
||||
setSelectedId(id);
|
||||
} catch (e: unknown) {
|
||||
setLaunchError(e instanceof Error ? e.message : 'Failed to start');
|
||||
} finally {
|
||||
setLaunching(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<AdminShell>
|
||||
<div className="space-y-6 max-w-4xl">
|
||||
<div>
|
||||
<h1 className="text-xl font-semibold">Simulations</h1>
|
||||
<p className="text-sm text-gray-500 mt-1">
|
||||
Compare recommendation policies offline using synthetic users and LLM-judged reactions.
|
||||
ml/serving must be running.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Launch form */}
|
||||
<div className="bg-gray-900 border border-gray-800 rounded-lg p-5 space-y-4">
|
||||
<h2 className="text-sm font-semibold text-gray-300">New simulation</h2>
|
||||
<div className="grid grid-cols-2 gap-4 sm:grid-cols-3">
|
||||
<Field label="Policy A">
|
||||
<select value={policyA} onChange={(e) => setPolicyA(e.target.value)} className={selectCls}>
|
||||
{KNOWN_POLICIES.map((p) => <option key={p}>{p}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Policy B">
|
||||
<select value={policyB} onChange={(e) => setPolicyB(e.target.value)} className={selectCls}>
|
||||
{KNOWN_POLICIES.map((p) => <option key={p}>{p}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Users">
|
||||
<input type="number" min={1} max={20} value={nUsers}
|
||||
onChange={(e) => setNUsers(Number(e.target.value))} className={inputCls} />
|
||||
</Field>
|
||||
<Field label="Rounds">
|
||||
<input type="number" min={5} max={100} value={nRounds}
|
||||
onChange={(e) => setNRounds(Number(e.target.value))} className={inputCls} />
|
||||
</Field>
|
||||
<Field label="Tasks/round">
|
||||
<input type="number" min={3} max={20} value={tasksPerRound}
|
||||
onChange={(e) => setTasksPerRound(Number(e.target.value))} className={inputCls} />
|
||||
</Field>
|
||||
<Field label="Judge">
|
||||
<label className="flex items-center gap-2 cursor-pointer mt-1">
|
||||
<input type="checkbox" checked={useLlm} onChange={(e) => setUseLlm(e.target.checked)}
|
||||
className="accent-indigo-500" />
|
||||
<span className="text-sm text-gray-300">Claude Haiku</span>
|
||||
</label>
|
||||
{!useLlm && <p className="text-[10px] text-gray-500 mt-0.5">Deterministic rule judge</p>}
|
||||
{useLlm && <p className="text-[10px] text-yellow-500 mt-0.5">Requires ANTHROPIC_API_KEY</p>}
|
||||
</Field>
|
||||
</div>
|
||||
{launchError && <p className="text-red-400 text-xs">{launchError}</p>}
|
||||
<button
|
||||
onClick={handleStart}
|
||||
disabled={launching}
|
||||
className="bg-indigo-600 hover:bg-indigo-500 disabled:opacity-50 text-white rounded px-4 py-1.5 text-sm"
|
||||
>
|
||||
{launching ? 'Starting…' : 'Run simulation'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Runs list */}
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-sm font-semibold text-gray-300">Past runs</h2>
|
||||
<button onClick={loadRuns} className="text-xs text-gray-500 hover:text-white">Refresh</button>
|
||||
</div>
|
||||
|
||||
{loading && <p className="text-gray-500 text-sm">Loading…</p>}
|
||||
{error && <p className="text-red-400 text-sm">{error}</p>}
|
||||
|
||||
{runs.length === 0 && !loading && (
|
||||
<p className="text-gray-600 text-sm">No simulation runs yet.</p>
|
||||
)}
|
||||
|
||||
<div className="space-y-1">
|
||||
{runs.map((run) => (
|
||||
<button
|
||||
key={run.id}
|
||||
onClick={() => setSelectedId(run.id)}
|
||||
className="w-full text-left bg-gray-900 hover:bg-gray-800 border border-gray-800 rounded px-4 py-3 flex items-center justify-between gap-4"
|
||||
>
|
||||
<div className="flex items-center gap-3 min-w-0">
|
||||
<StatusBadge status={run.status} />
|
||||
<span className="text-sm text-gray-300 font-mono truncate">{run.id}</span>
|
||||
<span className="text-xs text-gray-500 hidden sm:inline">
|
||||
{run.policyA} vs {run.policyB}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-4 flex-shrink-0">
|
||||
{run.winner && (
|
||||
<span className="text-xs text-indigo-400">→ {run.winner}</span>
|
||||
)}
|
||||
<span className="text-xs text-gray-600">{run.nUsers}u × {run.nRounds}r</span>
|
||||
<span className="text-xs text-gray-600">
|
||||
{new Date(run.createdAt).toLocaleString()}
|
||||
</span>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{selectedId && (
|
||||
<RunDetail runId={selectedId} onClose={() => setSelectedId(null)} />
|
||||
)}
|
||||
</AdminShell>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Small helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
const inputCls =
|
||||
'w-full bg-gray-800 border border-gray-700 rounded px-2.5 py-1.5 text-sm text-gray-200 focus:outline-none focus:border-indigo-500';
|
||||
const selectCls =
|
||||
'w-full bg-gray-800 border border-gray-700 rounded px-2.5 py-1.5 text-sm text-gray-200 focus:outline-none focus:border-indigo-500';
|
||||
|
||||
function Field({ label, children }: { label: string; children: React.ReactNode }) {
|
||||
return (
|
||||
<div>
|
||||
<label className="block text-xs text-gray-500 mb-1">{label}</label>
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -11,6 +11,7 @@ const NAV = [
|
||||
{ href: '/tips', label: 'Rec log' },
|
||||
{ href: '/reward-analytics', label: 'Rewards' },
|
||||
{ href: '/experiments', label: 'Experiments' },
|
||||
{ href: '/simulations', label: 'Simulations' },
|
||||
{ href: '/models', label: 'Models' },
|
||||
{ href: '/data-quality', label: 'Data quality' },
|
||||
{ href: '/ops', label: 'Ops' },
|
||||
|
||||
@@ -220,3 +220,67 @@ export function saveQuery(name: string, querySql: string) {
|
||||
export function deleteSavedQuery(id: string) {
|
||||
return apiFetch<{ ok: boolean }>(`/admin/saved-queries/${id}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
// ── Simulation ─────────────────────────────────────────────────────────────
|
||||
|
||||
export interface PolicySummary {
|
||||
total_reward: number;
|
||||
mean_reward: number;
|
||||
n_pulls: number;
|
||||
cumulative_rewards: number[];
|
||||
action_counts: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface SimRun {
|
||||
id: string;
|
||||
policyA: string;
|
||||
policyB: string;
|
||||
nUsers: number;
|
||||
nRounds: number;
|
||||
tasksPerRound: number;
|
||||
useLlm: boolean;
|
||||
status: 'pending' | 'running' | 'done' | 'failed';
|
||||
summaryJson: string | null;
|
||||
winner: string | null;
|
||||
personaBreakdownJson: string | null;
|
||||
createdAt: string;
|
||||
finishedAt: string | null;
|
||||
isRunning?: boolean;
|
||||
}
|
||||
|
||||
export interface SimEvent {
|
||||
id: string;
|
||||
runId: string;
|
||||
round: number;
|
||||
userId: string;
|
||||
persona: string;
|
||||
policy: string;
|
||||
tipContent: string;
|
||||
priority: number;
|
||||
isOverdue: boolean;
|
||||
action: string;
|
||||
rewardMilli: number;
|
||||
hour: number;
|
||||
dayOfWeek: number;
|
||||
}
|
||||
|
||||
export function startSimulation(params: {
|
||||
nUsers: number;
|
||||
nRounds: number;
|
||||
tasksPerRound: number;
|
||||
useLlm: boolean;
|
||||
policies: string[];
|
||||
}) {
|
||||
return apiFetch<{ id: string; status: string }>('/admin/simulate/start', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(params),
|
||||
});
|
||||
}
|
||||
|
||||
export function getSimRuns() {
|
||||
return apiFetch<{ runs: SimRun[] }>('/admin/simulate/runs');
|
||||
}
|
||||
|
||||
export function getSimRun(id: string) {
|
||||
return apiFetch<{ run: SimRun; events: SimEvent[] }>(`/admin/simulate/${id}`);
|
||||
}
|
||||
|
||||
12
apps/admin/tailwind.config.ts
Normal file
12
apps/admin/tailwind.config.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import type { Config } from 'tailwindcss';
|
||||
|
||||
const config: Config = {
|
||||
content: [
|
||||
'./src/**/*.{ts,tsx}',
|
||||
'./node_modules/@tremor/**/*.{js,jsx,ts,tsx}',
|
||||
],
|
||||
theme: { extend: {} },
|
||||
plugins: [],
|
||||
};
|
||||
|
||||
export default config;
|
||||
23
apps/admin/tsconfig.json
Normal file
23
apps/admin/tsconfig.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2017",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "preserve",
|
||||
"incremental": true,
|
||||
"plugins": [{ "name": "next" }],
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
1
apps/admin/tsconfig.tsbuildinfo
Normal file
1
apps/admin/tsconfig.tsbuildinfo
Normal file
File diff suppressed because one or more lines are too long
11
apps/web/e2e/sign-in.spec.ts
Normal file
11
apps/web/e2e/sign-in.spec.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { test, expect } from '@playwright/test';
|
||||
|
||||
test('sign-in page loads and shows Google button', async ({ page }) => {
|
||||
await page.goto('/sign-in');
|
||||
await expect(page.getByRole('link', { name: /google/i })).toBeVisible();
|
||||
});
|
||||
|
||||
test('unauthenticated root redirects to sign-in', async ({ page }) => {
|
||||
await page.goto('/');
|
||||
await expect(page).toHaveURL(/sign-in/);
|
||||
});
|
||||
@@ -7,6 +7,10 @@
|
||||
"build": "next build",
|
||||
"start": "next start -p 3079",
|
||||
"lint": "next lint",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:e2e": "playwright test",
|
||||
"test:e2e:ui": "playwright test --ui",
|
||||
"type-check": "tsc --noEmit",
|
||||
"clean": "rm -rf .next"
|
||||
},
|
||||
@@ -17,9 +21,17 @@
|
||||
"react-dom": "^19.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.59.1",
|
||||
"@testing-library/jest-dom": "^6.9.1",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@testing-library/user-event": "^14.6.1",
|
||||
"@types/node": "^22.10.5",
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
"@types/node": "^22.10.5",
|
||||
"typescript": "^5.7.3"
|
||||
"@vitejs/plugin-react": "^6.0.1",
|
||||
"@vitest/coverage-v8": "^4.1.4",
|
||||
"jsdom": "^29.0.2",
|
||||
"typescript": "^5.7.3",
|
||||
"vitest": "^4.1.4"
|
||||
}
|
||||
}
|
||||
|
||||
24
apps/web/playwright.config.ts
Normal file
24
apps/web/playwright.config.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
|
||||
export default defineConfig({
|
||||
testDir: './e2e',
|
||||
fullyParallel: true,
|
||||
forbidOnly: !!process.env.CI,
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
reporter: 'html',
|
||||
use: {
|
||||
baseURL: process.env.BASE_URL ?? 'http://localhost:3079',
|
||||
trace: 'on-first-retry',
|
||||
},
|
||||
projects: [
|
||||
{ name: 'chromium', use: { ...devices['Desktop Chrome'] } },
|
||||
],
|
||||
// Start dev server automatically in CI; locally, run `pnpm dev` first
|
||||
webServer: process.env.CI
|
||||
? {
|
||||
command: 'pnpm build && pnpm start',
|
||||
url: 'http://localhost:3079',
|
||||
reuseExistingServer: false,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
131
apps/web/src/components/__tests__/TipPage.test.tsx
Normal file
131
apps/web/src/components/__tests__/TipPage.test.tsx
Normal file
@@ -0,0 +1,131 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { render, screen, waitFor, act, fireEvent } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
|
||||
// Mock the API module — we test UI behaviour, not network calls
|
||||
vi.mock('@/lib/api', () => ({
|
||||
getRecommendation: vi.fn(),
|
||||
sendFeedback: vi.fn().mockResolvedValue(undefined),
|
||||
getVapidPublicKey: vi.fn(),
|
||||
subscribePush: vi.fn(),
|
||||
}));
|
||||
|
||||
import { getRecommendation, sendFeedback } from '@/lib/api';
|
||||
import TipPage from '@/app/tip/page';
|
||||
|
||||
const mockGetRec = getRecommendation as ReturnType<typeof vi.fn>;
|
||||
const mockSendFeedback = sendFeedback as ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('TipPage — empty / error states', () => {
|
||||
it('shows "All clear." when no tip is returned', async () => {
|
||||
mockGetRec.mockResolvedValue(null);
|
||||
render(<TipPage />);
|
||||
await waitFor(() => expect(screen.getByText('All clear.')).toBeInTheDocument());
|
||||
});
|
||||
|
||||
it('shows "All clear." when getRecommendation throws', async () => {
|
||||
mockGetRec.mockRejectedValue(Object.assign(new Error('Network error'), { status: 503 }));
|
||||
render(<TipPage />);
|
||||
await waitFor(() => expect(screen.getByText('All clear.')).toBeInTheDocument());
|
||||
});
|
||||
|
||||
it('"Check again" button re-calls getRecommendation', async () => {
|
||||
mockGetRec.mockResolvedValue(null);
|
||||
render(<TipPage />);
|
||||
await waitFor(() => screen.getByText('Check again'));
|
||||
|
||||
mockGetRec.mockResolvedValue({
|
||||
tip: { id: 'todoist:2', content: 'New tip', source: 'todoist', createdAt: '' },
|
||||
});
|
||||
fireEvent.click(screen.getByText('Check again'));
|
||||
await waitFor(() => expect(mockGetRec).toHaveBeenCalledTimes(2));
|
||||
});
|
||||
});
|
||||
|
||||
describe('TipPage — tip display', () => {
|
||||
it('renders tip content after loading', async () => {
|
||||
mockGetRec.mockResolvedValue({
|
||||
tip: { id: 'todoist:1', content: 'Write the test', source: 'todoist', createdAt: '' },
|
||||
});
|
||||
render(<TipPage />);
|
||||
await waitFor(() => expect(screen.getByText('Write the test')).toBeInTheDocument());
|
||||
});
|
||||
|
||||
it('shows "hold to act" hint when tip is displayed', async () => {
|
||||
mockGetRec.mockResolvedValue({
|
||||
tip: { id: 'todoist:3', content: 'Do the thing', source: 'todoist', createdAt: '' },
|
||||
});
|
||||
render(<TipPage />);
|
||||
await waitFor(() => expect(screen.getByText(/hold to act/i)).toBeInTheDocument());
|
||||
});
|
||||
|
||||
it('shows "reading you…" while loading', async () => {
|
||||
// Never resolves during this assertion
|
||||
mockGetRec.mockReturnValue(new Promise(() => {}));
|
||||
render(<TipPage />);
|
||||
expect(screen.getByText(/reading you/i)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('TipPage — action sheet', () => {
|
||||
// Render with real timers, THEN switch to fake for hold simulation
|
||||
async function renderTipAndHold(id: string, content: string) {
|
||||
mockGetRec.mockResolvedValue({ tip: { id, content, source: 'todoist', createdAt: '' } });
|
||||
render(<TipPage />);
|
||||
// Wait for tip to appear (real timers — no deadlock)
|
||||
await screen.findByText(content);
|
||||
const main = screen.getByRole('main');
|
||||
|
||||
// Switch to fake timers now that the component is fully loaded
|
||||
vi.useFakeTimers();
|
||||
act(() => { main.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true })); });
|
||||
act(() => { vi.advanceTimersByTime(650); });
|
||||
vi.useRealTimers();
|
||||
|
||||
// Wait for action sheet
|
||||
await screen.findByText('Done ✓');
|
||||
return main;
|
||||
}
|
||||
|
||||
it('action sheet appears after a long press (600 ms)', async () => {
|
||||
await renderTipAndHold('tip:lp', 'Hold me');
|
||||
expect(screen.getByText('Done ✓')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('action sheet does not appear on short press (<600 ms)', async () => {
|
||||
mockGetRec.mockResolvedValue({ tip: { id: 'tip:sp', content: 'Short press', source: 'todoist', createdAt: '' } });
|
||||
render(<TipPage />);
|
||||
await screen.findByText('Short press');
|
||||
const main = screen.getByRole('main');
|
||||
|
||||
vi.useFakeTimers();
|
||||
act(() => { main.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true })); });
|
||||
act(() => { vi.advanceTimersByTime(200); });
|
||||
act(() => { main.dispatchEvent(new PointerEvent('pointerup', { bubbles: true })); });
|
||||
vi.useRealTimers();
|
||||
|
||||
expect(screen.queryByText('Done ✓')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('clicking "Done ✓" calls sendFeedback with action=done', async () => {
|
||||
await renderTipAndHold('tip:d', 'Do it');
|
||||
await act(async () => { fireEvent.click(screen.getByText('Done ✓')); });
|
||||
expect(mockSendFeedback).toHaveBeenCalledWith('tip:d', { action: 'done' });
|
||||
});
|
||||
|
||||
it('clicking "Dismiss" calls sendFeedback with action=dismiss', async () => {
|
||||
await renderTipAndHold('tip:dis', 'Dismiss me');
|
||||
await act(async () => { fireEvent.click(screen.getByText('Dismiss')); });
|
||||
expect(mockSendFeedback).toHaveBeenCalledWith('tip:dis', { action: 'dismiss' });
|
||||
});
|
||||
|
||||
it('clicking "Helpful" calls sendFeedback with action=helpful (non-navigating)', async () => {
|
||||
await renderTipAndHold('tip:help', 'Helpful tip');
|
||||
await act(async () => { fireEvent.click(screen.getByText('Helpful')); });
|
||||
expect(mockSendFeedback).toHaveBeenCalledWith('tip:help', { action: 'helpful' });
|
||||
});
|
||||
});
|
||||
1
apps/web/src/test/setup.ts
Normal file
1
apps/web/src/test/setup.ts
Normal file
@@ -0,0 +1 @@
|
||||
import '@testing-library/jest-dom';
|
||||
23
apps/web/vitest.config.ts
Normal file
23
apps/web/vitest.config.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
import react from '@vitejs/plugin-react';
|
||||
import { resolve } from 'path';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'jsdom',
|
||||
setupFiles: ['./src/test/setup.ts'],
|
||||
exclude: ['e2e/**', 'node_modules/**'],
|
||||
coverage: {
|
||||
provider: 'v8',
|
||||
reporter: ['text', 'lcov'],
|
||||
include: ['src/**'],
|
||||
},
|
||||
},
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': resolve(__dirname, 'src'),
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user