alvis/oO
1
0
Fork 0
Code Issues 34 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(admin): LLM tip quality dashboard — per-model/prompt/kind breakdowns

Browse Source 
/admin/reward-analytics now surfaces served count, reaction rate, and avg
reward grouped by llm_model, prompt_version, and tip_kind — closing the
loop so model/prompt iterations in M2 are legible next to the bandit
policy view. Data comes from the tip_scores columns added in ffdf707 and
tip_feedback.reward_milli; bandit-only tips show as "(bandit-only)".

Closes #92.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
alvis
2026-04-24 15:24:52 +00:00
parent 75d0e89906
commit aa4bdd8f09
7 changed files with 227 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
apps/admin/src/app/reward-analytics/page.tsx
View File

@@ -2,7 +2,7 @@
import { useEffect, useState } from 'react';
import { AdminShell } from '@/components/AdminShell';
import { getRewardAnalytics } from '@/lib/api';
import { getRewardAnalytics, type QualityBreakdownRow } from '@/lib/api';
const ACTION_COLORS: Record<string, string> = {
done: 'bg-green-500',
@@ -12,6 +12,53 @@ const ACTION_COLORS: Record<string, string> = {
dismiss: 'bg-red-500',
};
function QualityBreakdown({ title, dimension, rows, emptyLabel }: {
title: string;
dimension: string;
rows: QualityBreakdownRow[];
emptyLabel: string; // shown when a row's key is null (e.g. bandit-only tips have no llm_model)
}) {
if (rows.length === 0) return null;
const totalServed = rows.reduce((sum, r) => sum + r.served, 0);
return (
<div className="space-y-2">
<h2 className="text-sm font-medium text-gray-400">{title}</h2>
<table className="w-full text-xs">
<thead>
<tr className="border-b border-gray-800 text-gray-500 text-left">
<th className="py-2 pr-4">{dimension}</th>
<th className="py-2 pr-4">served</th>
<th className="py-2 pr-4">reaction rate</th>
<th className="py-2 pr-4">avg reward</th>
{['done', 'helpful', 'snooze', 'not_helpful', 'dismiss'].map((a) => (
<th key={a} className="py-2 pr-4">{a}</th>
))}
</tr>
</thead>
<tbody>
{rows.map((r) => {
const reacted = r.done + r.snooze + r.dismiss + r.helpful + r.not_helpful;
const reactionRate = r.served > 0 ? (reacted / r.served) * 100 : 0;
const avgReward = r.avgRewardMilli == null ? null : r.avgRewardMilli / 1000;
return (
<tr key={r.key ?? '__null__'} className="border-b border-gray-800/50">
<td className="py-2 pr-4 font-medium text-indigo-300">{r.key ?? <span className="text-gray-500 italic">{emptyLabel}</span>}</td>
<td className="py-2 pr-4 text-gray-300">{r.served}</td>
<td className="py-2 pr-4 text-gray-300">{reactionRate.toFixed(1)}%</td>
<td className="py-2 pr-4 text-gray-300">{avgReward == null ? '—' : avgReward.toFixed(2)}</td>
{(['done', 'helpful', 'snooze', 'not_helpful', 'dismiss'] as const).map((a) => (
<td key={a} className="py-2 pr-4 text-gray-300">{r[a]}</td>
))}
</tr>
);
})}
</tbody>
</table>
<p className="text-xs text-gray-600">{totalServed} tips served total.</p>
</div>
);
}
export default function RewardAnalyticsPage() {
const [days, setDays] = useState(30);
const [data, setData] = useState<Awaited<ReturnType<typeof getRewardAnalytics>> | null>(null);
@@ -108,6 +155,30 @@ export default function RewardAnalyticsPage() {
</div>
)}
{/* LLM quality breakdowns (#92) */}
{data && (
<>
<QualityBreakdown
title="Per LLM model"
dimension="llm_model"
rows={data.byModel ?? []}
emptyLabel="(bandit-only)"
/>
<QualityBreakdown
title="Per prompt version"
dimension="prompt_version"
rows={data.byPromptVersion ?? []}
emptyLabel="(unset)"
/>
<QualityBreakdown
title="Per tip kind"
dimension="tip_kind"
rows={data.byKind ?? []}
emptyLabel="(unset)"
/>
</>
)}
{/* Daily table */}
{(data?.daily?.length ?? 0) > 0 && (
<div className="space-y-2">