feat(admin): profile freshness panel in data-quality (#81 phase B.4)
Adds a per-feature freshness summary to /admin/data-quality so the admin can spot features that are systematically stale or never computed: totalEligible — distinct users with tip_views in the last 30 days missing — eligible users with no row stored for the feature stale — eligible users whose stored row is past its TTL Backend exposes summarizeProfileFreshness() in profile/builder.ts; one query per feature joins eligible users LEFT JOIN profile rows. Coverage = (eligible − missing − stale) / eligible, colored green/yellow/red via the new PctGood helper (high-is-good, opposite of the existing Pct used for missing-feature/stale-token rates). Refs #81. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,19 @@ function Pct({ value }: { value: number }) {
|
|||||||
return <span className={color}>{pct}%</span>;
|
return <span className={color}>{pct}%</span>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function PctGood({ value }: { value: number }) {
|
||||||
|
const pct = (value * 100).toFixed(1);
|
||||||
|
const color = value > 0.95 ? 'text-green-400' : value > 0.8 ? 'text-yellow-400' : 'text-red-400';
|
||||||
|
return <span className={color}>{pct}%</span>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTtl(sec: number): string {
|
||||||
|
if (sec < 60) return `${sec}s`;
|
||||||
|
if (sec < 3600) return `${Math.round(sec / 60)}m`;
|
||||||
|
if (sec < 86400) return `${Math.round(sec / 3600)}h`;
|
||||||
|
return `${Math.round(sec / 86400)}d`;
|
||||||
|
}
|
||||||
|
|
||||||
export default function DataQualityPage() {
|
export default function DataQualityPage() {
|
||||||
const [data, setData] = useState<Awaited<ReturnType<typeof getDataQuality>> | null>(null);
|
const [data, setData] = useState<Awaited<ReturnType<typeof getDataQuality>> | null>(null);
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
@@ -50,6 +63,45 @@ export default function DataQualityPage() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Profile freshness — #81 phase B.4 */}
|
||||||
|
<div className="space-y-2">
|
||||||
|
<h2 className="text-sm font-medium text-gray-400">Profile feature freshness</h2>
|
||||||
|
<p className="text-xs text-gray-600">
|
||||||
|
Eligible = users with any tip activity in the last 30 days. Stale = stored row past its TTL. Missing = no row computed yet.
|
||||||
|
</p>
|
||||||
|
<table className="w-full text-xs">
|
||||||
|
<thead>
|
||||||
|
<tr className="border-b border-gray-800 text-gray-500 text-left">
|
||||||
|
<th className="py-2 pr-4">Feature</th>
|
||||||
|
<th className="py-2 pr-4">TTL</th>
|
||||||
|
<th className="py-2 pr-4">Eligible</th>
|
||||||
|
<th className="py-2 pr-4">Missing</th>
|
||||||
|
<th className="py-2 pr-4">Stale</th>
|
||||||
|
<th className="py-2">Coverage</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{data.profileFreshness.map((r) => {
|
||||||
|
const fresh = r.totalEligible - r.missing - r.stale;
|
||||||
|
const coverage = r.totalEligible > 0 ? fresh / r.totalEligible : 0;
|
||||||
|
return (
|
||||||
|
<tr key={r.feature} className="border-b border-gray-800/50">
|
||||||
|
<td className="py-1.5 pr-4 font-mono text-gray-400">{r.feature}</td>
|
||||||
|
<td className="py-1.5 pr-4 text-gray-500 tabular-nums">{formatTtl(r.ttlSec)}</td>
|
||||||
|
<td className="py-1.5 pr-4 text-gray-300 tabular-nums">{r.totalEligible}</td>
|
||||||
|
<td className={`py-1.5 pr-4 tabular-nums ${r.missing > 0 ? 'text-orange-400' : 'text-gray-500'}`}>{r.missing}</td>
|
||||||
|
<td className={`py-1.5 pr-4 tabular-nums ${r.stale > 0 ? 'text-yellow-400' : 'text-gray-500'}`}>{r.stale}</td>
|
||||||
|
<td className="py-1.5"><PctGood value={coverage} /></td>
|
||||||
|
</tr>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
{data.profileFreshness.length === 0 && (
|
||||||
|
<tr><td colSpan={6} className="py-4 text-center text-gray-600">No features registered</td></tr>
|
||||||
|
)}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="space-y-2">
|
<div className="space-y-2">
|
||||||
<h2 className="text-sm font-medium text-gray-400">Daily feature completeness (14d)</h2>
|
<h2 className="text-sm font-medium text-gray-400">Daily feature completeness (14d)</h2>
|
||||||
<table className="w-full text-xs">
|
<table className="w-full text-xs">
|
||||||
|
|||||||
@@ -199,6 +199,14 @@ export function getRewardAnalytics(days = 30) {
|
|||||||
}>(`/admin/reward-analytics?days=${days}`);
|
}>(`/admin/reward-analytics?days=${days}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface FeatureFreshnessRow {
|
||||||
|
feature: string;
|
||||||
|
ttlSec: number;
|
||||||
|
totalEligible: number;
|
||||||
|
missing: number;
|
||||||
|
stale: number;
|
||||||
|
}
|
||||||
|
|
||||||
export function getDataQuality() {
|
export function getDataQuality() {
|
||||||
return apiFetch<{
|
return apiFetch<{
|
||||||
scoringCallsLast30d: number;
|
scoringCallsLast30d: number;
|
||||||
@@ -207,6 +215,7 @@ export function getDataQuality() {
|
|||||||
totalTokens: number;
|
totalTokens: number;
|
||||||
staleTokens: number;
|
staleTokens: number;
|
||||||
dailyQuality: { date: string; total: number; withFeatures: number; avgCandidates: number }[];
|
dailyQuality: { date: string; total: number; withFeatures: number; avgCandidates: number }[];
|
||||||
|
profileFreshness: FeatureFreshnessRow[];
|
||||||
}>('/admin/data-quality');
|
}>('/admin/data-quality');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -120,6 +120,58 @@ export interface ProfileFeatureView {
|
|||||||
description: string;
|
description: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface FeatureFreshnessSummary {
|
||||||
|
feature: string;
|
||||||
|
ttlSec: number;
|
||||||
|
/** Distinct users with tip activity in the last 30 days. */
|
||||||
|
totalEligible: number;
|
||||||
|
/** Eligible users with no row stored at all for this feature. */
|
||||||
|
missing: number;
|
||||||
|
/** Eligible users whose stored row is past its TTL. */
|
||||||
|
stale: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-feature staleness summary across all eligible users (anyone with a tip
|
||||||
|
* served in the last 30 days). Used by `/admin/data-quality` so the admin can
|
||||||
|
* spot features that are systematically stale or never computed.
|
||||||
|
*
|
||||||
|
* Hand-written SQL because this joins (eligible_users LEFT JOIN profile_rows)
|
||||||
|
* with conditional aggregations — drizzle's query builder is more pain than
|
||||||
|
* value here, and the column allowlist is the registry.
|
||||||
|
*/
|
||||||
|
export function summarizeProfileFreshness(): FeatureFreshnessSummary[] {
|
||||||
|
return FEATURES.map((f) => {
|
||||||
|
const row = rawSqlite
|
||||||
|
.prepare(`
|
||||||
|
WITH eligible AS (
|
||||||
|
SELECT DISTINCT user_id
|
||||||
|
FROM tip_views
|
||||||
|
WHERE served_at >= datetime('now', '-30 days')
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
COUNT(*) AS total_eligible,
|
||||||
|
SUM(CASE WHEN upf.user_id IS NULL THEN 1 ELSE 0 END) AS missing,
|
||||||
|
SUM(CASE WHEN upf.user_id IS NOT NULL
|
||||||
|
AND upf.ttl_sec > 0
|
||||||
|
AND (julianday('now') - julianday(upf.updated_at)) * 86400.0 > upf.ttl_sec
|
||||||
|
THEN 1 ELSE 0 END) AS stale
|
||||||
|
FROM eligible e
|
||||||
|
LEFT JOIN user_profile_features upf
|
||||||
|
ON upf.user_id = e.user_id AND upf.name = ?
|
||||||
|
`)
|
||||||
|
.get(f.name) as { total_eligible: number; missing: number; stale: number } | undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
feature: f.name,
|
||||||
|
ttlSec: f.ttlSec,
|
||||||
|
totalEligible: Number(row?.total_eligible ?? 0),
|
||||||
|
missing: Number(row?.missing ?? 0),
|
||||||
|
stale: Number(row?.stale ?? 0),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Inspection helper for the admin UI: returns one row per registered feature,
|
* Inspection helper for the admin UI: returns one row per registered feature,
|
||||||
* joining stored value + metadata. No compute — surface staleness; rebuild is
|
* joining stored value + metadata. No compute — surface staleness; rebuild is
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import { describe, it, expect, vi, beforeAll } from 'vitest';
|
|||||||
import express from 'express';
|
import express from 'express';
|
||||||
import * as http from 'http';
|
import * as http from 'http';
|
||||||
import { makeTestDb } from '../../test/db.js';
|
import { makeTestDb } from '../../test/db.js';
|
||||||
import { users, integrationTokens, tipViews, tipFeedback, tipScores } from '../../db/schema.js';
|
import { users, integrationTokens, tipViews, tipFeedback, tipScores, userProfileFeatures } from '../../db/schema.js';
|
||||||
|
|
||||||
// ---- in-memory DB ----
|
// ---- in-memory DB ----
|
||||||
const testDb = makeTestDb();
|
const testDb = makeTestDb();
|
||||||
@@ -422,6 +422,56 @@ describe('GET /api/admin/users/:id — edge cases', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('GET /api/admin/data-quality — #81 phase B.4 profile freshness', () => {
|
||||||
|
type Body = {
|
||||||
|
profileFreshness: Array<{
|
||||||
|
feature: string;
|
||||||
|
ttlSec: number;
|
||||||
|
totalEligible: number;
|
||||||
|
missing: number;
|
||||||
|
stale: number;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
|
it('reports each registered feature once', async () => {
|
||||||
|
const { server, call } = await startServer(buildApp());
|
||||||
|
try {
|
||||||
|
const { status, body } = await call('GET', '/api/admin/data-quality');
|
||||||
|
expect(status).toBe(200);
|
||||||
|
const b = body as Body;
|
||||||
|
const names = b.profileFreshness.map((r) => r.feature).sort();
|
||||||
|
expect(names).toEqual([
|
||||||
|
'completion_rate_30d',
|
||||||
|
'dismiss_rate_30d',
|
||||||
|
'mean_dwell_ms_30d',
|
||||||
|
'preferred_hour',
|
||||||
|
'tip_volume_30d',
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
server.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('counts eligible users (with tip_views in 30d) and flags missing rows', async () => {
|
||||||
|
// Reset profile rows so this test is independent of any previous rebuild test
|
||||||
|
// that may have populated rows for some users.
|
||||||
|
await testDb.delete(userProfileFeatures);
|
||||||
|
const { server, call } = await startServer(buildApp());
|
||||||
|
try {
|
||||||
|
const { body } = await call('GET', '/api/admin/data-quality');
|
||||||
|
const b = body as Body;
|
||||||
|
// Seed has tip_views for user-1 and user-2 within 30d → 2 eligible.
|
||||||
|
const completion = b.profileFreshness.find((r) => r.feature === 'completion_rate_30d')!;
|
||||||
|
expect(completion.totalEligible).toBe(2);
|
||||||
|
// No profile rows seeded → both missing
|
||||||
|
expect(completion.missing).toBe(2);
|
||||||
|
expect(completion.stale).toBe(0);
|
||||||
|
} finally {
|
||||||
|
server.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('GET /api/admin/reward-analytics — #92 quality breakdowns', () => {
|
describe('GET /api/admin/reward-analytics — #92 quality breakdowns', () => {
|
||||||
type Row = {
|
type Row = {
|
||||||
key: string | null;
|
key: string | null;
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import { nanoid } from 'nanoid';
|
|||||||
import { bus } from '../events/bus.js';
|
import { bus } from '../events/bus.js';
|
||||||
import { config } from '../config.js';
|
import { config } from '../config.js';
|
||||||
import { getShadowPolicies, setPolicyActive } from './recommender.js';
|
import { getShadowPolicies, setPolicyActive } from './recommender.js';
|
||||||
import { inspectProfile, rebuildProfile } from '../profile/builder.js';
|
import { inspectProfile, rebuildProfile, summarizeProfileFreshness } from '../profile/builder.js';
|
||||||
import { spawn } from 'child_process';
|
import { spawn } from 'child_process';
|
||||||
import { existsSync, readFileSync, unlinkSync } from 'fs';
|
import { existsSync, readFileSync, unlinkSync } from 'fs';
|
||||||
import { resolve, dirname } from 'path';
|
import { resolve, dirname } from 'path';
|
||||||
@@ -514,6 +514,7 @@ router.get('/data-quality', async (req: AuthenticatedRequest, res: Response) =>
|
|||||||
totalTokens,
|
totalTokens,
|
||||||
staleTokens,
|
staleTokens,
|
||||||
dailyQuality,
|
dailyQuality,
|
||||||
|
profileFreshness: summarizeProfileFreshness(),
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user