// Produces a QA Lab UX Matrix evidence bundle through the script scenario contract. import { execFile } from "node:child_process"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { toRepoRelativePath } from "../../extensions/qa-lab/src/cli-paths.js"; import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND, QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, resolveQaEvidenceEnvironment, validateQaEvidenceSummaryJson, type QaEvidenceStatus, type QaEvidenceSummaryEntry, type QaEvidenceSummaryJson, } from "../../extensions/qa-lab/src/evidence-summary.js"; import { ensurePlaywrightChromium, resolveSystemChromiumExecutablePath, } from "../ensure-playwright-chromium.mjs"; const execFileAsync = promisify(execFile); const SCENARIO_ID = "ux-matrix-evidence-dashboard"; const SOURCE_PATH = "scripts/qa/ux-matrix-evidence-producer.ts"; const SUITE_COMMAND = `pnpm openclaw qa suite --scenario ${SCENARIO_ID}`; type MatrixCell = { artifacts: Array<{ kind: string; path: string }>; coverageIds: string[]; failureReason?: string; stage: string; status: QaEvidenceStatus; surface: string; title: string; wallMs: number; }; type ChromiumLauncher = Awaited["chromium"]; type ChromiumBrowser = Awaited>; export type ProducerOptions = { artifactBase: string; repoRoot: string; skipVisualProof: boolean; }; function usage() { return `Usage: node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base [options] Produces a QA Lab UX Matrix evidence bundle. Options: --artifact-base Evidence artifact directory --repo-root Repository root --skip-visual-proof Use fixture visual evidence instead of Playwright screenshots -h, --help Show this help `; } function readOptionValue(argv: readonly string[], index: number, arg: string) { const value = argv[index + 1] ?? ""; if (!value || value.startsWith("-")) { throw new Error(`${arg} requires a value`); } return value; } function isHelpRequest(argv: readonly string[]) { for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--artifact-base" || arg === "--repo-root") { index += 1; continue; } if (arg === "--help" || arg === "-h") { return true; } } return false; } function parseOptions(argv: readonly string[]): ProducerOptions { let artifactBase = ""; let repoRoot = process.cwd(); let skipVisualProof = false; const seen = new Set(); const recordOnce = (flag: string) => { if (seen.has(flag)) { throw new Error(`${flag} was provided more than once`); } seen.add(flag); }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--artifact-base") { const value = readOptionValue(argv, index, arg); recordOnce(arg); artifactBase = value; index += 1; continue; } if (arg === "--repo-root") { const value = readOptionValue(argv, index, arg); recordOnce(arg); repoRoot = value; index += 1; continue; } if (arg === "--skip-visual-proof") { recordOnce(arg); skipVisualProof = true; continue; } throw new Error(`unsupported UX Matrix producer arg: ${arg}`); } if (!artifactBase.trim()) { throw new Error("--artifact-base is required"); } if (!repoRoot.trim()) { throw new Error("--repo-root must not be empty"); } return { artifactBase: path.resolve(repoRoot, artifactBase), repoRoot: path.resolve(repoRoot), skipVisualProof, }; } async function writeJson(filePath: string, value: unknown) { await fs.mkdir(path.dirname(filePath), { recursive: true }); await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } async function writeText(filePath: string, value: string) { await fs.mkdir(path.dirname(filePath), { recursive: true }); await fs.writeFile(filePath, value, "utf8"); } function cellDir(artifactBase: string, cell: Pick) { return path.join(artifactBase, "surfaces", cell.surface, "stages", cell.stage); } function relativeToArtifactBase(artifactBase: string, filePath: string) { return path.relative(artifactBase, filePath).split(path.sep).join("/"); } function sanitizeArtifactText( value: string, params: { artifactBase?: string; repoRoot: string; }, ) { const roots = [ { from: path.resolve(params.repoRoot), to: "" }, { from: pathToFileURL(path.resolve(params.repoRoot)).href, to: "file://" }, ...(params.artifactBase ? [ { from: path.resolve(params.artifactBase), to: "" }, { from: pathToFileURL(path.resolve(params.artifactBase)).href, to: "file://", }, ] : []), { from: os.homedir(), to: "" }, { from: pathToFileURL(os.homedir()).href, to: "file://" }, ].filter((entry) => entry.from && entry.from !== path.parse(entry.from).root); return roots .toSorted((a, b) => b.from.length - a.from.length) .reduce((text, entry) => text.replaceAll(entry.from, entry.to), value); } function buildExecution(params: { artifacts: MatrixCell["artifacts"]; repoRoot: string; source: string; }): QaEvidenceSummaryEntry["execution"] { return { runner: "ux-matrix-script-producer", environment: resolveQaEvidenceEnvironment({ env: process.env, repoRoot: params.repoRoot, }), provider: { id: "ux-matrix", live: false, model: { name: null, ref: null, }, fixture: "local-qa-lab-script-producer", }, packageSource: { kind: "source-checkout", }, artifacts: params.artifacts.map((artifact) => ({ ...artifact, source: params.source, })), }; } function buildEvidenceEntry(cell: MatrixCell, repoRoot: string): QaEvidenceSummaryEntry { const source = `ux-matrix:${cell.surface}:${cell.stage}`; return { test: { kind: "ux-matrix-cell", id: `ux-matrix.${cell.surface}.${cell.stage}`, title: cell.title, source: { path: SOURCE_PATH }, }, coverage: cell.coverageIds.map((id, index) => ({ id, role: index === 0 ? "primary" : "secondary", })), refs: [ { kind: "code", path: SOURCE_PATH }, { kind: "docs", path: "docs/concepts/qa-e2e-automation.md" }, ], execution: buildExecution({ artifacts: cell.artifacts, repoRoot, source, }), result: { status: cell.status, ...(cell.status === "pass" ? {} : { failure: { class: cell.status, reason: cell.failureReason ?? `${cell.status} UX Matrix cell`, }, }), timing: { wallMs: Math.max(1, cell.wallMs), }, }, }; } function buildEvidenceSummary(params: { cells: readonly MatrixCell[]; generatedAt: string; repoRoot: string; }): QaEvidenceSummaryJson { return validateQaEvidenceSummaryJson({ kind: QA_EVIDENCE_SUMMARY_KIND, schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, generatedAt: params.generatedAt, evidenceMode: "full", entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot)), }); } async function runCommandForCell(params: { args: string[]; artifactBase: string; command: string; cwd: string; logPath: string; timeoutMs: number; }) { const startedAt = Date.now(); const commandLine = [params.command, ...params.args].join(" "); try { const { stdout, stderr } = await execFileAsync(params.command, params.args, { cwd: params.cwd, timeout: params.timeoutMs, env: process.env, maxBuffer: 1024 * 1024, }); await writeText( params.logPath, sanitizeArtifactText(`$ ${commandLine}\n${stdout}${stderr}`, { artifactBase: params.artifactBase, repoRoot: params.cwd, }), ); return { status: "pass" as const, wallMs: Date.now() - startedAt, }; } catch (error) { const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), { artifactBase: params.artifactBase, repoRoot: params.cwd, }); await writeText(params.logPath, `$ ${commandLine}\nblocked: ${details}\n`); return { failureReason: details, status: "blocked" as const, wallMs: Date.now() - startedAt, }; } } async function writePreflight(artifactBase: string) { await writeText( path.join(artifactBase, "preflight", "runtime.txt"), [ `platform=${process.platform}`, `arch=${process.arch}`, `node=${process.version}`, `freeMemoryBytes=${os.freemem()}`, `totalMemoryBytes=${os.totalmem()}`, ].join("\n") + "\n", ); } function isMissingManagedPlaywrightBrowser(error: unknown) { const message = error instanceof Error ? error.message : String(error); return ( message.includes("Executable doesn't exist") && message.includes(".cache/ms-playwright") && message.includes("playwright install") ); } export async function launchUxMatrixChromium(params?: { chromium?: Pick; systemExecutablePath?: string; }): Promise<{ browser: ChromiumBrowser; usedSystemExecutablePath?: string }> { const chromium = params?.chromium ?? (await import("playwright")).chromium; try { return { browser: await chromium.launch() }; } catch (error) { if (!isMissingManagedPlaywrightBrowser(error)) { throw error; } const executablePath = params?.systemExecutablePath ?? resolveSystemChromiumExecutablePath(); if (!executablePath) { throw error; } return { browser: await chromium.launch({ executablePath }), usedSystemExecutablePath: executablePath, }; } } export function ensureUxMatrixVideoDependencies(params: { ensureChromium?: typeof ensurePlaywrightChromium; usedSystemExecutablePath?: string; }) { if (!params.usedSystemExecutablePath) { return; } const ensureChromium = params.ensureChromium ?? ensurePlaywrightChromium; const status = ensureChromium({ ensureFfmpeg: true, systemExecutablePath: params.usedSystemExecutablePath, }); if (status !== 0) { throw new Error(`Playwright ffmpeg install failed with status ${status}`); } } async function captureControlUiScreenshot(params: { artifactBase: string; htmlPath: string; logPath: string; repoRoot: string; screenshotPath: string; }) { const startedAt = Date.now(); try { const { browser } = await launchUxMatrixChromium(); try { const page = await browser.newPage({ viewport: { width: 1024, height: 720 } }); await page.goto(pathToFileURL(params.htmlPath).href); await page.screenshot({ path: params.screenshotPath, fullPage: true }); } finally { await browser.close(); } await writeText( params.logPath, `Captured ${relativeToArtifactBase(params.artifactBase, params.screenshotPath)}\n`, ); return { status: "pass" as const, wallMs: Date.now() - startedAt, }; } catch (error) { const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), { artifactBase: params.artifactBase, repoRoot: params.repoRoot, }); await writeText(params.logPath, `blocked: ${details}\n`); return { failureReason: details, status: "blocked" as const, wallMs: Date.now() - startedAt, }; } } function escapeHtml(value: string) { return value .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll('"', """); } async function writeProducerArtifactFixtureHtml(params: { artifactBase: string; evidence: QaEvidenceSummaryJson; htmlPath: string; logPreview: string; }) { const previewArtifacts = params.evidence.entries .filter((entry) => entry.test.id !== "ux-matrix.qa-lab.producer-artifact-fixture") .flatMap((entry) => entry.execution?.artifacts ?? []); const screenshotArtifact = previewArtifacts.find((artifact) => artifact.kind === "screenshot"); const logArtifact = previewArtifacts.find((artifact) => artifact.kind === "log"); const screenshotPath = screenshotArtifact ? relativeToArtifactBase( path.dirname(params.htmlPath), path.join(params.artifactBase, screenshotArtifact.path), ) : ""; const entryRows = params.evidence.entries .map( (entry) => `
  • ${escapeHtml(entry.test.id)} - ${escapeHtml( entry.result.status, )} - ${entry.coverage.map((coverage) => escapeHtml(coverage.id)).join(", ")}
  • `, ) .join(""); await writeText( params.htmlPath, ` UX Matrix producer artifact fixture

    UX Matrix Producer Artifact Fixture

    Script-produced ${escapeHtml(QA_EVIDENCE_FILENAME)} for ${escapeHtml( SCENARIO_ID, )}; this fixture is not the QA Lab Evidence Archive UI.

    UX Matrix entries

      ${entryRows}

    Artifact preview

    ${escapeHtml(logArtifact?.path ?? "no log")} - ${escapeHtml( screenshotArtifact?.path ?? "no screenshot", )}

    Choose an artifact to preview.
    `, ); } async function captureProducerArtifactFixtureProof(params: { artifactBase: string; htmlPath: string; logPath: string; repoRoot: string; screenshotPath: string; skipVisualProof: boolean; videoPath: string; }) { const startedAt = Date.now(); if (params.skipVisualProof) { await writeText(params.logPath, "blocked: --skip-visual-proof was set\n"); return { failureReason: "--skip-visual-proof was set", status: "blocked" as const, wallMs: Date.now() - startedAt, }; } try { const videoDir = path.join(path.dirname(params.videoPath), "recording"); await fs.mkdir(videoDir, { recursive: true }); const { browser, usedSystemExecutablePath } = await launchUxMatrixChromium(); let recordedVideo: string | undefined; try { ensureUxMatrixVideoDependencies({ usedSystemExecutablePath }); const context = await browser.newContext({ viewport: { width: 1280, height: 820 }, recordVideo: { dir: videoDir, size: { width: 1280, height: 820 }, }, }); const page = await context.newPage(); const video = page.video(); await page.goto(pathToFileURL(params.htmlPath).href); await page.locator("#preview-screenshot").click(); await page.locator("#preview img").waitFor({ state: "visible", timeout: 5_000 }); await page.screenshot({ path: params.screenshotPath, fullPage: true }); await page.waitForTimeout(350); await page.locator("#preview-log").click(); await page.waitForTimeout(350); await context.close(); recordedVideo = await video?.path(); } finally { await browser.close(); } if (!recordedVideo) { throw new Error("Playwright did not provide a recording path"); } await fs.mkdir(path.dirname(params.videoPath), { recursive: true }); await fs.copyFile(recordedVideo, params.videoPath); await writeText( params.logPath, [ `Captured screenshot ${path.basename(params.screenshotPath)}`, `Captured recording ${path.basename(params.videoPath)}`, ].join("\n") + "\n", ); return { status: "pass" as const, wallMs: Date.now() - startedAt, }; } catch (error) { const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), { artifactBase: params.artifactBase, repoRoot: params.repoRoot, }); await writeText(params.logPath, `blocked: ${details}\n`); return { failureReason: details, status: "blocked" as const, wallMs: Date.now() - startedAt, }; } } async function writeProducerMetadata(params: { artifactBase: string; cells: readonly MatrixCell[]; repoRoot: string; }) { const counts = params.cells.reduce>((acc, cell) => { acc[cell.status] = (acc[cell.status] ?? 0) + 1; return acc; }, {}); await writeJson(path.join(params.artifactBase, "manifest.json"), { kind: "openclaw.qa.ux-matrix", run: { scenarioId: SCENARIO_ID, status: counts.fail ? "fail" : counts.blocked ? "blocked" : "pass", }, }); await writeJson(path.join(params.artifactBase, "matrix.json"), { cells: params.cells.map((cell) => ({ coverageIds: cell.coverageIds, stage: cell.stage, status: cell.status, surface: cell.surface, })), counts, }); await writeJson(path.join(params.artifactBase, "release-ledger.json"), { entries: params.cells.map((cell) => ({ coverageIds: cell.coverageIds, stage: cell.stage, status: cell.status, surface: cell.surface, })), kind: "openclaw.qa.ux-matrix.release-ledger", }); await writeText( path.join(params.artifactBase, "commands.txt"), [ `${SUITE_COMMAND} --output-dir ${toRepoRelativePath(params.repoRoot, params.artifactBase)}`, `node --import tsx ${SOURCE_PATH} --artifact-base ${toRepoRelativePath( params.repoRoot, params.artifactBase, )}`, ].join("\n") + "\n", ); await writeText( path.join(params.artifactBase, "scorecard.md"), ["# UX Matrix", "", ...Object.entries(counts).map(([status, count]) => `- ${status}: ${count}`)] .join("\n") .trimEnd() + "\n", ); } export async function runUxMatrixEvidenceProducer(options: ProducerOptions) { await fs.mkdir(options.artifactBase, { recursive: true }); await writePreflight(options.artifactBase); const cliLogPath = path.join( cellDir(options.artifactBase, { surface: "cli", stage: "entrypoint-help" }), "logs.txt", ); const cliResult = await runCommandForCell({ args: ["openclaw.mjs", "--help"], artifactBase: options.artifactBase, command: process.execPath, cwd: options.repoRoot, logPath: cliLogPath, timeoutMs: 30_000, }); const screenshotCellDir = cellDir(options.artifactBase, { surface: "control-ui", stage: "screenshot-artifact", }); const matrixHtmlPath = path.join(screenshotCellDir, "matrix-preview.html"); await writeText( matrixHtmlPath, 'UX Matrix

    UX Matrix

    Control UI artifact preview fixture generated by the scenario.

    ', ); const matrixScreenshotPath = path.join(screenshotCellDir, "screenshot.png"); const matrixScreenshotResult = await captureControlUiScreenshot({ artifactBase: options.artifactBase, htmlPath: matrixHtmlPath, logPath: path.join(screenshotCellDir, "logs.txt"), repoRoot: options.repoRoot, screenshotPath: matrixScreenshotPath, }); const initialCells: MatrixCell[] = [ { artifacts: [ { kind: "log", path: relativeToArtifactBase( options.artifactBase, path.join(screenshotCellDir, "logs.txt"), ), }, ...(matrixScreenshotResult.status === "pass" ? [ { kind: "screenshot", path: relativeToArtifactBase(options.artifactBase, matrixScreenshotPath), }, ] : []), ], coverageIds: ["ui.control", "gateway.control-ui-hosting"], failureReason: matrixScreenshotResult.failureReason, stage: "screenshot-artifact", status: matrixScreenshotResult.status, surface: "control-ui", title: "UX Matrix: screenshot artifact", wallMs: matrixScreenshotResult.wallMs, }, { artifacts: [{ kind: "log", path: relativeToArtifactBase(options.artifactBase, cliLogPath) }], coverageIds: ["cli.entrypoint", "cli.status-snapshots"], failureReason: cliResult.failureReason, stage: "entrypoint-help", status: cliResult.status, surface: "cli", title: "UX Matrix: CLI entrypoint help", wallMs: cliResult.wallMs, }, ]; const fixtureProofDir = cellDir(options.artifactBase, { surface: "qa-lab", stage: "producer-artifact-fixture", }); const fixtureHtmlPath = path.join(fixtureProofDir, "producer-artifact-fixture.html"); const previewEvidence = buildEvidenceSummary({ cells: initialCells, generatedAt: new Date().toISOString(), repoRoot: options.repoRoot, }); const screenshotLog = await fs.readFile(path.join(screenshotCellDir, "logs.txt"), "utf8"); await writeProducerArtifactFixtureHtml({ artifactBase: options.artifactBase, evidence: previewEvidence, htmlPath: fixtureHtmlPath, logPreview: screenshotLog, }); const fixtureProofResult = await captureProducerArtifactFixtureProof({ artifactBase: options.artifactBase, htmlPath: fixtureHtmlPath, logPath: path.join(fixtureProofDir, "logs.txt"), repoRoot: options.repoRoot, screenshotPath: path.join(fixtureProofDir, "producer-artifact-fixture.png"), skipVisualProof: options.skipVisualProof, videoPath: path.join(fixtureProofDir, "producer-artifact-fixture.webm"), }); const cells: MatrixCell[] = [ { artifacts: [ { kind: "html", path: relativeToArtifactBase(options.artifactBase, fixtureHtmlPath) }, { kind: "log", path: relativeToArtifactBase( options.artifactBase, path.join(fixtureProofDir, "logs.txt"), ), }, ...(fixtureProofResult.status === "pass" ? [ { kind: "screenshot", path: relativeToArtifactBase( options.artifactBase, path.join(fixtureProofDir, "producer-artifact-fixture.png"), ), }, { kind: "video", path: relativeToArtifactBase( options.artifactBase, path.join(fixtureProofDir, "producer-artifact-fixture.webm"), ), }, ] : []), ], coverageIds: ["qa.artifact-safety", "tools.evidence", "workspace.artifacts"], failureReason: fixtureProofResult.failureReason, stage: "producer-artifact-fixture", status: fixtureProofResult.status, surface: "qa-lab", title: "UX Matrix: producer artifact fixture", wallMs: fixtureProofResult.wallMs, }, ...initialCells, ]; const evidence = buildEvidenceSummary({ cells, generatedAt: new Date().toISOString(), repoRoot: options.repoRoot, }); await writeProducerArtifactFixtureHtml({ artifactBase: options.artifactBase, evidence, htmlPath: fixtureHtmlPath, logPreview: screenshotLog, }); await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence); await writeJson(path.join(options.artifactBase, "latest-run.json"), { qaEvidence: QA_EVIDENCE_FILENAME, }); await writeProducerMetadata({ artifactBase: options.artifactBase, cells, repoRoot: options.repoRoot, }); return { artifactBase: options.artifactBase, evidence, }; } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { (async () => { const cliArgs = process.argv.slice(2); if (isHelpRequest(cliArgs)) { console.log(usage()); return; } const result = await runUxMatrixEvidenceProducer(parseOptions(cliArgs)); console.log(`UX Matrix evidence: ${path.join(result.artifactBase, QA_EVIDENCE_FILENAME)}`); console.log(`UX Matrix entries: ${result.evidence.entries.length}`); })() .then(() => { process.exitCode = 0; }) .catch((error: unknown) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }); }