Vendor OpenClaw source as Adolf fork baseline

Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11),
free to diverge. Tree copied sans upstream .git; upstream remote added for
future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19.
Preserves docs/ARCHITECTURE.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2

extensions/qa-lab/api.ts

@@ -0,0 +1,122 @@
+// Qa Lab API module exposes the plugin public contract.
+export {
+  buildQaBusSnapshot,
+  cloneEvent,
+  cloneMessage,
+  DEFAULT_ACCOUNT_ID,
+  normalizeAccountId,
+  normalizeConversationFromTarget,
+  pollQaBusEvents,
+  readQaBusMessage,
+  searchQaBusMessages,
+} from "./src/bus-queries.js";
+export {
+  closeQaHttpServer,
+  createQaBusServer,
+  handleQaBusRequest,
+  startQaBusServer,
+  writeError,
+  writeJson,
+} from "./src/bus-server.js";
+export { createQaBusState, type QaBusState } from "./src/bus-state.js";
+export {
+  createQaBusWaiterStore,
+  DEFAULT_WAIT_TIMEOUT_MS,
+  type QaBusWaitMatch,
+} from "./src/bus-waiters.js";
+export { isQaLabCliAvailable, registerQaLabCli } from "./src/cli.js";
+export { createQaRunnerRuntime } from "./src/harness-runtime.js";
+export {
+  buildScriptEvidenceSummary,
+  QA_EVIDENCE_FILENAME,
+  type QaEvidencePackageSource,
+  type QaEvidenceStatus,
+  type QaEvidenceSummaryJson,
+  validateQaEvidenceSummaryJson,
+} from "./src/evidence-summary.js";
+export type { QaProviderMode } from "./src/providers/index.js";
+export {
+  type QaLabLatestReport,
+  type QaLabScenarioOutcome,
+  type QaLabScenarioRun,
+  type QaLabServerHandle,
+  type QaLabServerStartParams,
+  startQaLabServer,
+} from "./src/lab-server.js";
+export { buildQaDockerHarnessImage, writeQaDockerHarnessFiles } from "./src/docker-harness.js";
+export {
+  buildQaScenarioPlanMarkdown,
+  readQaAgentIdentityMarkdown,
+} from "./src/qa-agent-bootstrap.js";
+export { seedQaAgentWorkspace } from "./src/qa-agent-workspace.js";
+export {
+  buildQaGatewayConfig,
+  DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS,
+  mergeQaControlUiAllowedOrigins,
+  normalizeQaThinkingLevel,
+  QA_BASE_RUNTIME_PLUGIN_IDS,
+  type QaThinkingLevel,
+} from "./src/qa-gateway-config.js";
+export {
+  renderQaMarkdownReport,
+  type QaReportCheck,
+  type QaReportScenario,
+} from "openclaw/plugin-sdk/qa-runtime";
+export {
+  type QaScenarioDefinition,
+  type QaScenarioResult,
+  type QaScenarioStep,
+  type QaScenarioStepContext,
+  type QaScenarioStepResult,
+  runQaScenario,
+} from "./src/scenario.js";
+export {
+  DEFAULT_QA_AGENT_IDENTITY_MARKDOWN,
+  hasQaScenarioPack,
+  listQaScenarioYamlPaths,
+  type QaBootstrapScenarioCatalog,
+  type QaScenarioExecution,
+  type QaScenarioFlow,
+  type QaScenarioPack,
+  type QaSeedScenario,
+  type QaSeedScenarioWithSource,
+  readQaBootstrapScenarioCatalog,
+  readQaScenarioById,
+  readQaScenarioExecutionConfig,
+  readQaScenarioOverviewMarkdown,
+  readQaScenarioPack,
+  readQaScenarioPackYamlSource,
+  validateQaScenarioExecutionConfig,
+} from "./src/scenario-catalog.js";
+export { createQaSelfCheckScenario } from "./src/self-check-scenario.js";
+export {
+  isQaSelfCheckSuccessful,
+  type QaSelfCheckResult,
+  resolveQaSelfCheckOutputPath,
+  runQaSelfCheckAgainstState,
+} from "./src/self-check.js";
+export { runQaE2eSelfCheck, runQaLabSelfCheck } from "./src/self-check-runner.js";
+export {
+  testing,
+  testing as __testing,
+  buildQaRuntimeEnv,
+  type QaCliBackendAuthMode,
+  type QaGatewayChildListeningContext,
+  type QaGatewayChildCommand,
+  type QaGatewayChildStateMutationContext,
+  resolveQaControlUiRoot,
+  resolveQaGatewayChildProviderMode,
+  startQaGatewayChild,
+} from "./src/gateway-child.js";
+export {
+  buildQaSuiteSummaryJson,
+  qaSuiteProgressTesting,
+  type QaSuiteResult,
+  type QaSuiteRunParams,
+  type QaSuiteScenarioResult,
+  type QaSuiteStartLabFn,
+  type QaSuiteSummaryJson,
+  type QaSuiteSummaryJsonParams,
+  runQaFlowSuite,
+} from "./src/suite.js";
+export { runQaSuite, type QaSuiteRuntimeResult } from "./src/suite-launch.runtime.js";

extensions/qa-lab/bus-api.ts

@@ -0,0 +1,3 @@
+// Qa Lab API module exposes the plugin public contract.
+export { startQaBusServer } from "./src/bus-server.js";
+export { createQaBusState } from "./src/bus-state.js";

extensions/qa-lab/cli-metadata.ts

@@ -0,0 +1,19 @@
+// Qa Lab plugin module implements cli metadata behavior.
+import { definePluginEntry } from "openclaw/plugin-sdk/core";
+
+export default definePluginEntry({
+  id: "qa-lab",
+  name: "QA Lab",
+  description: "Private QA automation harness and debugger UI",
+  register(api) {
+    api.registerCli(() => {}, {
+      descriptors: [
+        {
+          name: "qa",
+          description: "Run QA scenarios and launch the private QA debugger UI",
+          hasSubcommands: true,
+        },
+      ],
+    });
+  },
+});

extensions/qa-lab/cli.ts

@@ -0,0 +1,2 @@
+// Qa Lab plugin module implements cli behavior.
+export { isQaLabCliAvailable, registerQaLabCli } from "./src/cli.js";

extensions/qa-lab/confidence-profiles/codex-100.json

@@ -0,0 +1,168 @@
+{
+  "version": 1,
+  "profile": "codex-100",
+  "lanes": [
+    {
+      "id": "tool-defaults-direct",
+      "title": "Tool-defaults direct runtime parity",
+      "kind": "qa-suite-summary",
+      "artifact": "tool-defaults-direct/qa-suite-summary.json",
+      "required": true,
+      "productImpact": "P2",
+      "qaImpact": "P0",
+      "issue": "https://github.com/openclaw/openclaw/issues/80319",
+      "ownerAction": "Fix product or harness before claiming the tool-defaults gate is trusted.",
+      "labels": ["qa-lab", "runtime-parity", "codex"]
+    },
+    {
+      "id": "openclaw-dynamic-tools-direct",
+      "title": "OpenClaw dynamic integration tools direct runtime parity",
+      "kind": "qa-suite-summary",
+      "artifact": "openclaw-dynamic-tools-direct/qa-suite-summary.json",
+      "required": true,
+      "productImpact": "P1",
+      "qaImpact": "P0",
+      "issue": "https://github.com/openclaw/openclaw/issues/80319",
+      "ownerAction": "Investigate any hard failure as an OpenClaw dynamic integration or QA loading regression.",
+      "labels": ["qa-lab", "runtime-parity", "openclaw-dynamic-tools"]
+    },
+    {
+      "id": "tool-defaults-searchable",
+      "title": "Tool-defaults searchable runtime parity",
+      "kind": "qa-suite-summary",
+      "artifact": "tool-defaults-searchable/qa-suite-summary.json",
+      "required": true,
+      "failureVerdict": "mock-limitation",
+      "skipBackfillLane": "openclaw-dynamic-tools-searchable-live",
+      "productImpact": "P4",
+      "qaImpact": "P2",
+      "issue": "https://github.com/openclaw/openclaw/issues/80319",
+      "ownerAction": "Keep as report-only until searchable/deferred tool modeling has no mock-only ambiguity.",
+      "labels": ["qa-lab", "runtime-parity", "searchable-tools"]
+    },
+    {
+      "id": "first-hour-20-direct",
+      "title": "First-hour 20-turn direct runtime parity",
+      "kind": "qa-suite-summary",
+      "artifact": "first-hour-20-direct/qa-suite-summary.json",
+      "required": true,
+      "skipBackfillLane": "codex-native-live",
+      "productImpact": "P1",
+      "qaImpact": "P0",
+      "ownerAction": "Triage row-by-row; do not file product bugs unless live/native proof reproduces.",
+      "labels": ["qa-lab", "runtime-parity", "first-hour"]
+    },
+    {
+      "id": "mock-token-efficiency",
+      "title": "Mock assistant-message token efficiency estimate",
+      "kind": "token-efficiency-summary",
+      "artifact": "first-hour-20-direct-report/qa-runtime-token-efficiency-summary.json",
+      "required": true,
+      "expectedTokenUsageSource": "mock-estimate",
+      "productImpact": "P4",
+      "qaImpact": "P1",
+      "ownerAction": "Fix labeling before trusting token-efficiency comparisons.",
+      "labels": ["qa-lab", "runtime-parity", "token-efficiency"]
+    },
+    {
+      "id": "fault-injection-mock",
+      "title": "Mock fault-injection runtime parity",
+      "kind": "qa-suite-summary",
+      "artifact": "fault-injection-mock/qa-suite-summary.json",
+      "required": true,
+      "skipBackfillLane": "codex-native-live",
+      "productImpact": "P2",
+      "qaImpact": "P0",
+      "ownerAction": "Treat failures as retry/recovery regressions unless evidence shows fixture drift.",
+      "labels": ["qa-lab", "runtime-parity", "fault-injection"]
+    },
+    {
+      "id": "jsonl-expanded",
+      "title": "Expanded curated JSONL replay",
+      "kind": "jsonl-replay-summary",
+      "artifact": "jsonl-expanded/qa-jsonl-replay-summary.json",
+      "required": true,
+      "productImpact": "P2",
+      "qaImpact": "P0",
+      "ownerAction": "Inspect first drift turn and transcript class before filing any product issue.",
+      "labels": ["qa-lab", "runtime-parity", "jsonl-replay"]
+    },
+    {
+      "id": "confidence-self-test",
+      "title": "Seeded confidence negative controls",
+      "kind": "self-test-summary",
+      "artifact": "confidence-self-test/qa-confidence-self-test-summary.json",
+      "required": true,
+      "productImpact": "P4",
+      "qaImpact": "P0",
+      "ownerAction": "Fix the harness before trusting any green parity result.",
+      "labels": ["qa-lab", "confidence-gate", "negative-controls"]
+    },
+    {
+      "id": "codex-native-live",
+      "title": "Codex-native live workspace capability proof",
+      "kind": "qa-suite-summary",
+      "artifact": "codex-native-live/qa-suite-summary.json",
+      "required": true,
+      "missingVerdict": "environment-blocked",
+      "missingReason": "Live/OAuth runner or OpenAI credentials were unavailable for this proof bundle.",
+      "productImpact": "P1",
+      "qaImpact": "P1",
+      "ownerAction": "Run with live-frontier OAuth before using this lane as product proof.",
+      "labels": ["qa-lab", "runtime-parity", "live-proof"]
+    },
+    {
+      "id": "first-hour-live",
+      "title": "Live first-hour capability proof",
+      "kind": "qa-suite-summary",
+      "artifact": "first-hour-live/qa-suite-summary.json",
+      "required": true,
+      "missingVerdict": "environment-blocked",
+      "missingReason": "Live/OAuth runner or OpenAI credentials were unavailable for this proof bundle.",
+      "productImpact": "P1",
+      "qaImpact": "P1",
+      "ownerAction": "Run with live-frontier OAuth before claiming live first-hour coverage.",
+      "labels": ["qa-lab", "runtime-parity", "live-proof"]
+    },
+    {
+      "id": "openclaw-dynamic-tools-searchable-live",
+      "title": "Live OpenClaw dynamic tools searchable proof",
+      "kind": "qa-suite-summary",
+      "artifact": "openclaw-dynamic-tools-searchable-live/qa-suite-summary.json",
+      "required": true,
+      "missingVerdict": "environment-blocked",
+      "missingReason": "Live/OAuth runner or OpenAI credentials were unavailable for this proof bundle.",
+      "productImpact": "P1",
+      "qaImpact": "P1",
+      "ownerAction": "Run with live-frontier OAuth before claiming production-shaped searchable OpenClaw dynamic tool coverage.",
+      "labels": ["qa-lab", "runtime-parity", "searchable-tools", "live-proof"]
+    },
+    {
+      "id": "live-token-efficiency",
+      "title": "Live assistant-message token efficiency",
+      "kind": "token-efficiency-summary",
+      "artifact": "live-token-efficiency/qa-runtime-token-efficiency-summary.json",
+      "required": true,
+      "expectedTokenUsageSource": "live-usage",
+      "missingVerdict": "environment-blocked",
+      "missingReason": "Live/OAuth runner or OpenAI credentials were unavailable for this proof bundle.",
+      "productImpact": "P3",
+      "qaImpact": "P1",
+      "ownerAction": "Run a live-frontier runtime parity summary and regenerate token efficiency.",
+      "labels": ["qa-lab", "runtime-parity", "token-efficiency"]
+    },
+    {
+      "id": "soak-100",
+      "title": "Optional 100-turn soak",
+      "kind": "qa-suite-summary",
+      "artifact": "soak-100/qa-suite-summary.json",
+      "required": true,
+      "missingVerdict": "environment-blocked",
+      "missingReason": "Scheduled/Testbox soak runner did not upload artifacts for this proof bundle.",
+      "productImpact": "P3",
+      "qaImpact": "P2",
+      "ownerAction": "Run remotely with a long timeout or record the runner budget blocker.",
+      "labels": ["qa-lab", "runtime-parity", "soak"]
+    }
+  ]
+}

extensions/qa-lab/index.ts

@@ -0,0 +1,27 @@
+// Qa Lab plugin entrypoint registers its OpenClaw integration.
+import { definePluginEntry } from "./runtime-api.js";
+import { registerQaLabCli } from "./src/cli.js";
+import { createQaLabWebSearchProvider } from "./src/qa-web-search-provider.js";
+
+export default definePluginEntry({
+  id: "qa-lab",
+  name: "QA Lab",
+  description: "Private QA automation harness and debugger UI",
+  register(api) {
+    api.registerWebSearchProvider(createQaLabWebSearchProvider());
+    api.registerCli(
+      async ({ program }) => {
+        registerQaLabCli(program);
+      },
+      {
+        descriptors: [
+          {
+            name: "qa",
+            description: "Run QA scenarios and launch the private QA debugger UI",
+            hasSubcommands: true,
+          },
+        ],
+      },
+    );
+  },
+});

extensions/qa-lab/model-selection.ts

@@ -0,0 +1,2 @@
+// Qa Lab plugin module implements model selection behavior.
+export * from "./src/model-selection.js";

extensions/qa-lab/openclaw.plugin.json

@@ -0,0 +1,15 @@
+{
+  "id": "qa-lab",
+  "description": "OpenClaw QA lab plugin with private debugger UI and scenario runner.",
+  "activation": {
+    "onStartup": false
+  },
+  "contracts": {
+    "webSearchProviders": ["qa-lab-search"]
+  },
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {}
+  }
+}

extensions/qa-lab/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@openclaw/qa-lab",
+  "version": "2026.6.11",
+  "private": true,
+  "description": "OpenClaw QA lab plugin with private debugger UI and scenario runner",
+  "type": "module",
+  "dependencies": {
+    "@copilotkit/aimock": "1.35.0",
+    "@modelcontextprotocol/sdk": "1.29.0",
+    "playwright-core": "1.61.1",
+    "yaml": "2.9.0",
+    "zod": "4.4.3"
+  },
+  "devDependencies": {
+    "@openclaw/crabline": "0.1.9",
+    "@openclaw/discord": "workspace:*",
+    "@openclaw/plugin-sdk": "workspace:*",
+    "@openclaw/slack": "workspace:*",
+    "@openclaw/whatsapp": "workspace:*",
+    "openclaw": "workspace:*"
+  },
+  "peerDependencies": {
+    "openclaw": ">=2026.6.11"
+  },
+  "peerDependenciesMeta": {
+    "openclaw": {
+      "optional": true
+    }
+  },
+  "openclaw": {
+    "extensions": [
+      "./index.ts"
+    ],
+    "compat": {
+      "pluginApi": ">=2026.6.11"
+    }
+  }
+}

extensions/qa-lab/runtime-api.ts

@@ -0,0 +1,42 @@
+// Qa Lab API module exposes the plugin public contract.
+export {
+  buildQaTarget,
+  callGatewayFromCli,
+  type Command,
+  createQaBusThread,
+  defaultQaRuntimeModelForMode,
+  definePluginEntry,
+  deleteQaBusMessage,
+  editQaBusMessage,
+  getQaBusState,
+  injectQaBusInboundMessage,
+  normalizeQaTarget,
+  type OpenClawConfig,
+  parseQaTarget,
+  type PluginRuntime,
+  pollQaBus,
+  type QaBusAttachment,
+  type QaBusConversation,
+  type QaBusCreateThreadInput,
+  type QaBusDeleteMessageInput,
+  type QaBusEditMessageInput,
+  type QaBusEvent,
+  type QaBusInboundMessageInput,
+  type QaBusMessage,
+  type QaBusOutboundMessageInput,
+  type QaBusPollInput,
+  type QaBusPollResult,
+  type QaBusReactToMessageInput,
+  type QaBusReadMessageInput,
+  type QaBusSearchMessagesInput,
+  type QaBusStateSnapshot,
+  type QaBusThread,
+  type QaBusWaitForInput,
+  qaChannelPlugin,
+  reactToQaBusMessage,
+  readQaBusMessage,
+  searchQaBusMessages,
+  sendQaBusMessage,
+  setQaChannelRuntime,
+} from "./src/runtime-api.js";
+export { startQaLiveLaneGateway } from "./src/live-transports/shared/live-gateway.runtime.js";

extensions/qa-lab/shared/evidence-gallery-types.ts

@@ -0,0 +1,88 @@
+export type QaEvidenceGalleryStatus = "pass" | "fail" | "blocked" | "skipped";
+
+export type QaEvidenceCoverageView = {
+  id: string;
+  role: string;
+};
+
+export type QaEvidenceProducerContextFile = {
+  href: string;
+  path: string;
+  preview: string | null;
+};
+
+export type QaEvidenceMatrixCellView = {
+  artifactKinds: string[];
+  artifactPaths: string[];
+  coverageIds: string[];
+  runner: {
+    availability: string | null;
+    command: string | null;
+    lane: string | null;
+    workflow: string | null;
+  } | null;
+  stage: string;
+  status: string;
+  surface: string;
+  testId: string | null;
+  title: string | null;
+};
+
+export type QaEvidenceArtifactView = {
+  exists: boolean;
+  error: string | null;
+  href: string | null;
+  kind: string;
+  mediaKind: "image" | "video" | "json" | "text" | "file";
+  path: string;
+  preview: string | null;
+  source: string;
+};
+
+export type QaEvidenceGalleryEntryView = {
+  artifacts: QaEvidenceArtifactView[];
+  coverage: QaEvidenceCoverageView[];
+  failureReason: string | null;
+  id: string;
+  kind: string;
+  sourcePath: string | null;
+  status: QaEvidenceGalleryStatus;
+  title: string;
+};
+
+export type QaEvidenceProducerContext = {
+  commands: QaEvidenceProducerContextFile | null;
+  kind: "ux-matrix";
+  manifest:
+    | (QaEvidenceProducerContextFile & {
+        path: string;
+        runStatus: string | null;
+        runId: string | null;
+      })
+    | null;
+  matrix: {
+    cells: QaEvidenceMatrixCellView[];
+    counts: Record<string, number>;
+    path: string;
+    stages: string[];
+    surfaces: string[];
+  } | null;
+  preflight: {
+    adbDevices: QaEvidenceProducerContextFile | null;
+    memory: QaEvidenceProducerContextFile | null;
+  };
+  releaseLedger: (QaEvidenceProducerContextFile & { counts: Record<string, number> }) | null;
+  rootPath: string;
+  scorecard: QaEvidenceProducerContextFile | null;
+};
+
+export type QaEvidenceGalleryModel = {
+  counts: Record<QaEvidenceGalleryStatus, number>;
+  entries: QaEvidenceGalleryEntryView[];
+  evidenceMode: string;
+  evidencePath: string;
+  generatedAt: string;
+  profile: string | null;
+  producerContext: QaEvidenceProducerContext | null;
+  schemaVersion: number;
+};

extensions/qa-lab/src/agentic-parity-report.test.ts

@@ -0,0 +1,999 @@
+// Qa Lab tests cover agentic parity report plugin behavior.
+import { describe, expect, it } from "vitest";
+import {
+  buildQaAgenticParityComparison,
+  buildQaRuntimeParityReport,
+  computeQaAgenticParityMetrics,
+  QaParityLabelMismatchError,
+  renderQaAgenticParityMarkdownReport,
+  renderQaRuntimeParityMarkdownReport,
+  type QaParityReportScenario,
+  type QaParitySuiteSummary,
+  type QaRuntimeParitySuiteSummary,
+} from "./agentic-parity-report.js";
+
+const FULL_PARITY_PASS_SCENARIOS: QaParityReportScenario[] = [
+  { name: "Approval turn tool followthrough", status: "pass" },
+  { name: "Compaction retry after mutating tool", status: "pass" },
+  { name: "Model switch with tool continuity", status: "pass" },
+  { name: "Source and docs discovery report", status: "pass" },
+  { name: "Image understanding from attachment", status: "pass" },
+  { name: "Subagent handoff", status: "pass" },
+  { name: "Subagent fanout synthesis", status: "pass" },
+  { name: "Subagent stale child links", status: "pass" },
+  { name: "Memory recall after context switch", status: "pass" },
+  { name: "Thread memory isolation", status: "pass" },
+  { name: "Config restart capability flip", status: "pass" },
+  { name: "Instruction followthrough repo contract", status: "pass" },
+];
+
+function withScenarioOverride(name: string, override: Partial<QaParityReportScenario>) {
+  return FULL_PARITY_PASS_SCENARIOS.map((scenario) =>
+    scenario.name === name ? { ...scenario, ...override } : scenario,
+  );
+}
+
+function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary {
+  return {
+    scenarios: [
+      {
+        name: "Approval turn tool followthrough",
+        status: "pass",
+        steps: [],
+        runtimeParity: {
+          scenarioId: "approval-turn-tool-followthrough",
+          drift: "none",
+          cells: {
+            openclaw: {
+              runtime: "openclaw",
+              transcriptBytes: '{"role":"assistant"}\n',
+              toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
+              finalText: "done",
+              usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+              wallClockMs: 20,
+              bootStateLines: [],
+            },
+            codex: {
+              runtime: "codex",
+              transcriptBytes: '{"role":"assistant"}\n',
+              toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
+              finalText: "done",
+              usage: { inputTokens: 8, outputTokens: 4, totalTokens: 12 },
+              wallClockMs: 18,
+              bootStateLines: [],
+            },
+          },
+        },
+      },
+      {
+        name: "Compaction retry after mutating tool",
+        status: "pass",
+        steps: [],
+        runtimeParity: {
+          scenarioId: "compaction-retry-after-mutating-tool",
+          drift: "tool-call-shape",
+          driftDetails: "tool call 1 differs",
+          cells: {
+            openclaw: {
+              runtime: "openclaw",
+              transcriptBytes: '{"role":"assistant"}\n',
+              toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }],
+              finalText: "done",
+              usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+              wallClockMs: 20,
+              bootStateLines: [],
+            },
+            codex: {
+              runtime: "codex",
+              transcriptBytes: '{"role":"assistant"}\n',
+              toolCalls: [{ tool: "read_file", argsHash: "b", resultHash: "r" }],
+              finalText: "done",
+              usage: { inputTokens: 9, outputTokens: 4, totalTokens: 13 },
+              wallClockMs: 19,
+              bootStateLines: [],
+            },
+          },
+        },
+      },
+    ],
+    counts: {
+      total: 2,
+      passed: 2,
+      failed: 0,
+    },
+    run: {
+      providerMode: "mock-openai",
+      primaryModel: "openai/gpt-5.5",
+      runtimePair: ["openclaw", "codex"],
+    },
+  };
+}
+
+function firstRuntimeParityScenario() {
+  const scenario = makeRuntimeParitySummary().scenarios[0];
+  if (!scenario) {
+    throw new Error("missing runtime parity scenario fixture");
+  }
+  return scenario;
+}
+
+describe("qa agentic parity report", () => {
+  it("computes first-wave parity metrics from suite summaries", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        { name: "Approval turn tool followthrough", status: "pass" },
+        {
+          name: "Compaction retry after mutating tool",
+          status: "fail",
+          details: "incomplete turn detected",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary)).toEqual({
+      totalScenarios: 2,
+      passedScenarios: 1,
+      failedScenarios: 1,
+      completionRate: 0.5,
+      unintendedStopCount: 1,
+      unintendedStopRate: 0.5,
+      validToolCallCount: 1,
+      validToolCallRate: 0.5,
+      fakeSuccessCount: 0,
+    });
+  });
+
+  it("uses scenario rows rather than stale summary counts for parity metrics", () => {
+    const summary: QaParitySuiteSummary = {
+      counts: {
+        total: 2,
+        passed: 2,
+        failed: 0,
+      },
+      scenarios: [
+        { name: "Approval turn tool followthrough", status: "pass" },
+        { name: "Compaction retry after mutating tool", status: "fail" },
+      ],
+    };
+
+    const metrics = computeQaAgenticParityMetrics(summary);
+
+    expect(metrics.totalScenarios).toBe(2);
+    expect(metrics.passedScenarios).toBe(1);
+    expect(metrics.failedScenarios).toBe(1);
+    expect(metrics.completionRate).toBe(0.5);
+  });
+
+  it("keeps non-tool scenarios out of the valid-tool-call metric", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        { name: "Approval turn tool followthrough", status: "pass" },
+        { name: "Memory recall after context switch", status: "pass" },
+        { name: "Image understanding from attachment", status: "pass" },
+      ],
+    };
+
+    const metrics = computeQaAgenticParityMetrics(summary);
+    expect(metrics.totalScenarios).toBe(3);
+    expect(metrics.passedScenarios).toBe(3);
+    expect(metrics.validToolCallCount).toBe(1);
+    expect(metrics.validToolCallRate).toBe(1);
+  });
+
+  it("does not count passing runtime parity scenarios without tool-call evidence", () => {
+    const summary: QaRuntimeParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Approval turn tool followthrough",
+          status: "pass",
+          steps: [],
+          runtimeParity: {
+            scenarioId: "approval-turn-tool-followthrough",
+            drift: "none",
+            cells: {
+              openclaw: {
+                runtime: "openclaw",
+                transcriptBytes: '{"role":"assistant"}\n',
+                toolCalls: [],
+                finalText: "done",
+                usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+                wallClockMs: 10,
+                bootStateLines: [],
+              },
+              codex: {
+                runtime: "codex",
+                transcriptBytes: '{"role":"assistant"}\n',
+                toolCalls: [],
+                finalText: "done",
+                usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+                wallClockMs: 10,
+                bootStateLines: [],
+              },
+            },
+          },
+        },
+      ],
+    };
+
+    const metrics = computeQaAgenticParityMetrics(summary);
+
+    expect(metrics.passedScenarios).toBe(1);
+    expect(metrics.validToolCallCount).toBe(0);
+    expect(metrics.validToolCallRate).toBe(0);
+  });
+
+  it("counts passing runtime parity scenarios with tool calls in both runtimes", () => {
+    const metrics = computeQaAgenticParityMetrics({
+      scenarios: [firstRuntimeParityScenario()],
+    });
+
+    expect(metrics.validToolCallCount).toBe(1);
+    expect(metrics.validToolCallRate).toBe(1);
+  });
+
+  it("fails the parity gate when the candidate regresses against baseline", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          {
+            name: "Compaction retry after mutating tool",
+            status: "fail",
+            details: "timed out before it continued",
+          },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      baselineSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          { name: "Compaction retry after mutating tool", status: "pass" },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "openai/gpt-5.5 completion rate 80.0% is below anthropic/claude-opus-4-8 100.0%.",
+    );
+    expect(comparison.failures).toContain(
+      "openai/gpt-5.5 unintended-stop rate 20.0% exceeds anthropic/claude-opus-4-8 0.0%.",
+    );
+  });
+
+  it("fails the parity gate when candidate and baseline cover different non-parity scenarios", () => {
+    const passScenario = (name: string): QaParityReportScenario => ({ name, status: "pass" });
+    const baselineScenarios = [
+      passScenario("Approval turn tool followthrough"),
+      passScenario("Compaction retry after mutating tool"),
+      passScenario("Model switch with tool continuity"),
+      passScenario("Source and docs discovery report"),
+      passScenario("Image understanding from attachment"),
+      passScenario("Extra non-parity lane"),
+    ];
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: {
+        scenarios: baselineScenarios.filter(
+          (scenario) => scenario.name !== "Extra non-parity lane",
+        ),
+      },
+      baselineSummary: { scenarios: baselineScenarios },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Scenario coverage mismatch for Extra non-parity lane: openai/gpt-5.5=missing, anthropic/claude-opus-4-8=pass.",
+    );
+  });
+
+  it("reports each missing required parity scenario exactly once (no double-counting)", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      baselineSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    const missingScenario = "Image understanding from attachment";
+    const requiredLines = comparison.failures.filter((failure) =>
+      failure.includes(`Missing required parity scenario coverage for ${missingScenario}:`),
+    );
+    const mismatchLines = comparison.failures.filter((failure) =>
+      failure.includes(`Scenario coverage mismatch for ${missingScenario}:`),
+    );
+    expect(requiredLines).toHaveLength(1);
+    expect(mismatchLines).toHaveLength(0);
+  });
+
+  it("scopes parity metrics to declared parity scenarios even when extra lanes are present", () => {
+    const scopedSummary: QaParitySuiteSummary = {
+      scenarios: [
+        { name: "Approval turn tool followthrough", status: "pass" },
+        { name: "Compaction retry after mutating tool", status: "pass" },
+        { name: "Model switch with tool continuity", status: "pass" },
+        { name: "Source and docs discovery report", status: "pass" },
+        { name: "Image understanding from attachment", status: "pass" },
+      ],
+    };
+    const summaryWithExtras: QaParitySuiteSummary = {
+      scenarios: [
+        ...scopedSummary.scenarios,
+        { name: "Extra lane A", status: "fail", details: "timed out" },
+        { name: "Extra lane B", status: "fail", details: "timed out" },
+      ],
+    };
+
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: summaryWithExtras,
+      baselineSummary: scopedSummary,
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    // Extra lanes must not drag the candidate's completion rate below baseline
+    // and must not generate unintended-stop or fake-success hits.
+    expect(comparison.candidateMetrics.totalScenarios).toBe(5);
+    expect(comparison.candidateMetrics.completionRate).toBe(1);
+    expect(comparison.candidateMetrics.unintendedStopRate).toBe(0);
+    expect(comparison.candidateMetrics.fakeSuccessCount).toBe(0);
+    // The pass/fail verdict here still depends only on the parity pack itself.
+    const regressionFailures = comparison.failures.filter((failure) =>
+      failure.includes("completion rate"),
+    );
+    expect(regressionFailures).toStrictEqual([]);
+  });
+
+  it("fails the parity gate when required parity scenarios are missing on both sides", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      baselineSummary: {
+        scenarios: [{ name: "Approval turn tool followthrough", status: "pass" }],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Missing required parity scenario coverage for Image understanding from attachment: openai/gpt-5.5=missing, anthropic/claude-opus-4-8=missing.",
+    );
+  });
+
+  it("fails the parity gate when required parity scenarios are skipped", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          { name: "Compaction retry after mutating tool", status: "skip" },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      baselineSummary: {
+        scenarios: [
+          { name: "Approval turn tool followthrough", status: "pass" },
+          { name: "Compaction retry after mutating tool", status: "skip" },
+          { name: "Model switch with tool continuity", status: "pass" },
+          { name: "Source and docs discovery report", status: "pass" },
+          { name: "Image understanding from attachment", status: "pass" },
+        ],
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Missing required parity scenario coverage for Compaction retry after mutating tool: openai/gpt-5.5=skip, anthropic/claude-opus-4-8=skip.",
+    );
+  });
+
+  it("fails the parity gate when a required parity scenario fails on both sides", () => {
+    // Regression for the loop-7 Codex-connector P1 finding: without this
+    // check, a required parity scenario that fails on both candidate and
+    // baseline still produces pass=true because the downstream metric
+    // comparisons are purely relative (candidate vs baseline). Cover the
+    // whole parity pack as pass on both sides except the one scenario we
+    // deliberately fail on both sides, so the assertion can pin the
+    // isolated gate failure under test.
+    const scenariosWithBothFail = withScenarioOverride("Approval turn tool followthrough", {
+      status: "fail",
+    });
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: { scenarios: scenariosWithBothFail },
+      baselineSummary: { scenarios: scenariosWithBothFail },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Required parity scenario Approval turn tool followthrough failed: openai/gpt-5.5=fail, anthropic/claude-opus-4-8=fail.",
+    );
+    // Metric comparisons are relative, so a same-on-both-sides failure
+    // must not appear as a relative metric failure. The required-scenario
+    // failure line is the only thing keeping the gate honest here.
+    expect(comparison.failures.filter((failure) => failure.includes("completion rate"))).toEqual(
+      [],
+    );
+  });
+
+  it("fails the parity gate when a required parity scenario fails on the candidate only", () => {
+    // A candidate regression below a passing baseline is already caught
+    // by the relative completion-rate comparison, but surface it as a
+    // named required-scenario failure too so operators see a concrete
+    // scenario name alongside the rate differential.
+    const candidateWithOneFail = withScenarioOverride("Approval turn tool followthrough", {
+      status: "fail",
+    });
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: { scenarios: candidateWithOneFail },
+      baselineSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toContain(
+      "Required parity scenario Approval turn tool followthrough failed: openai/gpt-5.5=fail, anthropic/claude-opus-4-8=pass.",
+    );
+  });
+
+  it("fails the parity gate when the baseline contains suspicious pass results", () => {
+    // Cover the full second-wave pack on both sides so the suspicious-pass assertion
+    // below is the isolated gate failure under test (no coverage-gap noise).
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+      },
+      baselineSummary: {
+        scenarios: withScenarioOverride("Approval turn tool followthrough", {
+          details: "timed out before it continued",
+        }),
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(false);
+    expect(comparison.failures).toEqual([
+      "anthropic/claude-opus-4-8 produced 1 suspicious pass result(s); baseline fake-success count must also be 0.",
+    ]);
+  });
+
+  it("ignores neutral Failed and Blocked headings in passing protocol reports", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Source and docs discovery report",
+          status: "pass",
+          details: `Worked:
+- Read the seeded QA material.
+Failed:
+- None observed.
+Blocked:
+- No live provider evidence in this lane.
+Follow-up:
+- Re-run with a real provider if needed.`,
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("ignores neutral error-budget and no-errors-observed phrasing in passing reports", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Source and docs discovery report",
+          status: "pass",
+          details: `Worked:
+- Scenario finished with Error budget: 0.
+- No errors found in the seeded material.
+- Errors: none observed.`,
+        },
+        {
+          name: "Image understanding from attachment",
+          status: "pass",
+          details: "Error: none. The attached image analysis completed without incident.",
+        },
+      ],
+    };
+
+    // Bare "error"/"Error" in narration is not a suspicious-pass signal on its own.
+    // Only phrases like "error occurred" or "an error was ..." should count.
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("still flags genuine error-narration suspicious passes", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Approval turn tool followthrough",
+          status: "pass",
+          details: "Tool call completed, but an error occurred mid-turn and no retry happened.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(1);
+  });
+
+  it("does not flag positive-tone prose as fake success (positive-tone detection removed)", () => {
+    // Positive-tone detection was removed because for passing runs the
+    // `details` field is the model's prose, which never contains tool-call
+    // evidence. Criterion 2 is enforced by per-scenario tool-call assertions.
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Subagent handoff",
+          status: "pass",
+          details: "Successfully completed the delegation. The subagent returned its result.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("does not flag bare 'Done.' prose as fake success", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Approval turn tool followthrough",
+          status: "pass",
+          details: "Done.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("does not flag structured status lines that end in `done`", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Compaction retry after mutating tool",
+          status: "pass",
+          details: `Confirmed, replay unsafe after write.
+compactionCount=0
+status=done`,
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("does not flag positive-tone passes when the scenario shows real tool-call evidence", () => {
+    // A legitimate tool-mediated pass that happens to include
+    // "successfully" in its prose must not be flagged. The
+    // `plannedToolName` evidence (or any of the other tool-call
+    // evidence patterns) exempts the scenario from positive-tone
+    // detection. Without this exemption, real tool-backed passes with
+    // self-congratulatory prose would count as fake successes and break
+    // the gate.
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Source and docs discovery report",
+          status: "pass",
+          details:
+            "Successfully completed the report. plannedToolName=read recorded via /debug/requests.",
+        },
+      ],
+    };
+
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(0);
+  });
+
+  it("only flags failure-tone passes, not positive-tone", () => {
+    const summary: QaParitySuiteSummary = {
+      scenarios: [
+        {
+          name: "Approval turn tool followthrough",
+          status: "pass",
+          details: "Task executed successfully without errors.",
+        },
+        {
+          name: "Subagent handoff",
+          status: "pass",
+          details: "Tool call completed, but an error occurred mid-turn.",
+        },
+      ],
+    };
+
+    // Only the failure-tone scenario ("error occurred") counts.
+    // The positive-tone one ("successfully") is not flagged.
+    expect(computeQaAgenticParityMetrics(summary).fakeSuccessCount).toBe(1);
+  });
+
+  it("throws QaParityLabelMismatchError when the candidate run.primaryProvider does not match the label", () => {
+    // Regression for the gate footgun: if an operator swaps the
+    // --candidate-summary and --baseline-summary paths, the gate would
+    // silently produce a reversed verdict. PR L #64789 ships the `run`
+    // block on every summary so the parity report can verify it against
+    // the caller-supplied label; this test pins the precondition check.
+    const parityPassScenarios: QaParityReportScenario[] = [
+      { name: "Approval turn tool followthrough", status: "pass" },
+      { name: "Compaction retry after mutating tool", status: "pass" },
+      { name: "Model switch with tool continuity", status: "pass" },
+      { name: "Source and docs discovery report", status: "pass" },
+      { name: "Image understanding from attachment", status: "pass" },
+    ];
+
+    expect(() =>
+      buildQaAgenticParityComparison({
+        candidateLabel: "openai/gpt-5.5",
+        baselineLabel: "anthropic/claude-opus-4-8",
+        candidateSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "anthropic", primaryModel: "claude-opus-4-8" },
+        },
+        baselineSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "anthropic", primaryModel: "claude-opus-4-8" },
+        },
+        comparedAt: "2026-04-11T00:00:00.000Z",
+      }),
+    ).toThrow(QaParityLabelMismatchError);
+  });
+
+  it("throws QaParityLabelMismatchError when the baseline run.primaryProvider does not match the label", () => {
+    const parityPassScenarios: QaParityReportScenario[] = [
+      { name: "Approval turn tool followthrough", status: "pass" },
+    ];
+
+    expect(() =>
+      buildQaAgenticParityComparison({
+        candidateLabel: "openai/gpt-5.5",
+        baselineLabel: "anthropic/claude-opus-4-8",
+        candidateSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "openai" },
+        },
+        baselineSummary: {
+          scenarios: parityPassScenarios,
+          run: { primaryProvider: "openai", primaryModel: "gpt-5.5" },
+        },
+        comparedAt: "2026-04-11T00:00:00.000Z",
+      }),
+    ).toThrow(
+      /baseline summary run\.primaryProvider=openai and run\.primaryModel=gpt-5\.5 do not match --baseline-label/,
+    );
+  });
+
+  it("accepts matching run.primaryProvider labels without throwing", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.5",
+          primaryModelName: "gpt-5.5",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-8",
+          primaryModelName: "claude-opus-4-8",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("skips run.primaryProvider verification when the summary is missing a run block (legacy summaries)", () => {
+    // Pre-PR-L summaries don't carry a `run` block. The gate must still
+    // work against those, trusting the caller-supplied label.
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      baselineSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("skips provider verification for arbitrary display labels when run metadata is present", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "GPT-5.5 candidate",
+      baselineLabel: "Opus 4.8 baseline",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.5",
+          primaryModelName: "gpt-5.5",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-8",
+          primaryModelName: "claude-opus-4-8",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("skips provider verification for mixed-case or decorated display labels", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "Candidate: GPT-5.5",
+      baselineLabel: "Opus 4.8 / baseline",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.5",
+          primaryModelName: "gpt-5.5",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-8",
+          primaryModelName: "claude-opus-4-8",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("throws when a structured label mismatches the recorded model even if the provider matches", () => {
+    expect(() =>
+      buildQaAgenticParityComparison({
+        candidateLabel: "openai/gpt-5.5",
+        baselineLabel: "anthropic/claude-opus-4-8",
+        candidateSummary: {
+          scenarios: FULL_PARITY_PASS_SCENARIOS,
+          run: {
+            primaryProvider: "openai",
+            primaryModel: "openai/gpt-5.5-alt",
+            primaryModelName: "gpt-5.5-alt",
+          },
+        },
+        baselineSummary: {
+          scenarios: FULL_PARITY_PASS_SCENARIOS,
+          run: {
+            primaryProvider: "anthropic",
+            primaryModel: "anthropic/claude-opus-4-8",
+            primaryModelName: "claude-opus-4-8",
+          },
+        },
+        comparedAt: "2026-04-11T00:00:00.000Z",
+      }),
+    ).toThrow(
+      /candidate summary run\.primaryProvider=openai and run\.primaryModel=openai\/gpt-5\.5-alt do not match --candidate-label=openai\/gpt-5\.5/,
+    );
+  });
+
+  it("accepts colon-delimited structured labels when provider and model both match", () => {
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai:gpt-5.5",
+      baselineLabel: "anthropic:claude-opus-4-8",
+      candidateSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "openai",
+          primaryModel: "openai/gpt-5.5",
+          primaryModelName: "gpt-5.5",
+        },
+      },
+      baselineSummary: {
+        scenarios: FULL_PARITY_PASS_SCENARIOS,
+        run: {
+          primaryProvider: "anthropic",
+          primaryModel: "anthropic/claude-opus-4-8",
+          primaryModelName: "claude-opus-4-8",
+        },
+      },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    expect(comparison.pass).toBe(true);
+  });
+
+  it("renders a readable markdown parity report", () => {
+    // Cover the full parity pack on both sides so the pass
+    // verdict is not disrupted by required-scenario coverage failures
+    // added by the second-wave expansion.
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5",
+      baselineLabel: "anthropic/claude-opus-4-8",
+      candidateSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      baselineSummary: { scenarios: FULL_PARITY_PASS_SCENARIOS },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+
+    const report = renderQaAgenticParityMarkdownReport(comparison);
+
+    expect(report).toContain(
+      "# OpenClaw Agentic Parity Report — openai/gpt-5.5 vs anthropic/claude-opus-4-8",
+    );
+    expect(report).toContain("| Completion rate | 100.0% | 100.0% |");
+    expect(report).toContain("### Approval turn tool followthrough");
+    expect(report).toContain("- Verdict: pass");
+  });
+
+  it("parametrizes the markdown header from the comparison labels", () => {
+    // Regression for the loop-7 Copilot finding: callers that configure
+    // non-gpt-5.5 / non-opus labels (for example an internal candidate vs
+    // another candidate) must see the labels in the rendered H1 instead of
+    // the hardcoded "GPT-5.5 / Opus 4.8" title that would otherwise confuse
+    // readers of saved reports.
+    const comparison = buildQaAgenticParityComparison({
+      candidateLabel: "openai/gpt-5.5-alt",
+      baselineLabel: "openai/gpt-5.5",
+      candidateSummary: { scenarios: [] },
+      baselineSummary: { scenarios: [] },
+      comparedAt: "2026-04-11T00:00:00.000Z",
+    });
+    const report = renderQaAgenticParityMarkdownReport(comparison);
+    expect(report).toContain(
+      "# OpenClaw Agentic Parity Report — openai/gpt-5.5-alt vs openai/gpt-5.5",
+    );
+  });
+
+  it("builds a runtime parity report from suite summaries", () => {
+    const report = buildQaRuntimeParityReport({
+      summary: makeRuntimeParitySummary(),
+      comparedAt: "2026-05-10T00:00:00.000Z",
+    });
+
+    expect(report.runtimePair).toEqual(["openclaw", "codex"]);
+    expect(report.pass).toBe(true);
+    expect(report.driftCounts.none).toBe(1);
+    expect(report.driftCounts["tool-call-shape"]).toBe(1);
+    expect(report.failures).toEqual([]);
+  });
+
+  it("fails runtime parity reports when a runtime cell has a hard failure", () => {
+    const summary = makeRuntimeParitySummary();
+    const scenario = summary.scenarios[1];
+    if (!scenario?.runtimeParity) {
+      throw new Error("runtime parity fixture missing");
+    }
+    scenario.status = "fail";
+    scenario.runtimeParity.cells.codex.runtimeErrorClass = "auth";
+
+    const report = buildQaRuntimeParityReport({
+      summary,
+      comparedAt: "2026-05-10T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.failedScenarios).toBe(1);
+    expect(report.failures).toContain(
+      "Compaction retry after mutating tool drift=tool-call-shape (tool call 1 differs).",
+    );
+  });
+
+  it("passes runtime parity reports with controlled tool-error cells and advisory drift", () => {
+    const summary = makeRuntimeParitySummary();
+    const scenario = summary.scenarios[1];
+    if (!scenario?.runtimeParity) {
+      throw new Error("runtime parity fixture missing");
+    }
+    scenario.runtimeParity.cells.codex.runtimeErrorClass = "tool-error";
+
+    const report = buildQaRuntimeParityReport({
+      summary,
+      comparedAt: "2026-05-10T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.failedScenarios).toBe(0);
+    expect(report.failures).toEqual([]);
+  });
+
+  it("fails live runtime parity reports when assistant-message usage is missing", () => {
+    const summary = makeRuntimeParitySummary();
+    summary.run = {
+      ...summary.run,
+      providerMode: "live-frontier",
+    };
+    const scenario = summary.scenarios[0];
+    if (!scenario?.runtimeParity) {
+      throw new Error("runtime parity fixture missing");
+    }
+    scenario.runtimeParity.cells.openclaw.usage = {
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0,
+    };
+    scenario.runtimeParity.cells.codex.usage = {
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0,
+    };
+
+    const report = buildQaRuntimeParityReport({
+      summary,
+      comparedAt: "2026-05-10T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.failedScenarios).toBe(1);
+    expect(report.failures).toContain(
+      "Approval turn tool followthrough missing live assistant-message usage (openclaw=0, codex=0).",
+    );
+    expect(report.scenarios[0]?.status).toBe("fail");
+  });
+
+  it("fails runtime parity reports with no executed scenarios", () => {
+    const report = buildQaRuntimeParityReport({
+      summary: {
+        scenarios: [],
+        counts: {
+          total: 0,
+          passed: 0,
+          failed: 0,
+        },
+        run: {
+          providerMode: "live-frontier",
+          runtimePair: ["openclaw", "codex"],
+        },
+      },
+      comparedAt: "2026-05-10T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.failures).toContain("Runtime parity report has no executed scenarios.");
+  });
+
+  it("renders a readable runtime parity markdown report", () => {
+    const report = renderQaRuntimeParityMarkdownReport(
+      buildQaRuntimeParityReport({
+        summary: makeRuntimeParitySummary(),
+        comparedAt: "2026-05-10T00:00:00.000Z",
+      }),
+    );
+
+    expect(report).toContain("# OpenClaw Runtime Parity Report — openclaw vs codex");
+    expect(report).toContain("| Tool-call-shape drift | 1 |");
+    expect(report).toContain("### Compaction retry after mutating tool");
+    expect(report).toContain("- drift: tool-call-shape");
+  });
+});

extensions/qa-lab/src/agentic-parity-report.ts

@@ -0,0 +1,780 @@
+// Qa Lab plugin module implements agentic parity report behavior.
+import {
+  QA_AGENTIC_PARITY_SCENARIO_TITLES,
+  QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
+} from "./agentic-parity.js";
+import type { RuntimeId, RuntimeParityDrift, RuntimeParityResult } from "./runtime-parity.js";
+import { isRuntimeParityResultPass, runtimeParityCellStatus } from "./runtime-parity.js";
+
+type QaParityReportStep = {
+  name: string;
+  status: "pass" | "fail" | "skip";
+  details?: string;
+};
+
+export type QaParityReportScenario = {
+  name: string;
+  status: "pass" | "fail" | "skip";
+  details?: string;
+  steps?: QaParityReportStep[];
+};
+
+/**
+ * Optional self-describing run metadata written by PR L (#64789). Before
+ * that PR merges, older summaries only have `scenarios` + `counts`; the
+ * parity report treats a missing `run` block as "unknown provenance" and
+ * skips the label-match verification for backwards compatibility
+ * with legacy summaries that predate the run metadata block.
+ */
+type QaParityRunBlock = {
+  primaryProvider?: string;
+  primaryModel?: string;
+  primaryModelName?: string;
+  providerMode?: string;
+  scenarioIds?: readonly string[] | null;
+  runtimePair?: [RuntimeId, RuntimeId] | null;
+};
+
+export type QaParitySuiteSummary = {
+  scenarios: QaParityReportScenario[];
+  counts?: {
+    total?: number;
+    passed?: number;
+    failed?: number;
+  };
+  /** Self-describing run metadata — see PR L #64789 for the writer side. */
+  run?: QaParityRunBlock;
+};
+
+type QaRuntimeParitySuiteScenario = QaParityReportScenario & {
+  runtimeParity?: RuntimeParityResult;
+};
+
+export type QaRuntimeParitySuiteSummary = Omit<QaParitySuiteSummary, "scenarios"> & {
+  scenarios: QaRuntimeParitySuiteScenario[];
+};
+
+type QaRuntimeParityScenarioReport = {
+  name: string;
+  status: "pass" | "fail";
+  drift: RuntimeParityDrift | "missing";
+  driftDetails?: string;
+  openclawStatus: "pass" | "fail" | "missing";
+  codexStatus: "pass" | "fail" | "missing";
+  openclawTokens: number;
+  codexTokens: number;
+  openclawToolCalls: number;
+  codexToolCalls: number;
+};
+
+export type QaRuntimeParityReport = {
+  runtimePair: [RuntimeId, RuntimeId];
+  comparedAt: string;
+  providerMode?: string;
+  primaryModel?: string;
+  totalScenarios: number;
+  passedScenarios: number;
+  failedScenarios: number;
+  driftCounts: Record<RuntimeParityDrift, number>;
+  scenarios: QaRuntimeParityScenarioReport[];
+  pass: boolean;
+  failures: string[];
+  notes: string[];
+};
+
+type QaAgenticParityMetrics = {
+  totalScenarios: number;
+  passedScenarios: number;
+  failedScenarios: number;
+  completionRate: number;
+  unintendedStopCount: number;
+  unintendedStopRate: number;
+  validToolCallCount: number;
+  validToolCallRate: number;
+  fakeSuccessCount: number;
+};
+
+type QaAgenticParityScenarioComparison = {
+  name: string;
+  candidateStatus: "pass" | "fail" | "skip" | "missing";
+  baselineStatus: "pass" | "fail" | "skip" | "missing";
+  candidateDetails?: string;
+  baselineDetails?: string;
+};
+
+type QaAgenticParityComparison = {
+  candidateLabel: string;
+  baselineLabel: string;
+  comparedAt: string;
+  candidateMetrics: QaAgenticParityMetrics;
+  baselineMetrics: QaAgenticParityMetrics;
+  scenarioComparisons: QaAgenticParityScenarioComparison[];
+  pass: boolean;
+  failures: string[];
+  notes: string[];
+};
+
+const UNINTENDED_STOP_PATTERNS = [
+  /incomplete turn/i,
+  /\btimed out\b/i,
+  /\btimeout\b/i,
+  /\bstopped\b/i,
+  /\bblocked\b/i,
+  /\babandoned\b/i,
+  /did not continue/i,
+] as const;
+
+// Failure-tone patterns: a passing scenario whose details text matches any
+// of these is treated as a "fake success" — the scenario is marked pass but
+// the supporting text reveals something went wrong. Adding new patterns here
+// widens the net for bad prose that correlates with runtime failure modes.
+const SUSPICIOUS_PASS_FAILURE_TONE_PATTERNS = [
+  /incomplete turn/i,
+  /\btimed out\b/i,
+  /\btimeout\b/i,
+  /\bfailed to\b/i,
+  /\bcould not\b/i,
+  /\bunable to\b/i,
+  /did not continue/i,
+  /error occurred/i,
+  /an error was/i,
+] as const;
+
+// Positive-tone patterns (e.g. "Successfully completed", "Done.") are NOT
+// checked in fakeSuccessCount. For passing runs, `details` is the model's
+// outbound prose, which never contains tool-call evidence strings, so a
+// tool-call-evidence exemption would false-positive on every legitimate
+// pass. Criterion 2 ("no fake progress") is enforced by per-scenario
+// `/debug/requests` tool-call assertions in the YAML flows (PR J) instead.
+
+function normalizeScenarioStatus(status: string | undefined): "pass" | "fail" | "skip" {
+  return status === "pass" || status === "fail" || status === "skip" ? status : "fail";
+}
+
+function scenarioText(scenario: QaParityReportScenario) {
+  const parts = [scenario.details ?? ""];
+  for (const step of scenario.steps ?? []) {
+    parts.push(step.details ?? "");
+  }
+  return parts.filter(Boolean).join("\n");
+}
+
+function scenarioHasPattern(
+  scenario: QaParityReportScenario,
+  patterns: readonly RegExp[],
+): boolean {
+  const text = scenarioText(scenario);
+  return text.length > 0 && patterns.some((pattern) => pattern.test(text));
+}
+
+function scenarioRuntimeParity(scenario: QaParityReportScenario): RuntimeParityResult | undefined {
+  return (scenario as QaRuntimeParitySuiteScenario).runtimeParity;
+}
+
+function scenarioHasRuntimeToolCallEvidence(scenario: QaParityReportScenario): boolean {
+  const parity = scenarioRuntimeParity(scenario);
+  if (!parity) {
+    return scenario.status === "pass";
+  }
+  return (
+    scenario.status === "pass" &&
+    isRuntimeParityResultPass(parity) &&
+    parity.cells.openclaw.toolCalls.length > 0 &&
+    parity.cells.codex.toolCalls.length > 0
+  );
+}
+
+export function computeQaAgenticParityMetrics(
+  summary: QaParitySuiteSummary,
+): QaAgenticParityMetrics {
+  const scenarios = summary.scenarios.map((scenario) => ({
+    ...scenario,
+    status: normalizeScenarioStatus(scenario.status),
+  }));
+  const toolBackedTitleSet: ReadonlySet<string> = new Set(
+    QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
+  );
+  const totalScenarios = scenarios.length;
+  const passedScenarios = scenarios.filter((scenario) => scenario.status === "pass").length;
+  const failedScenarios = scenarios.filter((scenario) => scenario.status === "fail").length;
+  const unintendedStopCount = scenarios.filter(
+    (scenario) =>
+      scenario.status !== "pass" && scenarioHasPattern(scenario, UNINTENDED_STOP_PATTERNS),
+  ).length;
+  const fakeSuccessCount = scenarios.filter((scenario) => {
+    if (scenario.status !== "pass") {
+      return false;
+    }
+    // Failure-tone patterns catch obviously-broken passes regardless of
+    // whether the scenario shows tool-call evidence — "timed out" under a
+    // pass is always fake.
+    if (scenarioHasPattern(scenario, SUSPICIOUS_PASS_FAILURE_TONE_PATTERNS)) {
+      return true;
+    }
+    // Positive-tone patterns (like "Successfully completed") are NOT checked
+    // here because for passing runs the `details` field is the model's
+    // outbound prose, which never contains tool-call evidence strings.
+    // The `scenarioLacksToolCallEvidence` check would return true for ALL
+    // passes and false-positive on legitimate completions. Criterion 2
+    // ("no fake tool completion") is instead enforced by the per-scenario
+    // `/debug/requests` tool-call assertions from the scenario YAML flows.
+    return false;
+  }).length;
+
+  // Count only the scenarios that are supposed to exercise a real tool,
+  // subagent, or capability invocation. Memory recall and image-only
+  // understanding lanes stay in the parity pack, but they should not inflate
+  // the tool-call metric just by passing.
+  const toolBackedScenarioCount = scenarios.filter((scenario) =>
+    toolBackedTitleSet.has(scenario.name),
+  ).length;
+  const validToolCallCount = scenarios.filter(
+    (scenario) =>
+      toolBackedTitleSet.has(scenario.name) && scenarioHasRuntimeToolCallEvidence(scenario),
+  ).length;
+
+  const rate = (value: number) => (totalScenarios > 0 ? value / totalScenarios : 0);
+  const toolRate = (value: number) =>
+    toolBackedScenarioCount > 0 ? value / toolBackedScenarioCount : 0;
+  return {
+    totalScenarios,
+    passedScenarios,
+    failedScenarios,
+    completionRate: rate(passedScenarios),
+    unintendedStopCount,
+    unintendedStopRate: rate(unintendedStopCount),
+    validToolCallCount,
+    validToolCallRate: toolRate(validToolCallCount),
+    fakeSuccessCount,
+  };
+}
+
+function formatPercent(value: number) {
+  return `${(value * 100).toFixed(1)}%`;
+}
+
+function buildRuntimeParityDriftCounts(): Record<RuntimeParityDrift, number> {
+  return {
+    none: 0,
+    "text-only": 0,
+    "tool-call-shape": 0,
+    "tool-result-shape": 0,
+    structural: 0,
+    "failure-mode": 0,
+  };
+}
+
+function isLiveProviderMode(providerMode: string | undefined) {
+  return providerMode?.startsWith("live-") === true;
+}
+
+function describeLiveUsageFailure(scenarioName: string, scenario: QaRuntimeParityScenarioReport) {
+  const missing = [
+    scenario.openclawTokens > 0
+      ? undefined
+      : `${scenario.openclawStatus === "pass" ? "openclaw" : "openclaw failed"}=0`,
+    scenario.codexTokens > 0
+      ? undefined
+      : `${scenario.codexStatus === "pass" ? "codex" : "codex failed"}=0`,
+  ].filter((entry): entry is string => Boolean(entry));
+  if (missing.length === 0) {
+    return undefined;
+  }
+  return `${scenarioName} missing live assistant-message usage (${missing.join(", ")}).`;
+}
+
+function normalizeRuntimePair(
+  pair: [RuntimeId, RuntimeId] | null | undefined,
+): [RuntimeId, RuntimeId] {
+  if (pair?.[0] && pair?.[1]) {
+    return pair;
+  }
+  return ["openclaw", "codex"];
+}
+
+function requiredCoverageStatus(
+  scenario: QaParityReportScenario | undefined,
+): "pass" | "fail" | "skip" | "missing" {
+  return scenario ? normalizeScenarioStatus(scenario.status) : "missing";
+}
+
+function scopeSummaryToParityPack(
+  summary: QaParitySuiteSummary,
+  parityTitleSet: ReadonlySet<string>,
+): QaParitySuiteSummary {
+  // The parity verdict must only consider the declared parity scenarios
+  // (the full first-wave + second-wave pack from QA_AGENTIC_PARITY_SCENARIOS).
+  // Drop `counts` so the metric helper recomputes totals from the filtered
+  // scenario list instead of inheriting the caller's full-suite counters.
+  return {
+    scenarios: summary.scenarios.filter((scenario) => parityTitleSet.has(scenario.name)),
+    ...(summary.run ? { run: summary.run } : {}),
+  };
+}
+
+type StructuredQaParityLabel = {
+  provider: string;
+  model: string;
+};
+
+/**
+ * Only treat caller labels as provenance-checked identifiers when they are
+ * exact lower-case provider/model refs. Human-facing display labels like
+ * "GPT-5.5 candidate" or "Candidate: GPT-5.5" should render in the report
+ * without being misread as structured provider ids.
+ */
+function parseStructuredLabelRef(label: string): StructuredQaParityLabel | null {
+  const trimmed = label.trim();
+  if (trimmed.length === 0) {
+    return null;
+  }
+  if (trimmed !== trimmed.toLowerCase()) {
+    return null;
+  }
+  const separatorMatch = /^([a-z0-9][a-z0-9-]*)[/:]([a-z0-9][a-z0-9._-]*)$/.exec(trimmed);
+  if (!separatorMatch) {
+    return null;
+  }
+  return {
+    provider: separatorMatch[1] ?? "",
+    model: separatorMatch[2] ?? "",
+  };
+}
+
+/**
+ * Verify the `run.primaryProvider` + `run.primaryModel` fields on a summary
+ * match the caller-supplied label when that label is a structured
+ * `provider/model` or `provider:model` ref. PR L #64789 ships the `run`
+ * block; before it lands, older summaries don't have the field and this check
+ * is a no-op.
+ *
+ * Throws `QaParityLabelMismatchError` when the summary reports a different
+ * provider/model than the caller claimed — this catches the "swapped
+ * candidate and baseline summary paths" footgun the earlier adversarial
+ * review flagged. Returns silently when the fields are absent (legacy
+ * summaries) or when the fields match.
+ */
+function verifySummaryLabelMatch(params: {
+  summary: QaParitySuiteSummary;
+  label: string;
+  role: "candidate" | "baseline";
+}): void {
+  const runProvider = params.summary.run?.primaryProvider?.trim();
+  const runModel = params.summary.run?.primaryModel?.trim();
+  const runModelName = params.summary.run?.primaryModelName?.trim();
+  if (!runProvider || !runModel) {
+    return;
+  }
+  const labelRef = parseStructuredLabelRef(params.label);
+  if (!labelRef) {
+    return;
+  }
+  const normalizedRunModel = runModel.toLowerCase();
+  const normalizedRunModelName = runModelName?.toLowerCase();
+  const normalizedLabelModel = labelRef.model;
+  if (
+    runProvider.toLowerCase() === labelRef.provider &&
+    (normalizedRunModel === normalizedLabelModel ||
+      normalizedRunModelName === normalizedLabelModel ||
+      normalizedRunModel === `${labelRef.provider}/${normalizedLabelModel}`)
+  ) {
+    return;
+  }
+  throw new QaParityLabelMismatchError({
+    role: params.role,
+    label: params.label,
+    runProvider,
+    runModel,
+  });
+}
+
+export class QaParityLabelMismatchError extends Error {
+  readonly role: "candidate" | "baseline";
+  readonly label: string;
+  readonly runProvider: string;
+  readonly runModel: string;
+
+  constructor(params: {
+    role: "candidate" | "baseline";
+    label: string;
+    runProvider: string;
+    runModel: string;
+  }) {
+    super(
+      `${params.role} summary run.primaryProvider=${params.runProvider} and run.primaryModel=${params.runModel} do not match --${params.role}-label=${params.label}. ` +
+        `Check that the --candidate-summary / --baseline-summary paths weren't swapped.`,
+    );
+    this.name = "QaParityLabelMismatchError";
+    this.role = params.role;
+    this.label = params.label;
+    this.runProvider = params.runProvider;
+    this.runModel = params.runModel;
+  }
+}
+
+export function buildQaAgenticParityComparison(params: {
+  candidateLabel: string;
+  baselineLabel: string;
+  candidateSummary: QaParitySuiteSummary;
+  baselineSummary: QaParitySuiteSummary;
+  comparedAt?: string;
+}): QaAgenticParityComparison {
+  // Precondition: verify the `run.primaryProvider` field on each summary
+  // matches the caller-supplied label (when the `run` block is present).
+  // Throws `QaParityLabelMismatchError` on mismatch so the release gate
+  // fails loudly instead of silently producing a reversed verdict when an
+  // operator swaps the --candidate-summary and --baseline-summary paths.
+  // Legacy summaries without a `run` block are accepted as-is.
+  verifySummaryLabelMatch({
+    summary: params.candidateSummary,
+    label: params.candidateLabel,
+    role: "candidate",
+  });
+  verifySummaryLabelMatch({
+    summary: params.baselineSummary,
+    label: params.baselineLabel,
+    role: "baseline",
+  });
+  const parityTitleSet: ReadonlySet<string> = new Set<string>(QA_AGENTIC_PARITY_SCENARIO_TITLES);
+  // Rates and fake-success counts are computed from the parity-scoped summaries only,
+  // so extra non-parity scenarios in the input (for example when a caller feeds a full
+  // qa-suite-summary.json rather than a --parity-pack agentic run) cannot influence
+  // the gate verdict.
+  const candidateMetrics = computeQaAgenticParityMetrics(
+    scopeSummaryToParityPack(params.candidateSummary, parityTitleSet),
+  );
+  const baselineMetrics = computeQaAgenticParityMetrics(
+    scopeSummaryToParityPack(params.baselineSummary, parityTitleSet),
+  );
+
+  const scenarioNames = new Set([
+    ...QA_AGENTIC_PARITY_SCENARIO_TITLES,
+    ...params.candidateSummary.scenarios.map((scenario) => scenario.name),
+    ...params.baselineSummary.scenarios.map((scenario) => scenario.name),
+  ]);
+  const candidateByName = new Map(
+    params.candidateSummary.scenarios.map((scenario) => [scenario.name, scenario]),
+  );
+  const baselineByName = new Map(
+    params.baselineSummary.scenarios.map((scenario) => [scenario.name, scenario]),
+  );
+
+  const scenarioComparisons = [...scenarioNames]
+    .toSorted((left, right) => left.localeCompare(right))
+    .map((name) => {
+      const candidate = candidateByName.get(name);
+      const baseline = baselineByName.get(name);
+      const candidateStatus = candidate ? normalizeScenarioStatus(candidate.status) : "missing";
+      const baselineStatus = baseline ? normalizeScenarioStatus(baseline.status) : "missing";
+      const comparison: QaAgenticParityScenarioComparison = {
+        name,
+        candidateStatus,
+        baselineStatus,
+      };
+      if (candidate?.details) {
+        comparison.candidateDetails = candidate.details;
+      }
+      if (baseline?.details) {
+        comparison.baselineDetails = baseline.details;
+      }
+      return comparison;
+    });
+
+  const failures: string[] = [];
+  const requiredScenarioStatuses = QA_AGENTIC_PARITY_SCENARIO_TITLES.map((name) => {
+    const candidate = candidateByName.get(name);
+    const baseline = baselineByName.get(name);
+    return {
+      name,
+      candidateStatus: requiredCoverageStatus(candidate),
+      baselineStatus: requiredCoverageStatus(baseline),
+    };
+  });
+  const requiredScenarioCoverage = requiredScenarioStatuses.filter(
+    (scenario) =>
+      scenario.candidateStatus === "missing" ||
+      scenario.baselineStatus === "missing" ||
+      scenario.candidateStatus === "skip" ||
+      scenario.baselineStatus === "skip",
+  );
+  for (const scenario of requiredScenarioCoverage) {
+    failures.push(
+      `Missing required parity scenario coverage for ${scenario.name}: ${params.candidateLabel}=${scenario.candidateStatus}, ${params.baselineLabel}=${scenario.baselineStatus}.`,
+    );
+  }
+  // Required parity scenarios that ran on both sides but FAILED also fail
+  // the gate. Without this check, a run where both models fail the same
+  // required scenarios still produced pass=true, because the downstream
+  // metric comparisons are purely relative (candidate vs baseline) and
+  // the suspicious-pass fake-success check only catches passes that carry
+  // failure-sounding details. Excluding missing/skip here keeps operator
+  // output from double-counting the same scenario with two lines.
+  const requiredScenarioFailures = requiredScenarioStatuses.filter(
+    (scenario) =>
+      scenario.candidateStatus !== "missing" &&
+      scenario.baselineStatus !== "missing" &&
+      scenario.candidateStatus !== "skip" &&
+      scenario.baselineStatus !== "skip" &&
+      (scenario.candidateStatus === "fail" || scenario.baselineStatus === "fail"),
+  );
+  for (const scenario of requiredScenarioFailures) {
+    failures.push(
+      `Required parity scenario ${scenario.name} failed: ${params.candidateLabel}=${scenario.candidateStatus}, ${params.baselineLabel}=${scenario.baselineStatus}.`,
+    );
+  }
+  // Required parity scenarios are already reported via `requiredScenarioCoverage`
+  // above; excluding them here keeps the operator-facing failure list from
+  // double-counting the same missing scenario (one "Missing required parity scenario
+  // coverage for X" line plus a "Scenario coverage mismatch for X" line on the same
+  // scenario).
+  const coverageMismatch = scenarioComparisons.filter(
+    (scenario) =>
+      !parityTitleSet.has(scenario.name) &&
+      (scenario.candidateStatus === "missing" || scenario.baselineStatus === "missing"),
+  );
+  for (const scenario of coverageMismatch) {
+    failures.push(
+      `Scenario coverage mismatch for ${scenario.name}: ${params.candidateLabel}=${scenario.candidateStatus}, ${params.baselineLabel}=${scenario.baselineStatus}.`,
+    );
+  }
+  if (candidateMetrics.completionRate < baselineMetrics.completionRate) {
+    failures.push(
+      `${params.candidateLabel} completion rate ${formatPercent(candidateMetrics.completionRate)} is below ${params.baselineLabel} ${formatPercent(baselineMetrics.completionRate)}.`,
+    );
+  }
+  if (candidateMetrics.unintendedStopRate > baselineMetrics.unintendedStopRate) {
+    failures.push(
+      `${params.candidateLabel} unintended-stop rate ${formatPercent(candidateMetrics.unintendedStopRate)} exceeds ${params.baselineLabel} ${formatPercent(baselineMetrics.unintendedStopRate)}.`,
+    );
+  }
+  if (candidateMetrics.validToolCallRate < baselineMetrics.validToolCallRate) {
+    failures.push(
+      `${params.candidateLabel} valid-tool-call rate ${formatPercent(candidateMetrics.validToolCallRate)} is below ${params.baselineLabel} ${formatPercent(baselineMetrics.validToolCallRate)}.`,
+    );
+  }
+  if (candidateMetrics.fakeSuccessCount > 0) {
+    failures.push(
+      `${params.candidateLabel} produced ${candidateMetrics.fakeSuccessCount} suspicious pass result(s); fake-success count must be 0.`,
+    );
+  }
+  if (baselineMetrics.fakeSuccessCount > 0) {
+    failures.push(
+      `${params.baselineLabel} produced ${baselineMetrics.fakeSuccessCount} suspicious pass result(s); baseline fake-success count must also be 0.`,
+    );
+  }
+
+  return {
+    candidateLabel: params.candidateLabel,
+    baselineLabel: params.baselineLabel,
+    comparedAt: params.comparedAt ?? new Date().toISOString(),
+    candidateMetrics,
+    baselineMetrics,
+    scenarioComparisons,
+    pass: failures.length === 0,
+    failures,
+    notes: [
+      "First-wave valid-tool-call rate is scenario-level and uses passing tool-mediated scenarios as the verified numerator.",
+      "Auth/proxy/DNS correctness is intentionally out of scope for this parity report and should be gated by the deterministic runtime-truthfulness suites.",
+    ],
+  };
+}
+
+export function renderQaAgenticParityMarkdownReport(comparison: QaAgenticParityComparison): string {
+  // Title is parametrized from the candidate / baseline labels so reports
+  // for any candidate/baseline pair (not only gpt-5.5 vs opus 4.6) render
+  // with an accurate header. The default CLI labels are still
+  // openai/gpt-5.5 vs anthropic/claude-opus-4-8, but the helper works for
+  // any parity comparison a caller configures.
+  const lines = [
+    `# OpenClaw Agentic Parity Report — ${comparison.candidateLabel} vs ${comparison.baselineLabel}`,
+    "",
+    `- Compared at: ${comparison.comparedAt}`,
+    `- Candidate: ${comparison.candidateLabel}`,
+    `- Baseline: ${comparison.baselineLabel}`,
+    `- Verdict: ${comparison.pass ? "pass" : "fail"}`,
+    "",
+    "## Aggregate Metrics",
+    "",
+    "| Metric | Candidate | Baseline |",
+    "| --- | ---: | ---: |",
+    `| Completion rate | ${formatPercent(comparison.candidateMetrics.completionRate)} | ${formatPercent(comparison.baselineMetrics.completionRate)} |`,
+    `| Unintended-stop rate | ${formatPercent(comparison.candidateMetrics.unintendedStopRate)} | ${formatPercent(comparison.baselineMetrics.unintendedStopRate)} |`,
+    `| Valid-tool-call rate | ${formatPercent(comparison.candidateMetrics.validToolCallRate)} | ${formatPercent(comparison.baselineMetrics.validToolCallRate)} |`,
+    `| Fake-success count | ${comparison.candidateMetrics.fakeSuccessCount} | ${comparison.baselineMetrics.fakeSuccessCount} |`,
+    "",
+  ];
+
+  if (comparison.failures.length > 0) {
+    lines.push("## Gate Failures", "");
+    for (const failure of comparison.failures) {
+      lines.push(`- ${failure}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Scenario Comparison", "");
+  for (const scenario of comparison.scenarioComparisons) {
+    lines.push(`### ${scenario.name}`, "");
+    lines.push(`- ${comparison.candidateLabel}: ${scenario.candidateStatus}`);
+    lines.push(`- ${comparison.baselineLabel}: ${scenario.baselineStatus}`);
+    if (scenario.candidateDetails) {
+      lines.push(`- ${comparison.candidateLabel} details: ${scenario.candidateDetails}`);
+    }
+    if (scenario.baselineDetails) {
+      lines.push(`- ${comparison.baselineLabel} details: ${scenario.baselineDetails}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Notes", "");
+  for (const note of comparison.notes) {
+    lines.push(`- ${note}`);
+  }
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+export function buildQaRuntimeParityReport(params: {
+  summary: QaRuntimeParitySuiteSummary;
+  comparedAt?: string;
+}): QaRuntimeParityReport {
+  const runtimePair = normalizeRuntimePair(params.summary.run?.runtimePair);
+  const providerMode = params.summary.run?.providerMode;
+  const requiresLiveUsage = isLiveProviderMode(providerMode);
+  const driftCounts = buildRuntimeParityDriftCounts();
+  const failures: string[] = [];
+  const scenarios: QaRuntimeParityScenarioReport[] = params.summary.scenarios.map((scenario) => {
+    const parity = scenario.runtimeParity;
+    if (!parity) {
+      failures.push(`Missing runtime parity capture for ${scenario.name}.`);
+      return {
+        name: scenario.name,
+        status: scenario.status === "pass" ? "pass" : "fail",
+        drift: "missing",
+        driftDetails: scenario.details,
+        openclawStatus: "missing",
+        codexStatus: "missing",
+        openclawTokens: 0,
+        codexTokens: 0,
+        openclawToolCalls: 0,
+        codexToolCalls: 0,
+      } satisfies QaRuntimeParityScenarioReport;
+    }
+    driftCounts[parity.drift] += 1;
+    const openclawCell = parity.cells.openclaw;
+    const codexCell = parity.cells.codex;
+    const openclawStatus = runtimeParityCellStatus(openclawCell);
+    const codexStatus = runtimeParityCellStatus(codexCell);
+    const parityStatus = isRuntimeParityResultPass(parity) ? "pass" : "fail";
+    const reportScenario = {
+      name: scenario.name,
+      status: parityStatus,
+      drift: parity.drift,
+      driftDetails: parity.driftDetails,
+      openclawStatus,
+      codexStatus,
+      openclawTokens: openclawCell.usage.totalTokens,
+      codexTokens: codexCell.usage.totalTokens,
+      openclawToolCalls: openclawCell.toolCalls.length,
+      codexToolCalls: codexCell.toolCalls.length,
+    } satisfies QaRuntimeParityScenarioReport;
+    if (parityStatus === "fail") {
+      failures.push(
+        `${scenario.name} drift=${parity.drift}${parity.driftDetails ? ` (${parity.driftDetails})` : ""}.`,
+      );
+    }
+    const usageFailure = requiresLiveUsage
+      ? describeLiveUsageFailure(scenario.name, reportScenario)
+      : undefined;
+    if (usageFailure) {
+      failures.push(usageFailure);
+      return { ...reportScenario, status: "fail" };
+    }
+    return reportScenario;
+  });
+
+  const totalScenarios = params.summary.counts?.total ?? scenarios.length;
+  const passedScenarios = scenarios.filter((scenario) => scenario.status === "pass").length;
+  const failedScenarios = scenarios.filter((scenario) => scenario.status === "fail").length;
+  if (scenarios.length === 0 || totalScenarios <= 0) {
+    failures.push("Runtime parity report has no executed scenarios.");
+  }
+
+  return {
+    runtimePair,
+    comparedAt: params.comparedAt ?? new Date().toISOString(),
+    providerMode,
+    primaryModel: params.summary.run?.primaryModel,
+    totalScenarios,
+    passedScenarios,
+    failedScenarios,
+    driftCounts,
+    scenarios,
+    pass: failures.length === 0 && failedScenarios === 0,
+    failures,
+    notes: [
+      "Runtime parity fails runtime, transport, and failure-mode drift; structural and tool-shape drift is recorded as advisory when both runtimes complete.",
+      "Token totals here are assistant-message usage captured from the normalized transcript, not provider transport payloads.",
+    ],
+  };
+}
+
+export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityReport): string {
+  const lines = [
+    `# OpenClaw Runtime Parity Report — ${report.runtimePair[0]} vs ${report.runtimePair[1]}`,
+    "",
+    `- Compared at: ${report.comparedAt}`,
+    `- Provider mode: ${report.providerMode ?? "unknown"}`,
+    `- Primary model: ${report.primaryModel ?? "unknown"}`,
+    `- Verdict: ${report.pass ? "pass" : "fail"}`,
+    "",
+    "## Aggregate Metrics",
+    "",
+    "| Metric | Value |",
+    "| --- | ---: |",
+    `| Total scenarios | ${report.totalScenarios} |`,
+    `| Passed scenarios | ${report.passedScenarios} |`,
+    `| Failed scenarios | ${report.failedScenarios} |`,
+    `| No drift | ${report.driftCounts.none} |`,
+    `| Text-only drift | ${report.driftCounts["text-only"]} |`,
+    `| Tool-call-shape drift | ${report.driftCounts["tool-call-shape"]} |`,
+    `| Tool-result-shape drift | ${report.driftCounts["tool-result-shape"]} |`,
+    `| Structural drift | ${report.driftCounts.structural} |`,
+    `| Failure-mode drift | ${report.driftCounts["failure-mode"]} |`,
+    "",
+  ];
+
+  if (report.failures.length > 0) {
+    lines.push("## Gate Failures", "");
+    for (const failure of report.failures) {
+      lines.push(`- ${failure}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Scenario Comparison", "");
+  for (const scenario of report.scenarios) {
+    lines.push(`### ${scenario.name}`, "");
+    lines.push(`- status: ${scenario.status}`);
+    lines.push(`- drift: ${scenario.drift}`);
+    lines.push(
+      `- openclaw: ${scenario.openclawStatus} (${scenario.openclawToolCalls} tool calls, ${scenario.openclawTokens} tokens)`,
+    );
+    lines.push(
+      `- codex: ${scenario.codexStatus} (${scenario.codexToolCalls} tool calls, ${scenario.codexTokens} tokens)`,
+    );
+    if (scenario.driftDetails) {
+      lines.push(`- details: ${scenario.driftDetails}`);
+    }
+    lines.push("");
+  }
+
+  lines.push("## Notes", "");
+  for (const note of report.notes) {
+    lines.push(`- ${note}`);
+  }
+  lines.push("");
+
+  return lines.join("\n");
+}

extensions/qa-lab/src/agentic-parity.ts

@@ -0,0 +1,93 @@
+// Qa Lab plugin module implements agentic parity behavior.
+import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
+
+const QA_AGENTIC_PARITY_PACK = "agentic";
+
+const QA_AGENTIC_PARITY_SCENARIOS = [
+  {
+    id: "approval-turn-tool-followthrough",
+    title: "Approval turn tool followthrough",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "model-switch-tool-continuity",
+    title: "Model switch with tool continuity",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "source-docs-discovery-report",
+    title: "Source and docs discovery report",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "image-understanding-attachment",
+    title: "Image understanding from attachment",
+    countsTowardValidToolCallRate: false,
+  },
+  {
+    id: "compaction-retry-mutating-tool",
+    title: "Compaction retry after mutating tool",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "subagent-handoff",
+    title: "Subagent handoff",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "subagent-fanout-synthesis",
+    title: "Subagent fanout synthesis",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "subagent-stale-child-links",
+    title: "Subagent stale child links",
+    countsTowardValidToolCallRate: false,
+  },
+  {
+    id: "memory-recall",
+    title: "Memory recall after context switch",
+    countsTowardValidToolCallRate: false,
+  },
+  {
+    id: "thread-memory-isolation",
+    title: "Thread memory isolation",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "config-restart-capability-flip",
+    title: "Config restart capability flip",
+    countsTowardValidToolCallRate: true,
+  },
+  {
+    id: "instruction-followthrough-repo-contract",
+    title: "Instruction followthrough repo contract",
+    countsTowardValidToolCallRate: true,
+  },
+] as const;
+
+export const QA_AGENTIC_PARITY_SCENARIO_IDS = QA_AGENTIC_PARITY_SCENARIOS.map(({ id }) => id);
+export const QA_AGENTIC_PARITY_SCENARIO_TITLES = QA_AGENTIC_PARITY_SCENARIOS.map(
+  ({ title }) => title,
+);
+export const QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES = QA_AGENTIC_PARITY_SCENARIOS.filter(
+  ({ countsTowardValidToolCallRate }) => countsTowardValidToolCallRate,
+).map(({ title }) => title);
+
+export function resolveQaParityPackScenarioIds(params: {
+  parityPack?: string;
+  scenarioIds?: string[];
+}): string[] {
+  const normalizedPack = params.parityPack?.trim().toLowerCase();
+  const explicitScenarioIds = uniqueStrings(params.scenarioIds ?? []);
+  if (!normalizedPack) {
+    return explicitScenarioIds;
+  }
+  if (normalizedPack !== QA_AGENTIC_PARITY_PACK) {
+    throw new Error(
+      `--parity-pack must be "${QA_AGENTIC_PARITY_PACK}", got "${params.parityPack}"`,
+    );
+  }
+
+  return uniqueStrings([...explicitScenarioIds, ...QA_AGENTIC_PARITY_SCENARIO_IDS]);
+}

extensions/qa-lab/src/artifact-assertion.ts

@@ -0,0 +1,18 @@
+import fs from "node:fs/promises";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { QaSuiteArtifactError } from "./errors.js";
+
+export async function assertQaSuiteArtifactWritten(
+  kind: "evidence" | "report" | "summary",
+  filePath: string,
+) {
+  try {
+    await fs.access(filePath);
+  } catch (error) {
+    throw new QaSuiteArtifactError(
+      `${kind}_missing`,
+      `QA suite did not produce ${kind} artifact at ${filePath}: ${formatErrorMessage(error)}`,
+      { cause: error },
+    );
+  }
+}

extensions/qa-lab/src/artifact-run-id.ts

@@ -0,0 +1,6 @@
+// Qa Lab plugin helper creates collision-resistant artifact run identifiers.
+import { randomUUID } from "node:crypto";
+
+export function createQaArtifactRunId(): string {
+  return `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
+}

extensions/qa-lab/src/auth-profile.fixture.ts

@@ -0,0 +1,178 @@
+// Qa Lab plugin module implements auth profile.fixture behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export const QA_CODEX_OAUTH_PROFILE_ID = "openai:qa-oauth";
+export const QA_OPENAI_API_KEY_PROFILE_ID = "openai:media-api";
+export const QA_AUTH_PROFILE_STORE_VERSION = 1;
+
+export type QaAuthProfileShape = "oauth-only" | "apikey-only" | "mixed";
+
+export type QaApiKeyAuthProfile = {
+  type: "api_key";
+  provider: "openai";
+  key: string;
+  displayName: string;
+};
+
+export type QaOAuthAuthProfile = {
+  type: "oauth";
+  provider: "openai";
+  access: string;
+  refresh: string;
+  expires: number;
+  email: string;
+  displayName: string;
+};
+
+export type QaAuthProfile = QaApiKeyAuthProfile | QaOAuthAuthProfile;
+
+export type QaAuthProfileSnapshot = {
+  version: number;
+  profiles: Record<string, QaAuthProfile>;
+};
+
+export type QaCodexAuthProfileSelection =
+  | {
+      status: "ready";
+      profileId: string;
+      provider: "openai";
+      mode: "oauth";
+    }
+  | {
+      status: "blocked";
+      remediation: string;
+    };
+
+const QA_FIXED_OAUTH_EXPIRY_MS = Date.UTC(2036, 0, 1);
+
+function authProfilesPath(agentDir: string) {
+  return path.join(agentDir, "auth-profiles.json");
+}
+
+function buildCodexOAuthProfile(): QaOAuthAuthProfile {
+  return {
+    type: "oauth",
+    provider: "openai",
+    access: "qa-codex-oauth-access-placeholder",
+    refresh: "qa-codex-oauth-refresh-placeholder",
+    expires: QA_FIXED_OAUTH_EXPIRY_MS,
+    email: "qa-codex@example.test",
+    displayName: "QA Codex OAuth profile",
+  };
+}
+
+function buildOpenAiApiKeyProfile(): QaApiKeyAuthProfile {
+  return {
+    type: "api_key",
+    provider: "openai",
+    key: "qa-openai-not-a-real-key",
+    displayName: "QA OpenAI API-key profile",
+  };
+}
+
+function buildProfileMap(shape: QaAuthProfileShape): Record<string, QaAuthProfile> {
+  switch (shape) {
+    case "oauth-only":
+      return {
+        [QA_CODEX_OAUTH_PROFILE_ID]: buildCodexOAuthProfile(),
+      };
+    case "apikey-only":
+      return {
+        [QA_OPENAI_API_KEY_PROFILE_ID]: buildOpenAiApiKeyProfile(),
+      };
+    case "mixed":
+      return {
+        [QA_CODEX_OAUTH_PROFILE_ID]: buildCodexOAuthProfile(),
+        [QA_OPENAI_API_KEY_PROFILE_ID]: buildOpenAiApiKeyProfile(),
+      };
+  }
+  const exhaustive: never = shape;
+  return exhaustive;
+}
+
+function isQaAuthProfile(value: unknown): value is QaAuthProfile {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const record = value as Record<string, unknown>;
+  return (
+    (record.type === "oauth" && record.provider === "openai") ||
+    (record.type === "api_key" && record.provider === "openai")
+  );
+}
+
+function normalizeAuthProfileSnapshot(value: unknown): QaAuthProfileSnapshot {
+  if (!value || typeof value !== "object") {
+    return { version: QA_AUTH_PROFILE_STORE_VERSION, profiles: {} };
+  }
+  const record = value as Record<string, unknown>;
+  const profilesRecord =
+    record.profiles && typeof record.profiles === "object"
+      ? (record.profiles as Record<string, unknown>)
+      : {};
+  const profiles = Object.fromEntries(
+    Object.entries(profilesRecord)
+      .filter((entry): entry is [string, QaAuthProfile] => isQaAuthProfile(entry[1]))
+      .toSorted(([left], [right]) => left.localeCompare(right)),
+  );
+  return {
+    version:
+      typeof record.version === "number" && Number.isFinite(record.version)
+        ? record.version
+        : QA_AUTH_PROFILE_STORE_VERSION,
+    profiles,
+  };
+}
+
+export async function seedAuthProfiles(
+  shape: QaAuthProfileShape,
+  agentDir: string,
+): Promise<QaAuthProfileSnapshot> {
+  const snapshot = {
+    version: QA_AUTH_PROFILE_STORE_VERSION,
+    profiles: buildProfileMap(shape),
+  };
+  await fs.mkdir(agentDir, { recursive: true });
+  await fs.writeFile(authProfilesPath(agentDir), `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
+  return snapshot;
+}
+
+export async function snapshotAuthProfiles(agentDir: string): Promise<QaAuthProfileSnapshot> {
+  const raw = await fs.readFile(authProfilesPath(agentDir), "utf8").catch((error: unknown) => {
+    if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") {
+      return null;
+    }
+    throw error;
+  });
+  if (!raw) {
+    return { version: QA_AUTH_PROFILE_STORE_VERSION, profiles: {} };
+  }
+  return normalizeAuthProfileSnapshot(JSON.parse(raw) as unknown);
+}
+
+export function resolveCodexAuthProfile(
+  snapshot: QaAuthProfileSnapshot,
+): QaCodexAuthProfileSelection {
+  const profileId = Object.keys(snapshot.profiles)
+    .toSorted((left, right) => left.localeCompare(right))
+    .find((candidate) => {
+      const profile = snapshot.profiles[candidate];
+      return profile?.type === "oauth" && profile.provider === "openai";
+    });
+
+  if (!profileId) {
+    return {
+      status: "blocked",
+      remediation:
+        'Codex app-server auth requires an openai OAuth profile. Run "openclaw doctor --fix" to repair Codex auth routing before retrying.',
+    };
+  }
+
+  return {
+    status: "ready",
+    profileId,
+    provider: "openai",
+    mode: "oauth",
+  };
+}

extensions/qa-lab/src/browser-runtime.test.ts

@@ -0,0 +1,198 @@
+// Qa Lab tests cover browser runtime plugin behavior.
+import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  callQaBrowserRequest,
+  qaBrowserAct,
+  qaBrowserOpenTab,
+  qaBrowserSnapshot,
+  waitForQaBrowserReady,
+} from "./browser-runtime.js";
+
+function createEnv() {
+  return {
+    gateway: {
+      call: vi.fn(async () => ({ ok: true })),
+    },
+  };
+}
+
+describe("browser-runtime", () => {
+  beforeEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("sends normalized browser.request payloads through the gateway", async () => {
+    const env = createEnv();
+
+    const result = await callQaBrowserRequest(env, {
+      method: "GET",
+      path: "/snapshot",
+      query: {
+        format: "ai",
+        targetId: "tab-1",
+        skip: undefined,
+        limit: 50,
+      },
+      timeoutMs: 12_345,
+    });
+
+    expect(result).toEqual({ ok: true });
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "GET",
+        path: "/snapshot",
+        query: {
+          format: "ai",
+          targetId: "tab-1",
+          limit: "50",
+        },
+        body: undefined,
+        timeoutMs: 12_345,
+      },
+      { timeoutMs: 12_345 },
+    );
+  });
+
+  it("opens tabs through the browser proxy", async () => {
+    const env = createEnv();
+
+    await qaBrowserOpenTab(env, {
+      url: "http://127.0.0.1:43124/control-ui/chat?session=test",
+      profile: "openclaw",
+    });
+
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "POST",
+        path: "/tabs/open",
+        query: {
+          profile: "openclaw",
+        },
+        body: {
+          url: "http://127.0.0.1:43124/control-ui/chat?session=test",
+        },
+        timeoutMs: 20_000,
+      },
+      { timeoutMs: 20_000 },
+    );
+  });
+
+  it("captures snapshots with query options", async () => {
+    const env = createEnv();
+
+    await qaBrowserSnapshot(env, {
+      targetId: "tab-1",
+      interactive: true,
+      labels: true,
+      maxChars: 4_000,
+    });
+
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "GET",
+        path: "/snapshot",
+        query: {
+          targetId: "tab-1",
+          format: "ai",
+          interactive: "true",
+          labels: "true",
+          maxChars: "4000",
+        },
+        body: undefined,
+        timeoutMs: 20_000,
+      },
+      { timeoutMs: 20_000 },
+    );
+  });
+
+  it("runs browser act requests through /act", async () => {
+    const env = createEnv();
+
+    await qaBrowserAct(env, {
+      profile: "openclaw",
+      request: {
+        kind: "type",
+        ref: "12",
+        text: "hello",
+        submit: true,
+      },
+      timeoutMs: 9_000,
+    });
+
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "POST",
+        path: "/act",
+        query: {
+          profile: "openclaw",
+        },
+        body: {
+          kind: "type",
+          ref: "12",
+          text: "hello",
+          submit: true,
+        },
+        timeoutMs: 9_000,
+      },
+      { timeoutMs: 9_000 },
+    );
+  });
+
+  it("caps oversized browser request timeouts", async () => {
+    const env = createEnv();
+
+    await callQaBrowserRequest(env, {
+      method: "GET",
+      path: "/snapshot",
+      timeoutMs: Number.MAX_SAFE_INTEGER,
+    });
+
+    expect(env.gateway.call).toHaveBeenCalledWith(
+      "browser.request",
+      {
+        method: "GET",
+        path: "/snapshot",
+        query: undefined,
+        body: undefined,
+        timeoutMs: MAX_TIMER_TIMEOUT_MS,
+      },
+      { timeoutMs: MAX_TIMER_TIMEOUT_MS },
+    );
+  });
+
+  it("waits until browser control reports a ready profile", async () => {
+    const env = createEnv();
+    env.gateway.call = vi
+      .fn()
+      .mockResolvedValueOnce({ enabled: true, running: false, cdpReady: false })
+      .mockResolvedValueOnce({ enabled: true, running: true, cdpReady: true });
+
+    const status = await waitForQaBrowserReady(env, {
+      profile: "user",
+      timeoutMs: 5_000,
+      intervalMs: 1,
+      sleepImpl: async () => {},
+    });
+
+    expect(status).toEqual({ enabled: true, running: true, cdpReady: true });
+    expect(env.gateway.call).toHaveBeenNthCalledWith(
+      1,
+      "browser.request",
+      {
+        method: "GET",
+        path: "/",
+        query: {
+          profile: "user",
+        },
+        body: undefined,
+        timeoutMs: 5_000,
+      },
+      { timeoutMs: 5_000 },
+    );
+  });
+});

extensions/qa-lab/src/browser-runtime.ts

@@ -0,0 +1,210 @@
+// Qa Lab plugin module implements browser runtime behavior.
+import { resolvePositiveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
+import { sleep } from "openclaw/plugin-sdk/runtime-env";
+
+type QaBrowserGateway = {
+  call: (
+    method: string,
+    params: Record<string, unknown>,
+    opts?: { timeoutMs?: number },
+  ) => Promise<unknown>;
+};
+
+type QaBrowserEnv = {
+  gateway: QaBrowserGateway;
+};
+
+type QaBrowserRequestParams = {
+  method: "GET" | "POST" | "DELETE";
+  path: string;
+  query?: Record<string, string | number | boolean | undefined>;
+  body?: unknown;
+  timeoutMs?: number;
+};
+
+type QaBrowserOpenTabParams = {
+  url: string;
+  profile?: string;
+  timeoutMs?: number;
+};
+
+type QaBrowserSnapshotParams = {
+  profile?: string;
+  targetId?: string;
+  format?: "ai" | "aria";
+  limit?: number;
+  interactive?: boolean;
+  compact?: boolean;
+  depth?: number;
+  selector?: string;
+  frame?: string;
+  labels?: boolean;
+  mode?: "efficient";
+  maxChars?: number;
+  timeoutMs?: number;
+};
+
+type QaBrowserActRequest = {
+  kind: string;
+  targetId?: string;
+  ref?: string;
+  doubleClick?: boolean;
+  button?: string;
+  modifiers?: string[];
+  text?: string;
+  submit?: boolean;
+  slowly?: boolean;
+  key?: string;
+  delayMs?: number;
+  startRef?: string;
+  endRef?: string;
+  values?: string[];
+  fields?: Array<Record<string, unknown>>;
+  width?: number;
+  height?: number;
+  timeMs?: number;
+  selector?: string;
+  url?: string;
+  loadState?: string;
+  textGone?: string;
+  timeoutMs?: number;
+  fn?: string;
+};
+
+type QaBrowserActParams = {
+  profile?: string;
+  request: QaBrowserActRequest;
+  timeoutMs?: number;
+};
+
+type QaBrowserStatus = {
+  enabled?: boolean;
+  running?: boolean;
+  cdpReady?: boolean;
+};
+
+type QaBrowserReadyParams = {
+  profile?: string;
+  timeoutMs?: number;
+  intervalMs?: number;
+  sleepImpl?: (ms: number) => Promise<unknown>;
+};
+
+function normalizeBrowserQuery(
+  query: QaBrowserRequestParams["query"],
+): Record<string, string> | undefined {
+  if (!query) {
+    return undefined;
+  }
+  const normalized = Object.fromEntries(
+    Object.entries(query)
+      .filter(([, value]) => value !== undefined)
+      .map(([key, value]) => [key, String(value)]),
+  );
+  return Object.keys(normalized).length > 0 ? normalized : undefined;
+}
+
+function resolveBrowserTimeoutMs(timeoutMs: number | undefined, fallbackMs: number) {
+  return resolvePositiveTimerTimeoutMs(timeoutMs, fallbackMs);
+}
+
+export async function callQaBrowserRequest<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserRequestParams,
+): Promise<T> {
+  const timeoutMs = resolveBrowserTimeoutMs(params.timeoutMs, 20_000);
+  const payload = await env.gateway.call(
+    "browser.request",
+    {
+      method: params.method,
+      path: params.path,
+      query: normalizeBrowserQuery(params.query),
+      body: params.body,
+      timeoutMs,
+    },
+    { timeoutMs },
+  );
+  return payload as T;
+}
+
+export async function qaBrowserOpenTab<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserOpenTabParams,
+): Promise<T> {
+  return await callQaBrowserRequest<T>(env, {
+    method: "POST",
+    path: "/tabs/open",
+    query: params.profile ? { profile: params.profile } : undefined,
+    body: { url: params.url },
+    timeoutMs: resolveBrowserTimeoutMs(params.timeoutMs, 20_000),
+  });
+}
+
+export async function qaBrowserSnapshot<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserSnapshotParams = {},
+): Promise<T> {
+  return await callQaBrowserRequest<T>(env, {
+    method: "GET",
+    path: "/snapshot",
+    query: {
+      profile: params.profile,
+      targetId: params.targetId,
+      format: params.format ?? "ai",
+      limit: params.limit,
+      interactive: params.interactive,
+      compact: params.compact,
+      depth: params.depth,
+      selector: params.selector,
+      frame: params.frame,
+      labels: params.labels,
+      mode: params.mode,
+      maxChars: params.maxChars,
+    },
+    timeoutMs: resolveBrowserTimeoutMs(params.timeoutMs, 20_000),
+  });
+}
+
+export async function qaBrowserAct<T = unknown>(
+  env: QaBrowserEnv,
+  params: QaBrowserActParams,
+): Promise<T> {
+  return await callQaBrowserRequest<T>(env, {
+    method: "POST",
+    path: "/act",
+    query: params.profile ? { profile: params.profile } : undefined,
+    body: params.request,
+    timeoutMs: resolveBrowserTimeoutMs(params.timeoutMs, 20_000),
+  });
+}
+
+function isQaBrowserReady(status: QaBrowserStatus | null | undefined) {
+  return status?.enabled === true && status?.running === true && status?.cdpReady === true;
+}
+
+export async function waitForQaBrowserReady<T extends QaBrowserStatus = QaBrowserStatus>(
+  env: QaBrowserEnv,
+  params: QaBrowserReadyParams = {},
+): Promise<T> {
+  const timeoutMs = resolveBrowserTimeoutMs(params.timeoutMs, 20_000);
+  const intervalMs = resolveBrowserTimeoutMs(params.intervalMs, 250);
+  const startedAt = Date.now();
+  let lastStatus: QaBrowserStatus | null = null;
+  while (Date.now() - startedAt < timeoutMs) {
+    lastStatus = await callQaBrowserRequest<QaBrowserStatus>(env, {
+      method: "GET",
+      path: "/",
+      query: params.profile ? { profile: params.profile } : undefined,
+      timeoutMs: Math.min(timeoutMs, 5_000),
+    });
+    if (isQaBrowserReady(lastStatus)) {
+      return lastStatus as T;
+    }
+    await (params.sleepImpl ?? sleep)(intervalMs);
+  }
+  throw new Error(
+    `browser control not ready after ${timeoutMs}ms${
+      lastStatus ? ` (${JSON.stringify(lastStatus)})` : ""
+    }`,
+  );
+}

extensions/qa-lab/src/bundled-plugin-staging.ts

@@ -0,0 +1,463 @@
+// Qa Lab plugin module implements bundled plugin staging behavior.
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
+import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
+
+const QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS = Object.freeze([
+  "image-generation-core",
+  "media-understanding-core",
+  "speech-core",
+]);
+const QA_OPENAI_PLUGIN_ID = "openai";
+const QA_BUNDLED_PLUGIN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+const QA_CLI_METADATA_ENTRY_BASENAMES = Object.freeze([
+  "cli-metadata.ts",
+  "cli-metadata.js",
+  "cli-metadata.mjs",
+  "cli-metadata.cjs",
+]);
+
+function assertSafeQaBundledPluginId(pluginId: string) {
+  if (!QA_BUNDLED_PLUGIN_ID_PATTERN.test(pluginId)) {
+    throw new Error(`invalid QA bundled plugin id: ${pluginId}`);
+  }
+}
+
+function parseStableSemverFloor(value: string | undefined) {
+  if (!value) {
+    return null;
+  }
+  const match = value.trim().match(/(\d+)\.(\d+)\.(\d+)/);
+  if (!match) {
+    return null;
+  }
+  return {
+    major: Number.parseInt(match[1] ?? "", 10),
+    minor: Number.parseInt(match[2] ?? "", 10),
+    patch: Number.parseInt(match[3] ?? "", 10),
+    label: `${match[1]}.${match[2]}.${match[3]}`,
+  };
+}
+
+function compareSemverFloors(
+  left: ReturnType<typeof parseStableSemverFloor>,
+  right: ReturnType<typeof parseStableSemverFloor>,
+) {
+  if (!left && !right) {
+    return 0;
+  }
+  if (!left) {
+    return -1;
+  }
+  if (!right) {
+    return 1;
+  }
+  if (left.major !== right.major) {
+    return left.major - right.major;
+  }
+  if (left.minor !== right.minor) {
+    return left.minor - right.minor;
+  }
+  return left.patch - right.patch;
+}
+
+function isQaOpenAiResponsesProviderConfig(config: ModelProviderConfig) {
+  return (
+    config.api === "openai-responses" ||
+    config.models.some((model) => model.api === "openai-responses")
+  );
+}
+
+export function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) {
+  assertSafeQaBundledPluginId(params.pluginId);
+  const candidates = [
+    path.join(params.repoRoot, "dist", "extensions", params.pluginId),
+    path.join(params.repoRoot, "dist-runtime", "extensions", params.pluginId),
+    path.join(params.repoRoot, "extensions", params.pluginId),
+  ];
+  const existingCandidates = candidates.filter((candidate) => existsSync(candidate));
+  const manifestCandidates = findQaBundledPluginDirsByManifestId(params);
+  const allCandidates = uniqueStrings([...existingCandidates, ...manifestCandidates]);
+  if (allCandidates.length === 0) {
+    return null;
+  }
+  const cliMetadataCandidate = allCandidates.find((candidate) =>
+    QA_CLI_METADATA_ENTRY_BASENAMES.some((basename) => existsSync(path.join(candidate, basename))),
+  );
+  if (cliMetadataCandidate) {
+    return cliMetadataCandidate;
+  }
+  return allCandidates[0] ?? null;
+}
+
+function resolveQaBundledPluginScanRoots(repoRoot: string) {
+  const candidates = [
+    path.join(repoRoot, "dist", "extensions"),
+    path.join(repoRoot, "dist-runtime", "extensions"),
+    path.join(repoRoot, "extensions"),
+  ];
+  return uniqueStrings(candidates.filter((candidate) => existsSync(candidate)));
+}
+
+function readQaBundledManifestId(manifestPath: string): string | null {
+  try {
+    const parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as { id?: unknown };
+    return typeof parsed.id === "string" ? parsed.id.trim() || null : null;
+  } catch {
+    return null;
+  }
+}
+
+function findQaBundledPluginDirsByManifestId(params: {
+  repoRoot: string;
+  pluginId: string;
+}): string[] {
+  const candidates: string[] = [];
+  for (const sourceRoot of resolveQaBundledPluginScanRoots(params.repoRoot)) {
+    for (const entry of readdirSync(sourceRoot, { withFileTypes: true }).toSorted((left, right) =>
+      left.name.localeCompare(right.name),
+    )) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      const candidate = path.join(sourceRoot, entry.name);
+      const manifestId = readQaBundledManifestId(path.join(candidate, "openclaw.plugin.json"));
+      if (manifestId === params.pluginId) {
+        candidates.push(candidate);
+      }
+    }
+  }
+  return candidates;
+}
+
+export async function resolveQaOwnerPluginIdsForProviderIds(params: {
+  repoRoot: string;
+  providerIds: readonly string[];
+  providerConfigs?: Record<string, ModelProviderConfig>;
+}) {
+  const providerIds = uniqueStrings(normalizeStringEntries(params.providerIds));
+  if (providerIds.length === 0) {
+    return [];
+  }
+  const remainingProviderIds = new Set(providerIds);
+  const ownerPluginIds = new Set<string>();
+  const visitedPluginIds = new Set<string>();
+  for (const sourceRoot of resolveQaBundledPluginScanRoots(params.repoRoot)) {
+    for (const entry of await fs.readdir(sourceRoot, { withFileTypes: true })) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      const manifestPath = path.join(sourceRoot, entry.name, "openclaw.plugin.json");
+      if (!existsSync(manifestPath)) {
+        continue;
+      }
+      const manifest = JSON.parse(await fs.readFile(manifestPath, "utf8")) as {
+        id?: unknown;
+        providers?: unknown;
+        cliBackends?: unknown;
+      };
+      const pluginId = typeof manifest.id === "string" ? manifest.id.trim() : entry.name;
+      if (!pluginId || visitedPluginIds.has(pluginId)) {
+        continue;
+      }
+      visitedPluginIds.add(pluginId);
+      const ownedIds = new Set(
+        [
+          pluginId,
+          ...(Array.isArray(manifest.providers) ? manifest.providers : []),
+          ...(Array.isArray(manifest.cliBackends) ? manifest.cliBackends : []),
+        ].filter((ownedId): ownedId is string => typeof ownedId === "string"),
+      );
+      for (const providerId of providerIds) {
+        if (!ownedIds.has(providerId)) {
+          continue;
+        }
+        ownerPluginIds.add(pluginId);
+        remainingProviderIds.delete(providerId);
+      }
+    }
+  }
+  for (const providerId of remainingProviderIds) {
+    const providerConfig = params.providerConfigs?.[providerId];
+    if (providerConfig && isQaOpenAiResponsesProviderConfig(providerConfig)) {
+      ownerPluginIds.add(QA_OPENAI_PLUGIN_ID);
+      continue;
+    }
+    ownerPluginIds.add(providerId);
+  }
+  return [...ownerPluginIds];
+}
+
+function collectQaBundledPluginIds(params: {
+  repoRoot: string;
+  allowedPluginIds: readonly string[];
+}) {
+  const pluginIds = new Set<string>();
+  for (const pluginId of params.allowedPluginIds) {
+    assertSafeQaBundledPluginId(pluginId);
+    if (resolveQaBundledPluginSourceDir({ repoRoot: params.repoRoot, pluginId })) {
+      pluginIds.add(pluginId);
+    }
+  }
+  for (const pluginId of QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS) {
+    if (
+      resolveQaBundledPluginSourceDir({
+        repoRoot: params.repoRoot,
+        pluginId,
+      })
+    ) {
+      pluginIds.add(pluginId);
+    }
+  }
+  return [...pluginIds];
+}
+
+function resolveQaStagedBundledTreeName(repoRoot: string) {
+  if (existsSync(path.join(repoRoot, "dist"))) {
+    return "dist";
+  }
+  if (existsSync(path.join(repoRoot, "dist-runtime"))) {
+    return "dist-runtime";
+  }
+  return "dist";
+}
+
+function resolveQaBuiltBundledPluginTreeRoot(params: { repoRoot: string; sourceDir: string }) {
+  const sourceDir = path.resolve(params.sourceDir);
+  for (const treeName of ["dist", "dist-runtime"] as const) {
+    const extensionsRoot = path.join(params.repoRoot, treeName, "extensions");
+    const relativeSourceDir = path.relative(extensionsRoot, sourceDir);
+    if (
+      relativeSourceDir.length > 0 &&
+      !relativeSourceDir.startsWith("..") &&
+      !path.isAbsolute(relativeSourceDir)
+    ) {
+      return path.join(params.repoRoot, treeName);
+    }
+  }
+  return null;
+}
+
+async function symlinkQaStagedDirEntry(params: {
+  sourcePath: string;
+  targetPath: string;
+  directory?: boolean;
+}) {
+  await fs.symlink(
+    params.sourcePath,
+    params.targetPath,
+    params.directory ? (process.platform === "win32" ? "junction" : "dir") : "file",
+  );
+}
+
+async function resolveQaStagedDirEntryDirectory(params: {
+  sourcePath: string;
+  entry?: {
+    isDirectory(): boolean;
+    isSymbolicLink(): boolean;
+  };
+}) {
+  if (params.entry?.isDirectory()) {
+    return true;
+  }
+  if (params.entry?.isSymbolicLink()) {
+    return (await fs.stat(params.sourcePath)).isDirectory();
+  }
+  if (params.entry) {
+    return false;
+  }
+  return (await fs.lstat(params.sourcePath)).isDirectory();
+}
+
+async function seedQaStagedNodeModules(params: { repoRoot: string; stagedRoot: string }) {
+  const sourceNodeModulesDir = path.join(params.repoRoot, "node_modules");
+  if (!existsSync(sourceNodeModulesDir)) {
+    return;
+  }
+  const stagedNodeModulesDir = path.join(params.stagedRoot, "node_modules");
+  await fs.mkdir(stagedNodeModulesDir, { recursive: true });
+  for (const entry of await fs.readdir(sourceNodeModulesDir, { withFileTypes: true })) {
+    if (entry.name === "openclaw") {
+      continue;
+    }
+    await symlinkQaStagedDirEntry({
+      sourcePath: path.join(sourceNodeModulesDir, entry.name),
+      targetPath: path.join(stagedNodeModulesDir, entry.name),
+      directory: await resolveQaStagedDirEntryDirectory({
+        sourcePath: path.join(sourceNodeModulesDir, entry.name),
+        entry,
+      }),
+    });
+  }
+}
+
+function collectQaBuiltTreeRoots(params: {
+  repoRoot: string;
+  stagedPluginIds: readonly string[];
+  stagedTreeName: string;
+}) {
+  const treeRoots = new Set<string>();
+  treeRoots.add(path.join(params.repoRoot, params.stagedTreeName));
+  for (const pluginId of params.stagedPluginIds) {
+    const sourceDir = resolveQaBundledPluginSourceDir({
+      repoRoot: params.repoRoot,
+      pluginId,
+    });
+    if (!sourceDir) {
+      continue;
+    }
+    const builtTreeRoot = resolveQaBuiltBundledPluginTreeRoot({
+      repoRoot: params.repoRoot,
+      sourceDir,
+    });
+    if (builtTreeRoot) {
+      treeRoots.add(builtTreeRoot);
+    }
+  }
+  return [...treeRoots];
+}
+
+async function seedQaStagedBuiltTreeRoots(params: {
+  stagedTreeRoot: string;
+  sourceTreeRoots: readonly string[];
+}) {
+  for (const sourceTreeRoot of params.sourceTreeRoots) {
+    if (!existsSync(sourceTreeRoot)) {
+      continue;
+    }
+    for (const entry of await fs.readdir(sourceTreeRoot, { withFileTypes: true })) {
+      if (entry.name === "extensions") {
+        continue;
+      }
+      const targetPath = path.join(params.stagedTreeRoot, entry.name);
+      if (existsSync(targetPath)) {
+        continue;
+      }
+      await symlinkQaStagedDirEntry({
+        sourcePath: path.join(sourceTreeRoot, entry.name),
+        targetPath,
+        directory: await resolveQaStagedDirEntryDirectory({
+          sourcePath: path.join(sourceTreeRoot, entry.name),
+          entry,
+        }),
+      });
+    }
+  }
+}
+
+export async function resolveQaRuntimeHostVersion(params: {
+  repoRoot: string;
+  allowedPluginIds: readonly string[];
+}) {
+  const rootPackageRaw = await fs.readFile(path.join(params.repoRoot, "package.json"), "utf8");
+  const rootPackage = JSON.parse(rootPackageRaw) as { version?: string };
+  let selected = parseStableSemverFloor(rootPackage.version);
+  const stagedPluginIds = collectQaBundledPluginIds({
+    repoRoot: params.repoRoot,
+    allowedPluginIds: params.allowedPluginIds,
+  });
+
+  for (const pluginId of stagedPluginIds) {
+    const sourceDir = resolveQaBundledPluginSourceDir({
+      repoRoot: params.repoRoot,
+      pluginId,
+    });
+    if (!sourceDir) {
+      continue;
+    }
+    const packagePath = path.join(sourceDir, "package.json");
+    if (!existsSync(packagePath)) {
+      continue;
+    }
+    const packageRaw = await fs.readFile(packagePath, "utf8");
+    const packageJson = JSON.parse(packageRaw) as {
+      openclaw?: {
+        install?: {
+          minHostVersion?: string;
+        };
+      };
+    };
+    const candidate = parseStableSemverFloor(packageJson.openclaw?.install?.minHostVersion);
+    if (compareSemverFloors(candidate, selected) > 0) {
+      selected = candidate;
+    }
+  }
+
+  return selected?.label;
+}
+
+export async function createQaBundledPluginsDir(params: {
+  repoRoot: string;
+  tempRoot: string;
+  allowedPluginIds: readonly string[];
+}) {
+  const stagedPluginIds = collectQaBundledPluginIds({
+    repoRoot: params.repoRoot,
+    allowedPluginIds: params.allowedPluginIds,
+  });
+  const stagedRoot = path.join(
+    params.repoRoot,
+    ".artifacts",
+    "qa-runtime",
+    path.basename(params.tempRoot),
+  );
+  await fs.rm(stagedRoot, { recursive: true, force: true });
+  await fs.mkdir(stagedRoot, { recursive: true });
+  await fs.copyFile(
+    path.join(params.repoRoot, "package.json"),
+    path.join(stagedRoot, "package.json"),
+  );
+  await seedQaStagedNodeModules({
+    repoRoot: params.repoRoot,
+    stagedRoot,
+  });
+  const stagedOpenClawPackageDir = path.join(stagedRoot, "node_modules", "openclaw");
+  await fs.mkdir(stagedOpenClawPackageDir, { recursive: true });
+  await fs.copyFile(
+    path.join(params.repoRoot, "package.json"),
+    path.join(stagedOpenClawPackageDir, "package.json"),
+  );
+  const stagedTreeName = resolveQaStagedBundledTreeName(params.repoRoot);
+  const stagedTreeRoot = path.join(stagedRoot, stagedTreeName);
+  await fs.mkdir(stagedTreeRoot, { recursive: true });
+  await seedQaStagedBuiltTreeRoots({
+    stagedTreeRoot,
+    sourceTreeRoots: collectQaBuiltTreeRoots({
+      repoRoot: params.repoRoot,
+      stagedPluginIds,
+      stagedTreeName,
+    }),
+  });
+  if (stagedTreeName === "dist-runtime" && !existsSync(path.join(stagedRoot, "dist"))) {
+    const repoDistDir = path.join(params.repoRoot, "dist");
+    const stagedDistTarget = existsSync(repoDistDir) ? repoDistDir : stagedTreeRoot;
+    await symlinkQaStagedDirEntry({
+      sourcePath: stagedDistTarget,
+      targetPath: path.join(stagedRoot, "dist"),
+      directory: true,
+    });
+  }
+  const bundledPluginsDir = path.join(stagedTreeRoot, "extensions");
+  await fs.mkdir(bundledPluginsDir, { recursive: true });
+  for (const pluginId of stagedPluginIds) {
+    const sourceDir = resolveQaBundledPluginSourceDir({
+      repoRoot: params.repoRoot,
+      pluginId,
+    });
+    if (!sourceDir) {
+      throw new Error(`qa bundled plugin not found: ${pluginId}`);
+    }
+    await fs.cp(sourceDir, path.join(bundledPluginsDir, pluginId), { recursive: true });
+  }
+  await symlinkQaStagedDirEntry({
+    sourcePath: path.join(stagedRoot, "dist"),
+    targetPath: path.join(stagedOpenClawPackageDir, "dist"),
+    directory: true,
+  });
+  return {
+    bundledPluginsDir,
+    stagedRoot,
+  };
+}

extensions/qa-lab/src/bus-queries.ts

@@ -0,0 +1,192 @@
+// Qa Lab plugin module implements bus queries behavior.
+import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/string-coerce-runtime";
+import type {
+  QaBusAttachment,
+  QaBusConversation,
+  QaBusEvent,
+  QaBusMessage,
+  QaBusPollInput,
+  QaBusPollResult,
+  QaBusReadMessageInput,
+  QaBusSearchMessagesInput,
+  QaBusStateSnapshot,
+  QaBusThread,
+  QaBusToolCall,
+} from "./runtime-api.js";
+
+export const DEFAULT_ACCOUNT_ID = "default";
+
+export function normalizeAccountId(raw?: string): string {
+  const trimmed = raw?.trim();
+  return trimmed || DEFAULT_ACCOUNT_ID;
+}
+
+export function normalizeConversationFromTarget(target: string): {
+  conversation: QaBusConversation;
+  threadId?: string;
+} {
+  const trimmed = target.trim();
+  if (trimmed.startsWith("thread:")) {
+    const rest = trimmed.slice("thread:".length);
+    const slash = rest.indexOf("/");
+    if (slash > 0) {
+      return {
+        conversation: { id: rest.slice(0, slash), kind: "channel" },
+        threadId: rest.slice(slash + 1),
+      };
+    }
+  }
+  if (trimmed.startsWith("channel:")) {
+    return {
+      conversation: { id: trimmed.slice("channel:".length), kind: "channel" },
+    };
+  }
+  if (trimmed.startsWith("group:")) {
+    return {
+      conversation: { id: trimmed.slice("group:".length), kind: "group" },
+    };
+  }
+  if (trimmed.startsWith("dm:")) {
+    return {
+      conversation: { id: trimmed.slice("dm:".length), kind: "direct" },
+    };
+  }
+  return {
+    conversation: { id: trimmed, kind: "direct" },
+  };
+}
+
+export function cloneMessage(message: QaBusMessage): QaBusMessage {
+  return {
+    ...message,
+    conversation: { ...message.conversation },
+    attachments: (message.attachments ?? []).map((attachment) => cloneAttachment(attachment)),
+    ...(message.nativeCommand ? { nativeCommand: { ...message.nativeCommand } } : {}),
+    toolCalls: message.toolCalls?.map((toolCall) => cloneToolCall(toolCall)),
+    reactions: message.reactions.map((reaction) => ({ ...reaction })),
+  };
+}
+
+function cloneAttachment(attachment: QaBusAttachment): QaBusAttachment {
+  return { ...attachment };
+}
+
+function cloneToolCall(toolCall: QaBusToolCall): QaBusToolCall {
+  return {
+    name: toolCall.name,
+    ...(toolCall.arguments ? { arguments: structuredClone(toolCall.arguments) } : {}),
+  };
+}
+
+export function cloneEvent(event: QaBusEvent): QaBusEvent {
+  switch (event.kind) {
+    case "inbound-message":
+    case "outbound-message":
+    case "message-edited":
+    case "message-deleted":
+    case "reaction-added":
+      return { ...event, message: cloneMessage(event.message) };
+    case "thread-created":
+      return { ...event, thread: { ...event.thread } };
+  }
+  throw new Error("Unsupported QA bus event kind");
+}
+
+export function buildQaBusSnapshot(params: {
+  cursor: number;
+  conversations: Map<string, QaBusConversation>;
+  threads: Map<string, QaBusThread>;
+  messages: Map<string, QaBusMessage>;
+  events: QaBusEvent[];
+}): QaBusStateSnapshot {
+  return {
+    cursor: params.cursor,
+    conversations: Array.from(params.conversations.values()).map((conversation) =>
+      Object.assign({}, conversation),
+    ),
+    threads: Array.from(params.threads.values()).map((thread) => Object.assign({}, thread)),
+    messages: Array.from(params.messages.values()).map((message) => cloneMessage(message)),
+    events: params.events.map((event) => cloneEvent(event)),
+  };
+}
+
+export function readQaBusMessage(params: {
+  messages: Map<string, QaBusMessage>;
+  input: QaBusReadMessageInput;
+}) {
+  const message = params.messages.get(params.input.messageId);
+  if (!message) {
+    throw new Error(`qa-bus message not found: ${params.input.messageId}`);
+  }
+  return cloneMessage(message);
+}
+
+export function searchQaBusMessages(params: {
+  messages: Map<string, QaBusMessage>;
+  input: QaBusSearchMessagesInput;
+}) {
+  const accountId = normalizeAccountId(params.input.accountId);
+  const limit = Math.max(1, Math.min(params.input.limit ?? 20, 100));
+  const query = normalizeOptionalLowercaseString(params.input.query);
+  return Array.from(params.messages.values())
+    .filter((message) => message.accountId === accountId)
+    .filter((message) =>
+      params.input.conversationId ? message.conversation.id === params.input.conversationId : true,
+    )
+    .filter((message) =>
+      params.input.threadId ? message.threadId === params.input.threadId : true,
+    )
+    .filter((message) => {
+      if (!query) {
+        return true;
+      }
+      const attachmentHaystack = message.attachments ?? [];
+      const searchableAttachmentText = attachmentHaystack
+        .flatMap((attachment) => [
+          attachment.fileName,
+          attachment.altText,
+          attachment.transcript,
+          attachment.mimeType,
+        ])
+        .filter((value): value is string => Boolean(value))
+        .join(" ")
+        .toLowerCase();
+      const messageText = normalizeOptionalLowercaseString(message.text) ?? "";
+      const searchableToolText = (message.toolCalls ?? [])
+        .map((toolCall) => toolCall.name)
+        .join(" ")
+        .toLowerCase();
+      return `${messageText} ${searchableAttachmentText} ${searchableToolText}`.includes(query);
+    })
+    .slice(-limit)
+    .map((message) => cloneMessage(message));
+}
+
+export function resolveQaBusPollStartCursor(params: {
+  currentCursor: number;
+  requestedCursor?: number;
+}): number {
+  const requestedCursor = params.requestedCursor ?? 0;
+  return params.currentCursor < requestedCursor ? 0 : requestedCursor;
+}
+
+export function pollQaBusEvents(params: {
+  events: QaBusEvent[];
+  cursor: number;
+  input?: QaBusPollInput;
+}): QaBusPollResult {
+  const accountId = normalizeAccountId(params.input?.accountId);
+  const effectiveStartCursor = resolveQaBusPollStartCursor({
+    currentCursor: params.cursor,
+    requestedCursor: params.input?.cursor,
+  });
+  const limit = Math.max(1, Math.min(params.input?.limit ?? 100, 500));
+  const matches = params.events
+    .filter((event) => event.accountId === accountId && event.cursor > effectiveStartCursor)
+    .slice(0, limit)
+    .map((event) => cloneEvent(event));
+  return {
+    cursor: params.cursor,
+    events: matches,
+  };
+}

extensions/qa-lab/src/bus-server.test.ts

@@ -0,0 +1,252 @@
+// Qa Lab tests cover bus server plugin behavior.
+import { Agent, createServer, request } from "node:http";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { closeQaHttpServer, handleQaBusRequest, startQaBusServer } from "./bus-server.js";
+import { createQaBusState } from "./bus-state.js";
+import type { QaBusPollResult } from "./runtime-api.js";
+
+async function listenOnLoopback(server: ReturnType<typeof createServer>): Promise<number> {
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => resolve());
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("expected server to bind a TCP port");
+  }
+  return address.port;
+}
+
+async function requestOnce(params: { port: number; agent: Agent }): Promise<void> {
+  await new Promise<void>((resolve, reject) => {
+    const req = request(
+      {
+        host: "127.0.0.1",
+        port: params.port,
+        path: "/",
+        agent: params.agent,
+      },
+      (res) => {
+        res.resume();
+        res.on("end", resolve);
+        res.on("error", reject);
+      },
+    );
+    req.on("error", reject);
+    req.end();
+  });
+}
+
+async function pollQaBus(params: {
+  baseUrl: string;
+  accountId: string;
+  cursor: number;
+  timeoutMs: number;
+}): Promise<QaBusPollResult> {
+  const response = await fetch(`${params.baseUrl}/v1/poll`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({
+      accountId: params.accountId,
+      cursor: params.cursor,
+      timeoutMs: params.timeoutMs,
+    }),
+  });
+  if (!response.ok) {
+    throw new Error(`qa-bus request failed: ${response.status}`);
+  }
+  return (await response.json()) as QaBusPollResult;
+}
+
+async function postQaBusJson(baseUrl: string, path: string, body: unknown) {
+  return await postQaBusRawJson(baseUrl, path, JSON.stringify(body));
+}
+
+async function postQaBusRawJson(baseUrl: string, path: string, body: string) {
+  return await fetch(`${baseUrl}${path}`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+    },
+    body,
+  });
+}
+
+describe("closeQaHttpServer", () => {
+  it("closes idle keep-alive sockets so suite processes can exit", async () => {
+    const server = createServer((_req, res) => {
+      res.writeHead(200, {
+        "content-type": "text/plain",
+        connection: "keep-alive",
+      });
+      res.end("ok");
+    });
+    const agent = new Agent({ keepAlive: true });
+    const port = await listenOnLoopback(server);
+
+    try {
+      await requestOnce({ port, agent });
+      const startedAt = Date.now();
+      await closeQaHttpServer(server);
+      expect(Date.now() - startedAt).toBeLessThan(1_000);
+    } finally {
+      agent.destroy();
+      server.closeAllConnections?.();
+    }
+  });
+});
+
+describe("qa-bus server", () => {
+  const stops: Array<() => Promise<void>> = [];
+
+  beforeEach(() => {
+    vi.useRealTimers();
+  });
+
+  afterEach(async () => {
+    await Promise.all(stops.splice(0).map((stop) => stop()));
+  });
+
+  it("wakes stale-cursor long polls as soon as matching account traffic arrives", async () => {
+    const state = createQaBusState();
+    const bus = await startQaBusServer({ state });
+    stops.push(bus["stop"]);
+
+    const pending = pollQaBus({
+      baseUrl: bus.baseUrl,
+      accountId: "acct-a",
+      cursor: 999,
+      timeoutMs: 500,
+    });
+
+    state.addInboundMessage({
+      accountId: "acct-a",
+      conversation: { id: "target", kind: "direct" },
+      senderId: "acct-a-user",
+      text: "fresh event",
+    });
+
+    const result = await pending;
+    expect(result.events).toHaveLength(1);
+    expect(result.events[0]).toMatchObject({
+      accountId: "acct-a",
+      cursor: 1,
+      kind: "inbound-message",
+    });
+  });
+
+  it("rejects malformed poll numeric fields before long-polling", async () => {
+    const state = createQaBusState();
+    const bus = await startQaBusServer({ state });
+    stops.push(bus["stop"]);
+
+    const startedAt = Date.now();
+    const response = await postQaBusJson(bus.baseUrl, "/v1/poll", {
+      accountId: "acct-a",
+      cursor: "999",
+      timeoutMs: 500,
+    });
+
+    expect(Date.now() - startedAt).toBeLessThan(300);
+    expect(response.status).toBe(400);
+    await expect(response.json()).resolves.toEqual({
+      error: "poll cursor must be an integer at least 0.",
+    });
+  });
+
+  it("rejects malformed search limits before querying state", async () => {
+    const state = createQaBusState();
+    const bus = await startQaBusServer({ state });
+    stops.push(bus["stop"]);
+
+    const response = await postQaBusJson(bus.baseUrl, "/v1/actions/search", {
+      limit: "all",
+      query: "anything",
+    });
+
+    expect(response.status).toBe(400);
+    await expect(response.json()).resolves.toEqual({
+      error: "search limit must be an integer at least 1.",
+    });
+  });
+
+  it("keeps oversized numeric poll and search fields bounded", async () => {
+    const state = createQaBusState();
+    const bus = await startQaBusServer({ state });
+    stops.push(bus["stop"]);
+
+    const message = state.addInboundMessage({
+      accountId: "acct-a",
+      conversation: { id: "target", kind: "direct" },
+      senderId: "acct-a-user",
+      text: "bounded numeric fields",
+    });
+
+    const pollResponse = await postQaBusJson(bus.baseUrl, "/v1/poll", {
+      accountId: "acct-a",
+      cursor: 0,
+      limit: 10_000,
+      timeoutMs: 60_000,
+    });
+    expect(pollResponse.status).toBe(200);
+    await expect(pollResponse.json()).resolves.toMatchObject({
+      events: [{ message: { id: message.id } }],
+    });
+
+    const searchResponse = await postQaBusJson(bus.baseUrl, "/v1/actions/search", {
+      accountId: "acct-a",
+      limit: 10_000,
+      query: "bounded",
+    });
+    expect(searchResponse.status).toBe(200);
+    await expect(searchResponse.json()).resolves.toMatchObject({
+      messages: [{ id: message.id }],
+    });
+
+    const extremeSearchResponse = await postQaBusRawJson(
+      bus.baseUrl,
+      "/v1/actions/search",
+      `{"accountId":"acct-a","limit":1e309,"query":"bounded"}`,
+    );
+    expect(extremeSearchResponse.status).toBe(200);
+    await expect(extremeSearchResponse.json()).resolves.toMatchObject({
+      messages: [{ id: message.id }],
+    });
+  });
+});
+
+describe("handleQaBusRequest", () => {
+  it("returns a controlled error when a v1 POST body exceeds the limit", async () => {
+    const req = {
+      method: "POST",
+      url: "/v1/reset",
+      headers: { "content-length": String(1024 * 1024 + 1) },
+      destroyed: false,
+      destroy() {
+        this.destroyed = true;
+      },
+    };
+    const res = {
+      statusCode: 0,
+      body: "",
+      writeHead(statusCode: number) {
+        this.statusCode = statusCode;
+      },
+      end(payload: string) {
+        this.body = payload;
+      },
+    };
+
+    const handled = await handleQaBusRequest({
+      req: req as never,
+      res: res as never,
+      state: createQaBusState(),
+    });
+
+    expect(handled).toBe(true);
+    expect(res.statusCode).toBe(413);
+    expect(JSON.parse(res.body)).toEqual({ error: "Payload too large" });
+  });
+});

extensions/qa-lab/src/bus-server.ts

@@ -0,0 +1,287 @@
+// Qa Lab plugin module implements bus server behavior.
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import {
+  isRequestBodyLimitError,
+  readRequestBodyWithLimit,
+  requestBodyErrorToText,
+} from "openclaw/plugin-sdk/webhook-ingress";
+import { normalizeAccountId, resolveQaBusPollStartCursor } from "./bus-queries.js";
+import type { QaBusState } from "./bus-state.js";
+import type {
+  QaBusCreateThreadInput,
+  QaBusDeleteMessageInput,
+  QaBusEditMessageInput,
+  QaBusInboundMessageInput,
+  QaBusOutboundMessageInput,
+  QaBusPollInput,
+  QaBusReactToMessageInput,
+  QaBusReadMessageInput,
+  QaBusSearchMessagesInput,
+  QaBusWaitForInput,
+} from "./runtime-api.js";
+
+const QA_HTTP_JSON_MAX_BODY_BYTES = 1024 * 1024;
+const QA_HTTP_JSON_BODY_TIMEOUT_MS = 5_000;
+const QA_BUS_POLL_TIMEOUT_MAX_MS = 30_000;
+const QA_BUS_POLL_LIMIT_MAX = 500;
+const QA_BUS_SEARCH_LIMIT_MAX = 100;
+
+export async function readQaJsonBody(req: IncomingMessage): Promise<unknown> {
+  const text = (
+    await readRequestBodyWithLimit(req, {
+      maxBytes: QA_HTTP_JSON_MAX_BODY_BYTES,
+      timeoutMs: QA_HTTP_JSON_BODY_TIMEOUT_MS,
+    })
+  ).trim();
+  return text ? (JSON.parse(text) as unknown) : {};
+}
+
+export function writeJson(res: ServerResponse, statusCode: number, body: unknown) {
+  const payload = JSON.stringify(body);
+  res.writeHead(statusCode, {
+    "content-type": "application/json; charset=utf-8",
+    "content-length": Buffer.byteLength(payload),
+  });
+  res.end(payload);
+}
+
+export function writeError(res: ServerResponse, statusCode: number, error: unknown) {
+  writeJson(res, statusCode, {
+    error: formatErrorMessage(error),
+  });
+}
+
+export function writeQaRequestBodyLimitError(res: ServerResponse, error: unknown): boolean {
+  if (!isRequestBodyLimitError(error)) {
+    return false;
+  }
+  writeError(res, error.statusCode, requestBodyErrorToText(error.code));
+  return true;
+}
+
+function readOptionalIntegerField(
+  input: Record<string, unknown>,
+  field: string,
+  opts: {
+    label: string;
+    max?: number;
+    min: number;
+  },
+): number | undefined {
+  const value = input[field];
+  if (value === undefined) {
+    return undefined;
+  }
+  if (typeof value !== "number" || value < opts.min) {
+    throw new Error(`${opts.label} must be an integer at least ${opts.min}.`);
+  }
+  if (opts.max !== undefined && value > opts.max) {
+    return opts.max;
+  }
+  if (!Number.isSafeInteger(value)) {
+    throw new Error(`${opts.label} must be an integer at least ${opts.min}.`);
+  }
+  return opts.max === undefined ? value : Math.min(value, opts.max);
+}
+
+function normalizeQaBusPollInput(input: Record<string, unknown>): QaBusPollInput {
+  const cursor = readOptionalIntegerField(input, "cursor", {
+    label: "poll cursor",
+    min: 0,
+  });
+  const limit = readOptionalIntegerField(input, "limit", {
+    label: "poll limit",
+    max: QA_BUS_POLL_LIMIT_MAX,
+    min: 1,
+  });
+  const timeoutMs = readOptionalIntegerField(input, "timeoutMs", {
+    label: "poll timeoutMs",
+    max: QA_BUS_POLL_TIMEOUT_MAX_MS,
+    min: 0,
+  });
+  return {
+    ...input,
+    ...(cursor !== undefined ? { cursor } : {}),
+    ...(limit !== undefined ? { limit } : {}),
+    ...(timeoutMs !== undefined ? { timeoutMs } : {}),
+  } as QaBusPollInput;
+}
+
+function normalizeQaBusSearchInput(input: Record<string, unknown>): QaBusSearchMessagesInput {
+  const limit = readOptionalIntegerField(input, "limit", {
+    label: "search limit",
+    max: QA_BUS_SEARCH_LIMIT_MAX,
+    min: 1,
+  });
+  return {
+    ...input,
+    ...(limit !== undefined ? { limit } : {}),
+  } as QaBusSearchMessagesInput;
+}
+
+export async function closeQaHttpServer(server: Server): Promise<void> {
+  let forceCloseTimer: NodeJS.Timeout | undefined;
+  try {
+    await new Promise<void>((resolve, reject) => {
+      server.close((error) => (error ? reject(error) : resolve()));
+      server.closeIdleConnections?.();
+      forceCloseTimer = setTimeout(() => {
+        server.closeAllConnections?.();
+      }, 250);
+      forceCloseTimer.unref();
+    });
+  } finally {
+    if (forceCloseTimer) {
+      clearTimeout(forceCloseTimer);
+    }
+  }
+}
+
+export async function handleQaBusRequest(params: {
+  req: IncomingMessage;
+  res: ServerResponse;
+  state: QaBusState;
+}): Promise<boolean> {
+  const method = params.req.method ?? "GET";
+  const url = new URL(params.req.url ?? "/", "http://127.0.0.1");
+
+  if (method === "GET" && url.pathname === "/health") {
+    writeJson(params.res, 200, { ok: true });
+    return true;
+  }
+
+  if (method === "GET" && url.pathname === "/v1/state") {
+    writeJson(params.res, 200, params.state.getSnapshot());
+    return true;
+  }
+
+  if (!url.pathname.startsWith("/v1/")) {
+    return false;
+  }
+
+  if (method !== "POST") {
+    writeError(params.res, 405, "method not allowed");
+    return true;
+  }
+
+  try {
+    const body = (await readQaJsonBody(params.req)) as Record<string, unknown>;
+    switch (url.pathname) {
+      case "/v1/reset":
+        params.state.reset();
+        writeJson(params.res, 200, { ok: true });
+        return true;
+      case "/v1/inbound/message":
+        writeJson(params.res, 200, {
+          message: params.state.addInboundMessage(body as unknown as QaBusInboundMessageInput),
+        });
+        return true;
+      case "/v1/outbound/message":
+        writeJson(params.res, 200, {
+          message: params.state.addOutboundMessage(body as unknown as QaBusOutboundMessageInput),
+        });
+        return true;
+      case "/v1/actions/thread-create":
+        writeJson(params.res, 200, {
+          thread: params.state.createThread(body as unknown as QaBusCreateThreadInput),
+        });
+        return true;
+      case "/v1/actions/react":
+        writeJson(params.res, 200, {
+          message: params.state.reactToMessage(body as unknown as QaBusReactToMessageInput),
+        });
+        return true;
+      case "/v1/actions/edit":
+        writeJson(params.res, 200, {
+          message: params.state.editMessage(body as unknown as QaBusEditMessageInput),
+        });
+        return true;
+      case "/v1/actions/delete":
+        writeJson(params.res, 200, {
+          message: params.state.deleteMessage(body as unknown as QaBusDeleteMessageInput),
+        });
+        return true;
+      case "/v1/actions/read":
+        writeJson(params.res, 200, {
+          message: params.state.readMessage(body as unknown as QaBusReadMessageInput),
+        });
+        return true;
+      case "/v1/actions/search":
+        writeJson(params.res, 200, {
+          messages: params.state.searchMessages(normalizeQaBusSearchInput(body)),
+        });
+        return true;
+      case "/v1/poll": {
+        const input = normalizeQaBusPollInput(body);
+        const timeoutMs = input.timeoutMs ?? 0;
+        const accountId = normalizeAccountId(input.accountId);
+        const initial = params.state.poll(input);
+        const effectiveStartCursor = resolveQaBusPollStartCursor({
+          currentCursor: initial.cursor,
+          requestedCursor: input.cursor,
+        });
+        if (initial.events.length > 0 || timeoutMs === 0) {
+          writeJson(params.res, 200, initial);
+          return true;
+        }
+        try {
+          await params.state.waitForCursorAdvance(effectiveStartCursor, timeoutMs, (snapshot) => {
+            return snapshot.events.some(
+              (event) => event.accountId === accountId && event.cursor > effectiveStartCursor,
+            );
+          });
+        } catch {
+          // timeout ok for long-poll
+        }
+        writeJson(params.res, 200, params.state.poll(input));
+        return true;
+      }
+      case "/v1/wait":
+        writeJson(params.res, 200, {
+          match: await params.state.waitFor(body as unknown as QaBusWaitForInput),
+        });
+        return true;
+      default:
+        writeError(params.res, 404, "not found");
+        return true;
+    }
+  } catch (error) {
+    if (writeQaRequestBodyLimitError(params.res, error)) {
+      return true;
+    }
+    writeError(params.res, 400, error);
+    return true;
+  }
+}
+
+export function createQaBusServer(state: QaBusState): Server {
+  return createServer((req, res) => {
+    void (async () => {
+      const handled = await handleQaBusRequest({ req, res, state });
+      if (!handled) {
+        writeError(res, 404, "not found");
+      }
+    })();
+  });
+}
+
+export async function startQaBusServer(params: { state: QaBusState; port?: number }) {
+  const server = createQaBusServer(params.state);
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(params.port ?? 0, "127.0.0.1", () => resolve());
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("qa-bus failed to bind");
+  }
+  return {
+    server,
+    port: address.port,
+    baseUrl: `http://127.0.0.1:${address.port}`,
+    async stop() {
+      await closeQaHttpServer(server);
+    },
+  };
+}

extensions/qa-lab/src/bus-state.test.ts

@@ -0,0 +1,241 @@
+// Qa Lab tests cover bus state plugin behavior.
+import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
+import { describe, expect, it, vi } from "vitest";
+import { createQaBusState } from "./bus-state.js";
+
+describe("qa-bus state", () => {
+  it("records inbound and outbound traffic in cursor order", () => {
+    const state = createQaBusState();
+
+    const inbound = state.addInboundMessage({
+      conversation: { id: "alice", kind: "direct" },
+      senderId: "alice",
+      text: "hello",
+    });
+    const outbound = state.addOutboundMessage({
+      to: "dm:alice",
+      text: "hi",
+    });
+
+    const snapshot = state.getSnapshot();
+    expect(snapshot.cursor).toBe(2);
+    expect(snapshot.events.map((event) => event.kind)).toEqual([
+      "inbound-message",
+      "outbound-message",
+    ]);
+    expect(snapshot.messages.map((message) => message.id)).toEqual([inbound.id, outbound.id]);
+  });
+
+  it("creates threads and mutates message state", () => {
+    const state = createQaBusState();
+
+    const thread = state.createThread({
+      conversationId: "qa-room",
+      title: "QA thread",
+    });
+    const message = state.addOutboundMessage({
+      to: `thread:qa-room/${thread.id}`,
+      text: "inside thread",
+      threadId: thread.id,
+    });
+
+    state.reactToMessage({
+      messageId: message.id,
+      emoji: "eyes",
+      senderId: "alice",
+    });
+    state.editMessage({
+      messageId: message.id,
+      text: "inside thread (edited)",
+    });
+    state.deleteMessage({
+      messageId: message.id,
+    });
+
+    const snapshot = state.getSnapshot();
+    expect(snapshot.threads).toHaveLength(1);
+    expect(snapshot.threads[0]?.id).toBe(thread.id);
+    expect(snapshot.threads[0]?.conversationId).toBe("qa-room");
+    expect(snapshot.threads[0]?.title).toBe("QA thread");
+    expect(snapshot.messages[0]?.id).toBe(message.id);
+    expect(snapshot.messages[0]?.text).toBe("inside thread (edited)");
+    expect(snapshot.messages[0]?.deleted).toBe(true);
+    expect(snapshot.messages[0]?.reactions).toHaveLength(1);
+    expect(snapshot.messages[0]?.reactions[0]?.emoji).toBe("eyes");
+    expect(snapshot.messages[0]?.reactions[0]?.senderId).toBe("alice");
+    expect(typeof snapshot.messages[0]?.reactions[0]?.timestamp).toBe("number");
+  });
+
+  it("waits for a text match and rejects on timeout", async () => {
+    const state = createQaBusState();
+    const pending = state.waitFor({
+      kind: "message-text",
+      textIncludes: "needle",
+      timeoutMs: 500,
+    });
+
+    setTimeout(() => {
+      state.addOutboundMessage({
+        to: "dm:alice",
+        text: "haystack + needle",
+      });
+    }, 20);
+
+    const matched = await pending;
+    expect("text" in matched && matched.text).toContain("needle");
+
+    await expect(
+      state.waitFor({
+        kind: "message-text",
+        textIncludes: "missing",
+        timeoutMs: 20,
+      }),
+    ).rejects.toThrow("qa-bus wait timeout");
+  });
+
+  it("caps oversized wait timers", async () => {
+    vi.useFakeTimers();
+    const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
+    try {
+      const state = createQaBusState();
+      const pendingMessage = state.waitFor({
+        kind: "message-text",
+        textIncludes: "missing",
+        timeoutMs: Number.MAX_SAFE_INTEGER,
+      });
+      const pendingCursor = state.waitForCursorAdvance(0, Number.MAX_SAFE_INTEGER);
+
+      expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS);
+      expect(timeoutSpy).toHaveBeenCalledTimes(2);
+
+      pendingMessage.catch(() => undefined);
+      pendingCursor.catch(() => undefined);
+    } finally {
+      timeoutSpy.mockRestore();
+      vi.useRealTimers();
+    }
+  });
+
+  it("keeps account-scoped cursor waits blocked on unrelated account traffic", async () => {
+    const state = createQaBusState();
+    const pending = state.waitForCursorAdvance(0, 500, (snapshot) => {
+      return snapshot.events.some((event) => event.accountId === "acct-a" && event.cursor > 0);
+    });
+
+    state.addInboundMessage({
+      accountId: "acct-b",
+      conversation: { id: "other", kind: "direct" },
+      senderId: "acct-b-user",
+      text: "unrelated",
+    });
+
+    const beforeMatch = await Promise.race([
+      pending.then(() => "resolved"),
+      new Promise((resolve) => {
+        setTimeout(() => resolve("still-waiting"), 20);
+      }),
+    ]);
+    expect(beforeMatch).toBe("still-waiting");
+
+    state.addInboundMessage({
+      accountId: "acct-a",
+      conversation: { id: "target", kind: "direct" },
+      senderId: "acct-a-user",
+      text: "matched",
+    });
+
+    await expect(pending).resolves.toBeUndefined();
+  });
+
+  it("wakes default-account cursor waits when accountId is omitted", async () => {
+    const state = createQaBusState();
+    const pending = state.waitForCursorAdvance(0, 500, (snapshot) => {
+      return snapshot.events.some((event) => event.accountId === "default" && event.cursor > 0);
+    });
+
+    state.addInboundMessage({
+      conversation: { id: "target", kind: "direct" },
+      senderId: "default-user",
+      text: "matched",
+    });
+
+    await expect(pending).resolves.toBeUndefined();
+  });
+
+  it("preserves inline attachments and lets search match attachment metadata", () => {
+    const state = createQaBusState();
+
+    const outbound = state.addOutboundMessage({
+      to: "dm:alice",
+      text: "artifact attached",
+      attachments: [
+        {
+          id: "image-1",
+          kind: "image",
+          mimeType: "image/png",
+          fileName: "qa-screenshot.png",
+          altText: "QA dashboard screenshot",
+          contentBase64: "aGVsbG8=",
+        },
+      ],
+    });
+
+    const readback = state.readMessage({ messageId: outbound.id });
+    expect(readback.attachments).toHaveLength(1);
+    const attachment = readback.attachments?.[0];
+    expect(attachment?.kind).toBe("image");
+    expect(attachment?.fileName).toBe("qa-screenshot.png");
+    expect(attachment?.altText).toBe("QA dashboard screenshot");
+
+    const byFilename = state.searchMessages({
+      query: "screenshot",
+    });
+    expect(byFilename.map((message) => message.id)).toContain(outbound.id);
+
+    const byAltText = state.searchMessages({
+      query: "dashboard",
+    });
+    expect(byAltText.map((message) => message.id)).toContain(outbound.id);
+  });
+
+  it("preserves sanitized tool-call traces on bus messages", () => {
+    const state = createQaBusState();
+
+    const outbound = state.addOutboundMessage({
+      to: "dm:alice",
+      text: "used a tool",
+      toolCalls: [
+        {
+          name: "exec",
+          arguments: {
+            command: "pwd",
+            apiToken: "secret-token",
+          },
+        },
+      ],
+    });
+
+    const readback = state.readMessage({ messageId: outbound.id });
+    expect(readback.toolCalls).toEqual([
+      {
+        name: "exec",
+        arguments: {
+          command: "[redacted]",
+          apiToken: "[redacted]",
+        },
+      },
+    ]);
+    expect(state.searchMessages({ query: "exec" }).map((message) => message.id)).toContain(
+      outbound.id,
+    );
+
+    const readbackArguments = readback.toolCalls?.[0]?.arguments;
+    if (!readbackArguments) {
+      throw new Error("expected tool-call arguments");
+    }
+    readbackArguments.command = "mutated";
+    expect(state.readMessage({ messageId: outbound.id }).toolCalls?.[0]?.arguments?.command).toBe(
+      "[redacted]",
+    );
+  });
+});

extensions/qa-lab/src/bus-state.ts

@@ -0,0 +1,307 @@
+// Qa Lab plugin module implements bus state behavior.
+import { randomUUID } from "node:crypto";
+import { sanitizeQaBusToolCalls } from "openclaw/plugin-sdk/qa-channel-protocol";
+import {
+  buildQaBusSnapshot,
+  cloneMessage,
+  normalizeAccountId,
+  normalizeConversationFromTarget,
+  pollQaBusEvents,
+  readQaBusMessage,
+  searchQaBusMessages,
+} from "./bus-queries.js";
+import { createQaBusWaiterStore } from "./bus-waiters.js";
+import type {
+  QaBusAttachment,
+  QaBusConversation,
+  QaBusCreateThreadInput,
+  QaBusDeleteMessageInput,
+  QaBusEditMessageInput,
+  QaBusEvent,
+  QaBusInboundMessageInput,
+  QaBusMessage,
+  QaBusOutboundMessageInput,
+  QaBusPollInput,
+  QaBusReadMessageInput,
+  QaBusReactToMessageInput,
+  QaBusSearchMessagesInput,
+  QaBusStateSnapshot,
+  QaBusThread,
+  QaBusToolCall,
+  QaBusWaitForInput,
+} from "./runtime-api.js";
+
+const DEFAULT_BOT_ID = "openclaw";
+const DEFAULT_BOT_NAME = "OpenClaw QA";
+
+type QaBusEventSeed =
+  | {
+      kind: "inbound-message";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "outbound-message";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "thread-created";
+      accountId: string;
+      thread: QaBusThread;
+    }
+  | {
+      kind: "message-edited";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "message-deleted";
+      accountId: string;
+      message: QaBusMessage;
+    }
+  | {
+      kind: "reaction-added";
+      accountId: string;
+      message: QaBusMessage;
+      emoji: string;
+      senderId: string;
+    };
+
+export function createQaBusState() {
+  const conversations = new Map<string, QaBusConversation>();
+  const threads = new Map<string, QaBusThread>();
+  const messages = new Map<string, QaBusMessage>();
+  const events: QaBusEvent[] = [];
+  let cursor = 0;
+  const waiters = createQaBusWaiterStore(() =>
+    buildQaBusSnapshot({
+      cursor,
+      conversations,
+      threads,
+      messages,
+      events,
+    }),
+  );
+
+  const pushEvent = (event: QaBusEventSeed | ((cursor: number) => QaBusEventSeed)): QaBusEvent => {
+    cursor += 1;
+    const next = typeof event === "function" ? event(cursor) : event;
+    const finalized = { cursor, ...next } as QaBusEvent;
+    events.push(finalized);
+    waiters.settle();
+    return finalized;
+  };
+
+  const ensureConversation = (conversation: QaBusConversation): QaBusConversation => {
+    const existing = conversations.get(conversation.id);
+    if (existing) {
+      if (!existing.title && conversation.title) {
+        existing.title = conversation.title;
+      }
+      return existing;
+    }
+    const created = { ...conversation };
+    conversations.set(created.id, created);
+    return created;
+  };
+
+  const createMessage = (params: {
+    direction: QaBusMessage["direction"];
+    accountId: string;
+    conversation: QaBusConversation;
+    senderId: string;
+    senderName?: string;
+    text: string;
+    timestamp?: number;
+    threadId?: string;
+    threadTitle?: string;
+    replyToId?: string;
+    attachments?: QaBusAttachment[];
+    nativeCommand?: QaBusInboundMessageInput["nativeCommand"];
+    toolCalls?: QaBusToolCall[];
+  }): QaBusMessage => {
+    const conversation = ensureConversation(params.conversation);
+    const toolCalls = sanitizeQaBusToolCalls(params.toolCalls);
+    const message: QaBusMessage = {
+      id: randomUUID(),
+      accountId: params.accountId,
+      direction: params.direction,
+      conversation,
+      senderId: params.senderId,
+      senderName: params.senderName,
+      text: params.text,
+      timestamp: params.timestamp ?? Date.now(),
+      threadId: params.threadId,
+      threadTitle: params.threadTitle,
+      replyToId: params.replyToId,
+      attachments: params.attachments?.map((attachment) => ({ ...attachment })) ?? [],
+      ...(params.nativeCommand ? { nativeCommand: { ...params.nativeCommand } } : {}),
+      ...(toolCalls ? { toolCalls } : {}),
+      reactions: [],
+    };
+    messages.set(message.id, message);
+    return message;
+  };
+
+  return {
+    reset() {
+      conversations.clear();
+      threads.clear();
+      messages.clear();
+      events.length = 0;
+      // Keep the cursor monotonic across resets so long-poll clients do not
+      // miss fresh events after the bus is cleared mid-session.
+      waiters.reset();
+    },
+    getSnapshot() {
+      return buildQaBusSnapshot({
+        cursor,
+        conversations,
+        threads,
+        messages,
+        events,
+      });
+    },
+    addInboundMessage(input: QaBusInboundMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = createMessage({
+        direction: "inbound",
+        accountId,
+        conversation: input.conversation,
+        senderId: input.senderId,
+        senderName: input.senderName,
+        text: input.text,
+        timestamp: input.timestamp,
+        threadId: input.threadId,
+        threadTitle: input.threadTitle,
+        replyToId: input.replyToId,
+        attachments: input.attachments,
+        nativeCommand: input.nativeCommand,
+        toolCalls: input.toolCalls,
+      });
+      pushEvent({
+        kind: "inbound-message",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    addOutboundMessage(input: QaBusOutboundMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const { conversation, threadId } = normalizeConversationFromTarget(input.to);
+      const message = createMessage({
+        direction: "outbound",
+        accountId,
+        conversation,
+        senderId: input.senderId?.trim() || DEFAULT_BOT_ID,
+        senderName: input.senderName?.trim() || DEFAULT_BOT_NAME,
+        text: input.text,
+        timestamp: input.timestamp,
+        threadId: input.threadId ?? threadId,
+        replyToId: input.replyToId,
+        attachments: input.attachments,
+        toolCalls: input.toolCalls,
+      });
+      pushEvent({
+        kind: "outbound-message",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    createThread(input: QaBusCreateThreadInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const thread: QaBusThread = {
+        id: `thread-${randomUUID()}`,
+        accountId,
+        conversationId: input.conversationId,
+        title: input.title,
+        createdAt: input.timestamp ?? Date.now(),
+        createdBy: input.createdBy?.trim() || DEFAULT_BOT_ID,
+      };
+      threads.set(thread.id, thread);
+      ensureConversation({
+        id: input.conversationId,
+        kind: "channel",
+      });
+      pushEvent({
+        kind: "thread-created",
+        accountId,
+        thread: { ...thread },
+      });
+      return { ...thread };
+    },
+    reactToMessage(input: QaBusReactToMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = messages.get(input.messageId);
+      if (!message) {
+        throw new Error(`qa-bus message not found: ${input.messageId}`);
+      }
+      const reaction = {
+        emoji: input.emoji,
+        senderId: input.senderId?.trim() || DEFAULT_BOT_ID,
+        timestamp: input.timestamp ?? Date.now(),
+      };
+      message.reactions.push(reaction);
+      pushEvent({
+        kind: "reaction-added",
+        accountId,
+        message: cloneMessage(message),
+        emoji: reaction.emoji,
+        senderId: reaction.senderId,
+      });
+      return cloneMessage(message);
+    },
+    editMessage(input: QaBusEditMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = messages.get(input.messageId);
+      if (!message) {
+        throw new Error(`qa-bus message not found: ${input.messageId}`);
+      }
+      message.text = input.text;
+      message.editedAt = input.timestamp ?? Date.now();
+      pushEvent({
+        kind: "message-edited",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    deleteMessage(input: QaBusDeleteMessageInput) {
+      const accountId = normalizeAccountId(input.accountId);
+      const message = messages.get(input.messageId);
+      if (!message) {
+        throw new Error(`qa-bus message not found: ${input.messageId}`);
+      }
+      message.deleted = true;
+      pushEvent({
+        kind: "message-deleted",
+        accountId,
+        message: cloneMessage(message),
+      });
+      return cloneMessage(message);
+    },
+    readMessage(input: QaBusReadMessageInput) {
+      return readQaBusMessage({ messages, input });
+    },
+    searchMessages(input: QaBusSearchMessagesInput) {
+      return searchQaBusMessages({ messages, input });
+    },
+    poll(input: QaBusPollInput = {}) {
+      return pollQaBusEvents({ events, cursor, input });
+    },
+    async waitFor(input: QaBusWaitForInput) {
+      return await waiters.waitFor(input);
+    },
+    async waitForCursorAdvance(
+      afterCursor: number,
+      timeoutMs: number,
+      shouldResolve?: (snapshot: QaBusStateSnapshot) => boolean,
+    ) {
+      return await waiters.waitForCursorAdvance(afterCursor, timeoutMs, shouldResolve);
+    },
+  };
+}
+
+export type QaBusState = ReturnType<typeof createQaBusState>;

extensions/qa-lab/src/bus-waiters.ts

@@ -0,0 +1,138 @@
+// Qa Lab plugin module implements bus waiters behavior.
+import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
+import type {
+  QaBusEvent,
+  QaBusMessage,
+  QaBusStateSnapshot,
+  QaBusThread,
+  QaBusWaitForInput,
+} from "./runtime-api.js";
+
+export const DEFAULT_WAIT_TIMEOUT_MS = 5_000;
+
+export type QaBusWaitMatch = QaBusEvent | QaBusMessage | QaBusThread;
+
+type Waiter = {
+  resolve: (event: QaBusWaitMatch) => void;
+  reject: (error: Error) => void;
+  timer: NodeJS.Timeout;
+  matcher: (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null;
+};
+
+type CursorWaiter = {
+  resolve: () => void;
+  reject: (error: Error) => void;
+  timer: NodeJS.Timeout;
+  afterCursor: number;
+  shouldResolve?: (snapshot: QaBusStateSnapshot) => boolean;
+};
+
+function createQaBusMatcher(
+  input: QaBusWaitForInput,
+): (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null {
+  return (snapshot) => {
+    if (input.kind === "event-kind") {
+      return snapshot.events.find((event) => event.kind === input.eventKind) ?? null;
+    }
+    if (input.kind === "thread-id") {
+      return snapshot.threads.find((thread) => thread.id === input.threadId) ?? null;
+    }
+    return (
+      snapshot.messages.find(
+        (message) =>
+          (!input.direction || message.direction === input.direction) &&
+          message.text.includes(input.textIncludes),
+      ) ?? null
+    );
+  };
+}
+
+export function createQaBusWaiterStore(getSnapshot: () => QaBusStateSnapshot) {
+  const waiters = new Set<Waiter>();
+  const cursorWaiters = new Set<CursorWaiter>();
+
+  return {
+    reset(reason = "qa-bus reset") {
+      for (const waiter of waiters) {
+        clearTimeout(waiter.timer);
+        waiter.reject(new Error(reason));
+      }
+      waiters.clear();
+      for (const waiter of cursorWaiters) {
+        clearTimeout(waiter.timer);
+        waiter.reject(new Error(reason));
+      }
+      cursorWaiters.clear();
+    },
+    settle() {
+      if (waiters.size === 0 && cursorWaiters.size === 0) {
+        return;
+      }
+      const snapshot = getSnapshot();
+      for (const waiter of Array.from(waiters)) {
+        const match = waiter.matcher(snapshot);
+        if (!match) {
+          continue;
+        }
+        clearTimeout(waiter.timer);
+        waiters.delete(waiter);
+        waiter.resolve(match);
+      }
+      for (const waiter of Array.from(cursorWaiters)) {
+        if (snapshot.cursor <= waiter.afterCursor) {
+          continue;
+        }
+        if (waiter.shouldResolve && !waiter.shouldResolve(snapshot)) {
+          continue;
+        }
+        clearTimeout(waiter.timer);
+        cursorWaiters.delete(waiter);
+        waiter.resolve();
+      }
+    },
+    async waitFor(input: QaBusWaitForInput) {
+      const matcher = createQaBusMatcher(input);
+      const immediate = matcher(getSnapshot());
+      if (immediate) {
+        return immediate;
+      }
+      return await new Promise<QaBusWaitMatch>((resolve, reject) => {
+        const timeoutMs = resolveTimerTimeoutMs(input.timeoutMs, DEFAULT_WAIT_TIMEOUT_MS, 0);
+        const waiter: Waiter = {
+          resolve,
+          reject,
+          matcher,
+          timer: setTimeout(() => {
+            waiters.delete(waiter);
+            reject(new Error(`qa-bus wait timeout after ${timeoutMs}ms`));
+          }, timeoutMs),
+        };
+        waiters.add(waiter);
+      });
+    },
+    async waitForCursorAdvance(
+      afterCursor: number,
+      timeoutMs: number,
+      shouldResolve?: (snapshot: QaBusStateSnapshot) => boolean,
+    ) {
+      const snapshot = getSnapshot();
+      if (snapshot.cursor > afterCursor && (!shouldResolve || shouldResolve(snapshot))) {
+        return;
+      }
+      return await new Promise<void>((resolve, reject) => {
+        const resolvedTimeoutMs = resolveTimerTimeoutMs(timeoutMs, DEFAULT_WAIT_TIMEOUT_MS, 0);
+        const waiter: CursorWaiter = {
+          resolve,
+          reject,
+          afterCursor,
+          shouldResolve,
+          timer: setTimeout(() => {
+            cursorWaiters.delete(waiter);
+            reject(new Error(`qa-bus wait timeout after ${resolvedTimeoutMs}ms`));
+          }, resolvedTimeoutMs),
+        };
+        cursorWaiters.add(waiter);
+      });
+    },
+  };
+}

extensions/qa-lab/src/character-eval.test.ts

@@ -0,0 +1,729 @@
+// Qa Lab tests cover character eval plugin behavior.
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  runQaCharacterEval,
+  type QaCharacterEvalJudgment,
+  type QaCharacterEvalParams,
+} from "./character-eval.js";
+import type { QaSuiteResult } from "./suite.js";
+
+type CharacterRunSuiteParams = Parameters<NonNullable<QaCharacterEvalParams["runSuite"]>>[0];
+type CharacterRunJudgeParams = Parameters<NonNullable<QaCharacterEvalParams["runJudge"]>>[0];
+type TestJudgeRanking = Pick<QaCharacterEvalJudgment, "model" | "rank" | "score" | "summary"> &
+  Partial<Pick<QaCharacterEvalJudgment, "strengths" | "weaknesses">>;
+
+function makeJudgeReply(rankings: TestJudgeRanking[]) {
+  return JSON.stringify({ rankings });
+}
+
+function makeRunJudge(rankings: TestJudgeRanking[]) {
+  return vi.fn(async (_params: CharacterRunJudgeParams) => makeJudgeReply(rankings));
+}
+
+function defaultModelTranscript(model: string) {
+  return `USER Alice: hi\n\nASSISTANT openclaw: reply from ${model}`;
+}
+
+function makeReplySuiteResult(params: CharacterRunSuiteParams, transcript?: string) {
+  return makeSuiteResult({
+    outputDir: params.outputDir,
+    model: params.primaryModel,
+    transcript: transcript ?? defaultModelTranscript(params.primaryModel),
+  });
+}
+
+function makeRunSuite(transcriptForModel: (model: string) => string = defaultModelTranscript) {
+  return vi.fn(async (params: CharacterRunSuiteParams) =>
+    makeReplySuiteResult(params, transcriptForModel(params.primaryModel)),
+  );
+}
+
+function createConcurrencyGate(expectedActive: number) {
+  let active = 0;
+  let maxActive = 0;
+  let releaseStartedTasks = false;
+  let resolveExpectedActive: () => void = () => {};
+  const expectedActiveReached = new Promise<void>((resolve) => {
+    resolveExpectedActive = resolve;
+  });
+  const taskReleases: Array<() => void> = [];
+  const releaseQueuedTasks = () => {
+    if (!releaseStartedTasks) {
+      return;
+    }
+    let releaseTask: (() => void) | undefined;
+    while ((releaseTask = taskReleases.shift())) {
+      releaseTask();
+    }
+  };
+
+  return {
+    get maxActive() {
+      return maxActive;
+    },
+    async run<T>(work: () => T | Promise<T>): Promise<T> {
+      active += 1;
+      maxActive = Math.max(maxActive, active);
+      if (active >= expectedActive) {
+        resolveExpectedActive();
+      }
+      await new Promise<void>((resolve) => {
+        taskReleases.push(resolve);
+        releaseQueuedTasks();
+      });
+      try {
+        return await work();
+      } finally {
+        active -= 1;
+      }
+    },
+    async waitForExpectedActive(): Promise<void> {
+      await expectedActiveReached;
+    },
+    releaseStartedTasks(): void {
+      releaseStartedTasks = true;
+      releaseQueuedTasks();
+    },
+  };
+}
+
+async function makeSuiteResult(params: {
+  outputDir: string;
+  model: string;
+  transcript: string;
+  resultStatus?: "pass" | "fail";
+  summaryStatus?: "pass" | "fail";
+  summaryFailedCount?: number;
+}) {
+  const resultStatus = params.resultStatus ?? "pass";
+  const summaryStatus = params.summaryStatus ?? resultStatus;
+  const summaryFailedCount = params.summaryFailedCount ?? (summaryStatus === "fail" ? 1 : 0);
+  const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
+  await fs.mkdir(params.outputDir, { recursive: true });
+  await fs.writeFile(
+    summaryPath,
+    `${JSON.stringify(
+      {
+        counts: {
+          total: 1,
+          passed: summaryFailedCount > 0 ? 0 : 1,
+          failed: summaryFailedCount,
+        },
+        scenarios: [
+          {
+            name: "Character vibes",
+            status: summaryStatus,
+            steps: [],
+          },
+        ],
+      },
+      null,
+      2,
+    )}\n`,
+    "utf8",
+  );
+  return {
+    outputDir: params.outputDir,
+    evidencePath: path.join(params.outputDir, "qa-evidence.json"),
+    reportPath: path.join(params.outputDir, "qa-suite-report.md"),
+    summaryPath,
+    report: "# report",
+    watchUrl: "http://127.0.0.1:43124",
+    scenarios: [
+      {
+        name: "Character vibes",
+        status: resultStatus,
+        steps: [
+          {
+            name: `transcript for ${params.model}`,
+            status: "pass",
+            details: params.transcript,
+          },
+        ],
+      },
+    ],
+  } satisfies QaSuiteResult;
+}
+
+function requireRunSuiteParams(runSuite: ReturnType<typeof vi.fn>, index = 0) {
+  const params = runSuite.mock.calls[index]?.[0] as CharacterRunSuiteParams | undefined;
+  if (!params) {
+    throw new Error(`runSuite call ${index} missing`);
+  }
+  return params;
+}
+
+function requireRunJudgeParams(runJudge: ReturnType<typeof vi.fn>, index = 0) {
+  const params = runJudge.mock.calls[index]?.[0] as CharacterRunJudgeParams | undefined;
+  if (!params) {
+    throw new Error(`runJudge call ${index} missing`);
+  }
+  return params;
+}
+
+function expectFirstRunFailure(
+  result: Awaited<ReturnType<typeof runQaCharacterEval>>,
+  expected: { model: string; error: string },
+) {
+  const run = result.runs[0];
+  expect(run?.model).toBe(expected.model);
+  expect(run?.status).toBe("fail");
+  expect(run?.error).toBe(expected.error);
+}
+
+describe("runQaCharacterEval", () => {
+  let tempRoot: string;
+
+  beforeEach(async () => {
+    tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-character-eval-test-"));
+  });
+
+  afterEach(async () => {
+    await fs.rm(tempRoot, { recursive: true, force: true });
+  });
+
+  it("runs each requested model and writes a judged report with transcripts", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
+      const model = params.primaryModel;
+      const transcript = `USER Alice: prompt for ${model}\n\nASSISTANT openclaw: reply from ${model}`;
+      return makeSuiteResult({ outputDir: params.outputDir, model, transcript });
+    });
+    const runJudge = makeRunJudge([
+      {
+        model: "openai/gpt-5.5",
+        rank: 1,
+        score: 9.1,
+        summary: "Most natural.",
+        strengths: ["vivid"],
+        weaknesses: ["none"],
+      },
+      {
+        model: "codex-cli/test-model",
+        rank: 2,
+        score: 7,
+        summary: "Readable but flatter.",
+        strengths: ["coherent"],
+        weaknesses: ["less funny"],
+      },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5", "codex-cli/test-model", "openai/gpt-5.5"],
+      scenarioId: "character-vibes-gollum",
+      candidateFastMode: true,
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite).toHaveBeenCalledTimes(2);
+    const firstRunParams = requireRunSuiteParams(runSuite);
+    expect(firstRunParams.providerMode).toBe("live-frontier");
+    expect(firstRunParams.primaryModel).toBe("openai/gpt-5.5");
+    expect(firstRunParams.alternateModel).toBe("openai/gpt-5.5");
+    expect(firstRunParams.fastMode).toBe(true);
+    expect(firstRunParams.scenarioIds).toEqual(["character-vibes-gollum"]);
+    const judgeParams = requireRunJudgeParams(runJudge);
+    expect(judgeParams.judgeModel).toBe("openai/gpt-5.5");
+    expect(judgeParams.judgeThinkingDefault).toBe("xhigh");
+    expect(judgeParams.judgeFastMode).toBe(true);
+    expect(judgeParams.timeoutMs).toBe(300_000);
+    expect(result.judgments).toHaveLength(1);
+    expect(result.judgments[0]?.rankings.map((ranking) => ranking.model)).toEqual([
+      "openai/gpt-5.5",
+      "codex-cli/test-model",
+    ]);
+
+    const report = await fs.readFile(result.reportPath, "utf8");
+    expect(report).toContain("Execution: local QA gateway child processes, not Docker");
+    expect(report).toContain("Judges: openai/gpt-5.5");
+    expect(report).toContain("Judge model labels: visible");
+    expect(report).toContain("## Judge Rankings");
+    expect(report).toContain("### openai/gpt-5.5");
+    expect(report).toContain("reply from openai/gpt-5.5");
+    expect(report).toContain("reply from codex-cli/test-model");
+    expect(report).toContain("Judge thinking: xhigh");
+    expect(report).toContain("- Timeout: 5m");
+    expect(report).toContain("Fast mode: on");
+    expect(report).toContain("Duration:");
+    expect(report).not.toContain("Duration ms:");
+    expect(report).not.toContain("Judge Raw Reply");
+  });
+
+  it("creates a unique default output directory under repo artifacts", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "USER Alice: hi\n\nASSISTANT openclaw: default dir reply",
+      }),
+    );
+    const runJudge = makeRunJudge([
+      {
+        model: "openai/gpt-5.5",
+        rank: 1,
+        score: 8,
+        summary: "solid",
+        strengths: ["clear"],
+        weaknesses: [],
+      },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      models: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(path.dirname(result.outputDir)).toBe(path.join(tempRoot, ".artifacts", "qa-e2e"));
+    expect(path.basename(result.outputDir)).toMatch(
+      /^character-eval-[a-z0-9]+-[a-f0-9]{8}$/u,
+    );
+    await expect(fs.stat(result.reportPath).then((stats) => stats.isFile())).resolves.toBe(true);
+  });
+
+  it("can hide candidate model refs from judge prompts and map rankings back", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "USER Alice: hi\n\nASSISTANT openclaw: anonymous reply",
+      }),
+    );
+    const runJudge = vi.fn(async (params: CharacterRunJudgeParams) => {
+      expect(params.prompt).toContain("## CANDIDATE candidate-01");
+      expect(params.prompt).toContain("## CANDIDATE candidate-02");
+      expect(params.prompt).not.toContain("openai/gpt-5.5");
+      expect(params.prompt).not.toContain("codex-cli/test-model");
+      return makeJudgeReply([
+        {
+          model: "candidate-02",
+          rank: 1,
+          score: 9.1,
+          summary: "Better vibes.",
+        },
+        {
+          model: "candidate-01",
+          rank: 2,
+          score: 7.4,
+          summary: "Solid.",
+        },
+      ]);
+    });
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5", "codex-cli/test-model"],
+      judgeModels: ["openai/gpt-5.5"],
+      judgeBlindModels: true,
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.judgments[0]?.blindModels).toBe(true);
+    expect(result.judgments[0]?.rankings.map((ranking) => ranking.model)).toEqual([
+      "codex-cli/test-model",
+      "openai/gpt-5.5",
+    ]);
+    const report = await fs.readFile(result.reportPath, "utf8");
+    expect(report).toContain("Judge model labels: blind");
+    expect(report).toContain("1. codex-cli/test-model - 9.1 - Better vibes.");
+  });
+
+  it("defaults to the character eval model panel when no models are provided", async () => {
+    const runSuite = makeRunSuite();
+    const runJudge = makeRunJudge([
+      { model: "openai/gpt-5.5", rank: 1, score: 8, summary: "ok" },
+      { model: "openai/gpt-5.2", rank: 2, score: 7.5, summary: "ok" },
+      { model: "openai/gpt-5", rank: 3, score: 7.2, summary: "ok" },
+      { model: "anthropic/claude-opus-4-8", rank: 4, score: 7, summary: "ok" },
+      { model: "anthropic/claude-sonnet-4-6", rank: 5, score: 6.8, summary: "ok" },
+      { model: "zai/glm-5.1", rank: 6, score: 6.3, summary: "ok" },
+      { model: "moonshot/kimi-k2.5", rank: 7, score: 6.2, summary: "ok" },
+      { model: "google/gemini-3.1-pro-preview", rank: 8, score: 6, summary: "ok" },
+    ]);
+
+    await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: [],
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite).toHaveBeenCalledTimes(8);
+    expect(runSuite.mock.calls.map(([params]) => params.primaryModel)).toEqual([
+      "openai/gpt-5.5",
+      "openai/gpt-5.2",
+      "openai/gpt-5",
+      "anthropic/claude-opus-4-8",
+      "anthropic/claude-sonnet-4-6",
+      "zai/glm-5.1",
+      "moonshot/kimi-k2.5",
+      "google/gemini-3.1-pro-preview",
+    ]);
+    expect(runSuite.mock.calls.map(([params]) => params.thinkingDefault)).toEqual([
+      "medium",
+      "xhigh",
+      "xhigh",
+      "high",
+      "high",
+      "high",
+      "high",
+      "high",
+    ]);
+    expect(runSuite.mock.calls.map(([params]) => params.fastMode)).toEqual([
+      true,
+      true,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+    ]);
+    expect(runJudge).toHaveBeenCalledTimes(2);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeModel)).toEqual([
+      "openai/gpt-5.5",
+      "anthropic/claude-opus-4-8",
+    ]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeThinkingDefault)).toEqual([
+      "xhigh",
+      "high",
+    ]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeFastMode)).toEqual([true, false]);
+  });
+
+  it("runs candidate models with bounded concurrency while preserving result order", async () => {
+    const runGate = createConcurrencyGate(2);
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      runGate.run(() => makeReplySuiteResult(params)),
+    );
+    const runJudge = makeRunJudge([
+      { model: "openai/gpt-5.5", rank: 1, score: 8, summary: "ok" },
+      { model: "anthropic/claude-sonnet-4-6", rank: 2, score: 7, summary: "ok" },
+      { model: "moonshot/kimi-k2.5", rank: 3, score: 6, summary: "ok" },
+    ]);
+
+    const resultPromise = runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5", "anthropic/claude-sonnet-4-6", "moonshot/kimi-k2.5"],
+      candidateConcurrency: 2,
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    await runGate.waitForExpectedActive();
+    expect(runGate.maxActive).toBe(2);
+    runGate.releaseStartedTasks();
+    const result = await resultPromise;
+    expect(result.runs.map((run) => run.model)).toEqual([
+      "openai/gpt-5.5",
+      "anthropic/claude-sonnet-4-6",
+      "moonshot/kimi-k2.5",
+    ]);
+  });
+
+  it("defaults candidate and judge concurrency to sixteen", async () => {
+    const runGate = createConcurrencyGate(16);
+    const judgeGate = createConcurrencyGate(16);
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      runGate.run(() => makeReplySuiteResult(params)),
+    );
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) => {
+      return await judgeGate.run(() =>
+        makeJudgeReply(
+          Array.from({ length: 20 }, (_, index) => ({
+            model: `provider/model-${index + 1}`,
+            rank: index + 1,
+            score: 10 - index,
+            summary: "ok",
+          })),
+        ),
+      );
+    });
+
+    const resultPromise = runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: Array.from({ length: 20 }, (_, index) => `provider/model-${index + 1}`),
+      judgeModels: Array.from({ length: 20 }, (_, index) => `judge/model-${index + 1}`),
+      runSuite,
+      runJudge,
+    });
+
+    await runGate.waitForExpectedActive();
+    expect(runGate.maxActive).toBe(16);
+    runGate.releaseStartedTasks();
+    await judgeGate.waitForExpectedActive();
+    expect(judgeGate.maxActive).toBe(16);
+    judgeGate.releaseStartedTasks();
+    await resultPromise;
+  });
+
+  it("marks raw provider error transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "USER Alice: Are you awake?\n\nASSISTANT OpenClaw QA: 400 model `qwen3.6-plus` is not supported.",
+      }),
+    );
+    const runJudge = makeRunJudge([
+      { model: "qwen/qwen3.6-plus", rank: 1, score: 0.5, summary: "failed" },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["qwen/qwen3.6-plus"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expectFirstRunFailure(result, {
+      model: "qwen/qwen3.6-plus",
+      error: "model unsupported error leaked into transcript",
+    });
+  });
+
+  it("marks candidates failed when the suite summary has failed scenarios", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "USER Alice: hi\n\nASSISTANT openclaw: outwardly fine",
+        summaryStatus: "fail",
+        summaryFailedCount: 1,
+      }),
+    );
+    const runJudge = makeRunJudge([
+      { model: "openai/gpt-5.5", rank: 1, score: 0.5, summary: "failed" },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs[0]?.status).toBe("fail");
+    expect(result.runs[0]?.error).toBeUndefined();
+  });
+
+  it("marks raw tool failure transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "ASSISTANT OpenClaw QA: ⚠️ ✍️ Write: to /tmp/precious.html failed",
+      }),
+    );
+    const runJudge = makeRunJudge([
+      { model: "qwen/qwen3.5-plus", rank: 1, score: 0.5, summary: "failed" },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["qwen/qwen3.5-plus"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expectFirstRunFailure(result, {
+      model: "qwen/qwen3.5-plus",
+      error: "tool failure leaked into transcript",
+    });
+  });
+
+  it("marks generic channel fallback transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "ASSISTANT OpenClaw QA: ⚠️ Something went wrong while processing your request. Please try again, or use /new to start a fresh session.",
+      }),
+    );
+    const runJudge = makeRunJudge([
+      { model: "qa/generic-fallback-model", rank: 1, score: 0.5, summary: "failed" },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["qa/generic-fallback-model"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expectFirstRunFailure(result, {
+      model: "qa/generic-fallback-model",
+      error: "generic request failure leaked into transcript",
+    });
+  });
+
+  it("marks idle-timeout fallback transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "ASSISTANT OpenClaw QA: The model did not produce a response before the LLM idle timeout. Please try again, or increase `agents.defaults.llm.idleTimeoutSeconds` in your config.",
+      }),
+    );
+    const runJudge = makeRunJudge([
+      { model: "google/gemini-test", rank: 1, score: 0.5, summary: "failed" },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["google/gemini-test"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expectFirstRunFailure(result, {
+      model: "google/gemini-test",
+      error: "LLM timeout leaked into transcript",
+    });
+  });
+
+  it("marks leaked harness coordination transcripts as failed output", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript:
+          "ASSISTANT OpenClaw QA: checking thread context; then post a tight progress reply here.\nQA_LEAK_OK",
+      }),
+    );
+    const runJudge = makeRunJudge([
+      { model: "codex/gpt-5.5", rank: 1, score: 0.5, summary: "failed" },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["codex/gpt-5.5"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expectFirstRunFailure(result, {
+      model: "codex/gpt-5.5",
+      error: "internal harness/meta text leaked into transcript",
+    });
+  });
+
+  it("lets explicit candidate thinking override the default panel", async () => {
+    const runSuite = makeRunSuite();
+    const runJudge = makeRunJudge([
+      { model: "openai/gpt-5.5", rank: 1, score: 8, summary: "ok" },
+      { model: "moonshot/kimi-k2.5", rank: 2, score: 7, summary: "ok" },
+    ]);
+
+    await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5", "moonshot/kimi-k2.5"],
+      candidateThinkingDefault: "medium",
+      candidateThinkingByModel: { "moonshot/kimi-k2.5": "high" },
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite.mock.calls.map(([params]) => params.thinkingDefault)).toEqual([
+      "medium",
+      "high",
+    ]);
+  });
+
+  it("lets model-specific options override candidate and judge defaults", async () => {
+    const runSuite = makeRunSuite();
+    const runJudge = makeRunJudge([{ model: "openai/gpt-5.5", rank: 1, score: 8, summary: "ok" }]);
+
+    await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5", "moonshot/kimi-k2.5"],
+      candidateFastMode: true,
+      candidateThinkingDefault: "medium",
+      candidateModelOptions: {
+        "openai/gpt-5.5": { thinkingDefault: "xhigh", fastMode: false },
+      },
+      judgeModels: ["openai/gpt-5.5", "anthropic/claude-opus-4-8"],
+      judgeThinkingDefault: "medium",
+      judgeModelOptions: {
+        "openai/gpt-5.5": { thinkingDefault: "xhigh", fastMode: true },
+        "anthropic/claude-opus-4-8": { thinkingDefault: "high" },
+      },
+      runSuite,
+      runJudge,
+    });
+
+    expect(runSuite.mock.calls.map(([params]) => params.thinkingDefault)).toEqual([
+      "xhigh",
+      "medium",
+    ]);
+    expect(runSuite.mock.calls.map(([params]) => params.fastMode)).toEqual([false, true]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeThinkingDefault)).toEqual([
+      "xhigh",
+      "high",
+    ]);
+    expect(runJudge.mock.calls.map(([params]) => params.judgeFastMode)).toEqual([true, false]);
+  });
+
+  it("keeps failed model runs in the report for grader context", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
+      if (params.primaryModel === "codex-cli/test-model") {
+        throw new Error("backend unavailable");
+      }
+      return makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "USER Alice: hi\n\nASSISTANT openclaw: hello",
+      });
+    });
+    const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) =>
+      JSON.stringify({
+        rankings: [{ model: "openai/gpt-5.5", rank: 1, score: 8, summary: "ok" }],
+      }),
+    );
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5", "codex-cli/test-model"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs.map((run) => run.status)).toEqual(["pass", "fail"]);
+    expect(result.runs[1]?.error).toContain("backend unavailable");
+    const report = await fs.readFile(result.reportPath, "utf8");
+    expect(report).toContain("backend unavailable");
+  });
+});

extensions/qa-lab/src/character-eval.ts

@@ -0,0 +1,732 @@
+// Qa Lab plugin module implements character eval behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
+import { createQaArtifactRunId } from "./artifact-run-id.js";
+import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js";
+import {
+  QA_FRONTIER_CHARACTER_EVAL_MODELS,
+  QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS,
+  QA_FRONTIER_CHARACTER_JUDGE_MODELS,
+  QA_FRONTIER_CHARACTER_THINKING_BY_MODEL,
+} from "./providers/live-frontier/character-eval.js";
+import type { QaThinkingLevel } from "./qa-gateway-config.js";
+import { extractQaVisibleReplyLeakText } from "./reply-failure.js";
+import { readQaSuiteFailedScenarioCountFromFile } from "./suite-summary.js";
+import type { QaSuiteResult } from "./suite.js";
+
+const DEFAULT_CHARACTER_SCENARIO_ID = "character-vibes-gollum";
+const DEFAULT_CHARACTER_EVAL_MODELS = QA_FRONTIER_CHARACTER_EVAL_MODELS;
+const DEFAULT_CHARACTER_THINKING: QaThinkingLevel = "high";
+const DEFAULT_CHARACTER_EVAL_CONCURRENCY = 16;
+const DEFAULT_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
+  QA_FRONTIER_CHARACTER_THINKING_BY_MODEL;
+const DEFAULT_JUDGE_MODELS = QA_FRONTIER_CHARACTER_JUDGE_MODELS;
+const DEFAULT_JUDGE_THINKING: QaThinkingLevel = "xhigh";
+const DEFAULT_JUDGE_TIMEOUT_MS = 300_000;
+const DEFAULT_JUDGE_MODEL_OPTIONS: Readonly<Record<string, QaCharacterModelOptions>> =
+  QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS;
+
+type QaCharacterRunStatus = "pass" | "fail";
+
+export type QaCharacterModelOptions = {
+  thinkingDefault?: QaThinkingLevel;
+  fastMode?: boolean;
+};
+
+type QaCharacterEvalRun = {
+  model: string;
+  status: QaCharacterRunStatus;
+  durationMs: number;
+  outputDir: string;
+  thinkingDefault: QaThinkingLevel;
+  fastMode: boolean;
+  reportPath?: string;
+  summaryPath?: string;
+  transcript: string;
+  stats: {
+    transcriptChars: number;
+    transcriptLines: number;
+    userTurns: number;
+    assistantTurns: number;
+  };
+  error?: string;
+};
+
+export type QaCharacterEvalJudgment = {
+  model: string;
+  rank: number;
+  score: number;
+  summary: string;
+  strengths: string[];
+  weaknesses: string[];
+};
+
+type QaCharacterEvalResult = {
+  outputDir: string;
+  reportPath: string;
+  summaryPath: string;
+  runs: QaCharacterEvalRun[];
+  judgments: QaCharacterEvalJudgeResult[];
+};
+
+type QaCharacterEvalJudgeResult = {
+  model: string;
+  thinkingDefault: QaThinkingLevel;
+  fastMode: boolean;
+  blindModels: boolean;
+  timeoutMs: number;
+  durationMs: number;
+  rankings: QaCharacterEvalJudgment[];
+  error?: string;
+};
+
+type QaCharacterEvalProgressLogger = (message: string) => void;
+
+type RunSuiteFn = (params: {
+  repoRoot: string;
+  outputDir: string;
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+  fastMode?: boolean;
+  thinkingDefault?: QaThinkingLevel;
+  scenarioIds: string[];
+}) => Promise<QaSuiteResult>;
+
+type RunJudgeFn = (params: {
+  repoRoot: string;
+  judgeModel: string;
+  judgeThinkingDefault: QaThinkingLevel;
+  judgeFastMode: boolean;
+  prompt: string;
+  timeoutMs: number;
+}) => Promise<string | null>;
+
+export type QaCharacterEvalParams = {
+  repoRoot?: string;
+  outputDir?: string;
+  models: string[];
+  scenarioId?: string;
+  candidateFastMode?: boolean;
+  candidateThinkingDefault?: QaThinkingLevel;
+  candidateThinkingByModel?: Record<string, QaThinkingLevel>;
+  candidateModelOptions?: Record<string, QaCharacterModelOptions>;
+  judgeModel?: string;
+  judgeModels?: string[];
+  judgeThinkingDefault?: QaThinkingLevel;
+  judgeModelOptions?: Record<string, QaCharacterModelOptions>;
+  judgeTimeoutMs?: number;
+  judgeBlindModels?: boolean;
+  candidateConcurrency?: number;
+  judgeConcurrency?: number;
+  runSuite?: RunSuiteFn;
+  runJudge?: RunJudgeFn;
+  progress?: QaCharacterEvalProgressLogger;
+};
+
+function normalizeModelRefs(models: readonly string[]) {
+  return uniqueStrings(normalizeStringEntries(models));
+}
+
+function resolveCandidateThinkingDefault(params: {
+  model: string;
+  candidateThinkingDefault?: QaThinkingLevel;
+  candidateThinkingByModel?: Record<string, QaThinkingLevel>;
+  candidateModelOptions?: Record<string, QaCharacterModelOptions>;
+}) {
+  return (
+    params.candidateModelOptions?.[params.model]?.thinkingDefault ??
+    params.candidateThinkingByModel?.[params.model] ??
+    params.candidateThinkingDefault ??
+    DEFAULT_CHARACTER_THINKING_BY_MODEL[params.model] ??
+    DEFAULT_CHARACTER_THINKING
+  );
+}
+
+function resolveCandidateFastMode(params: {
+  model: string;
+  candidateFastMode?: boolean;
+  candidateModelOptions?: Record<string, QaCharacterModelOptions>;
+}) {
+  return (
+    params.candidateModelOptions?.[params.model]?.fastMode ??
+    params.candidateFastMode ??
+    isQaFastModeModelRef(params.model)
+  );
+}
+
+function resolveJudgeOptions(params: {
+  model: string;
+  judgeThinkingDefault?: QaThinkingLevel;
+  judgeModelOptions?: Record<string, QaCharacterModelOptions>;
+}) {
+  const modelDefaults = DEFAULT_JUDGE_MODEL_OPTIONS[params.model];
+  const modelOptions = params.judgeModelOptions?.[params.model];
+  return {
+    thinkingDefault:
+      modelOptions?.thinkingDefault ??
+      params.judgeThinkingDefault ??
+      modelDefaults?.thinkingDefault ??
+      DEFAULT_JUDGE_THINKING,
+    fastMode: modelOptions?.fastMode ?? modelDefaults?.fastMode ?? false,
+  };
+}
+
+function sanitizePathPart(value: string) {
+  const sanitized = value.replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "");
+  return sanitized || "model";
+}
+
+function normalizeConcurrency(value: number | undefined, fallback = 1) {
+  if (value === undefined) {
+    return fallback;
+  }
+  if (!Number.isFinite(value)) {
+    return fallback;
+  }
+  return Math.max(1, Math.floor(value));
+}
+
+async function mapWithConcurrency<T, U>(
+  items: readonly T[],
+  concurrency: number,
+  mapper: (item: T, index: number) => Promise<U>,
+) {
+  const results = Array.from<U>({ length: items.length });
+  let nextIndex = 0;
+  const workerCount = Math.min(normalizeConcurrency(concurrency), items.length);
+  const workers = Array.from({ length: workerCount }, async () => {
+    while (nextIndex < items.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      results[index] = await mapper(items[index], index);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}
+
+function extractTranscript(result: QaSuiteResult) {
+  let longestDetail: string | undefined;
+  for (const scenario of result.scenarios) {
+    for (const step of scenario.steps) {
+      const detail = step.details;
+      if (detail && (!longestDetail || detail.length > longestDetail.length)) {
+        longestDetail = detail;
+      }
+    }
+  }
+  return longestDetail ?? result.report;
+}
+
+function collectTranscriptStats(transcript: string) {
+  return {
+    transcriptChars: transcript.length,
+    transcriptLines: transcript.length === 0 ? 0 : transcript.split(/\r?\n/).length,
+    userTurns: transcript.match(/^USER\b/gm)?.length ?? 0,
+    assistantTurns: transcript.match(/^ASSISTANT\b/gm)?.length ?? 0,
+  };
+}
+
+function detectTranscriptFailure(transcript: string): string | undefined {
+  if (extractQaVisibleReplyLeakText(transcript)) {
+    return "internal harness/meta text leaked into transcript";
+  }
+  const checks: Array<[RegExp, string]> = [
+    [/\bmodel `[^`]+` is not supported\b/i, "model unsupported error leaked into transcript"],
+    [/\binsufficient account balance\b/i, "account balance error leaked into transcript"],
+    [/\b(?:backend|transport|internal) error\b/i, "backend error leaked into transcript"],
+    [
+      /\bsomething went wrong while processing your request\b/i,
+      "generic request failure leaked into transcript",
+    ],
+    [/\buse \/new to start a fresh session\b/i, "generic request failure leaked into transcript"],
+    [
+      /\bmodel did not produce a response before the LLM idle timeout\b/i,
+      "LLM timeout leaked into transcript",
+    ],
+    [/\btool failed\b/i, "tool failure leaked into transcript"],
+    [/\b(?:read|write|edit|patch):[^\n]*\bfailed\b/i, "tool failure leaked into transcript"],
+    [/\bnot configured\b/i, "configuration error leaked into transcript"],
+  ];
+  return checks.find(([pattern]) => pattern.test(transcript))?.[1];
+}
+
+function formatDuration(ms: number) {
+  if (!Number.isFinite(ms) || ms < 0) {
+    return "unknown";
+  }
+  if (ms < 1_000) {
+    return `${Math.round(ms)}ms`;
+  }
+  if (ms < 60_000) {
+    const seconds = ms / 1_000;
+    return `${seconds >= 10 ? Math.round(seconds) : Number(seconds.toFixed(1))}s`;
+  }
+  const totalSeconds = Math.round(ms / 1_000);
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  return seconds === 0 ? `${minutes}m` : `${minutes}m ${seconds}s`;
+}
+
+function logCharacterEvalProgress(
+  progress: QaCharacterEvalProgressLogger | undefined,
+  message: string,
+) {
+  progress?.(`[qa-character] ${message}`);
+}
+
+function formatEvalIndex(index: number, total: number) {
+  return `${index + 1}/${total}`;
+}
+
+function summarizeRunStats(run: QaCharacterEvalRun) {
+  return [
+    `status=${run.status}`,
+    `duration=${formatDuration(run.durationMs)}`,
+    `turns=${run.stats.userTurns}/${run.stats.assistantTurns}`,
+    `chars=${run.stats.transcriptChars}`,
+    ...(run.error ? [`error="${run.error}"`] : []),
+  ].join(" ");
+}
+
+function formatBlindCandidateLabel(index: number) {
+  return `candidate-${String(index + 1).padStart(2, "0")}`;
+}
+
+function buildJudgePrompt(params: {
+  scenarioId: string;
+  runs: readonly QaCharacterEvalRun[];
+  blindModels?: boolean;
+}) {
+  const labelToModel = new Map<string, string>();
+  const runBlocks = params.runs
+    .map((run, index) => {
+      const label = params.blindModels ? formatBlindCandidateLabel(index) : run.model;
+      labelToModel.set(label, run.model);
+      return `## CANDIDATE ${label}
+
+Status: ${run.status}
+Duration ms (not used for ranking): ${run.durationMs}
+Fast mode: ${run.fastMode ? "on" : "off"}
+Thinking: ${run.thinkingDefault}
+Transcript chars: ${run.stats.transcriptChars}
+Assistant turns: ${run.stats.assistantTurns}
+Error: ${run.error ?? "none"}
+
+\`\`\`text
+${run.transcript}
+\`\`\``;
+    })
+    .join("\n\n");
+
+  const prompt = `You are grading OpenClaw natural character conversation transcripts for naturalness, vibes, and funniness.
+
+Scenario id: ${params.scenarioId}
+
+Rank the models by:
+- natural conversational reaction
+- playful character commitment
+- funny, surprising details
+- coherence across turns
+- completing real user tasks without becoming generic
+- not sounding aware of an eval or test
+- avoiding tool/backend/error leakage
+
+Treat candidate labels as opaque identifiers. Do not assume quality from the label.
+Duration is recorded for separate benchmark analysis only. Do not rank models by speed.
+
+Return strict JSON only with this shape:
+{
+  "rankings": [
+    {
+      "model": "same candidate label",
+      "rank": 1,
+      "score": 9.2,
+      "summary": "one sentence",
+      "strengths": ["short"],
+      "weaknesses": ["short"]
+    }
+  ]
+}
+
+${runBlocks}`;
+  return { prompt, labelToModel };
+}
+
+function normalizeJudgment(value: unknown, allowedModels: Set<string>): QaCharacterEvalJudgment[] {
+  const payload = value && typeof value === "object" ? (value as Record<string, unknown>) : {};
+  const rankings = Array.isArray(payload.rankings) ? payload.rankings : [];
+  return rankings
+    .map((entry): QaCharacterEvalJudgment | null => {
+      if (!entry || typeof entry !== "object") {
+        return null;
+      }
+      const record = entry as Record<string, unknown>;
+      const model = typeof record.model === "string" ? record.model : "";
+      if (!allowedModels.has(model)) {
+        return null;
+      }
+      const rank = typeof record.rank === "number" ? record.rank : Number(record.rank);
+      const score = typeof record.score === "number" ? record.score : Number(record.score);
+      const summary = typeof record.summary === "string" ? record.summary : "";
+      const strengths = Array.isArray(record.strengths)
+        ? record.strengths.filter((item): item is string => typeof item === "string")
+        : [];
+      const weaknesses = Array.isArray(record.weaknesses)
+        ? record.weaknesses.filter((item): item is string => typeof item === "string")
+        : [];
+      if (!Number.isFinite(rank) || !Number.isFinite(score)) {
+        return null;
+      }
+      return { model, rank, score, summary, strengths, weaknesses };
+    })
+    .filter((entry): entry is QaCharacterEvalJudgment => Boolean(entry))
+    .toSorted((left, right) => left.rank - right.rank || right.score - left.score);
+}
+
+function parseJudgeReply(reply: string | null, allowedModels: Set<string>) {
+  if (!reply) {
+    throw new Error("judge did not return a reply");
+  }
+  const trimmed = reply.trim();
+  const jsonText =
+    trimmed.match(/```(?:json)?\s*([\s\S]*?)```/)?.[1]?.trim() ??
+    trimmed.match(/\{[\s\S]*\}/)?.[0]?.trim() ??
+    trimmed;
+  const parsed = JSON.parse(jsonText) as unknown;
+  const rankings = normalizeJudgment(parsed, allowedModels);
+  if (rankings.length === 0) {
+    throw new Error("judge reply did not contain valid rankings");
+  }
+  return rankings;
+}
+
+async function defaultRunJudge(params: {
+  repoRoot: string;
+  judgeModel: string;
+  judgeThinkingDefault: QaThinkingLevel;
+  judgeFastMode: boolean;
+  prompt: string;
+  timeoutMs: number;
+}) {
+  const { runQaManualLane } = await import("./manual-lane.runtime.js");
+  const result = await runQaManualLane({
+    repoRoot: params.repoRoot,
+    providerMode: "live-frontier",
+    primaryModel: params.judgeModel,
+    alternateModel: params.judgeModel,
+    fastMode: params.judgeFastMode,
+    thinkingDefault: params.judgeThinkingDefault,
+    message: params.prompt,
+    timeoutMs: params.timeoutMs,
+  });
+  return result.reply;
+}
+
+async function defaultRunSuite(params: Parameters<RunSuiteFn>[0]) {
+  const { runQaFlowSuiteFromRuntime } = await import("./suite-launch.runtime.js");
+  return await runQaFlowSuiteFromRuntime(params);
+}
+
+function renderCharacterEvalReport(params: {
+  scenarioId: string;
+  startedAt: Date;
+  finishedAt: Date;
+  runs: readonly QaCharacterEvalRun[];
+  judgments: readonly QaCharacterEvalJudgeResult[];
+}) {
+  const lines = [
+    "# OpenClaw Character Eval Report",
+    "",
+    `- Started: ${params.startedAt.toISOString()}`,
+    `- Finished: ${params.finishedAt.toISOString()}`,
+    `- Duration: ${formatDuration(params.finishedAt.getTime() - params.startedAt.getTime())}`,
+    `- Scenario: ${params.scenarioId}`,
+    "- Execution: local QA gateway child processes, not Docker",
+    `- Judges: ${params.judgments.map((judgment) => judgment.model).join(", ")}`,
+    `- Judge thinking: ${params.judgments[0]?.thinkingDefault ?? DEFAULT_JUDGE_THINKING}`,
+    `- Judge fast mode: ${params.judgments.every((judgment) => judgment.fastMode) ? "on" : "mixed"}`,
+    `- Judge model labels: ${params.judgments.every((judgment) => judgment.blindModels) ? "blind" : "visible"}`,
+    "",
+    "## Judge Rankings",
+    "",
+  ];
+
+  for (const judgment of params.judgments) {
+    lines.push(`### ${judgment.model}`, "");
+    lines.push(`- Duration: ${formatDuration(judgment.durationMs)}`, "");
+    lines.push(`- Timeout: ${formatDuration(judgment.timeoutMs)}`, "");
+    if (judgment.rankings.length > 0) {
+      for (const ranking of judgment.rankings) {
+        lines.push(
+          `${ranking.rank}. ${ranking.model} - ${ranking.score.toFixed(1)} - ${ranking.summary}`,
+        );
+        if (ranking.strengths.length > 0) {
+          lines.push(`   Strengths: ${ranking.strengths.join("; ")}`);
+        }
+        if (ranking.weaknesses.length > 0) {
+          lines.push(`   Weaknesses: ${ranking.weaknesses.join("; ")}`);
+        }
+      }
+    } else {
+      lines.push("- Judge ranking unavailable.");
+      if (judgment.error) {
+        lines.push(`- Judge error: ${judgment.error}`);
+      }
+    }
+    lines.push("");
+  }
+
+  lines.push("## Run Stats", "");
+  lines.push(
+    "| Model | Thinking | Fast mode | Status | Duration | User turns | Assistant turns | Transcript chars |",
+  );
+  lines.push("| --- | --- | --- | --- | ---: | ---: | ---: | ---: |");
+  for (const run of params.runs) {
+    lines.push(
+      `| ${run.model} | ${run.thinkingDefault} | ${run.fastMode ? "on" : "off"} | ${run.status} | ${formatDuration(run.durationMs)} | ${run.stats.userTurns} | ${run.stats.assistantTurns} | ${run.stats.transcriptChars} |`,
+    );
+  }
+
+  lines.push("", "## Transcripts", "");
+  for (const run of params.runs) {
+    lines.push(`### ${run.model}`, "");
+    lines.push(`- Status: ${run.status}`);
+    lines.push(`- Thinking: ${run.thinkingDefault}`);
+    lines.push(`- Fast mode: ${run.fastMode ? "on" : "off"}`);
+    lines.push(`- Duration: ${formatDuration(run.durationMs)}`);
+    lines.push(`- Report: ${run.reportPath ?? "unavailable"}`);
+    if (run.error) {
+      lines.push(`- Error: ${run.error}`);
+    }
+    lines.push("", "```text", run.transcript.trim() || "(empty transcript)", "```", "");
+  }
+
+  return `${lines.join("\n")}\n`;
+}
+
+export async function runQaCharacterEval(params: QaCharacterEvalParams) {
+  const startedAt = new Date();
+  const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
+  const scenarioId = params.scenarioId?.trim() || DEFAULT_CHARACTER_SCENARIO_ID;
+  const models = normalizeModelRefs(
+    params.models.length > 0 ? params.models : DEFAULT_CHARACTER_EVAL_MODELS,
+  );
+  if (models.length === 0) {
+    throw new Error("qa character-eval needs at least one --model <provider/model> ref");
+  }
+
+  const outputDir =
+    params.outputDir ??
+    path.join(repoRoot, ".artifacts", "qa-e2e", `character-eval-${createQaArtifactRunId()}`);
+  const runsDir = path.join(outputDir, "runs");
+  await fs.mkdir(runsDir, { recursive: true });
+
+  const runSuite = params.runSuite ?? defaultRunSuite;
+  const candidateConcurrency = normalizeConcurrency(
+    params.candidateConcurrency,
+    DEFAULT_CHARACTER_EVAL_CONCURRENCY,
+  );
+  logCharacterEvalProgress(
+    params.progress,
+    `start scenario=${scenarioId} candidates=${models.length} candidateConcurrency=${candidateConcurrency} output=${outputDir}`,
+  );
+  const candidatesStartedAt = Date.now();
+  const runs = await mapWithConcurrency(models, candidateConcurrency, async (model, index) => {
+    const thinkingDefault = resolveCandidateThinkingDefault({
+      model,
+      candidateThinkingDefault: params.candidateThinkingDefault,
+      candidateThinkingByModel: params.candidateThinkingByModel,
+      candidateModelOptions: params.candidateModelOptions,
+    });
+    const fastMode = resolveCandidateFastMode({
+      model,
+      candidateFastMode: params.candidateFastMode,
+      candidateModelOptions: params.candidateModelOptions,
+    });
+    const modelOutputDir = path.join(runsDir, sanitizePathPart(model));
+    const runStartedAt = Date.now();
+    logCharacterEvalProgress(
+      params.progress,
+      `candidate start ${formatEvalIndex(index, models.length)} model=${model} thinking=${thinkingDefault} fast=${fastMode ? "on" : "off"}`,
+    );
+    try {
+      const result = await runSuite({
+        repoRoot,
+        outputDir: modelOutputDir,
+        providerMode: "live-frontier",
+        primaryModel: model,
+        alternateModel: model,
+        fastMode,
+        thinkingDefault,
+        scenarioIds: [scenarioId],
+      });
+      const transcript = extractTranscript(result);
+      const transcriptFailure = detectTranscriptFailure(transcript);
+      const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
+      const status = failedScenarioCount > 0 || transcriptFailure ? "fail" : "pass";
+      const run = {
+        model,
+        status,
+        durationMs: Date.now() - runStartedAt,
+        outputDir: modelOutputDir,
+        thinkingDefault,
+        fastMode,
+        reportPath: result.reportPath,
+        summaryPath: result.summaryPath,
+        transcript,
+        stats: collectTranscriptStats(transcript),
+        ...(transcriptFailure ? { error: transcriptFailure } : {}),
+      } satisfies QaCharacterEvalRun;
+      logCharacterEvalProgress(
+        params.progress,
+        `candidate done ${formatEvalIndex(index, models.length)} model=${model} ${summarizeRunStats(run)}`,
+      );
+      return run;
+    } catch (error) {
+      const transcript = "";
+      const run = {
+        model,
+        status: "fail",
+        durationMs: Date.now() - runStartedAt,
+        outputDir: modelOutputDir,
+        thinkingDefault,
+        fastMode,
+        transcript,
+        stats: collectTranscriptStats(transcript),
+        error: formatErrorMessage(error),
+      } satisfies QaCharacterEvalRun;
+      logCharacterEvalProgress(
+        params.progress,
+        `candidate done ${formatEvalIndex(index, models.length)} model=${model} ${summarizeRunStats(run)}`,
+      );
+      return run;
+    }
+  });
+  const failedCandidateCount = runs.filter((run) => run.status === "fail").length;
+  logCharacterEvalProgress(
+    params.progress,
+    `candidates done pass=${runs.length - failedCandidateCount} fail=${failedCandidateCount} duration=${formatDuration(Date.now() - candidatesStartedAt)}`,
+  );
+
+  const judgeModels = normalizeModelRefs(
+    params.judgeModels && params.judgeModels.length > 0
+      ? params.judgeModels
+      : params.judgeModel
+        ? [params.judgeModel]
+        : DEFAULT_JUDGE_MODELS,
+  );
+  const runJudge = params.runJudge ?? defaultRunJudge;
+  const judgeConcurrency = normalizeConcurrency(
+    params.judgeConcurrency,
+    DEFAULT_CHARACTER_EVAL_CONCURRENCY,
+  );
+  const judgeTimeoutMs = params.judgeTimeoutMs ?? DEFAULT_JUDGE_TIMEOUT_MS;
+  logCharacterEvalProgress(
+    params.progress,
+    `judges start judges=${judgeModels.length} judgeConcurrency=${judgeConcurrency} timeout=${formatDuration(judgeTimeoutMs)} labels=${params.judgeBlindModels === true ? "blind" : "visible"}`,
+  );
+  const judgesStartedAt = Date.now();
+  const judgments = await mapWithConcurrency(
+    judgeModels,
+    judgeConcurrency,
+    async (judgeModel, index) => {
+      const judgeOptions = resolveJudgeOptions({
+        model: judgeModel,
+        judgeThinkingDefault: params.judgeThinkingDefault,
+        judgeModelOptions: params.judgeModelOptions,
+      });
+      let rankings: QaCharacterEvalJudgment[] = [];
+      let judgeError: string | undefined;
+      const judgeStartedAt = Date.now();
+      logCharacterEvalProgress(
+        params.progress,
+        `judge start ${formatEvalIndex(index, judgeModels.length)} model=${judgeModel} thinking=${judgeOptions.thinkingDefault} fast=${judgeOptions.fastMode ? "on" : "off"} timeout=${formatDuration(judgeTimeoutMs)}`,
+      );
+      try {
+        const judgePrompt = buildJudgePrompt({
+          scenarioId,
+          runs,
+          blindModels: params.judgeBlindModels,
+        });
+        const rawReply = await runJudge({
+          repoRoot,
+          judgeModel,
+          judgeThinkingDefault: judgeOptions.thinkingDefault,
+          judgeFastMode: judgeOptions.fastMode,
+          prompt: judgePrompt.prompt,
+          timeoutMs: judgeTimeoutMs,
+        });
+        rankings = parseJudgeReply(rawReply, new Set(judgePrompt.labelToModel.keys())).map(
+          (ranking) =>
+            Object.assign({}, ranking, {
+              model: judgePrompt.labelToModel.get(ranking.model) ?? ranking.model,
+            }),
+        );
+      } catch (error) {
+        judgeError = formatErrorMessage(error);
+      }
+
+      const judgment = {
+        model: judgeModel,
+        thinkingDefault: judgeOptions.thinkingDefault,
+        fastMode: judgeOptions.fastMode,
+        blindModels: params.judgeBlindModels === true,
+        timeoutMs: judgeTimeoutMs,
+        durationMs: Date.now() - judgeStartedAt,
+        rankings,
+        ...(judgeError ? { error: judgeError } : {}),
+      } satisfies QaCharacterEvalJudgeResult;
+      logCharacterEvalProgress(
+        params.progress,
+        `judge done ${formatEvalIndex(index, judgeModels.length)} model=${judgeModel} rankings=${rankings.length} duration=${formatDuration(judgment.durationMs)}${judgeError ? ` error="${judgeError}"` : ""}`,
+      );
+      return judgment;
+    },
+  );
+  const failedJudgeCount = judgments.filter((judgment) => judgment.rankings.length === 0).length;
+  logCharacterEvalProgress(
+    params.progress,
+    `judges done ranked=${judgments.length - failedJudgeCount} failed=${failedJudgeCount} duration=${formatDuration(Date.now() - judgesStartedAt)}`,
+  );
+
+  const finishedAt = new Date();
+  const report = renderCharacterEvalReport({
+    scenarioId,
+    startedAt,
+    finishedAt,
+    runs,
+    judgments,
+  });
+  const reportPath = path.join(outputDir, "character-eval-report.md");
+  const summaryPath = path.join(outputDir, "character-eval-summary.json");
+  await fs.writeFile(reportPath, report, "utf8");
+  await fs.writeFile(
+    summaryPath,
+    `${JSON.stringify(
+      {
+        scenarioId,
+        runs,
+        judgments,
+      },
+      null,
+      2,
+    )}\n`,
+    "utf8",
+  );
+  logCharacterEvalProgress(
+    params.progress,
+    `report written duration=${formatDuration(finishedAt.getTime() - startedAt.getTime())} report=${reportPath} summary=${summaryPath}`,
+  );
+
+  return {
+    outputDir,
+    reportPath,
+    summaryPath,
+    runs,
+    judgments,
+  } satisfies QaCharacterEvalResult;
+}

extensions/qa-lab/src/child-output.ts

@@ -0,0 +1,83 @@
+// Qa Lab plugin module implements child output behavior.
+export const QA_CHILD_STDOUT_MAX_BYTES = 1024 * 1024;
+export const QA_CHILD_STDERR_TAIL_BYTES = 64 * 1024;
+
+export type QaChildOutputCapture = {
+  chunks: Buffer[];
+  bytes: number;
+  exceeded: boolean;
+  maxBytes: number;
+};
+
+export type QaChildOutputTail = {
+  buffer: Buffer;
+  maxBytes: number;
+  truncated: boolean;
+};
+
+function toBuffer(chunk: unknown): Buffer {
+  return Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk));
+}
+
+export function createQaChildOutputCapture(maxBytes = QA_CHILD_STDOUT_MAX_BYTES) {
+  return {
+    chunks: [],
+    bytes: 0,
+    exceeded: false,
+    maxBytes,
+  } satisfies QaChildOutputCapture;
+}
+
+export function appendQaChildOutput(capture: QaChildOutputCapture, chunk: unknown) {
+  if (capture.exceeded) {
+    return;
+  }
+  const buffer = toBuffer(chunk);
+  const remainingBytes = capture.maxBytes - capture.bytes;
+  if (buffer.byteLength > remainingBytes) {
+    if (remainingBytes > 0) {
+      capture.chunks.push(Buffer.from(buffer.subarray(0, remainingBytes)));
+    }
+    capture.bytes = capture.maxBytes;
+    capture.exceeded = true;
+    return;
+  }
+  capture.chunks.push(Buffer.from(buffer));
+  capture.bytes += buffer.byteLength;
+}
+
+export function readQaChildOutput(capture: QaChildOutputCapture) {
+  return Buffer.concat(capture.chunks, capture.bytes).toString("utf8");
+}
+
+export function createQaChildOutputTail(maxBytes = QA_CHILD_STDERR_TAIL_BYTES) {
+  return {
+    buffer: Buffer.alloc(0),
+    maxBytes,
+    truncated: false,
+  } satisfies QaChildOutputTail;
+}
+
+export function appendQaChildOutputTail(tail: QaChildOutputTail, chunk: unknown) {
+  const buffer = toBuffer(chunk);
+  if (buffer.byteLength >= tail.maxBytes) {
+    tail.buffer = Buffer.from(buffer.subarray(buffer.byteLength - tail.maxBytes));
+    tail.truncated = true;
+    return;
+  }
+  const next = Buffer.concat([tail.buffer, buffer], tail.buffer.byteLength + buffer.byteLength);
+  if (next.byteLength <= tail.maxBytes) {
+    tail.buffer = next;
+    return;
+  }
+  tail.buffer = Buffer.from(next.subarray(next.byteLength - tail.maxBytes));
+  tail.truncated = true;
+}
+
+export function formatQaChildOutputTail(tail: QaChildOutputTail, label: string) {
+  const text = tail.buffer.toString("utf8").trim();
+  if (!text) {
+    return "";
+  }
+  return tail.truncated ? `[${label} truncated to last ${tail.maxBytes} bytes]\n${text}` : text;
+}

extensions/qa-lab/src/cli-options.ts

@@ -0,0 +1,5 @@
+// Qa Lab plugin module implements cli options behavior.
+export function collectString(value: string, previous: string[]) {
+  const trimmed = value.trim();
+  return trimmed ? [...previous, trimmed] : previous;
+}

extensions/qa-lab/src/cli-paths.ts

@@ -0,0 +1,99 @@
+// Qa Lab plugin module implements cli paths behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { assertNoSymlinkParents, pathScope } from "openclaw/plugin-sdk/security-runtime";
+
+export function toRepoPath(filePath: string): string {
+  return filePath.split(path.sep).join("/");
+}
+
+export function toRepoRelativePath(repoRoot: string, filePath: string): string {
+  return toRepoPath(path.relative(repoRoot, filePath));
+}
+
+export function isRepoRootRelativeRef(value: string) {
+  return !path.isAbsolute(value) && value.split(/[\\/]+/u).every((part) => part !== "..");
+}
+
+export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: string) {
+  if (!outputDir) {
+    return undefined;
+  }
+  if (path.isAbsolute(outputDir)) {
+    throw new Error("--output-dir must be a relative path inside the repo root.");
+  }
+  const resolved = pathScope(repoRoot, { label: "repo root" }).resolve(outputDir);
+  if (!resolved.ok) {
+    throw new Error("--output-dir must stay within the repo root.");
+  }
+  return resolved.path;
+}
+
+async function resolveNearestExistingPath(targetPath: string) {
+  let current = path.resolve(targetPath);
+  while (true) {
+    try {
+      await fs.lstat(current);
+      return current;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+        throw error;
+      }
+    }
+    const parent = path.dirname(current);
+    if (parent === current) {
+      throw new Error(`failed to resolve existing path for ${targetPath}`);
+    }
+    current = parent;
+  }
+}
+
+function assertRepoRelativePath(repoRoot: string, targetPath: string, label: string) {
+  const relative = path.relative(repoRoot, targetPath);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`${label} must stay within the repo root.`);
+  }
+  return relative;
+}
+
+async function assertNoSymlinkSegments(repoRoot: string, targetPath: string, label: string) {
+  assertRepoRelativePath(repoRoot, targetPath, label);
+  try {
+    await assertNoSymlinkParents({
+      rootDir: repoRoot,
+      targetPath,
+      messagePrefix: label,
+    });
+  } catch (error) {
+    if (error instanceof Error && error.message.includes("symlink")) {
+      throw new Error(`${label} must not traverse symlinks.`, { cause: error });
+    }
+    throw error;
+  }
+}
+
+export async function assertRepoBoundPath(repoRoot: string, targetPath: string, label: string) {
+  const repoRootResolved = path.resolve(repoRoot);
+  const targetResolved = path.resolve(targetPath);
+  assertRepoRelativePath(repoRootResolved, targetResolved, label);
+  await assertNoSymlinkSegments(repoRootResolved, targetResolved, label);
+  const repoRootReal = await fs.realpath(repoRootResolved);
+  const nearestExistingPath = await resolveNearestExistingPath(targetResolved);
+  const nearestExistingReal = await fs.realpath(nearestExistingPath);
+  assertRepoRelativePath(repoRootReal, nearestExistingReal, label);
+  return targetResolved;
+}
+
+export async function ensureRepoBoundDirectory(
+  repoRoot: string,
+  targetDir: string,
+  label: string,
+  opts?: { mode?: number },
+) {
+  await assertNoSymlinkSegments(path.resolve(repoRoot), path.resolve(targetDir), label);
+  const result = await pathScope(repoRoot, { label }).ensureDir(targetDir, { mode: opts?.mode });
+  if (!result.ok) {
+    throw new Error(`${label} must stay within the repo root.`);
+  }
+  return result.path;
+}

extensions/qa-lab/src/cli.test.ts

@@ -0,0 +1,997 @@
+// Qa Lab tests cover cli plugin behavior.
+import { Command } from "commander";
+import type { QaRunnerCliContribution } from "openclaw/plugin-sdk/qa-runner-runtime";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const TEST_QA_RUNNER = {
+  pluginId: "qa-runner-test",
+  commandName: "runner-test",
+  description: "Run the test live QA lane",
+} as const;
+
+function createAvailableQaRunnerContribution() {
+  return {
+    pluginId: TEST_QA_RUNNER.pluginId,
+    commandName: TEST_QA_RUNNER.commandName,
+    status: "available" as const,
+    registration: {
+      commandName: TEST_QA_RUNNER.commandName,
+      register: vi.fn((qa: Command) => {
+        qa.command(TEST_QA_RUNNER.commandName).action(() => undefined);
+      }),
+    },
+  } satisfies QaRunnerCliContribution;
+}
+
+function createBlockedQaRunnerContribution(): QaRunnerCliContribution {
+  return {
+    pluginId: TEST_QA_RUNNER.pluginId,
+    commandName: TEST_QA_RUNNER.commandName,
+    description: TEST_QA_RUNNER.description,
+    status: "blocked",
+  };
+}
+
+function createConflictingQaRunnerContribution(commandName: string): QaRunnerCliContribution {
+  return {
+    pluginId: TEST_QA_RUNNER.pluginId,
+    commandName,
+    description: TEST_QA_RUNNER.description,
+    status: "blocked",
+  };
+}
+
+const {
+  runQaCredentialsAddCommand,
+  runQaCredentialsListCommand,
+  runQaCredentialsRemoveCommand,
+  runQaCoverageReportCommand,
+  runQaJsonlReplayCommand,
+  runQaLabSelfCheckCommand,
+  runQaProfileCommand,
+  runQaProviderServerCommand,
+  runQaSuiteCommand,
+  runQaTelegramCommand,
+  runMantisBeforeAfterCommand,
+  runMantisDesktopBrowserSmokeCommand,
+  runMantisDiscordSmokeCommand,
+  runMantisSlackDesktopSmokeCommand,
+  runMantisTelegramDesktopBuilderCommand,
+} = vi.hoisted(() => ({
+  runQaCredentialsAddCommand: vi.fn(),
+  runQaCredentialsListCommand: vi.fn(),
+  runQaCredentialsRemoveCommand: vi.fn(),
+  runQaCoverageReportCommand: vi.fn(),
+  runQaJsonlReplayCommand: vi.fn(),
+  runQaLabSelfCheckCommand: vi.fn(),
+  runQaProfileCommand: vi.fn(),
+  runQaProviderServerCommand: vi.fn(),
+  runQaSuiteCommand: vi.fn(),
+  runQaTelegramCommand: vi.fn(),
+  runMantisBeforeAfterCommand: vi.fn(),
+  runMantisDesktopBrowserSmokeCommand: vi.fn(),
+  runMantisDiscordSmokeCommand: vi.fn(),
+  runMantisSlackDesktopSmokeCommand: vi.fn(),
+  runMantisTelegramDesktopBuilderCommand: vi.fn(),
+}));
+
+const { listQaRunnerCliContributions } = vi.hoisted(() => ({
+  listQaRunnerCliContributions: vi.fn<() => QaRunnerCliContribution[]>(() => [
+    createAvailableQaRunnerContribution(),
+  ]),
+}));
+
+function requireQaTelegramOptions() {
+  const [call] = runQaTelegramCommand.mock.calls;
+  if (!call) {
+    throw new Error("expected qa telegram command call");
+  }
+  const [options] = call;
+  return options;
+}
+
+function requireQaSuiteOptions() {
+  const [call] = runQaSuiteCommand.mock.calls;
+  if (!call) {
+    throw new Error("expected qa suite command call");
+  }
+  const [options] = call;
+  return options;
+}
+
+vi.mock("openclaw/plugin-sdk/qa-runner-runtime", () => ({
+  listQaRunnerCliContributions,
+}));
+
+vi.mock("./live-transports/telegram/cli.runtime.js", () => ({
+  runQaTelegramCommand,
+}));
+
+vi.mock("./mantis/cli.runtime.js", () => ({
+  runMantisBeforeAfterCommand,
+  runMantisDesktopBrowserSmokeCommand,
+  runMantisDiscordSmokeCommand,
+  runMantisSlackDesktopSmokeCommand,
+  runMantisTelegramDesktopBuilderCommand,
+}));
+
+vi.mock("./cli.runtime.js", () => ({
+  runQaCredentialsAddCommand,
+  runQaCredentialsListCommand,
+  runQaCredentialsRemoveCommand,
+  runQaCoverageReportCommand,
+  runQaJsonlReplayCommand,
+  runQaLabSelfCheckCommand,
+  runQaProfileCommand,
+  runQaProviderServerCommand,
+  runQaSuiteCommand,
+}));
+
+import { registerQaLabCli } from "./cli.js";
+
+describe("qa cli registration", () => {
+  let program: Command;
+
+  beforeEach(() => {
+    program = new Command();
+    runQaCredentialsAddCommand.mockReset();
+    runQaCredentialsListCommand.mockReset();
+    runQaCredentialsRemoveCommand.mockReset();
+    runQaCoverageReportCommand.mockReset();
+    runQaJsonlReplayCommand.mockReset();
+    runQaLabSelfCheckCommand.mockReset();
+    runQaProfileCommand.mockReset();
+    runQaProviderServerCommand.mockReset();
+    runQaSuiteCommand.mockReset();
+    runQaTelegramCommand.mockReset();
+    runMantisBeforeAfterCommand.mockReset();
+    runMantisDesktopBrowserSmokeCommand.mockReset();
+    runMantisDiscordSmokeCommand.mockReset();
+    runMantisSlackDesktopSmokeCommand.mockReset();
+    runMantisTelegramDesktopBuilderCommand.mockReset();
+    listQaRunnerCliContributions
+      .mockReset()
+      .mockReturnValue([createAvailableQaRunnerContribution()]);
+    registerQaLabCli(program);
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("registers discovered and built-in live transport subcommands", () => {
+    const qa = program.commands.find((command) => command.name() === "qa");
+    if (!qa) {
+      throw new Error("expected qa command");
+    }
+    const commandNames = qa.commands.map((command) => command.name());
+    expect(commandNames).toContain(TEST_QA_RUNNER.commandName);
+    expect(commandNames).toContain("telegram");
+    expect(commandNames).toContain("mantis");
+    expect(commandNames).toContain("credentials");
+    expect(commandNames).toContain("coverage");
+  });
+
+  it("does not expose a control-ui token flag on qa ui", () => {
+    const qa = program.commands.find((command) => command.name() === "qa");
+    const ui = qa?.commands.find((command) => command.name() === "ui");
+    if (!ui) {
+      throw new Error("expected qa ui command");
+    }
+
+    expect(ui.options.map((option) => option.long)).not.toContain("--control-ui-token");
+  });
+
+  it("keeps qa run without a profile on the self-check command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "run",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output",
+      ".artifacts/qa-self-check.md",
+    ]);
+
+    expect(runQaLabSelfCheckCommand).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      output: ".artifacts/qa-self-check.md",
+    });
+    expect(runQaProfileCommand).not.toHaveBeenCalled();
+  });
+
+  it("routes qa run qa-profile flags into the taxonomy-backed profile command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "run",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output-dir",
+      ".artifacts/qa-e2e/smoke-ci",
+      "--qa-profile",
+      "smoke-ci",
+      "--surface",
+      "channel-framework",
+      "--category",
+      "channel-framework.conversation-routing-and-delivery",
+      "--scenario",
+      "dm-chat-baseline",
+      "--evidence-mode",
+      "slim",
+      "--transport",
+      "qa-channel",
+      "--provider-mode",
+      "mock-openai",
+      "--model",
+      "openai/gpt-5.5",
+      "--alt-model",
+      "anthropic/claude-sonnet-4-6",
+      "--concurrency",
+      "2",
+      "--allow-failures",
+      "--fast",
+    ]);
+
+    expect(runQaProfileCommand).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      outputDir: ".artifacts/qa-e2e/smoke-ci",
+      profile: "smoke-ci",
+      surface: "channel-framework",
+      category: "channel-framework.conversation-routing-and-delivery",
+      scenarioIds: ["dm-chat-baseline"],
+      evidenceMode: "slim",
+      transportId: "qa-channel",
+      providerMode: "mock-openai",
+      primaryModel: "openai/gpt-5.5",
+      alternateModel: "anthropic/claude-sonnet-4-6",
+      concurrency: 2,
+      allowFailures: true,
+      fastMode: true,
+    });
+    expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    ["--output-dir", [".artifacts/qa-e2e/smoke-ci"]],
+    ["--surface", ["agent-runtime-and-provider-execution"]],
+    ["--category", ["channel-framework.conversation-routing-and-delivery"]],
+    ["--scenario", ["dm-chat-baseline"]],
+    ["--evidence-mode", ["slim"]],
+    ["--exclude-test-execution-evidence", []],
+    ["--transport", ["qa-channel"]],
+    ["--provider-mode", ["mock-openai"]],
+    ["--model", ["openai/gpt-5.5"]],
+    ["--alt-model", ["anthropic/claude-sonnet-4-6"]],
+    ["--concurrency", ["2"]],
+    ["--allow-failures", []],
+    ["--fast", []],
+  ])("rejects qa run profile-only flag %s without --qa-profile", async (flag, values) => {
+    await expect(
+      program.parseAsync(["node", "openclaw", "qa", "run", flag, ...values]),
+    ).rejects.toThrow(`qa run ${flag} requires --qa-profile`);
+
+    expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
+    expect(runQaProfileCommand).not.toHaveBeenCalled();
+  });
+
+  it.each([["--evidence-mode", "compact"], ["--exclude-test-execution-evidence"]])(
+    "maps deprecated compact evidence flag %s to slim",
+    async (...flags) => {
+      await program.parseAsync([
+        "node",
+        "openclaw",
+        "qa",
+        "run",
+        "--qa-profile",
+        "release",
+        ...flags.filter(Boolean),
+      ]);
+
+      expect(runQaProfileCommand).toHaveBeenCalledWith(
+        expect.objectContaining({
+          evidenceMode: "slim",
+          profile: "release",
+        }),
+      );
+    },
+  );
+
+  it("rejects conflicting deprecated evidence flags", async () => {
+    await expect(
+      program.parseAsync([
+        "node",
+        "openclaw",
+        "qa",
+        "run",
+        "--qa-profile",
+        "release",
+        "--evidence-mode",
+        "full",
+        "--exclude-test-execution-evidence",
+      ]),
+    ).rejects.toThrow("--exclude-test-execution-evidence conflicts with --evidence-mode full");
+
+    expect(runQaProfileCommand).not.toHaveBeenCalled();
+  });
+
+  it("rejects unknown qa evidence modes", async () => {
+    const invalidProgram = new Command();
+    invalidProgram.exitOverride();
+    invalidProgram.configureOutput({
+      writeErr: () => {},
+      writeOut: () => {},
+    });
+    registerQaLabCli(invalidProgram);
+
+    await expect(
+      invalidProgram.parseAsync([
+        "node",
+        "openclaw",
+        "qa",
+        "run",
+        "--qa-profile",
+        "smoke-ci",
+        "--evidence-mode",
+        "tiny",
+      ]),
+    ).rejects.toThrow("--evidence-mode must be one of full, slim.");
+
+    expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
+    expect(runQaProfileCommand).not.toHaveBeenCalled();
+  });
+
+  it("rejects an empty qa run --qa-profile instead of falling back to self-check", async () => {
+    await expect(
+      program.parseAsync(["node", "openclaw", "qa", "run", "--qa-profile", ""]),
+    ).rejects.toThrow("--qa-profile must not be empty.");
+
+    expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
+    expect(runQaProfileCommand).not.toHaveBeenCalled();
+  });
+
+  it("rejects self-check output flags in qa run profile mode", async () => {
+    await expect(
+      program.parseAsync([
+        "node",
+        "openclaw",
+        "qa",
+        "run",
+        "--qa-profile",
+        "smoke-ci",
+        "--output",
+        ".artifacts/qa-self-check.md",
+      ]),
+    ).rejects.toThrow("qa run --output is only valid for the self-check mode");
+
+    expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
+    expect(runQaProfileCommand).not.toHaveBeenCalled();
+  });
+
+  it("routes mantis discord-smoke flags into the mantis runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "mantis",
+      "discord-smoke",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output-dir",
+      ".artifacts/qa-e2e/mantis/discord-smoke",
+      "--guild-id",
+      "123456789012345678",
+      "--channel-id",
+      "223456789012345678",
+      "--token-file",
+      "/tmp/mantis-token",
+      "--message",
+      "hello from mantis",
+      "--skip-post",
+    ]);
+
+    expect(runMantisDiscordSmokeCommand).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      outputDir: ".artifacts/qa-e2e/mantis/discord-smoke",
+      guildId: "123456789012345678",
+      channelId: "223456789012345678",
+      tokenEnv: undefined,
+      tokenFile: "/tmp/mantis-token",
+      tokenFileEnv: undefined,
+      message: "hello from mantis",
+      skipPost: true,
+    });
+  });
+
+  it("routes mantis before/after flags into the mantis runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "mantis",
+      "run",
+      "--transport",
+      "discord",
+      "--scenario",
+      "discord-status-reactions-tool-only",
+      "--baseline",
+      "origin/main",
+      "--candidate",
+      "HEAD",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output-dir",
+      ".artifacts/qa-e2e/mantis/local-discord-status-reactions",
+      "--credential-source",
+      "convex",
+      "--credential-role",
+      "maintainer",
+      "--skip-install",
+      "--skip-build",
+    ]);
+
+    expect(runMantisBeforeAfterCommand).toHaveBeenCalledWith({
+      baseline: "origin/main",
+      candidate: "HEAD",
+      credentialRole: "maintainer",
+      credentialSource: "convex",
+      fastMode: true,
+      outputDir: ".artifacts/qa-e2e/mantis/local-discord-status-reactions",
+      providerMode: "live-frontier",
+      repoRoot: "/tmp/openclaw-repo",
+      scenario: "discord-status-reactions-tool-only",
+      skipBuild: true,
+      skipInstall: true,
+      transport: "discord",
+    });
+  });
+
+  it("routes mantis desktop browser smoke flags into the mantis runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "mantis",
+      "desktop-browser-smoke",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output-dir",
+      ".artifacts/qa-e2e/mantis/desktop-browser",
+      "--browser-url",
+      "https://openclaw.ai/docs",
+      "--html-file",
+      "qa-artifacts/timeline.html",
+      "--crabbox-bin",
+      "/tmp/crabbox",
+      "--provider",
+      "hetzner",
+      "--class",
+      "beast",
+      "--lease-id",
+      "cbx_123abc",
+      "--idle-timeout",
+      "30m",
+      "--ttl",
+      "90m",
+      "--keep-lease",
+    ]);
+
+    expect(runMantisDesktopBrowserSmokeCommand).toHaveBeenCalledWith({
+      browserUrl: "https://openclaw.ai/docs",
+      crabboxBin: "/tmp/crabbox",
+      htmlFile: "qa-artifacts/timeline.html",
+      idleTimeout: "30m",
+      keepLease: true,
+      leaseId: "cbx_123abc",
+      machineClass: "beast",
+      outputDir: ".artifacts/qa-e2e/mantis/desktop-browser",
+      provider: "hetzner",
+      repoRoot: "/tmp/openclaw-repo",
+      ttl: "90m",
+    });
+  });
+
+  it("does not shadow mantis desktop browser runtime env defaults", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "mantis",
+      "desktop-browser-smoke",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+    ]);
+
+    expect(runMantisDesktopBrowserSmokeCommand).toHaveBeenCalledWith({
+      browserUrl: undefined,
+      crabboxBin: undefined,
+      htmlFile: undefined,
+      idleTimeout: undefined,
+      keepLease: undefined,
+      leaseId: undefined,
+      machineClass: undefined,
+      outputDir: undefined,
+      provider: undefined,
+      repoRoot: "/tmp/openclaw-repo",
+      ttl: undefined,
+    });
+  });
+
+  it("routes mantis Slack desktop smoke flags into the mantis runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "mantis",
+      "slack-desktop-smoke",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output-dir",
+      ".artifacts/qa-e2e/mantis/slack-desktop",
+      "--crabbox-bin",
+      "/tmp/crabbox",
+      "--provider",
+      "hetzner",
+      "--market",
+      "on-demand",
+      "--machine-class",
+      "beast",
+      "--lease-id",
+      "cbx_123abc",
+      "--fresh-pr",
+      "openclaw/openclaw#85141",
+      "--idle-timeout",
+      "45m",
+      "--ttl",
+      "120m",
+      "--slack-url",
+      "https://app.slack.com/client/T123/C123",
+      "--provider-mode",
+      "live-frontier",
+      "--model",
+      "openai/gpt-5.5",
+      "--alt-model",
+      "openai/gpt-5.5",
+      "--scenario",
+      "slack-canary",
+      "--credential-source",
+      "env",
+      "--credential-role",
+      "maintainer",
+      "--fast",
+      "--keep-lease",
+    ]);
+
+    expect(runMantisSlackDesktopSmokeCommand).toHaveBeenCalledWith({
+      alternateModel: "openai/gpt-5.5",
+      crabboxBin: "/tmp/crabbox",
+      credentialRole: "maintainer",
+      credentialSource: "env",
+      fastMode: true,
+      freshPr: "openclaw/openclaw#85141",
+      gatewaySetup: undefined,
+      idleTimeout: "45m",
+      keepLease: true,
+      leaseId: "cbx_123abc",
+      machineClass: "beast",
+      market: "on-demand",
+      outputDir: ".artifacts/qa-e2e/mantis/slack-desktop",
+      primaryModel: "openai/gpt-5.5",
+      provider: "hetzner",
+      providerMode: "live-frontier",
+      repoRoot: "/tmp/openclaw-repo",
+      scenarioIds: ["slack-canary"],
+      slackChannelId: undefined,
+      slackUrl: "https://app.slack.com/client/T123/C123",
+      ttl: "120m",
+    });
+  });
+
+  it("routes mantis Telegram desktop builder flags into the mantis runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "mantis",
+      "telegram-desktop-builder",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output-dir",
+      ".artifacts/qa-e2e/mantis/telegram-desktop",
+      "--crabbox-bin",
+      "/tmp/crabbox",
+      "--provider",
+      "hetzner",
+      "--machine-class",
+      "beast",
+      "--lease-id",
+      "cbx_123abc",
+      "--idle-timeout",
+      "45m",
+      "--ttl",
+      "120m",
+      "--credential-source",
+      "convex",
+      "--credential-role",
+      "ci",
+      "--hydrate-mode",
+      "prehydrated",
+      "--telegram-profile-archive-env",
+      "TELEGRAM_PROFILE_TGZ_B64",
+      "--telegram-profile-dir",
+      "/home/crabbox/.local/share/TelegramDesktop",
+      "--no-gateway-setup",
+      "--keep-lease",
+    ]);
+
+    expect(runMantisTelegramDesktopBuilderCommand).toHaveBeenCalledWith({
+      crabboxBin: "/tmp/crabbox",
+      credentialRole: "ci",
+      credentialSource: "convex",
+      gatewaySetup: false,
+      hydrateMode: "prehydrated",
+      idleTimeout: "45m",
+      keepLease: true,
+      leaseId: "cbx_123abc",
+      machineClass: "beast",
+      outputDir: ".artifacts/qa-e2e/mantis/telegram-desktop",
+      provider: "hetzner",
+      repoRoot: "/tmp/openclaw-repo",
+      telegramProfileArchiveEnv: "TELEGRAM_PROFILE_TGZ_B64",
+      telegramProfileDir: "/home/crabbox/.local/share/TelegramDesktop",
+      ttl: "120m",
+    });
+  });
+
+  it("routes coverage report flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "coverage",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--output",
+      ".artifacts/qa-coverage.md",
+      "--json",
+    ]);
+
+    expect(runQaCoverageReportCommand).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      output: ".artifacts/qa-coverage.md",
+      json: true,
+      tools: false,
+      match: [],
+    });
+  });
+
+  it("routes tool coverage report flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "coverage",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--tools",
+      "--summary",
+      ".artifacts/runtime-summary.json",
+    ]);
+
+    expect(runQaCoverageReportCommand).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      tools: true,
+      json: false,
+      summary: ".artifacts/runtime-summary.json",
+      match: [],
+    });
+  });
+
+  it("routes coverage match queries into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "coverage",
+      "--match",
+      "image roundtrip",
+      "--match",
+      "native",
+    ]);
+
+    expect(runQaCoverageReportCommand).toHaveBeenCalledWith({
+      tools: false,
+      json: false,
+      match: ["image roundtrip", "native"],
+    });
+  });
+
+  it("routes JSONL replay flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "jsonl-replay",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--transcripts",
+      "qa/scenarios/jsonl-replay",
+      "--runtime-pair",
+      "openclaw,codex",
+      "--provider-mode",
+      "mock-openai",
+      "--output-dir",
+      ".artifacts/qa-e2e/jsonl-replay-test",
+    ]);
+
+    expect(runQaJsonlReplayCommand).toHaveBeenCalledWith({
+      repoRoot: "/tmp/openclaw-repo",
+      transcripts: "qa/scenarios/jsonl-replay",
+      runtimePair: "openclaw,codex",
+      providerMode: "mock-openai",
+      outputDir: ".artifacts/qa-e2e/jsonl-replay-test",
+    });
+  });
+
+  it("delegates discovered qa runner registration through the generic host seam", () => {
+    const [{ registration }] = listQaRunnerCliContributions.mock.results[0].value;
+    expect(registration.register).toHaveBeenCalledTimes(1);
+  });
+
+  it("keeps Telegram credential flags on the shared host CLI", () => {
+    const qa = program.commands.find((command) => command.name() === "qa");
+    const telegram = qa?.commands.find((command) => command.name() === "telegram");
+    const optionNames = telegram?.options.map((option) => option.long) ?? [];
+
+    expect(optionNames).toContain("--credential-source");
+    expect(optionNames).toContain("--credential-role");
+    expect(optionNames).toContain("--list-scenarios");
+  });
+
+  it("registers standalone provider server commands from the provider registry", async () => {
+    const qa = program.commands.find((command) => command.name() === "qa");
+    const commandNames = qa?.commands.map((command) => command.name()) ?? [];
+    expect(commandNames).toContain("mock-openai");
+    expect(commandNames).toContain("aimock");
+
+    await program.parseAsync(["node", "openclaw", "qa", "aimock", "--port", "44080"]);
+
+    expect(runQaProviderServerCommand).toHaveBeenCalledWith("aimock", {
+      host: "127.0.0.1",
+      port: 44080,
+    });
+  });
+
+  it("normalizes signed decimal QA numeric option values through the shared parser", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "aimock", "--port", "+044080"]);
+
+    expect(runQaProviderServerCommand).toHaveBeenCalledWith("aimock", {
+      host: "127.0.0.1",
+      port: 44080,
+    });
+  });
+
+  it.each([
+    [["qa", "suite", "--concurrency", "1.5"], "--concurrency must be a positive integer."],
+    [["qa", "suite", "--cpus", "0x4"], "--cpus must be a positive integer."],
+    [
+      ["qa", "manual", "--message", "hi", "--timeout-ms", "1e3"],
+      "--timeout-ms must be a positive integer.",
+    ],
+    [["qa", "credentials", "list", "--limit", "0x10"], "--limit must be a positive integer."],
+    [["qa", "ui", "--port", "1e4"], "--port must be a positive integer."],
+    [
+      ["qa", "docker-scaffold", "--output-dir", "/tmp/qa", "--gateway-port", "1.5"],
+      "--gateway-port must be a positive integer.",
+    ],
+    [["qa", "up", "--qa-lab-port", "0x43124"], "--qa-lab-port must be a positive integer."],
+    [["qa", "aimock", "--port", "1e4"], "--port must be a positive integer."],
+  ])("rejects non-decimal QA numeric option %j", async (args, message) => {
+    const invalidProgram = new Command();
+    invalidProgram.exitOverride();
+    invalidProgram.configureOutput({
+      writeErr: () => {},
+      writeOut: () => {},
+    });
+    registerQaLabCli(invalidProgram);
+
+    await expect(invalidProgram.parseAsync(["node", "openclaw", ...args])).rejects.toThrow(message);
+  });
+
+  it.each([
+    [["qa", "ui", "--port", "65536"], "--port must be a TCP port between 1 and 65535."],
+    [
+      ["qa", "ui", "--advertise-port", "999999"],
+      "--advertise-port must be a TCP port between 1 and 65535.",
+    ],
+    [
+      ["qa", "docker-scaffold", "--output-dir", "/tmp/qa", "--gateway-port", "65536"],
+      "--gateway-port must be a TCP port between 1 and 65535.",
+    ],
+    [
+      ["qa", "up", "--qa-lab-port", "65536"],
+      "--qa-lab-port must be a TCP port between 1 and 65535.",
+    ],
+    [["qa", "aimock", "--port", "65536"], "--port must be a TCP port between 1 and 65535."],
+  ])("rejects out-of-range QA port option %j", async (args, message) => {
+    const invalidProgram = new Command();
+    invalidProgram.exitOverride();
+    invalidProgram.configureOutput({
+      writeErr: () => {},
+      writeOut: () => {},
+    });
+    registerQaLabCli(invalidProgram);
+
+    await expect(invalidProgram.parseAsync(["node", "openclaw", ...args])).rejects.toThrow(message);
+  });
+
+  it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => {
+    listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]);
+    const blockedProgram = new Command();
+    registerQaLabCli(blockedProgram);
+
+    await expect(
+      blockedProgram.parseAsync(["node", "openclaw", "qa", TEST_QA_RUNNER.commandName]),
+    ).rejects.toThrow(`Enable or allow plugin "${TEST_QA_RUNNER.pluginId}"`);
+  });
+
+  it("rejects discovered runners that collide with built-in qa subcommands", () => {
+    listQaRunnerCliContributions
+      .mockReset()
+      .mockReturnValue([createConflictingQaRunnerContribution("manual")]);
+
+    expect(() => registerQaLabCli(new Command())).toThrow(
+      'QA runner command "manual" conflicts with an existing qa subcommand',
+    );
+  });
+
+  it("routes telegram CLI defaults into the lane runtime", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "telegram"]);
+
+    expect(runQaTelegramCommand).toHaveBeenCalledWith({
+      repoRoot: undefined,
+      outputDir: undefined,
+      providerMode: "live-frontier",
+      primaryModel: undefined,
+      alternateModel: undefined,
+      fastMode: false,
+      allowFailures: false,
+      scenarioIds: [],
+      listScenarios: false,
+      sutAccountId: "sut",
+      credentialSource: undefined,
+      credentialRole: undefined,
+    });
+  });
+
+  it("forwards --list-scenarios for telegram runs", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "telegram", "--list-scenarios"]);
+
+    const options = requireQaTelegramOptions();
+    expect(options.listScenarios).toBe(true);
+  });
+
+  it("forwards --allow-failures for telegram runs", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "telegram", "--allow-failures"]);
+
+    const options = requireQaTelegramOptions();
+    expect(options.allowFailures).toBe(true);
+  });
+
+  it("forwards --allow-failures for suite runs", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "suite", "--allow-failures"]);
+
+    const options = requireQaSuiteOptions();
+    expect(options.allowFailures).toBe(true);
+    expect(options.providerMode).toBeUndefined();
+  });
+
+  it("forwards --pack for suite runs", async () => {
+    await program.parseAsync(["node", "openclaw", "qa", "suite", "--pack", "personal-agent"]);
+
+    const options = requireQaSuiteOptions();
+    expect(options.pack).toBe("personal-agent");
+  });
+
+  it("forwards --runtime-parity-tier for suite runs", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "suite",
+      "--runtime-parity-tier",
+      "standard",
+      "--runtime-parity-tier",
+      "optional,soak",
+    ]);
+
+    const options = requireQaSuiteOptions();
+    expect(options.runtimeParityTier).toEqual(["standard", "optional,soak"]);
+  });
+
+  it("routes credential add flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "credentials",
+      "add",
+      "--kind",
+      "telegram",
+      "--payload-file",
+      "qa/payload.json",
+      "--repo-root",
+      "/tmp/openclaw-repo",
+      "--note",
+      "shared lane",
+      "--site-url",
+      "https://first-schnauzer-821.convex.site",
+      "--endpoint-prefix",
+      "/qa-credentials/v1",
+      "--actor-id",
+      "maintainer-local",
+      "--json",
+    ]);
+
+    expect(runQaCredentialsAddCommand).toHaveBeenCalledWith({
+      kind: "telegram",
+      payloadFile: "qa/payload.json",
+      repoRoot: "/tmp/openclaw-repo",
+      note: "shared lane",
+      siteUrl: "https://first-schnauzer-821.convex.site",
+      endpointPrefix: "/qa-credentials/v1",
+      actorId: "maintainer-local",
+      json: true,
+    });
+  });
+
+  it("routes credential remove flags into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "credentials",
+      "remove",
+      "--credential-id",
+      "j57b8k419ba7bcsfw99rg05c9184p8br",
+      "--site-url",
+      "https://first-schnauzer-821.convex.site",
+      "--actor-id",
+      "maintainer-local",
+      "--json",
+    ]);
+
+    expect(runQaCredentialsRemoveCommand).toHaveBeenCalledWith({
+      credentialId: "j57b8k419ba7bcsfw99rg05c9184p8br",
+      siteUrl: "https://first-schnauzer-821.convex.site",
+      actorId: "maintainer-local",
+      endpointPrefix: undefined,
+      json: true,
+    });
+  });
+
+  it("routes credential list defaults into the qa runtime command", async () => {
+    await program.parseAsync([
+      "node",
+      "openclaw",
+      "qa",
+      "credentials",
+      "list",
+      "--kind",
+      "telegram",
+    ]);
+
+    expect(runQaCredentialsListCommand).toHaveBeenCalledWith({
+      kind: "telegram",
+      status: "all",
+      limit: undefined,
+      showSecrets: false,
+      siteUrl: undefined,
+      endpointPrefix: undefined,
+      actorId: undefined,
+      json: false,
+    });
+  });
+});

extensions/qa-lab/src/codex-plugin-lifecycle.test.ts

@@ -0,0 +1,191 @@
+// Qa Lab tests cover codex plugin lifecycle plugin behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  QA_CODEX_OAUTH_PROFILE_ID,
+  QA_OPENAI_API_KEY_PROFILE_ID,
+  resolveCodexAuthProfile,
+  seedAuthProfiles,
+  snapshotAuthProfiles,
+} from "./auth-profile.fixture.js";
+import {
+  CODEX_PLUGIN_CURRENT_VERSION,
+  CODEX_PLUGIN_LIFECYCLE_MESSAGES,
+  createCodexPluginInstallGate,
+  evaluateCodexPluginLifecycle,
+  seedCodexPluginAt,
+  snapshotCodexPluginState,
+} from "./codex-plugin.fixture.js";
+import { createTempDirHarness } from "./temp-dir.test-helper.js";
+
+const tempDirs = createTempDirHarness();
+
+async function createAgentDir(prefix: string) {
+  const root = await tempDirs.makeTempDir(prefix);
+  const agentDir = path.join(root, "agents", "qa", "agent");
+  await fs.mkdir(agentDir, { recursive: true });
+  return agentDir;
+}
+
+afterEach(async () => {
+  await tempDirs.cleanup();
+});
+
+describe("codex plugin lifecycle: cold install", () => {
+  it("repairs a missing codex plugin before the retry succeeds without leaking to the API-key path", async () => {
+    const agentDir = await createAgentDir("qa-codex-plugin-cold-");
+    await seedCodexPluginAt("missing", agentDir);
+    await seedAuthProfiles("mixed", agentDir);
+
+    const missing = evaluateCodexPluginLifecycle({
+      plugin: await snapshotCodexPluginState(agentDir),
+      auth: await snapshotAuthProfiles(agentDir),
+      hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
+    });
+
+    expect(missing.status).toBe("repair-required");
+    expect(missing.remediation).toBe(CODEX_PLUGIN_LIFECYCLE_MESSAGES.missingPlugin);
+    expect(missing.selectedAuthProfileId).toBe(QA_CODEX_OAUTH_PROFILE_ID);
+    expect(missing.selectedAuthProfileId).not.toBe(QA_OPENAI_API_KEY_PROFILE_ID);
+
+    await seedCodexPluginAt("current", agentDir);
+    const repaired = evaluateCodexPluginLifecycle({
+      plugin: await snapshotCodexPluginState(agentDir),
+      auth: await snapshotAuthProfiles(agentDir),
+      hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
+    });
+
+    expect(repaired.status).toBe("ready");
+    expect(repaired.remediation).toBeUndefined();
+    expect(repaired.tokenRoute).toBe("codex-oauth");
+  });
+});
+
+describe("codex plugin lifecycle: OAuth-only with mixed profiles", () => {
+  it("selects openai OAuth when openai API-key profiles are present", async () => {
+    const agentDir = await createAgentDir("qa-codex-auth-mixed-");
+    await seedAuthProfiles("mixed", agentDir);
+
+    const selection = resolveCodexAuthProfile(await snapshotAuthProfiles(agentDir));
+
+    expect(selection.status).toBe("ready");
+    if (selection.status !== "ready") {
+      throw new Error(selection.remediation);
+    }
+    expect(selection.profileId).toBe(QA_CODEX_OAUTH_PROFILE_ID);
+    expect(selection.profileId).not.toBe(QA_OPENAI_API_KEY_PROFILE_ID);
+    expect(selection.provider).toBe("openai");
+    expect(selection.mode).toBe("oauth");
+  });
+});
+
+describe("codex plugin lifecycle: pinned-old codex plugin with new OpenClaw", () => {
+  it("blocks with a precise update remediation when the plugin is older than the host", async () => {
+    const agentDir = await createAgentDir("qa-codex-plugin-old-");
+    await seedCodexPluginAt("2026.5.19", agentDir);
+    await seedAuthProfiles("oauth-only", agentDir);
+
+    const result = evaluateCodexPluginLifecycle({
+      plugin: await snapshotCodexPluginState(agentDir),
+      auth: await snapshotAuthProfiles(agentDir),
+      hostVersion: "2026.5.21",
+    });
+
+    expect(result.status).toBe("blocked");
+    expect(result.remediation).toBe(
+      'Codex plugin version 2026.5.19 is older than OpenClaw 2026.5.21. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".',
+    );
+  });
+});
+
+describe("codex plugin lifecycle: pinned-new codex plugin with old OpenClaw", () => {
+  it("blocks with a precise host-upgrade remediation when the plugin is newer than the host", async () => {
+    const agentDir = await createAgentDir("qa-codex-plugin-new-");
+    await seedCodexPluginAt("2026.5.22", agentDir);
+    await seedAuthProfiles("oauth-only", agentDir);
+
+    const result = evaluateCodexPluginLifecycle({
+      plugin: await snapshotCodexPluginState(agentDir),
+      auth: await snapshotAuthProfiles(agentDir),
+      hostVersion: "2026.5.21",
+    });
+
+    expect(result.status).toBe("blocked");
+    expect(result.remediation).toBe(
+      "Codex plugin version 2026.5.22 requires a newer OpenClaw host than 2026.5.21. Upgrade OpenClaw or install a codex plugin version pinned to 2026.5.21.",
+    );
+  });
+});
+
+describe("codex plugin lifecycle: install racing first agent turn", () => {
+  it("gates the first turn on install completion without sleeps, lost tokens, or duplicate responses", async () => {
+    const gate = createCodexPluginInstallGate();
+    const turn = gate.runFirstTurnAfterInstall({
+      inputTokens: 17,
+      run: () => "QA_CODEX_PLUGIN_TURN_OK",
+    });
+
+    expect(gate.events).toEqual(["agent-turn:waiting-for-codex-plugin"]);
+
+    gate.markInstalled();
+    await expect(turn).resolves.toEqual({
+      text: "QA_CODEX_PLUGIN_TURN_OK",
+      inputTokens: 17,
+      responseCount: 1,
+    });
+    expect(gate.events).toEqual([
+      "agent-turn:waiting-for-codex-plugin",
+      "codex-plugin:installed",
+      "agent-turn:started",
+      "agent-turn:completed",
+    ]);
+  });
+});
+
+describe("codex plugin lifecycle: doctor migration safety matrix", () => {
+  it.each([
+    {
+      name: "oauth-only host",
+      profileShape: "oauth-only" as const,
+      config: {},
+    },
+    {
+      name: "mixed profile with no pin",
+      profileShape: "mixed" as const,
+      config: {},
+    },
+    {
+      name: "mixed profile with defaults OpenClaw pin",
+      profileShape: "mixed" as const,
+      config: { agents: { defaults: { agentRuntime: { id: "openclaw" } } } },
+      expectedRemovedRuntimePins: ["agentRuntime.id=openclaw"],
+    },
+    {
+      name: "mixed profile with main-agent OpenClaw pin",
+      profileShape: "mixed" as const,
+      config: { agents: { list: { main: { agentRuntime: { id: "openclaw" } } } } },
+      expectedRemovedRuntimePins: ["agentRuntime.id=openclaw"],
+    },
+  ])(
+    "keeps codex auth and strips stale OpenClaw runtime pins for $name",
+    async ({ profileShape, config, expectedRemovedRuntimePins = [] }) => {
+      const agentDir = await createAgentDir("qa-codex-doctor-matrix-");
+      await seedCodexPluginAt("current", agentDir);
+      await seedAuthProfiles(profileShape, agentDir);
+
+      const result = evaluateCodexPluginLifecycle({
+        plugin: await snapshotCodexPluginState(agentDir),
+        auth: await snapshotAuthProfiles(agentDir),
+        hostVersion: CODEX_PLUGIN_CURRENT_VERSION,
+        config,
+        doctorFix: true,
+      });
+
+      expect(result.status).toBe("ready");
+      expect(result.selectedAuthProfileId).toBe(QA_CODEX_OAUTH_PROFILE_ID);
+      expect(result.tokenRoute).toBe("codex-oauth");
+      expect(result.removedRuntimePins).toEqual(expectedRemovedRuntimePins);
+    },
+  );
+});

extensions/qa-lab/src/codex-plugin.fixture.ts

@@ -0,0 +1,289 @@
+// Qa Lab plugin module implements codex plugin.fixture behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { resolveCodexAuthProfile, type QaAuthProfileSnapshot } from "./auth-profile.fixture.js";
+
+export const CODEX_PLUGIN_CURRENT_VERSION = "2026.5.21";
+export const CODEX_PLUGIN_HEAD_VERSION = "head";
+export const CODEX_PLUGIN_ID = "codex";
+
+export const CODEX_PLUGIN_LIFECYCLE_MESSAGES = Object.freeze({
+  missingPlugin:
+    'Codex plugin is required for Codex runtime. Run "openclaw doctor --fix" to install @openclaw/codex, then retry.',
+});
+
+export type CodexPluginFixtureVersion = "missing" | "current" | "head" | (string & {});
+
+export type CodexPluginState = {
+  installed: boolean;
+  version?: string;
+};
+
+export type CodexPluginLifecycleStatus = "ready" | "repair-required" | "blocked";
+
+export type CodexPluginLifecycleResult = {
+  status: CodexPluginLifecycleStatus;
+  pluginState: CodexPluginState;
+  selectedAuthProfileId?: string;
+  tokenRoute?: "codex-oauth" | "unavailable";
+  remediation?: string;
+  removedRuntimePins: string[];
+};
+
+type CodexPluginPackageJson = {
+  name: "@openclaw/codex";
+  version: string;
+  openclaw: {
+    install: {
+      minHostVersion: string;
+    };
+    compat: {
+      pluginApi: string;
+    };
+  };
+};
+
+type ComparableVersion = {
+  major: number;
+  minor: number;
+  patch: number;
+};
+
+type CodexPluginInstallGateResult = {
+  text: string;
+  inputTokens: number;
+  responseCount: number;
+};
+
+function codexPluginDir(agentDir: string) {
+  return path.join(agentDir, "plugins", CODEX_PLUGIN_ID);
+}
+
+function resolveFixtureVersion(version: CodexPluginFixtureVersion): string {
+  if (version === "current") {
+    return CODEX_PLUGIN_CURRENT_VERSION;
+  }
+  return version;
+}
+
+function buildPackageJson(version: string): CodexPluginPackageJson {
+  return {
+    name: "@openclaw/codex",
+    version,
+    openclaw: {
+      install: {
+        minHostVersion: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`,
+      },
+      compat: {
+        pluginApi: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`,
+      },
+    },
+  };
+}
+
+function parseComparableVersion(value: string | undefined): ComparableVersion | null {
+  if (!value || value === CODEX_PLUGIN_HEAD_VERSION) {
+    return parseComparableVersion(CODEX_PLUGIN_CURRENT_VERSION);
+  }
+  const match = value.trim().match(/^(\d+)\.(\d+)\.(\d+)/);
+  if (!match) {
+    return null;
+  }
+  return {
+    major: Number.parseInt(match[1] ?? "0", 10),
+    minor: Number.parseInt(match[2] ?? "0", 10),
+    patch: Number.parseInt(match[3] ?? "0", 10),
+  };
+}
+
+function compareVersions(left: string | undefined, right: string): number {
+  const leftVersion = parseComparableVersion(left);
+  const rightVersion = parseComparableVersion(right);
+  if (!leftVersion || !rightVersion) {
+    return 0;
+  }
+  if (leftVersion.major !== rightVersion.major) {
+    return leftVersion.major - rightVersion.major;
+  }
+  if (leftVersion.minor !== rightVersion.minor) {
+    return leftVersion.minor - rightVersion.minor;
+  }
+  return leftVersion.patch - rightVersion.patch;
+}
+
+function formatPinnedOldRemediation(pluginVersion: string, hostVersion: string) {
+  return `Codex plugin version ${pluginVersion} is older than OpenClaw ${hostVersion}. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".`;
+}
+
+function formatPinnedNewRemediation(pluginVersion: string, hostVersion: string) {
+  return `Codex plugin version ${pluginVersion} requires a newer OpenClaw host than ${hostVersion}. Upgrade OpenClaw or install a codex plugin version pinned to ${hostVersion}.`;
+}
+
+function collectStaleLegacyRuntimePins(config: unknown): string[] {
+  if (!config || typeof config !== "object") {
+    return [];
+  }
+  const root = config as {
+    agents?: {
+      defaults?: { agentRuntime?: { id?: unknown } };
+      list?: Record<string, { agentRuntime?: { id?: unknown } }>;
+    };
+  };
+  const markers = new Set<string>();
+  const collectRuntimePin = (value: unknown) => {
+    if (value === "openclaw") {
+      markers.add(`agentRuntime.id=${value}`);
+    }
+  };
+  collectRuntimePin(root.agents?.defaults?.agentRuntime?.id);
+  for (const entry of Object.values(root.agents?.list ?? {})) {
+    collectRuntimePin(entry.agentRuntime?.id);
+  }
+  return [...markers].toSorted();
+}
+
+export async function seedCodexPluginAt(
+  version: CodexPluginFixtureVersion,
+  agentDir: string,
+): Promise<void> {
+  const targetDir = codexPluginDir(agentDir);
+  await fs.rm(targetDir, { recursive: true, force: true });
+  if (version === "missing") {
+    return;
+  }
+
+  const resolvedVersion = resolveFixtureVersion(version);
+  await fs.mkdir(targetDir, { recursive: true });
+  await fs.writeFile(
+    path.join(targetDir, "package.json"),
+    `${JSON.stringify(buildPackageJson(resolvedVersion), null, 2)}\n`,
+    "utf8",
+  );
+  await fs.writeFile(
+    path.join(targetDir, "openclaw.plugin.json"),
+    `${JSON.stringify({ id: CODEX_PLUGIN_ID, name: "Codex" }, null, 2)}\n`,
+    "utf8",
+  );
+}
+
+export async function snapshotCodexPluginState(agentDir: string): Promise<CodexPluginState> {
+  const packagePath = path.join(codexPluginDir(agentDir), "package.json");
+  const raw = await fs.readFile(packagePath, "utf8").catch((error: unknown) => {
+    if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") {
+      return null;
+    }
+    throw error;
+  });
+  if (!raw) {
+    return { installed: false };
+  }
+
+  const parsed = JSON.parse(raw) as { version?: unknown };
+  return {
+    installed: true,
+    ...(typeof parsed.version === "string" ? { version: parsed.version } : {}),
+  };
+}
+
+export function evaluateCodexPluginLifecycle(params: {
+  plugin: CodexPluginState;
+  auth: QaAuthProfileSnapshot;
+  hostVersion: string;
+  config?: unknown;
+  doctorFix?: boolean;
+}): CodexPluginLifecycleResult {
+  const authSelection = resolveCodexAuthProfile(params.auth);
+  const selectedAuthProfileId =
+    authSelection.status === "ready" ? authSelection.profileId : undefined;
+  const tokenRoute = authSelection.status === "ready" ? "codex-oauth" : "unavailable";
+  const removedRuntimePins = params.doctorFix ? collectStaleLegacyRuntimePins(params.config) : [];
+
+  if (!params.plugin.installed) {
+    return {
+      status: "repair-required",
+      pluginState: params.plugin,
+      ...(selectedAuthProfileId ? { selectedAuthProfileId } : {}),
+      tokenRoute,
+      remediation: CODEX_PLUGIN_LIFECYCLE_MESSAGES.missingPlugin,
+      removedRuntimePins,
+    };
+  }
+
+  if (authSelection.status === "blocked") {
+    return {
+      status: "blocked",
+      pluginState: params.plugin,
+      tokenRoute,
+      remediation: authSelection.remediation,
+      removedRuntimePins,
+    };
+  }
+
+  const versionDelta = compareVersions(params.plugin.version, params.hostVersion);
+  if (versionDelta < 0 && params.plugin.version) {
+    return {
+      status: "blocked",
+      pluginState: params.plugin,
+      selectedAuthProfileId,
+      tokenRoute,
+      remediation: formatPinnedOldRemediation(params.plugin.version, params.hostVersion),
+      removedRuntimePins,
+    };
+  }
+  if (versionDelta > 0 && params.plugin.version) {
+    return {
+      status: "blocked",
+      pluginState: params.plugin,
+      selectedAuthProfileId,
+      tokenRoute,
+      remediation: formatPinnedNewRemediation(params.plugin.version, params.hostVersion),
+      removedRuntimePins,
+    };
+  }
+
+  return {
+    status: "ready",
+    pluginState: params.plugin,
+    selectedAuthProfileId,
+    tokenRoute,
+    removedRuntimePins,
+  };
+}
+
+export function createCodexPluginInstallGate() {
+  const events: string[] = [];
+  let installed = false;
+  let resolveInstall: (() => void) | undefined;
+  const installedPromise = new Promise<void>((resolve) => {
+    resolveInstall = resolve;
+  });
+
+  return {
+    events,
+    markInstalled() {
+      if (installed) {
+        return;
+      }
+      installed = true;
+      events.push("codex-plugin:installed");
+      resolveInstall?.();
+    },
+    async runFirstTurnAfterInstall(params: {
+      inputTokens: number;
+      run: () => string | Promise<string>;
+    }): Promise<CodexPluginInstallGateResult> {
+      if (!installed) {
+        events.push("agent-turn:waiting-for-codex-plugin");
+        await installedPromise;
+      }
+      events.push("agent-turn:started");
+      const text = await params.run();
+      events.push("agent-turn:completed");
+      return {
+        text,
+        inputTokens: params.inputTokens,
+        responseCount: 1,
+      };
+    },
+  };
+}

extensions/qa-lab/src/confidence-report.test.ts

@@ -0,0 +1,978 @@
+// Qa Lab tests cover confidence report plugin behavior.
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+  buildQaConfidenceReport,
+  buildQaConfidenceSelfTestSummary,
+  renderQaConfidenceMarkdownReport,
+  writeQaConfidenceSelfTestArtifacts,
+  type QaConfidenceManifest,
+} from "./confidence-report.js";
+
+describe("qa confidence report", () => {
+  let tempRoot: string;
+
+  beforeEach(async () => {
+    tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "qa-confidence-"));
+  });
+
+  afterEach(async () => {
+    await fs.rm(tempRoot, { recursive: true, force: true });
+  });
+
+  async function writeJson(relativePath: string, payload: unknown) {
+    const filePath = path.join(tempRoot, relativePath);
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    await fs.writeFile(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
+    return filePath;
+  }
+
+  it("passes strict zero-unknowns when every lane passes or has an allowed blocked verdict", async () => {
+    await writeJson("tool-defaults/qa-suite-summary.json", {
+      counts: { total: 20, passed: 18, skipped: 2, failed: 0 },
+      scenarios: [],
+    });
+    await writeJson("token/qa-runtime-token-efficiency-summary.json", {
+      status: "estimated",
+      pass: true,
+      rows: [{ scenarioId: "one", usageSource: "mock-estimate" }],
+    });
+
+    const manifest: QaConfidenceManifest = {
+      version: 1,
+      profile: "codex-100",
+      lanes: [
+        {
+          id: "tool-defaults-direct",
+          title: "Tool defaults direct",
+          kind: "qa-suite-summary",
+          artifact: "tool-defaults/qa-suite-summary.json",
+          required: true,
+        },
+        {
+          id: "mock-token-efficiency",
+          title: "Mock token efficiency",
+          kind: "token-efficiency-summary",
+          artifact: "token/qa-runtime-token-efficiency-summary.json",
+          required: true,
+          expectedTokenUsageSource: "mock-estimate",
+        },
+        {
+          id: "live-token-efficiency",
+          title: "Live token efficiency",
+          kind: "token-efficiency-summary",
+          artifact: "live/qa-runtime-token-efficiency-summary.json",
+          required: true,
+          missingVerdict: "environment-blocked",
+          missingReason: "OPENAI OAuth credentials are not available in this runner.",
+        },
+      ],
+    };
+
+    const report = await buildQaConfidenceReport({
+      manifest,
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.globalPass).toBe(false);
+    expect(report.counts).toMatchObject({ passed: 2, blocked: 1, unknown: 0, failed: 0 });
+    expect(report.lanes.map((lane) => lane.verdict)).toEqual([
+      "pass",
+      "pass",
+      "environment-blocked",
+    ]);
+    expect(report.lanes[0]?.artifactPath).toBe("tool-defaults/qa-suite-summary.json");
+    expect(report.lanes[0]?.artifactPath).not.toContain(tempRoot);
+    expect(report.lanes[0]?.details).toContain("counts.skipped=2");
+    expect(renderQaConfidenceMarkdownReport(report)).toContain("Zero unknowns: yes");
+    expect(renderQaConfidenceMarkdownReport(report)).toContain("Global pass: no");
+  });
+
+  it("does not let optional lanes block strict gates", async () => {
+    await writeJson("required/qa-suite-summary.json", {
+      counts: { total: 1, passed: 1, skipped: 0, failed: 0 },
+      scenarios: [],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "required",
+            title: "Required",
+            kind: "qa-suite-summary",
+            artifact: "required/qa-suite-summary.json",
+            required: true,
+          },
+          {
+            id: "optional-missing",
+            title: "Optional missing",
+            kind: "qa-suite-summary",
+            artifact: "optional/qa-suite-summary.json",
+            required: false,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      strictGlobalPass: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.counts).toMatchObject({ total: 1, passed: 1, unknown: 0 });
+    expect(report.failures).toEqual([]);
+    expect(report.lanes[1]).toMatchObject({ id: "optional-missing", status: "missing" });
+  });
+
+  it("fails strict global pass when any lane is blocked, missing, unknown, or classified failed", async () => {
+    await writeJson("classified/qa-suite-summary.json", {
+      counts: { total: 1, passed: 0, skipped: 0, failed: 1 },
+      scenarios: [{ name: "classified", status: "fail" }],
+    });
+    await writeJson("unknown/qa-suite-summary.json", {
+      counts: { total: 1, passed: 0, skipped: 0, failed: 1 },
+      scenarios: [{ name: "unknown", status: "fail" }],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "blocked-live",
+            title: "Blocked live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+            missingVerdict: "environment-blocked",
+            missingReason: "OPENAI_API_KEY missing.",
+          },
+          {
+            id: "missing-soak",
+            title: "Missing soak",
+            kind: "qa-suite-summary",
+            artifact: "soak/qa-suite-summary.json",
+            required: true,
+          },
+          {
+            id: "classified-fixture",
+            title: "Classified fixture",
+            kind: "qa-suite-summary",
+            artifact: "classified/qa-suite-summary.json",
+            required: true,
+            failureVerdict: "fixture-bug",
+          },
+          {
+            id: "unknown-failure",
+            title: "Unknown failure",
+            kind: "qa-suite-summary",
+            artifact: "unknown/qa-suite-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      strictGlobalPass: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.zeroUnknowns).toBe(false);
+    expect(report.globalPass).toBe(false);
+    expect(report.counts).toMatchObject({
+      blocked: 1,
+      missing: 1,
+      failed: 1,
+      unknown: 2,
+    });
+    expect(report.failures).toEqual([
+      "blocked-live is blocked: OPENAI_API_KEY missing.",
+      "missing-soak is missing: artifact missing and no missingVerdict was configured",
+      "classified-fixture is classified fixture-bug: qa-suite-summary counts.failed=1 counts.total=1 counts.skipped=0",
+      "unknown-failure is unclassified: qa-suite-summary counts.failed=1 counts.total=1 counts.skipped=0",
+    ]);
+  });
+
+  it("fails strict global pass for skipped suite rows until a backfill lane passes", async () => {
+    await writeJson("report-only/qa-suite-summary.json", {
+      counts: { total: 3, passed: 2, skipped: 1, failed: 0 },
+      scenarios: [],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "report-only",
+            title: "Report-only",
+            kind: "qa-suite-summary",
+            artifact: "report-only/qa-suite-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      strictGlobalPass: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.zeroUnknowns).toBe(true);
+    expect(report.globalPass).toBe(false);
+    expect(report.failures).toEqual([
+      "report-only has 1 skipped row(s) with no passing backfill lane",
+    ]);
+  });
+
+  it("infers skipped suite rows from totals and scenario status", async () => {
+    for (const [artifact, expectedDetail] of [
+      [{ counts: { total: 3, passed: 2, failed: 0 }, scenarios: [] }, "counts.skipped=1"],
+      [
+        {
+          counts: { total: 2, passed: 2, failed: 0 },
+          scenarios: [
+            { name: "passing", status: "pass" },
+            { name: "skipped", status: "skip" },
+          ],
+        },
+        "counts.skipped=1",
+      ],
+    ] as const) {
+      await writeJson("report-only/qa-suite-summary.json", artifact);
+
+      const report = await buildQaConfidenceReport({
+        manifest: {
+          version: 1,
+          profile: "codex-100",
+          lanes: [
+            {
+              id: "report-only",
+              title: "Report-only",
+              kind: "qa-suite-summary",
+              artifact: "report-only/qa-suite-summary.json",
+              required: true,
+            },
+          ],
+        },
+        artifactRoot: tempRoot,
+        strictZeroUnknowns: true,
+        strictGlobalPass: true,
+        generatedAt: "2026-05-12T00:00:00.000Z",
+      });
+
+      expect(report.globalPass).toBe(false);
+      expect(report.failures).toEqual([
+        "report-only has 1 skipped row(s) with no passing backfill lane",
+      ]);
+      expect(report.lanes[0]).toMatchObject({ skippedCount: 1 });
+      expect(report.lanes[0]?.details).toContain(expectedDetail);
+    }
+  });
+
+  it("does not pass suite summaries with unsupported non-pass statuses", async () => {
+    for (const [artifact, expectedDetail] of [
+      [
+        {
+          counts: { total: 1, passed: 1, failed: 0, skipped: 0 },
+          scenarios: [{ name: "errored", status: "error" }],
+        },
+        "unsupported non-pass status",
+      ],
+      [
+        {
+          scenarios: [{ name: "timed out", status: "timeout" }],
+        },
+        "unsupported non-pass status",
+      ],
+    ] as const) {
+      await writeJson("report-only/qa-suite-summary.json", artifact);
+
+      const report = await buildQaConfidenceReport({
+        manifest: {
+          version: 1,
+          profile: "codex-100",
+          lanes: [
+            {
+              id: "report-only",
+              title: "Report-only",
+              kind: "qa-suite-summary",
+              artifact: "report-only/qa-suite-summary.json",
+              required: true,
+            },
+          ],
+        },
+        artifactRoot: tempRoot,
+        strictZeroUnknowns: true,
+        strictGlobalPass: true,
+        generatedAt: "2026-05-12T00:00:00.000Z",
+      });
+
+      expect(report.pass).toBe(false);
+      expect(report.globalPass).toBe(false);
+      expect(report.zeroUnknowns).toBe(false);
+      expect(report.lanes[0]).toMatchObject({
+        status: "unknown",
+      });
+      expect(report.lanes[0]?.details).toContain(expectedDetail);
+    }
+  });
+
+  it("rejects skipped token reports when a live usage source is required", async () => {
+    await writeJson("live-token/qa-runtime-token-efficiency-summary.json", {
+      status: "skipped",
+      pass: true,
+      rows: [],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "live-token-efficiency",
+            title: "Live token efficiency",
+            kind: "token-efficiency-summary",
+            artifact: "live-token/qa-runtime-token-efficiency-summary.json",
+            required: true,
+            expectedTokenUsageSource: "live-usage",
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.lanes[0]).toMatchObject({
+      status: "unknown",
+      details: "token summary has no live-usage rows",
+    });
+  });
+
+  it("preserves partial zero-unknown mode for classified failing lanes", async () => {
+    await writeJson("classified/qa-suite-summary.json", {
+      counts: { total: 1, passed: 0, skipped: 0, failed: 1 },
+      scenarios: [{ name: "classified", status: "fail" }],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "classified-fixture",
+            title: "Classified fixture",
+            kind: "qa-suite-summary",
+            artifact: "classified/qa-suite-summary.json",
+            required: true,
+            failureVerdict: "fixture-bug",
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.zeroUnknowns).toBe(true);
+    expect(report.globalPass).toBe(false);
+    expect(report.counts.failed).toBe(1);
+  });
+
+  it("passes strict global pass when skipped suite rows are backfilled by a passing lane", async () => {
+    await writeJson("report-only/qa-suite-summary.json", {
+      counts: { total: 3, passed: 2, skipped: 1, failed: 0 },
+      scenarios: [],
+    });
+    await writeJson("live-backfill/qa-suite-summary.json", {
+      counts: { total: 1, passed: 1, skipped: 0, failed: 0 },
+      scenarios: [],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "report-only",
+            title: "Report-only",
+            kind: "qa-suite-summary",
+            artifact: "report-only/qa-suite-summary.json",
+            required: true,
+            skipBackfillLane: "live-backfill",
+          },
+          {
+            id: "live-backfill",
+            title: "Live backfill",
+            kind: "qa-suite-summary",
+            artifact: "live-backfill/qa-suite-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      strictGlobalPass: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.zeroUnknowns).toBe(true);
+    expect(report.globalPass).toBe(true);
+    expect(report.lanes[0]).toMatchObject({
+      skippedCount: 1,
+      skipBackfillLane: "live-backfill",
+      skipBackfilled: true,
+    });
+  });
+
+  it("classifies environment-blocking gateway sentinels without turning them into unknowns", async () => {
+    await writeJson("live/qa-suite-summary.json", {
+      counts: { total: 1, passed: 1, skipped: 0, failed: 0 },
+      gatewayLogSentinels: [
+        {
+          kind: "live-quota-or-subscription",
+          verdict: "environment-blocked",
+          owner: "environment",
+          productImpact: "P4",
+          qaImpact: "P0",
+          line: 12,
+          text: "OpenAI quota exceeded",
+        },
+      ],
+      scenarios: [],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-live",
+            title: "First hour live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.globalPass).toBe(false);
+    expect(report.counts).toMatchObject({ blocked: 1, unknown: 0 });
+    expect(report.lanes[0]).toMatchObject({
+      status: "blocked",
+      verdict: "environment-blocked",
+    });
+  });
+
+  it("does not let environment sentinels hide separate suite failures", async () => {
+    await writeJson("live/qa-suite-summary.json", {
+      counts: { total: 2, passed: 1, skipped: 0, failed: 1 },
+      gatewayLogSentinels: [
+        {
+          kind: "live-quota-or-subscription",
+          verdict: "environment-blocked",
+          owner: "environment",
+          line: 12,
+          text: "OpenAI quota exceeded",
+        },
+      ],
+      scenarios: [
+        { name: "quota", status: "pass" },
+        { name: "unrelated-drift", status: "fail" },
+      ],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-live",
+            title: "First hour live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+            missingVerdict: "environment-blocked",
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.counts).toMatchObject({ blocked: 0, unknown: 1 });
+    expect(report.lanes[0]).toMatchObject({ status: "unknown" });
+    expect(report.lanes[0]?.details).toContain("suite also reports failures");
+  });
+
+  it("classifies product and plugin gateway sentinels as known failing lanes", async () => {
+    await writeJson("live/qa-suite-summary.json", {
+      counts: { total: 1, passed: 1, skipped: 0, failed: 0 },
+      scenarios: [
+        {
+          name: "plugin hook health sentinel",
+          status: "pass",
+          steps: [],
+          runtimeParity: {
+            scenarioId: "plugin-hook-health-sentinel",
+            drift: "none",
+            cells: {
+              pi: { sentinelFindings: [] },
+              codex: {
+                sentinelFindings: [
+                  {
+                    kind: "plugin-hook-failure",
+                    verdict: "qa-harness-bug",
+                    owner: "plugin",
+                    productImpact: "P1",
+                    qaImpact: "P0",
+                    line: 4,
+                    text: "before_prompt_build hook failed",
+                  },
+                ],
+              },
+            },
+          },
+        },
+      ],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-live",
+            title: "First hour live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.globalPass).toBe(false);
+    expect(report.counts).toMatchObject({ failed: 1, unknown: 0 });
+    expect(report.lanes[0]).toMatchObject({
+      status: "fail",
+      verdict: "qa-harness-bug",
+    });
+  });
+
+  it("treats corrupt artifacts as unknown instead of allowed missing lanes", async () => {
+    const artifactPath = path.join(tempRoot, "live", "qa-suite-summary.json");
+    await fs.mkdir(path.dirname(artifactPath), { recursive: true });
+    await fs.writeFile(artifactPath, "{not-json", "utf8");
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-live",
+            title: "First hour live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+            missingVerdict: "environment-blocked",
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.counts).toMatchObject({ blocked: 0, unknown: 1 });
+    expect(report.lanes[0]).toMatchObject({
+      status: "unknown",
+    });
+    expect(report.lanes[0]?.details).toContain("artifact unreadable");
+  });
+
+  it("treats schema-invalid suite artifacts as unknown", async () => {
+    await writeJson("live/qa-suite-summary.json", {});
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-live",
+            title: "First hour live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.counts.unknown).toBe(1);
+    expect(report.lanes[0]?.details).toContain("missing counts.failed and scenarios[]");
+  });
+
+  it("treats empty suite artifacts as unknown", async () => {
+    await writeJson("live/qa-suite-summary.json", {
+      counts: { total: 0, passed: 0, skipped: 0, failed: 0 },
+      scenarios: [],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-live",
+            title: "First hour live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+            failureVerdict: "qa-harness-bug",
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.counts).toMatchObject({ failed: 0, unknown: 1 });
+    expect(report.lanes[0]).toMatchObject({ status: "unknown" });
+    expect(report.lanes[0]?.details).toContain("no executed scenarios");
+  });
+
+  it("treats suite count and scenario mismatches as unknown", async () => {
+    await writeJson("live/qa-suite-summary.json", {
+      counts: { total: 2, passed: 2, skipped: 0, failed: 0 },
+      scenarios: [
+        { name: "passing", status: "pass" },
+        { name: "stale-failure", status: "fail" },
+      ],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-live",
+            title: "First hour live",
+            kind: "qa-suite-summary",
+            artifact: "live/qa-suite-summary.json",
+            required: true,
+            failureVerdict: "qa-harness-bug",
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.counts).toMatchObject({ failed: 0, unknown: 1 });
+    expect(report.lanes[0]).toMatchObject({ status: "unknown" });
+    expect(report.lanes[0]?.details).toContain("count/scenario mismatch");
+  });
+
+  it("treats impossible suite counts as unknown", async () => {
+    for (const [artifact, expectedDetail] of [
+      [
+        { counts: { total: 1, passed: -1, skipped: 0, failed: 0 } },
+        "counts.passed must be a non-negative integer",
+      ],
+      [
+        { counts: { total: 1, passed: 2, failed: 0 } },
+        "counts.total=1 is less than provided count sum=2",
+      ],
+      [
+        { counts: { total: 1, skipped: 2, failed: 0 } },
+        "counts.total=1 is less than provided count sum=2",
+      ],
+      [
+        { counts: { total: 5, passed: 2, skipped: 2, failed: 0 } },
+        "counts.total=5 does not match counts.passed+counts.failed+counts.skipped=4",
+      ],
+    ] as const) {
+      await writeJson("live/qa-suite-summary.json", artifact);
+
+      const report = await buildQaConfidenceReport({
+        manifest: {
+          version: 1,
+          profile: "codex-100",
+          lanes: [
+            {
+              id: "first-hour-live",
+              title: "First hour live",
+              kind: "qa-suite-summary",
+              artifact: "live/qa-suite-summary.json",
+              required: true,
+              failureVerdict: "qa-harness-bug",
+            },
+          ],
+        },
+        artifactRoot: tempRoot,
+        strictZeroUnknowns: true,
+        generatedAt: "2026-05-13T00:00:00.000Z",
+      });
+
+      expect(report.pass).toBe(false);
+      expect(report.counts).toMatchObject({ failed: 0, unknown: 1 });
+      expect(report.lanes[0]).toMatchObject({ status: "unknown" });
+      expect(report.lanes[0]?.details).toContain(expectedDetail);
+    }
+  });
+
+  it("requires generic summary lanes to expose an explicit pass signal", async () => {
+    await writeJson("runtime/qa-runtime-parity-summary.json", {});
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "runtime-parity",
+            title: "Runtime parity",
+            kind: "runtime-parity-summary",
+            artifact: "runtime/qa-runtime-parity-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-13T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.counts.unknown).toBe(1);
+    expect(report.lanes[0]?.details).toContain("explicit pass signal");
+  });
+
+  it("requires JSONL replay summaries to contain replayed user turns", async () => {
+    for (const [artifact, expectedDetail] of [
+      [{ transcripts: [] }, "no transcripts"],
+      [
+        { transcripts: [{ transcriptPath: "empty.jsonl", userTurnCount: 0, drift: [] }] },
+        "no replayed user turns",
+      ],
+      [
+        { transcripts: [{ transcriptPath: "missing-drift.jsonl", userTurnCount: 1 }] },
+        "missing drift array",
+      ],
+    ] as const) {
+      await writeJson("jsonl/qa-jsonl-replay-summary.json", artifact);
+
+      const report = await buildQaConfidenceReport({
+        manifest: {
+          version: 1,
+          profile: "codex-100",
+          lanes: [
+            {
+              id: "jsonl-expanded",
+              title: "Expanded JSONL replay",
+              kind: "jsonl-replay-summary",
+              artifact: "jsonl/qa-jsonl-replay-summary.json",
+              required: true,
+              failureVerdict: "fixture-bug",
+            },
+          ],
+        },
+        artifactRoot: tempRoot,
+        strictZeroUnknowns: true,
+        generatedAt: "2026-05-13T00:00:00.000Z",
+      });
+
+      expect(report.pass).toBe(false);
+      expect(report.counts).toMatchObject({ failed: 0, unknown: 1 });
+      expect(report.lanes[0]).toMatchObject({ status: "unknown" });
+      expect(report.lanes[0]?.details).toContain(expectedDetail);
+    }
+  });
+
+  it("requires confidence self-test summaries to contain every seeded canary", async () => {
+    for (const [artifact, expectedDetail] of [
+      [{ pass: true, canaries: [] }, "no canaries"],
+      [
+        { pass: true, canaries: [{ id: "prompt-drift", detected: true }] },
+        "missing expected canaries",
+      ],
+    ] as const) {
+      await writeJson("confidence-self-test/qa-confidence-self-test-summary.json", artifact);
+
+      const report = await buildQaConfidenceReport({
+        manifest: {
+          version: 1,
+          profile: "codex-100",
+          lanes: [
+            {
+              id: "confidence-self-test",
+              title: "Confidence self-test",
+              kind: "self-test-summary",
+              artifact: "confidence-self-test/qa-confidence-self-test-summary.json",
+              required: true,
+              failureVerdict: "qa-harness-bug",
+            },
+          ],
+        },
+        artifactRoot: tempRoot,
+        strictZeroUnknowns: true,
+        generatedAt: "2026-05-13T00:00:00.000Z",
+      });
+
+      expect(report.pass).toBe(false);
+      expect(report.counts).toMatchObject({ failed: 0, unknown: 1 });
+      expect(report.lanes[0]).toMatchObject({ status: "unknown" });
+      expect(report.lanes[0]?.details).toContain(expectedDetail);
+    }
+  });
+
+  it("fails strict zero-unknowns for an unclassified failing lane", async () => {
+    await writeJson("first-hour/qa-suite-summary.json", {
+      counts: { total: 18, passed: 17, failed: 1 },
+      scenarios: [{ name: "approval-turn-tool-followthrough", status: "fail", steps: [] }],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "first-hour-20-direct",
+            title: "First-hour 20 direct",
+            kind: "qa-suite-summary",
+            artifact: "first-hour/qa-suite-summary.json",
+            required: true,
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(false);
+    expect(report.counts.unknown).toBe(1);
+    expect(report.failures[0]).toContain("first-hour-20-direct is unclassified");
+  });
+
+  it("accepts a classified failing lane without treating it as unknown", async () => {
+    await writeJson("jsonl/qa-jsonl-replay-summary.json", {
+      transcripts: [
+        {
+          transcriptPath: "curated.jsonl",
+          userTurnCount: 2,
+          drift: ["none", "tool-result-shape"],
+          firstDriftAtTurn: 2,
+        },
+      ],
+    });
+
+    const report = await buildQaConfidenceReport({
+      manifest: {
+        version: 1,
+        profile: "codex-100",
+        lanes: [
+          {
+            id: "jsonl-expanded",
+            title: "Expanded JSONL replay",
+            kind: "jsonl-replay-summary",
+            artifact: "jsonl/qa-jsonl-replay-summary.json",
+            required: true,
+            failureVerdict: "fixture-bug",
+            productImpact: "P4",
+            qaImpact: "P1",
+          },
+        ],
+      },
+      artifactRoot: tempRoot,
+      strictZeroUnknowns: true,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.globalPass).toBe(false);
+    expect(report.counts.failed).toBe(1);
+    expect(report.counts.unknown).toBe(0);
+    expect(report.lanes[0]).toMatchObject({
+      status: "fail",
+      verdict: "fixture-bug",
+      productImpact: "P4",
+      qaImpact: "P1",
+    });
+  });
+
+  it("emits confidence self-test canaries for every drift class we need to catch", async () => {
+    const summary = await buildQaConfidenceSelfTestSummary("2026-05-12T00:00:00.000Z");
+
+    expect(summary.pass).toBe(true);
+    expect(summary.canaries.map((canary) => canary.id)).toEqual([
+      "prompt-drift",
+      "tool-description-schema-drift",
+      "runtime-tool-call-drop",
+      "tool-result-mismatch",
+      "failure-mode-drift",
+      "token-efficiency-regression",
+      "jsonl-replay-ordering-drift",
+    ]);
+    expect(summary.canaries.every((canary) => canary.detected)).toBe(true);
+  });
+
+  it("writes confidence self-test artifacts", async () => {
+    const result = await writeQaConfidenceSelfTestArtifacts({
+      outputDir: tempRoot,
+      generatedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    await expect(fs.stat(result.summaryPath)).resolves.toBeTruthy();
+    await expect(fs.stat(result.reportPath)).resolves.toBeTruthy();
+    const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { pass: boolean };
+    expect(summary.pass).toBe(true);
+  });
+});

extensions/qa-lab/src/coverage-report.test.ts

@@ -0,0 +1,658 @@
+// Qa Lab tests cover coverage report plugin behavior.
+import { describe, expect, it } from "vitest";
+import {
+  buildQaCoverageInventory,
+  findQaScenarioMatches,
+  renderQaCoverageMarkdownReport,
+  renderQaScenarioMatchesMarkdownReport,
+} from "./coverage-report.js";
+import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js";
+import { buildQaScorecardTaxonomyReport, type QaMaturityTaxonomy } from "./scorecard-taxonomy.js";
+
+const TEST_EXECUTABLE_CATEGORY_ID = "agent-runtime-and-provider-execution.agent-turn-execution";
+const TEST_EXECUTABLE_COVERAGE_ID = "channels.dm";
+const TEST_BROWSER_CATEGORY_ID = "browser-control-ui-and-webchat.browser-ui";
+const TEST_BROWSER_COVERAGE_ID = "ui.control";
+const TEST_WEBCHAT_COVERAGE_ID = "ui.webchat";
+const DOTTED_COVERAGE_ID_PATTERN = /^[a-z0-9][a-z0-9-]*(?:\.[a-z0-9][a-z0-9-]*)+$/;
+
+function testMaturityTaxonomy(params?: {
+  categoryId?: string;
+  coverageIds?: readonly string[];
+  featureCoverageIds?: readonly (readonly string[])[];
+  includeAllCategories?: boolean;
+  includeArchivedSurface?: boolean;
+  profileCategoryIds?: readonly string[];
+}): QaMaturityTaxonomy {
+  const categoryId = params?.categoryId ?? TEST_EXECUTABLE_CATEGORY_ID;
+  const firstDot = categoryId.indexOf(".");
+  const surfaceId = firstDot === -1 ? categoryId : categoryId.slice(0, firstDot);
+  const categoryLocalId = firstDot === -1 ? categoryId : categoryId.slice(firstDot + 1);
+  return {
+    version: 1 as const,
+    title: "Test taxonomy",
+    levels: [],
+    profiles: [
+      {
+        id: "smoke-ci",
+        description: "Test smoke profile.",
+        includeAllCategories: false,
+        channelDriver: "crabline" as const,
+        categoryIds: [categoryId],
+      },
+      {
+        id: "release",
+        description: "Test release profile.",
+        includeAllCategories: params?.includeAllCategories ?? false,
+        channelDriver: "qa-channel" as const,
+        categoryIds: [
+          ...(params?.includeAllCategories ? [] : (params?.profileCategoryIds ?? [categoryId])),
+        ],
+      },
+    ],
+    surfaces: [
+      {
+        id: surfaceId,
+        name: "Test surface",
+        family: "test",
+        level: "experimental",
+        categories: [
+          {
+            id: categoryLocalId,
+            name: "Test category",
+            category_note: "test-category.md",
+            docs: [],
+            search_anchors: [],
+            features: (
+              params?.featureCoverageIds ??
+              (params?.coverageIds ?? [TEST_EXECUTABLE_COVERAGE_ID]).map((coverageId) => [
+                coverageId,
+              ])
+            ).map((coverageIds) => ({
+              name: coverageIds.join(" + "),
+              coverageIds: [...coverageIds],
+            })),
+          },
+        ],
+      },
+      ...(params?.includeArchivedSurface
+        ? [
+            {
+              id: "archived-surface",
+              name: "Archived surface",
+              family: "test",
+              level: "experimental",
+              archived: true,
+              categories: [
+                {
+                  id: "legacy-category",
+                  name: "Legacy category",
+                  category_note: "legacy-category.md",
+                  docs: [],
+                  search_anchors: [],
+                  features: [{ name: "legacy.feature", coverageIds: ["legacy.feature"] }],
+                },
+              ],
+            },
+          ]
+        : []),
+    ],
+  };
+}
+
+function scenarioWithCoverage(params: {
+  primary?: readonly string[];
+  secondary?: readonly string[];
+  sourcePath?: string;
+  executionKind?: "flow" | "script" | "vitest" | "playwright";
+  executionPath?: string;
+}): QaSeedScenarioWithSource {
+  const execution =
+    params.executionKind === "script" ||
+    params.executionKind === "vitest" ||
+    params.executionKind === "playwright"
+      ? {
+          kind: params.executionKind,
+          path: params.executionPath ?? "src/test.test.ts",
+        }
+      : {
+          kind: "flow" as const,
+          flow: {
+            steps: [
+              {
+                name: "noop",
+                actions: [{ set: "ok", value: true }],
+              },
+            ],
+          },
+        };
+  return {
+    id: "test-scenario",
+    title: "Test scenario",
+    surface: "test",
+    coverage: {
+      primary: [...(params.primary ?? [])],
+      ...(params.secondary ? { secondary: [...params.secondary] } : {}),
+    },
+    objective: "Exercise test coverage.",
+    successCriteria: ["Evidence is recorded."],
+    sourcePath: params.sourcePath ?? "qa/scenarios/test/test-scenario.yaml",
+    execution,
+  };
+}
+
+describe("qa coverage report", () => {
+  it("groups scenario coverage metadata by theme and surface", () => {
+    const inventory = buildQaCoverageInventory(readQaScenarioPack().scenarios);
+
+    expect(inventory.scenarioCount).toBeGreaterThan(0);
+    expect(inventory.coverageIdCount).toBeGreaterThan(0);
+    expect(inventory.primaryCoverageIdCount).toBeGreaterThan(0);
+    expect(inventory.secondaryCoverageIdCount).toBeGreaterThan(0);
+    expect(inventory.overlappingCoverage.length).toBeGreaterThan(0);
+    expect(inventory.missingCoverage).toStrictEqual([]);
+    expect(inventory.liveTransportLanes.map((lane) => lane.transportId)).toEqual([
+      "discord",
+      "slack",
+      "telegram",
+      "whatsapp",
+    ]);
+    expect(inventory.scorecardTaxonomy.profileCount).toBe(3);
+    expect(
+      inventory.scorecardTaxonomy.profiles.find((profile) => profile.id === "smoke-ci"),
+    ).toMatchObject({
+      channelDriver: "crabline",
+      evidenceMode: "slim",
+    });
+    expect(
+      inventory.scorecardTaxonomy.profiles.find((profile) => profile.id === "release"),
+    ).toMatchObject({
+      channelDriver: "live",
+    });
+    expect(
+      inventory.scorecardTaxonomy.profiles.find((profile) => profile.id === "all"),
+    ).toMatchObject({
+      channelDriver: "live",
+      categoryIds: expect.arrayContaining([
+        "browser-automation-and-exec-sandbox-tools.tool-invocation-and-execution",
+      ]),
+    });
+    expect(inventory.scorecardTaxonomy.categoryCount).toBeGreaterThan(200);
+    expect(inventory.scorecardTaxonomy.requiredCategoryCount).toBeGreaterThan(0);
+    expect(inventory.scorecardTaxonomy.requiredCategoryCount).toBeLessThanOrEqual(
+      inventory.scorecardTaxonomy.categoryCount,
+    );
+    expect(inventory.scorecardTaxonomy.requiredCoverageIdCount).toBeGreaterThan(0);
+    expect(inventory.scorecardTaxonomy.fulfilledCoverageIdCount).toBeGreaterThan(0);
+    expect(inventory.scorecardTaxonomy.coverageIdFulfillmentPercent).toBeGreaterThan(0);
+    expect(inventory.scorecardTaxonomy.evidenceRefCount).toBeGreaterThan(0);
+    expect(inventory.scorecardTaxonomy.scenarioCoverageIdCount).toBeGreaterThan(0);
+    expect(inventory.scorecardTaxonomy.unknownCoverageIdCount).toBe(0);
+    expect(
+      inventory.scorecardTaxonomy.categories
+        .flatMap((category) => category.coverageIds)
+        .every((coverageId) => DOTTED_COVERAGE_ID_PATTERN.test(coverageId)),
+    ).toBe(true);
+    expect(inventory.scorecardTaxonomy.validationIssues.length).toBeGreaterThan(0);
+    expect(
+      inventory.scorecardTaxonomy.validationIssues.some((issue) =>
+        issue.code.endsWith("not-found"),
+      ),
+    ).toBe(false);
+    expect(
+      inventory.scorecardTaxonomy.validationIssues.some(
+        (issue) => issue.code === "coverage-id-missing-primary-evidence",
+      ),
+    ).toBe(true);
+    expect(
+      inventory.scorecardTaxonomy.categories.find(
+        (category) => category.id === TEST_BROWSER_CATEGORY_ID,
+      )?.evidence,
+    ).toContainEqual({
+      coverageId: TEST_BROWSER_COVERAGE_ID,
+      kind: "playwright",
+      path: "ui/src/e2e/chat-flow.e2e.test.ts",
+      role: "primary",
+      scenarioRefs: ["qa/scenarios/ui/control-ui-chat-flow-playwright.yaml"],
+    });
+    expect(inventory.scenarioPacks.map((pack) => pack.id)).toEqual([
+      "observability",
+      "personal-agent",
+    ]);
+    const personalPack = inventory.scenarioPacks.find((pack) => pack.id === "personal-agent");
+    const observabilityPack = inventory.scenarioPacks.find((pack) => pack.id === "observability");
+    expect(personalPack?.missingScenarioIds).toStrictEqual([]);
+    expect(personalPack?.scenarioIds).toContain("personal-share-safe-diagnostics-artifact");
+    expect(personalPack?.coverageIds).toContain("personal.redaction");
+    expect(observabilityPack?.missingScenarioIds).toStrictEqual([]);
+    expect(observabilityPack?.scenarioIds).toEqual(["otel-trace-smoke", "docker-prometheus-smoke"]);
+    expect(observabilityPack?.coverageIds).toContain("telemetry.prometheus");
+    expect(inventory.byTheme.memory.map((coverage) => coverage.id)).toContain("memory.recall");
+    expect(inventory.bySurface.memory.map((coverage) => coverage.id)).toContain("memory.recall");
+  });
+
+  it("rejects duplicate ownership across YAML and non-YAML catalogs", () => {
+    const scenario = scenarioWithCoverage({
+      primary: [TEST_EXECUTABLE_COVERAGE_ID],
+      executionKind: "script",
+      executionPath: "scripts/test-scenario.ts",
+    });
+
+    expect(() =>
+      buildQaCoverageInventory([scenario], {
+        nonYamlScenarios: [
+          {
+            id: scenario.id,
+            sourcePath: "extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts",
+          },
+        ],
+      }),
+    ).toThrow(
+      "duplicate qa scenario id(s): test-scenario (qa/scenarios/test/test-scenario.yaml, extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts)",
+    );
+  });
+
+  it("renders a compact markdown inventory", () => {
+    const report = renderQaCoverageMarkdownReport(
+      buildQaCoverageInventory(readQaScenarioPack().scenarios),
+    );
+
+    expect(report).toContain("# QA Coverage Inventory");
+    expect(report).toContain("- Missing coverage metadata: 0");
+    expect(report).toContain("- Overlapping coverage IDs:");
+    expect(report).toContain("memory.recall");
+    expect(report).toContain("primary: memory-recall (qa/scenarios/memory/memory-recall.yaml)");
+    expect(report).toContain("secondary: active-memory-preprompt-recall");
+    expect(report).toContain("## Scenario Packs");
+    expect(report).toContain(
+      "- personal-agent (Personal Agent Benchmark Pack): 10 scenarios; coverage IDs:",
+    );
+    expect(report).toContain(
+      "- observability (Observability Smoke Pack): 2 scenarios; coverage IDs:",
+    );
+    expect(report).toContain("otel-trace-smoke, docker-prometheus-smoke");
+    expect(report).toContain("personal-share-safe-diagnostics-artifact");
+    expect(report).toContain("## Live Transport Lanes");
+    expect(report).toContain(
+      "- telegram (telegram): canary: always-on, help-command: telegram-help-command, mention-gating: telegram-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume",
+    );
+    expect(report).toContain("thread-follow-up: slack-thread-follow-up");
+    expect(report).toContain("## Scorecard Taxonomy");
+    expect(report).toContain("- Taxonomy: taxonomy.yaml");
+    expect(report).toContain("- Fulfilled taxonomy categories:");
+    expect(report).toContain("- Fulfilled taxonomy coverage IDs:");
+    expect(report).toContain("- Evidence refs:");
+    expect(report).toContain("- Scenario coverage IDs:");
+    expect(report).toContain(
+      "- browser-automation-and-exec-sandbox-tools.tool-invocation-and-execution (browser-automation-and-exec-sandbox-tools / Tool Invocation and Execution; partial): profiles: all, release, smoke-ci; coverage IDs:",
+    );
+    expect(report).toContain("primary:playwright:ui/src/e2e/chat-flow.e2e.test.ts (ui.control)");
+    expect(report).not.toContain("### Unknown Scenario Coverage IDs");
+  });
+
+  it("renders Playwright matches as qa suite targets", () => {
+    const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, "chat-flow.e2e");
+    const report = renderQaScenarioMatchesMarkdownReport({
+      query: "chat-flow.e2e",
+      matches,
+    });
+
+    expect(report).toContain(
+      "- Suite command: `pnpm openclaw qa suite --scenario control-ui-chat-flow-playwright`",
+    );
+    expect(report).toContain("  - execution: playwright ui/src/e2e/chat-flow.e2e.test.ts");
+    expect(report).not.toContain("Native test refs");
+  });
+
+  it("splits qa suite targets when matches mix execution kinds", () => {
+    const playwrightExecutionPath = "ui/src/e2e/chat-flow.e2e.test.ts";
+    const flowScenario = scenarioWithCoverage({
+      primary: [TEST_EXECUTABLE_COVERAGE_ID],
+    });
+    const playwrightScenario = scenarioWithCoverage({
+      primary: [TEST_BROWSER_COVERAGE_ID],
+      executionKind: "playwright",
+      executionPath: playwrightExecutionPath,
+      sourcePath: "qa/scenarios/ui/control-ui-chat-flow-playwright.yaml",
+    });
+    const report = renderQaScenarioMatchesMarkdownReport({
+      query: "mixed",
+      matches: [
+        {
+          ...flowScenario,
+          id: "flow-proof",
+          theme: "test",
+          surfaces: [flowScenario.surface],
+          risk: "unassigned",
+          coverageIds: [
+            ...(flowScenario.coverage?.primary ?? []),
+            ...(flowScenario.coverage?.secondary ?? []),
+          ],
+          docsRefs: [],
+          codeRefs: [],
+          executionKind: flowScenario.execution.kind,
+        },
+        {
+          ...playwrightScenario,
+          id: "playwright-proof",
+          theme: "test",
+          surfaces: [playwrightScenario.surface],
+          risk: "unassigned",
+          coverageIds: [
+            ...(playwrightScenario.coverage?.primary ?? []),
+            ...(playwrightScenario.coverage?.secondary ?? []),
+          ],
+          docsRefs: [],
+          codeRefs: [],
+          executionKind: playwrightScenario.execution.kind,
+          executionPath: playwrightExecutionPath,
+        },
+      ],
+    });
+
+    expect(report).toContain("- Suite commands:");
+    expect(report).toContain("  - flow: `pnpm openclaw qa suite --scenario flow-proof`");
+    expect(report).toContain(
+      "  - playwright: `pnpm openclaw qa suite --scenario playwright-proof`",
+    );
+  });
+
+  it("reports missing taxonomy coverage refs without treating them as fulfilled", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy(),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: ["agent-runtime-and-provider-execution.agent-turn-execution.missing-coverage"],
+        }),
+      ],
+    });
+
+    expect(report.fulfilledCoverageIdCount).toBe(0);
+    expect(report.categories[0]?.coverageStatus).toBe("missing");
+    expect(report.validationIssues.map((issue) => issue.code)).toEqual([
+      "coverage-id-not-found",
+      "coverage-id-missing-primary-evidence",
+      "profile-category-missing-evidence",
+    ]);
+  });
+
+  it("uses explicit native test evidence as coverage fulfillment", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy({
+        categoryId: TEST_BROWSER_CATEGORY_ID,
+        coverageIds: [TEST_BROWSER_COVERAGE_ID],
+      }),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_BROWSER_COVERAGE_ID],
+          sourcePath: "qa/scenarios/ui/control-ui-chat-flow-playwright.yaml",
+          executionKind: "playwright",
+          executionPath: "ui/src/e2e/chat-flow.e2e.test.ts",
+        }),
+      ],
+    });
+
+    expect(report.validationIssues).toStrictEqual([]);
+    expect(report.fulfilledCategoryCount).toBe(1);
+    expect(report.fulfilledCoverageIdCount).toBe(1);
+    expect(report.categories[0]?.coverageStatus).toBe("covered");
+    expect(report.categories[0]?.scenarioRefs).toStrictEqual([
+      "qa/scenarios/ui/control-ui-chat-flow-playwright.yaml",
+    ]);
+    expect(report.categories[0]?.evidence).toStrictEqual([
+      {
+        coverageId: TEST_BROWSER_COVERAGE_ID,
+        kind: "playwright",
+        path: "ui/src/e2e/chat-flow.e2e.test.ts",
+        role: "primary",
+        scenarioRefs: ["qa/scenarios/ui/control-ui-chat-flow-playwright.yaml"],
+      },
+    ]);
+  });
+
+  it("counts partial coverage IDs proportionately for taxonomy fulfillment", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy({
+        featureCoverageIds: [[TEST_EXECUTABLE_COVERAGE_ID, TEST_WEBCHAT_COVERAGE_ID]],
+      }),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_EXECUTABLE_COVERAGE_ID],
+          secondary: [TEST_WEBCHAT_COVERAGE_ID],
+          sourcePath: "qa/scenarios/channels/dm-chat-baseline.yaml",
+        }),
+      ],
+    });
+
+    expect(report.fulfilledCategoryCount).toBe(0);
+    expect(report.requiredCoverageIdCount).toBe(2);
+    expect(report.fulfilledCoverageIdCount).toBe(1);
+    expect(report.coverageIdFulfillmentPercent).toBe(50);
+    expect(report.categories[0]?.coverageStatus).toBe("partial");
+    expect(report.categories[0]?.fulfilledCoverageIds).toStrictEqual([TEST_EXECUTABLE_COVERAGE_ID]);
+    expect(report.validationIssues).toContainEqual(
+      expect.objectContaining({
+        code: "coverage-id-missing-primary-evidence",
+        ref: TEST_WEBCHAT_COVERAGE_ID,
+      }),
+    );
+  });
+
+  it("counts each required taxonomy coverage ID once across categories", () => {
+    const taxonomy: QaMaturityTaxonomy = {
+      ...testMaturityTaxonomy(),
+      profiles: [
+        {
+          id: "release",
+          description: "Test release profile.",
+          includeAllCategories: false,
+          channelDriver: "qa-channel",
+          categoryIds: [
+            "agent-runtime-and-provider-execution.agent-turn-execution",
+            "agent-runtime-and-provider-execution.tool-execution-controls",
+          ],
+        },
+      ],
+      surfaces: [
+        {
+          id: "agent-runtime-and-provider-execution",
+          name: "Agent Runtime",
+          family: "test",
+          level: "experimental",
+          categories: [
+            {
+              id: "agent-turn-execution",
+              name: "Agent Turn Execution",
+              category_note: "agent-turn-execution.md",
+              docs: [],
+              search_anchors: [],
+              features: [
+                {
+                  name: "shared plus unique",
+                  coverageIds: [TEST_EXECUTABLE_COVERAGE_ID, TEST_WEBCHAT_COVERAGE_ID],
+                },
+              ],
+            },
+            {
+              id: "tool-execution-controls",
+              name: "Tool Execution Controls",
+              category_note: "tool-execution-controls.md",
+              docs: [],
+              search_anchors: [],
+              features: [
+                {
+                  name: "shared",
+                  coverageIds: [TEST_EXECUTABLE_COVERAGE_ID],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy,
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_EXECUTABLE_COVERAGE_ID],
+          secondary: [TEST_WEBCHAT_COVERAGE_ID],
+          sourcePath: "qa/scenarios/channels/dm-chat-baseline.yaml",
+        }),
+      ],
+    });
+
+    expect(report.requiredCoverageIdCount).toBe(2);
+    expect(report.fulfilledCoverageIdCount).toBe(1);
+    expect(report.coverageIdFulfillmentPercent).toBe(50);
+  });
+
+  it("uses script producer evidence as coverage fulfillment", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy({
+        categoryId: TEST_BROWSER_CATEGORY_ID,
+        coverageIds: [TEST_BROWSER_COVERAGE_ID],
+      }),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_BROWSER_COVERAGE_ID],
+          sourcePath: "qa/scenarios/ui/script-evidence-producer.yaml",
+          executionKind: "script",
+          executionPath: "scripts/check-no-conflict-markers.mjs",
+        }),
+      ],
+    });
+
+    expect(report.validationIssues).toStrictEqual([]);
+    expect(report.fulfilledCategoryCount).toBe(1);
+    expect(report.fulfilledCoverageIdCount).toBe(1);
+    expect(report.categories[0]?.evidence).toStrictEqual([
+      {
+        coverageId: TEST_BROWSER_COVERAGE_ID,
+        kind: "script",
+        path: "scripts/check-no-conflict-markers.mjs",
+        role: "primary",
+        scenarioRefs: ["qa/scenarios/ui/script-evidence-producer.yaml"],
+      },
+    ]);
+  });
+
+  it("reports profile membership refs missing from taxonomy categories", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy({
+        profileCategoryIds: ["missing.category"],
+      }),
+      repoRoot: process.cwd(),
+      scenarios: [],
+    });
+
+    expect(report.validationIssues.map((issue) => issue.code)).toContain(
+      "profile-category-ref-not-found",
+    );
+  });
+
+  it("resolves all-category profiles from taxonomy categories", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy({
+        includeAllCategories: true,
+        includeArchivedSurface: true,
+      }),
+      repoRoot: process.cwd(),
+      scenarios: [],
+    });
+
+    expect(report.profiles.find((profile) => profile.id === "release")?.categoryIds).toStrictEqual([
+      TEST_EXECUTABLE_CATEGORY_ID,
+    ]);
+    expect(report.requiredCategoryCount).toBe(1);
+    expect(report.categoryCount).toBe(1);
+    expect(report.profiles.find((profile) => profile.id === "release")?.categoryIds).not.toContain(
+      "archived-surface.legacy-category",
+    );
+  });
+
+  it("reports profile categories missing primary coverage evidence", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy(),
+      repoRoot: process.cwd(),
+      scenarios: [],
+    });
+
+    expect(report.validationIssues.map((issue) => issue.code)).toEqual([
+      "coverage-id-missing-primary-evidence",
+      "profile-category-missing-evidence",
+    ]);
+  });
+
+  it("reports native test evidence refs outside the repository", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy(),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_EXECUTABLE_COVERAGE_ID],
+          executionKind: "playwright",
+          executionPath: "../outside-openclaw.test.ts",
+        }),
+      ],
+    });
+
+    expect(report.validationIssues.map((issue) => issue.code)).toEqual([
+      "evidence-ref-not-found",
+      "coverage-id-missing-primary-evidence",
+      "profile-category-missing-evidence",
+    ]);
+  });
+
+  it("uses scenario coverage metadata as runnable scenario evidence", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy(),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_EXECUTABLE_COVERAGE_ID],
+          sourcePath: "qa/scenarios/channels/dm-chat-baseline.yaml",
+        }),
+      ],
+    });
+
+    expect(report.validationIssues).toStrictEqual([]);
+    expect(report.categories[0]?.scenarioRefs).toStrictEqual([
+      "qa/scenarios/channels/dm-chat-baseline.yaml",
+    ]);
+    expect(report.categories[0]?.evidence).toStrictEqual([
+      {
+        coverageId: TEST_EXECUTABLE_COVERAGE_ID,
+        kind: "qa-scenario",
+        path: null,
+        role: "primary",
+        scenarioRefs: ["qa/scenarios/channels/dm-chat-baseline.yaml"],
+      },
+    ]);
+  });
+
+  it("counts secondary scenario metadata as evidence but not fulfillment", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy(),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_WEBCHAT_COVERAGE_ID],
+          secondary: [TEST_EXECUTABLE_COVERAGE_ID],
+        }),
+      ],
+    });
+
+    expect(report.fulfilledCoverageIdCount).toBe(0);
+    expect(report.categories[0]?.coverageStatus).toBe("partial");
+    expect(report.validationIssues.map((issue) => issue.code)).toEqual([
+      "coverage-id-not-found",
+      "coverage-id-missing-primary-evidence",
+      "profile-category-missing-evidence",
+    ]);
+  });
+});

extensions/qa-lab/src/coverage-report.ts

@@ -0,0 +1,553 @@
+// Qa Lab plugin module implements coverage report behavior.
+import { normalizeStringEntriesLower } from "openclaw/plugin-sdk/string-coerce-runtime";
+import {
+  buildLiveTransportCoverageLaneSummaries,
+  type LiveTransportCoverageLaneSummary,
+} from "./live-transports/shared/live-transport-scenarios.js";
+import { QA_SCENARIO_PACKS, type QaSeedScenarioWithSource } from "./scenario-catalog.js";
+import {
+  readQaScorecardTaxonomyReport,
+  type QaScorecardTaxonomyReport,
+} from "./scorecard-taxonomy.js";
+
+type QaCoverageScenarioSummary = {
+  id: string;
+  title: string;
+  sourcePath: string;
+  theme: string;
+  surfaces: string[];
+  risk: string;
+};
+
+type QaScenarioSearchMatch = QaCoverageScenarioSummary & {
+  coverageIds: string[];
+  docsRefs: string[];
+  codeRefs: string[];
+  executionKind: QaSeedScenarioWithSource["execution"]["kind"];
+  executionPath?: string;
+  runtimeParityTier?: string;
+  requiredProviderMode?: string;
+  requiredChannelDriver?: string;
+  requiredProvider?: string;
+  requiredModel?: string;
+};
+
+type QaCoverageIntent = "primary" | "secondary";
+
+type QaCoverageScenarioReference = QaCoverageScenarioSummary & {
+  intent: QaCoverageIntent;
+};
+
+type QaCoverageIdSummary = {
+  id: string;
+  scenarios: QaCoverageScenarioReference[];
+};
+
+type QaCoverageScenarioPackSummary = {
+  id: string;
+  title: string;
+  scenarioIds: string[];
+  coverageIds: string[];
+  missingScenarioIds: string[];
+};
+
+type QaCoverageInventory = {
+  scenarioCount: number;
+  coverageIdCount: number;
+  primaryCoverageIdCount: number;
+  secondaryCoverageIdCount: number;
+  coverageIds: QaCoverageIdSummary[];
+  overlappingCoverage: QaCoverageIdSummary[];
+  missingCoverage: QaCoverageScenarioSummary[];
+  byTheme: Record<string, QaCoverageIdSummary[]>;
+  bySurface: Record<string, QaCoverageIdSummary[]>;
+  scenarioPacks: QaCoverageScenarioPackSummary[];
+  liveTransportLanes: LiveTransportCoverageLaneSummary[];
+  scorecardTaxonomy: QaScorecardTaxonomyReport;
+};
+
+function assertUniqueQaScenarioIds(
+  scenarios: readonly QaSeedScenarioWithSource[],
+  nonYamlScenarios: readonly { id: string; sourcePath: string }[],
+): void {
+  const sourcePathsById = new Map<string, string[]>();
+  for (const { id, sourcePath } of [...scenarios, ...nonYamlScenarios]) {
+    const sourcePaths = sourcePathsById.get(id) ?? [];
+    sourcePaths.push(sourcePath);
+    sourcePathsById.set(id, sourcePaths);
+  }
+  const duplicates = [...sourcePathsById.entries()]
+    .filter(([, sourcePaths]) => sourcePaths.length > 1)
+    .toSorted(([left], [right]) => left.localeCompare(right));
+  if (duplicates.length > 0) {
+    const details = duplicates
+      .map(([id, sourcePaths]) => `${id} (${sourcePaths.join(", ")})`)
+      .join("; ");
+    throw new Error(`duplicate qa scenario id(s): ${details}`);
+  }
+}
+
+function scenarioTheme(sourcePath: string) {
+  const parts = sourcePath.split("/");
+  return parts[2] ?? "unknown";
+}
+
+function scenarioSurfaces(scenario: QaSeedScenarioWithSource) {
+  return scenario.surfaces && scenario.surfaces.length > 0 ? scenario.surfaces : [scenario.surface];
+}
+
+function scenarioRisk(scenario: QaSeedScenarioWithSource) {
+  return scenario.risk ?? scenario.riskLevel ?? "unassigned";
+}
+
+function summarizeScenario(scenario: QaSeedScenarioWithSource): QaCoverageScenarioSummary {
+  return {
+    id: scenario.id,
+    title: scenario.title,
+    sourcePath: scenario.sourcePath,
+    theme: scenarioTheme(scenario.sourcePath),
+    surfaces: scenarioSurfaces(scenario),
+    risk: scenarioRisk(scenario),
+  };
+}
+
+function normalizeSearchText(value: string) {
+  return value.toLowerCase();
+}
+
+function tokenizeScenarioSearchQuery(query: string) {
+  return normalizeStringEntriesLower(query.split(/\s+/u));
+}
+
+function scenarioSearchText(scenario: QaSeedScenarioWithSource) {
+  const config = scenario.execution.config ?? {};
+  return normalizeSearchText(
+    [
+      scenario.id,
+      scenario.title,
+      scenario.sourcePath,
+      scenario.surface,
+      ...(scenario.surfaces ?? []),
+      scenario.category ?? "",
+      scenario.runtimeParityTier ?? "",
+      scenario.risk ?? "",
+      scenario.riskLevel ?? "",
+      scenario.objective,
+      ...scenario.successCriteria,
+      ...(scenario.capabilities ?? []),
+      ...(scenario.plugins ?? []),
+      ...(scenario.docsRefs ?? []),
+      ...(scenario.codeRefs ?? []),
+      ...(scenario.coverage?.primary ?? []),
+      ...(scenario.coverage?.secondary ?? []),
+      ...Object.entries(config).flatMap(([key, value]) => [
+        key,
+        typeof value === "string" ? value : "",
+      ]),
+    ].join("\n"),
+  );
+}
+
+function stringifyConfigValue(value: unknown) {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaScenarioSearchMatch {
+  const config = scenario.execution.config ?? {};
+  return {
+    ...summarizeScenario(scenario),
+    coverageIds: [
+      ...(scenario.coverage?.primary ?? []),
+      ...(scenario.coverage?.secondary ?? []),
+    ].toSorted((left, right) => left.localeCompare(right)),
+    docsRefs: [...(scenario.docsRefs ?? [])],
+    codeRefs: [...(scenario.codeRefs ?? [])],
+    executionKind: scenario.execution.kind,
+    ...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}),
+    runtimeParityTier: scenario.runtimeParityTier,
+    requiredProviderMode: stringifyConfigValue(config.requiredProviderMode),
+    requiredChannelDriver: stringifyConfigValue(config.requiredChannelDriver),
+    requiredProvider: stringifyConfigValue(config.requiredProvider),
+    requiredModel: stringifyConfigValue(config.requiredModel),
+  };
+}
+
+export function findQaScenarioMatches(
+  scenarios: readonly QaSeedScenarioWithSource[],
+  query: string,
+) {
+  const tokens = tokenizeScenarioSearchQuery(query);
+  if (tokens.length === 0) {
+    return [];
+  }
+  return scenarios
+    .filter((scenario) => {
+      const haystack = scenarioSearchText(scenario);
+      return tokens.every((token) => haystack.includes(token));
+    })
+    .map(summarizeScenarioSearchMatch)
+    .toSorted((left, right) => left.id.localeCompare(right.id));
+}
+
+function sortCoverageIds(coverageIds: readonly QaCoverageIdSummary[]) {
+  return coverageIds.toSorted((left, right) => left.id.localeCompare(right.id));
+}
+
+function buildScenarioPackSummaries(
+  scenarios: readonly QaSeedScenarioWithSource[],
+): QaCoverageScenarioPackSummary[] {
+  const scenariosById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
+  return QA_SCENARIO_PACKS.map((pack) => {
+    const coverageIds = new Set<string>();
+    const missingScenarioIds: string[] = [];
+    for (const scenarioId of pack.scenarioIds) {
+      const scenario = scenariosById.get(scenarioId);
+      if (!scenario) {
+        missingScenarioIds.push(scenarioId);
+        continue;
+      }
+      for (const coverageId of [
+        ...(scenario.coverage?.primary ?? []),
+        ...(scenario.coverage?.secondary ?? []),
+      ]) {
+        coverageIds.add(coverageId);
+      }
+    }
+    return {
+      id: pack.id,
+      title: pack.title,
+      scenarioIds: [...pack.scenarioIds],
+      coverageIds: [...coverageIds].toSorted(),
+      missingScenarioIds,
+    };
+  }).toSorted((left, right) => left.id.localeCompare(right.id));
+}
+
+export function buildQaCoverageInventory(
+  scenarios: readonly QaSeedScenarioWithSource[],
+  params?: { nonYamlScenarios?: readonly { id: string; sourcePath: string }[] },
+): QaCoverageInventory {
+  assertUniqueQaScenarioIds(scenarios, params?.nonYamlScenarios ?? []);
+  const byCoverageId = new Map<string, QaCoverageIdSummary>();
+  const primaryCoverageIds = new Set<string>();
+  const secondaryCoverageIds = new Set<string>();
+  const missingCoverage: QaCoverageScenarioSummary[] = [];
+
+  const addFeatureCoverage = (
+    scenario: QaSeedScenarioWithSource,
+    coverageIds: readonly string[] | undefined,
+    intent: QaCoverageIntent,
+  ) => {
+    const summary = summarizeScenario(scenario);
+    for (const coverageId of coverageIds ?? []) {
+      const coverage = byCoverageId.get(coverageId) ?? {
+        id: coverageId,
+        scenarios: [],
+      };
+      coverage.scenarios.push({ ...summary, intent });
+      byCoverageId.set(coverageId, coverage);
+      if (intent === "primary") {
+        primaryCoverageIds.add(coverageId);
+      } else {
+        secondaryCoverageIds.add(coverageId);
+      }
+    }
+  };
+
+  for (const scenario of scenarios) {
+    if (!scenario.coverage) {
+      missingCoverage.push(summarizeScenario(scenario));
+      continue;
+    }
+    addFeatureCoverage(scenario, scenario.coverage.primary, "primary");
+    addFeatureCoverage(scenario, scenario.coverage.secondary, "secondary");
+  }
+
+  const coverageIds = sortCoverageIds([...byCoverageId.values()]);
+  const overlappingCoverage = coverageIds.filter((coverage) => coverage.scenarios.length > 1);
+  const byTheme: Record<string, QaCoverageIdSummary[]> = {};
+  const bySurface: Record<string, QaCoverageIdSummary[]> = {};
+
+  for (const coverage of coverageIds) {
+    const themes = new Set(coverage.scenarios.map((scenario) => scenario.theme));
+    for (const theme of themes) {
+      byTheme[theme] ??= [];
+      byTheme[theme].push({
+        ...coverage,
+        scenarios: coverage.scenarios.filter((scenario) => scenario.theme === theme),
+      });
+    }
+    const surfaces = new Set(coverage.scenarios.flatMap((scenario) => scenario.surfaces));
+    for (const surface of surfaces) {
+      bySurface[surface] ??= [];
+      bySurface[surface].push({
+        ...coverage,
+        scenarios: coverage.scenarios.filter((scenario) => scenario.surfaces.includes(surface)),
+      });
+    }
+  }
+
+  return {
+    scenarioCount: scenarios.length,
+    coverageIdCount: coverageIds.length,
+    primaryCoverageIdCount: primaryCoverageIds.size,
+    secondaryCoverageIdCount: secondaryCoverageIds.size,
+    coverageIds,
+    overlappingCoverage,
+    missingCoverage,
+    byTheme,
+    bySurface,
+    scenarioPacks: buildScenarioPackSummaries(scenarios),
+    liveTransportLanes: buildLiveTransportCoverageLaneSummaries(),
+    scorecardTaxonomy: readQaScorecardTaxonomyReport(scenarios),
+  };
+}
+
+function pushCoverageIdLines(lines: string[], coverageIds: readonly QaCoverageIdSummary[]) {
+  for (const coverage of sortCoverageIds(coverageIds)) {
+    const scenarios = coverage.scenarios
+      .map((scenario) => `${scenario.intent}: ${scenario.id} (${scenario.sourcePath})`)
+      .join(", ");
+    lines.push(`- ${coverage.id}: ${scenarios}`);
+  }
+}
+
+function pushLiveTransportLines(
+  lines: string[],
+  lanes: readonly LiveTransportCoverageLaneSummary[],
+) {
+  for (const lane of lanes) {
+    const members = lane.members
+      .map((member) =>
+        member.scenarioId
+          ? `${member.standardId}: ${member.scenarioId}`
+          : `${member.standardId}: always-on`,
+      )
+      .join(", ");
+    const missing =
+      lane.baselineMissingStandardScenarioIds.length > 0
+        ? lane.baselineMissingStandardScenarioIds.join(", ")
+        : "none";
+    lines.push(
+      `- ${lane.transportId} (${lane.commandName}): ${members}; missing baseline: ${missing}`,
+    );
+  }
+}
+
+function pushScenarioPackLines(lines: string[], packs: readonly QaCoverageScenarioPackSummary[]) {
+  for (const pack of packs) {
+    const missing =
+      pack.missingScenarioIds.length > 0 ? pack.missingScenarioIds.join(", ") : "none";
+    lines.push(
+      `- ${pack.id} (${pack.title}): ${pack.scenarioIds.length} scenarios; coverage IDs: ${pack.coverageIds.join(", ")}; missing scenarios: ${missing}`,
+    );
+    lines.push(`  - scenarios: ${pack.scenarioIds.join(", ")}`);
+  }
+}
+
+function pushScorecardTaxonomyLines(lines: string[], report: QaScorecardTaxonomyReport) {
+  lines.push("## Scorecard Taxonomy", "");
+  lines.push(`- Taxonomy: ${report.taxonomyPath ?? "missing"}`);
+  lines.push(`- Categories: ${report.categoryCount}`);
+  lines.push(`- Profiles: ${report.profileCount}`);
+  lines.push(
+    `- Fulfilled taxonomy categories: ${report.fulfilledCategoryCount}/${report.requiredCategoryCount} (${report.categoryFulfillmentPercent}%)`,
+  );
+  lines.push(
+    `- Fulfilled taxonomy coverage IDs: ${report.fulfilledCoverageIdCount}/${report.requiredCoverageIdCount} (${report.coverageIdFulfillmentPercent}%)`,
+  );
+  lines.push(`- Evidence refs: ${report.evidenceRefCount}`);
+  lines.push(`- Scenario coverage IDs: ${report.scenarioCoverageIdCount}`);
+  lines.push(`- Unknown scenario coverage IDs: ${report.unknownCoverageIdCount}`);
+  lines.push(`- Validation warnings: ${report.validationIssueCount}`, "");
+
+  if (report.profiles.length > 0) {
+    lines.push("### Profiles", "");
+    for (const profile of report.profiles) {
+      const categories = profile.categoryIds.length > 0 ? profile.categoryIds.join(", ") : "none";
+      lines.push(`- ${profile.id}: ${profile.categoryIds.length} categories; ${categories}`);
+    }
+    lines.push("");
+  }
+
+  if (report.categories.length > 0) {
+    lines.push("### Category Coverage", "");
+    for (const category of report.categories) {
+      const coverageIds =
+        category.coverageIds.length > 0 ? category.coverageIds.join(", ") : "none";
+      const evidence =
+        category.evidence.length > 0
+          ? category.evidence
+              .map((ref) => {
+                const target = ref.path ?? (ref.scenarioRefs.join("|") || "discovered");
+                return `${ref.role}:${ref.kind}:${target} (${ref.coverageId})`;
+              })
+              .join(", ")
+          : "none";
+      const profiles = category.profiles.length > 0 ? category.profiles.join(", ") : "none";
+      lines.push(
+        `- ${category.id} (${category.taxonomySurfaceId} / ${category.taxonomyCategoryName}; ${category.coverageStatus}): profiles: ${profiles}; coverage IDs: ${coverageIds}; evidence: ${evidence}`,
+      );
+    }
+    lines.push("");
+  }
+
+  if (report.validationIssues.length > 0) {
+    lines.push("### Validation Warnings", "");
+    for (const issue of report.validationIssues) {
+      const category = issue.categoryId ? `${issue.categoryId}: ` : "";
+      lines.push(`- ${issue.code}: ${category}${issue.message}`);
+    }
+    lines.push("");
+  }
+
+  if (report.unknownCoverageIds.length > 0) {
+    lines.push("### Unknown Scenario Coverage IDs", "");
+    lines.push(report.unknownCoverageIds.join(", "));
+    lines.push("");
+  }
+}
+
+export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): string {
+  const lines: string[] = [
+    "# QA Coverage Inventory",
+    "",
+    `- Scenarios: ${inventory.scenarioCount}`,
+    `- Taxonomy coverage IDs: ${inventory.coverageIdCount}`,
+    `- Primary coverage IDs: ${inventory.primaryCoverageIdCount}`,
+    `- Secondary coverage IDs: ${inventory.secondaryCoverageIdCount}`,
+    `- Overlapping coverage IDs: ${inventory.overlappingCoverage.length}`,
+    `- Missing coverage metadata: ${inventory.missingCoverage.length}`,
+    "",
+  ];
+
+  if (inventory.scenarioPacks.length > 0) {
+    lines.push("## Scenario Packs", "");
+    pushScenarioPackLines(lines, inventory.scenarioPacks);
+    lines.push("");
+  }
+
+  lines.push("## By Theme", "");
+  for (const theme of Object.keys(inventory.byTheme).toSorted()) {
+    lines.push(`### ${theme}`, "");
+    pushCoverageIdLines(lines, inventory.byTheme[theme] ?? []);
+    lines.push("");
+  }
+
+  lines.push("## By Surface", "");
+  for (const surface of Object.keys(inventory.bySurface).toSorted()) {
+    lines.push(`### ${surface}`, "");
+    pushCoverageIdLines(lines, inventory.bySurface[surface] ?? []);
+    lines.push("");
+  }
+
+  if (inventory.liveTransportLanes.length > 0) {
+    lines.push("## Live Transport Lanes", "");
+    pushLiveTransportLines(lines, inventory.liveTransportLanes);
+    lines.push("");
+  }
+
+  pushScorecardTaxonomyLines(lines, inventory.scorecardTaxonomy);
+
+  if (inventory.overlappingCoverage.length > 0) {
+    lines.push("## Overlap", "");
+    pushCoverageIdLines(lines, inventory.overlappingCoverage);
+    lines.push("");
+  }
+
+  if (inventory.missingCoverage.length > 0) {
+    lines.push("## Missing Metadata", "");
+    for (const scenario of inventory.missingCoverage.toSorted((left, right) =>
+      left.id.localeCompare(right.id),
+    )) {
+      lines.push(`- ${scenario.id}: ${scenario.sourcePath}`);
+    }
+    lines.push("");
+  }
+
+  return `${lines.join("\n").trimEnd()}\n`;
+}
+
+function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) {
+  const metadata = [
+    match.runtimeParityTier ? `runtimeParityTier=${match.runtimeParityTier}` : "",
+    match.requiredProviderMode ? `providerMode=${match.requiredProviderMode}` : "",
+    match.requiredChannelDriver ? `channelDriver=${match.requiredChannelDriver}` : "",
+    match.requiredProvider ? `provider=${match.requiredProvider}` : "",
+    match.requiredModel ? `model=${match.requiredModel}` : "",
+  ].filter(Boolean);
+  return metadata.length > 0 ? metadata.join("; ") : "none";
+}
+
+function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) {
+  const scenarioArgs = matches.map((match) => `--scenario ${match.id}`).join(" ");
+  return `pnpm openclaw qa suite ${scenarioArgs}`;
+}
+
+function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) {
+  const groups = new Map<QaScenarioSearchMatch["executionKind"], QaScenarioSearchMatch[]>();
+  for (const match of matches) {
+    const group = groups.get(match.executionKind) ?? [];
+    group.push(match);
+    groups.set(match.executionKind, group);
+  }
+  const executionOrder: QaScenarioSearchMatch["executionKind"][] = [
+    "flow",
+    "script",
+    "vitest",
+    "playwright",
+  ];
+  return executionOrder.flatMap((executionKind) => {
+    const group = groups.get(executionKind);
+    return group && group.length > 0 ? [{ executionKind, matches: group }] : [];
+  });
+}
+
+export function renderQaScenarioMatchesMarkdownReport(params: {
+  query: string;
+  matches: readonly QaScenarioSearchMatch[];
+}) {
+  const commandGroups = scenarioMatchCommandGroups(params.matches);
+  const lines = [
+    "# QA Scenario Matches",
+    "",
+    `- Query: ${params.query}`,
+    `- Matches: ${params.matches.length}`,
+  ];
+
+  if (commandGroups.length === 1) {
+    lines.push(`- Suite command: \`${formatSuiteCommand(commandGroups[0].matches)}\``);
+  } else if (commandGroups.length > 1) {
+    lines.push("- Suite commands:");
+    for (const group of commandGroups) {
+      lines.push(`  - ${group.executionKind}: \`${formatSuiteCommand(group.matches)}\``);
+    }
+  }
+  lines.push("");
+
+  if (params.matches.length === 0) {
+    lines.push("No QA scenarios matched the query.", "");
+    return lines.join("\n");
+  }
+
+  for (const match of params.matches) {
+    lines.push(`- ${match.id}: ${match.title}`);
+    lines.push(`  - source: ${match.sourcePath}`);
+    lines.push(`  - surface: ${match.surfaces.join(", ")}`);
+    lines.push(
+      match.executionKind === "flow"
+        ? "  - execution: flow"
+        : `  - execution: ${match.executionKind} ${match.executionPath ?? "missing"}`,
+    );
+    lines.push(`  - coverage IDs: ${match.coverageIds.join(", ") || "none"}`);
+    lines.push(`  - live requirements: ${formatOptionalScenarioMetadata(match)}`);
+    if (match.codeRefs.length > 0) {
+      lines.push(`  - code refs: ${match.codeRefs.join(", ")}`);
+    }
+    if (match.docsRefs.length > 0) {
+      lines.push(`  - docs refs: ${match.docsRefs.join(", ")}`);
+    }
+  }
+
+  return `${lines.join("\n").trimEnd()}\n`;
+}

extensions/qa-lab/src/crabline-transport.test.ts

@@ -0,0 +1,858 @@
+// Qa Lab tests cover Crabline local-provider transport integration behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+import {
+  OPENCLAW_CRABLINE_MANIFEST_PATH,
+  type OpenClawCrablineChannelDriverSelection,
+} from "@openclaw/crabline";
+import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
+import { withTempDir } from "openclaw/plugin-sdk/test-env";
+import { describe, expect, it } from "vitest";
+import { createQaBusState } from "./bus-state.js";
+import { createQaCrablineTransportAdapter } from "./crabline-transport.js";
+
+function createSelection(channel: OpenClawCrablineChannelDriverSelection["channel"] = "telegram") {
+  return {
+    capabilityMatrixPath: "crabline-fake-provider-capabilities.json",
+    channel,
+    channelDriver: "crabline",
+    smokeArtifactPath: "crabline-fake-provider-smoke.json",
+  } as const;
+}
+
+describe("crabline transport", () => {
+  it("configures OpenClaw's Telegram plugin against a Crabline local provider server", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection(),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.id).toBe("crabline");
+        expect(transport.requiredPluginIds).toEqual(["telegram"]);
+        expect(transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" })).toMatchObject({
+          channels: {
+            telegram: {
+              apiRoot: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/u),
+              botToken: "424242:crabline-telegram-token",
+              dmPolicy: "open",
+              enabled: true,
+              groupPolicy: "open",
+            },
+          },
+        });
+        expect(transport.buildAgentDelivery({ target: "dm:alice" })).toEqual({
+          channel: "telegram",
+          to: "100001",
+          replyChannel: "telegram",
+          replyTo: "100001",
+        });
+
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          provider?: string;
+        };
+        expect(manifest.provider).toBe("telegram");
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("injects Telegram native commands through the shared transport adapter", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection(),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.sendNativeCommand).toBeTypeOf("function");
+        await transport.sendNativeCommand?.({
+          command: "stop",
+          conversation: { id: "alice", kind: "direct" },
+          senderId: "alice",
+          senderName: "Alice",
+        });
+
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          botToken: string;
+          endpoints: { apiRoot: string };
+        };
+        const response = await fetch(
+          `${manifest.endpoints.apiRoot}/bot${manifest.botToken}/getUpdates`,
+        );
+        await expect(response.json()).resolves.toMatchObject({
+          result: [
+            {
+              message: {
+                entities: [{ length: 5, offset: 0, type: "bot_command" }],
+                text: "/stop",
+              },
+            },
+          ],
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("observes Telegram preview edits through the shared transport adapter", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection(),
+        state: createQaBusState(),
+      });
+
+      try {
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          botToken: string;
+          endpoints: { apiRoot: string };
+        };
+        const postTelegram = async (method: string, body: Record<string, unknown>) => {
+          const response = await fetch(
+            `${manifest.endpoints.apiRoot}/bot${manifest.botToken}/${method}`,
+            {
+              body: JSON.stringify(body),
+              headers: { "content-type": "application/json" },
+              method: "POST",
+            },
+          );
+          expect(response.ok).toBe(true);
+          return (await response.json()) as { result: { message_id: number } };
+        };
+        const sent = await postTelegram("sendMessage", {
+          chat_id: "-1001234567890",
+          message_thread_id: 42,
+          text: "preview text",
+        });
+        expect(transport.state.searchMessages({ query: "preview text" })).toEqual([
+          expect.objectContaining({ text: "preview text" }),
+        ]);
+        await postTelegram("editMessageText", {
+          chat_id: "-1001234567890",
+          message_id: sent.result.message_id,
+          text: "final marker",
+        });
+
+        expect(transport.waitForOutboundSequence).toBeTypeOf("function");
+        await expect(
+          transport.waitForOutboundSequence!({
+            conversationId: "-1001234567890",
+            finalSettleMs: 0,
+            finalTextIncludes: "final marker",
+            minimumPreviewEvents: 1,
+            threadId: "42",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          events: [{ kind: "sent" }, { kind: "edited" }],
+          final: { text: "final marker", threadId: "42" },
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("configures OpenClaw's Slack plugin against a Crabline local provider server", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("slack"),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.id).toBe("crabline");
+        expect(transport.requiredPluginIds).toEqual(["slack"]);
+        expect(transport.sendNativeCommand).toBeUndefined();
+        expect(transport.waitForOutboundSequence).toBeUndefined();
+        expect(transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" })).toMatchObject({
+          channels: {
+            slack: {
+              botToken: "xoxb-crabline-slack-token",
+              enabled: true,
+              mode: "http",
+              signingSecret: "crabline-slack-signing-secret",
+            },
+          },
+        });
+        expect(transport.createRuntimeEnvPatch?.()).toMatchObject({
+          SLACK_API_URL: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+\/api\/$/u),
+          SLACK_BOT_TOKEN: "xoxb-crabline-slack-token",
+          SLACK_SIGNING_SECRET: "crabline-slack-signing-secret",
+        });
+
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          provider?: string;
+        };
+        expect(manifest.provider).toBe("slack");
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("injects inbound messages through Crabline and mirrors Slack sends into normalized state", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("slack"),
+        state: createQaBusState(),
+      });
+
+      try {
+        await transport.sendInbound({
+          conversation: {
+            id: "D12345678",
+            kind: "direct",
+          },
+          senderId: "U12345678",
+          senderName: "Alice",
+          text: "Slack baseline marker check.",
+        });
+
+        const env = transport.createRuntimeEnvPatch?.() ?? {};
+        expect(env.SLACK_API_URL).toBeTruthy();
+        expect(env.SLACK_BOT_TOKEN).toBeTruthy();
+        const { response, release } = await fetchWithSsrFGuard({
+          url: `${env.SLACK_API_URL}chat.postMessage`,
+          init: {
+            body: JSON.stringify({
+              channel: "D12345678",
+              text: "assistant via fake slack",
+            }),
+            headers: {
+              authorization: `Bearer ${env.SLACK_BOT_TOKEN}`,
+              "content-type": "application/json",
+            },
+            method: "POST",
+          },
+          policy: { allowPrivateNetwork: true },
+          auditContext: "qa-lab-crabline-slack-transport-test",
+        });
+        await release();
+        expect(response.ok).toBe(true);
+
+        await expect(
+          transport.waitForOutbound({
+            conversation: { id: "D12345678", kind: "direct" },
+            textIncludes: "assistant via fake slack",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: "D12345678", kind: "direct" },
+          text: "assistant via fake slack",
+        });
+
+        await expect(
+          transport.state.waitFor({
+            direction: "outbound",
+            kind: "message-text",
+            textIncludes: "assistant via fake slack",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: {
+            id: "D12345678",
+            kind: "direct",
+          },
+          direction: "outbound",
+          text: "assistant via fake slack",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("configures OpenClaw's WhatsApp plugin against a Crabline Baileys WebSocket server", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("whatsapp"),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.id).toBe("crabline");
+        expect(transport.requiredPluginIds).toEqual(["whatsapp"]);
+        expect(transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" })).toMatchObject({
+          channels: {
+            whatsapp: {
+              allowFrom: ["*"],
+              dmPolicy: "open",
+              enabled: true,
+              groupAllowFrom: ["*"],
+              groupPolicy: "open",
+            },
+          },
+        });
+        expect(transport.buildAgentDelivery({ target: "15551234567@s.whatsapp.net" })).toEqual({
+          channel: "whatsapp",
+          to: "15551234567@s.whatsapp.net",
+          replyChannel: "whatsapp",
+          replyTo: "15551234567@s.whatsapp.net",
+        });
+        const env = transport.createRuntimeEnvPatch?.() ?? {};
+        expect(env).toMatchObject({
+          CRABLINE_WHATSAPP_ADMIN_TOKEN: expect.any(String),
+          CRABLINE_WHATSAPP_RECORDER_PATH: expect.stringMatching(/whatsapp-fake-provider\.jsonl$/u),
+          CRABLINE_WHATSAPP_SELF_JID: "15550000000@s.whatsapp.net",
+          OPENCLAW_WHATSAPP_WEB_SOCKET_URL: expect.stringMatching(
+            /^ws:\/\/127\.0\.0\.1:\d+\/crabline\/whatsapp\/ws\/chat\?access_token=/u,
+          ),
+        });
+        expect(env.CRABLINE_WHATSAPP_ACCESS_TOKEN).toBeUndefined();
+        expect(env.CRABLINE_WHATSAPP_API_ROOT).toBeUndefined();
+
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          provider?: string;
+        };
+        expect(manifest.provider).toBe("whatsapp");
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("injects WhatsApp inbound messages through Crabline into normalized state", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("whatsapp"),
+        state: createQaBusState(),
+      });
+
+      try {
+        const message = await transport.state.addInboundMessage({
+          conversation: {
+            id: "15551234567@s.whatsapp.net",
+            kind: "direct",
+          },
+          senderId: "15557654321@s.whatsapp.net",
+          senderName: "Alice",
+          text: "WhatsApp baseline marker check.",
+        });
+        expect(message).toMatchObject({
+          conversation: {
+            id: "15551234567@s.whatsapp.net",
+            kind: "direct",
+          },
+          direction: "inbound",
+          senderId: "15557654321@s.whatsapp.net",
+          text: "WhatsApp baseline marker check.",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("binds Signal config and normalizes transport targets", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("signal"),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.requiredPluginIds).toEqual(["signal"]);
+        expect(transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" })).toMatchObject({
+          channels: {
+            signal: {
+              account: "+15550000000",
+              apiMode: "native",
+              autoStart: false,
+              enabled: true,
+              httpUrl: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/u),
+            },
+          },
+        });
+        expect(transport.createRuntimeEnvPatch?.()).toEqual({});
+        expect(transport.buildAgentDelivery({ target: "dm:alice" })).toMatchObject({
+          channel: "signal",
+          replyChannel: "signal",
+          replyTo: expect.stringMatching(/^\+1555\d{7}$/u),
+          to: expect.stringMatching(/^\+1555\d{7}$/u),
+        });
+
+        await expect(
+          transport.state.addInboundMessage({
+            conversation: { id: "alice", kind: "direct" },
+            senderId: "alice",
+            senderName: "Alice",
+            text: "Signal baseline marker check.",
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: "alice", kind: "direct" },
+          direction: "inbound",
+          senderId: "alice",
+          text: "Signal baseline marker check.",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("normalizes native Signal JSON-RPC sends into outbound state", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("signal"),
+        state: createQaBusState(),
+      });
+
+      try {
+        const delivery = transport.buildAgentDelivery({ target: "dm:alice" });
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          endpoints: { rpcUrl: string };
+        };
+        const { response, release } = await fetchWithSsrFGuard({
+          url: manifest.endpoints.rpcUrl,
+          init: {
+            body: JSON.stringify({
+              id: "qa-signal-send",
+              jsonrpc: "2.0",
+              method: "send",
+              params: {
+                message: "assistant via fake signal",
+                recipient: [delivery.to],
+              },
+            }),
+            headers: { "content-type": "application/json" },
+            method: "POST",
+          },
+          policy: { allowPrivateNetwork: true },
+          auditContext: "qa-lab-crabline-signal-transport-test",
+        });
+        await release();
+        expect(response.ok).toBe(true);
+
+        await expect(
+          transport.waitForOutbound({
+            conversation: { id: "alice", kind: "direct" },
+            textIncludes: "assistant via fake signal",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: "alice", kind: "direct" },
+          text: "assistant via fake signal",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("binds Mattermost config and normalizes transport targets", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("mattermost"),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.requiredPluginIds).toEqual(["mattermost"]);
+        expect(transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" })).toMatchObject({
+          channels: {
+            mattermost: {
+              baseUrl: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/u),
+              botToken: "crabline-mattermost-token",
+              enabled: true,
+              network: { dangerouslyAllowPrivateNetwork: true },
+            },
+          },
+        });
+        expect(transport.createRuntimeEnvPatch?.()).toMatchObject({
+          MATTERMOST_BOT_TOKEN: "crabline-mattermost-token",
+          MATTERMOST_URL: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/u),
+        });
+        expect(transport.buildAgentDelivery({ target: "group:qa-channel" })).toMatchObject({
+          channel: "mattermost",
+          replyChannel: "mattermost",
+          replyTo: expect.stringMatching(/^channel:[a-z0-9]{26}$/u),
+          to: expect.stringMatching(/^channel:[a-z0-9]{26}$/u),
+        });
+
+        await expect(
+          transport.state.addInboundMessage({
+            conversation: { id: "qa-channel", kind: "group" },
+            senderId: "alice",
+            senderName: "Alice",
+            text: "Mattermost baseline marker check.",
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: "qa-channel", kind: "group" },
+          direction: "inbound",
+          senderId: "alice",
+          text: "Mattermost baseline marker check.",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("normalizes native Mattermost post creation into outbound state", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("mattermost"),
+        state: createQaBusState(),
+      });
+
+      try {
+        await transport.state.addInboundMessage({
+          conversation: { id: "qa-channel", kind: "group" },
+          senderId: "alice",
+          senderName: "Alice",
+          text: "Mattermost baseline marker check.",
+        });
+        const delivery = transport.buildAgentDelivery({ target: "group:qa-channel" });
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          botToken: string;
+          endpoints: { apiRoot: string };
+        };
+        const { response, release } = await fetchWithSsrFGuard({
+          url: `${manifest.endpoints.apiRoot}/posts`,
+          init: {
+            body: JSON.stringify({
+              channel_id: delivery.to.replace(/^channel:/u, ""),
+              message: "assistant via fake mattermost",
+            }),
+            headers: {
+              authorization: `Bearer ${manifest.botToken}`,
+              "content-type": "application/json",
+            },
+            method: "POST",
+          },
+          policy: { allowPrivateNetwork: true },
+          auditContext: "qa-lab-crabline-mattermost-transport-test",
+        });
+        await release();
+        expect(response.ok).toBe(true);
+
+        await expect(
+          transport.waitForOutbound({
+            conversation: { id: "qa-channel", kind: "group" },
+            textIncludes: "assistant via fake mattermost",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: "qa-channel", kind: "group" },
+          text: "assistant via fake mattermost",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("binds Matrix config and normalizes transport targets", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("matrix"),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.requiredPluginIds).toEqual(["matrix"]);
+        expect(transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" })).toMatchObject({
+          channels: {
+            matrix: {
+              accessToken: expect.any(String),
+              enabled: true,
+              encryption: false,
+              homeserver: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/u),
+              network: { dangerouslyAllowPrivateNetwork: true },
+              userId: "@openclaw:matrix.test",
+            },
+          },
+        });
+        expect(transport.createRuntimeEnvPatch?.()).toMatchObject({
+          MATRIX_ACCESS_TOKEN: expect.any(String),
+          MATRIX_BASE_URL: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/u),
+          MATRIX_USER_ID: "@openclaw:matrix.test",
+        });
+
+        const roomId = "!qa:matrix.test";
+        expect(transport.buildAgentDelivery({ target: `group:${roomId}` })).toEqual({
+          channel: "matrix",
+          replyChannel: "matrix",
+          replyTo: `room:${roomId}`,
+          to: `room:${roomId}`,
+        });
+        await expect(
+          transport.state.addInboundMessage({
+            conversation: { id: roomId, kind: "group" },
+            senderId: "@alice:matrix.test",
+            senderName: "Alice",
+            text: "Matrix baseline marker check.",
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: roomId, kind: "group" },
+          direction: "inbound",
+          senderId: "@alice:matrix.test",
+          text: "Matrix baseline marker check.",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("normalizes native Matrix room message sends into outbound state", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("matrix"),
+        state: createQaBusState(),
+      });
+
+      try {
+        const roomId = "!qa:matrix.test";
+        await transport.state.addInboundMessage({
+          conversation: { id: roomId, kind: "group" },
+          senderId: "@alice:matrix.test",
+          senderName: "Alice",
+          text: "Matrix baseline marker check.",
+        });
+        const delivery = transport.buildAgentDelivery({ target: `group:${roomId}` });
+        const providerRoomId = delivery.to.replace(/^room:/u, "");
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          accessToken: string;
+          endpoints: { clientApiRoot: string };
+        };
+        const { response, release } = await fetchWithSsrFGuard({
+          url: `${manifest.endpoints.clientApiRoot}/rooms/${encodeURIComponent(providerRoomId)}/send/m.room.message/qa-matrix-send`,
+          init: {
+            body: JSON.stringify({ body: "assistant via fake matrix", msgtype: "m.text" }),
+            headers: {
+              authorization: `Bearer ${manifest.accessToken}`,
+              "content-type": "application/json",
+            },
+            method: "PUT",
+          },
+          policy: { allowPrivateNetwork: true },
+          auditContext: "qa-lab-crabline-matrix-transport-test",
+        });
+        await release();
+        expect(response.ok).toBe(true);
+
+        await expect(
+          transport.waitForOutbound({
+            conversation: { id: roomId, kind: "group" },
+            textIncludes: "assistant via fake matrix",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: roomId, kind: "group" },
+          text: "assistant via fake matrix",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("configures Zalo and normalizes native message sends", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection("zalo"),
+        state: createQaBusState(),
+      });
+
+      try {
+        expect(transport.requiredPluginIds).toEqual(["zalo"]);
+        expect(transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" })).toMatchObject({
+          channels: {
+            zalo: {
+              allowFrom: ["*"],
+              botToken: "crabline-zalo-bot-token",
+              dmPolicy: "open",
+              enabled: true,
+              groupAllowFrom: ["*"],
+              groupPolicy: "open",
+            },
+          },
+        });
+        expect(transport.createRuntimeEnvPatch?.()).toMatchObject({
+          ZALO_API_URL: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/u),
+          ZALO_BOT_TOKEN: "crabline-zalo-bot-token",
+        });
+
+        await transport.state.addInboundMessage({
+          conversation: { id: "qa-group", kind: "group" },
+          senderId: "alice",
+          senderName: "Alice",
+          text: "Zalo baseline marker check.",
+        });
+        const delivery = transport.buildAgentDelivery({ target: "group:qa-group" });
+        expect(delivery).toEqual({
+          channel: "zalo",
+          replyChannel: "zalo",
+          replyTo: "qa-group",
+          to: "qa-group",
+        });
+
+        const manifest = JSON.parse(
+          await fs.readFile(path.join(outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH), "utf8"),
+        ) as {
+          botToken: string;
+          endpoints: { apiRoot: string };
+        };
+        const { response, release } = await fetchWithSsrFGuard({
+          url: `${manifest.endpoints.apiRoot}/bot${manifest.botToken}/sendMessage`,
+          init: {
+            body: JSON.stringify({
+              chat_id: delivery.to,
+              text: "assistant via fake zalo",
+            }),
+            headers: { "content-type": "application/json" },
+            method: "POST",
+          },
+          policy: { allowPrivateNetwork: true },
+          auditContext: "qa-lab-crabline-zalo-transport-test",
+        });
+        await release();
+        expect(response.ok).toBe(true);
+
+        await expect(
+          transport.waitForOutbound({
+            conversation: { id: "qa-group", kind: "group" },
+            textIncludes: "assistant via fake zalo",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: { id: "qa-group", kind: "group" },
+          text: "assistant via fake zalo",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+
+  it("injects inbound messages through Crabline and mirrors Telegram sends into normalized state", async () => {
+    await withTempDir("qa-crabline-transport-", async (outputDir) => {
+      const transport = await createQaCrablineTransportAdapter({
+        outputDir,
+        selection: createSelection(),
+        state: createQaBusState(),
+      });
+
+      try {
+        await transport.state.addInboundMessage({
+          conversation: {
+            id: "alice",
+            kind: "direct",
+          },
+          senderId: "alice",
+          senderName: "Alice",
+          text: "DM baseline marker check.",
+        });
+
+        const config = transport.createGatewayConfig({ baseUrl: "http://127.0.0.1:1" });
+        const telegram = config.channels?.telegram as
+          | { apiRoot?: string; botToken?: string }
+          | undefined;
+        expect(telegram?.apiRoot).toBeTruthy();
+        expect(telegram?.botToken).toBeTruthy();
+        const { response, release } = await fetchWithSsrFGuard({
+          url: `${telegram?.apiRoot}/bot${telegram?.botToken}/sendMessage`,
+          init: {
+            body: JSON.stringify({
+              chat_id: "100001",
+              text: "assistant via fake telegram",
+            }),
+            headers: { "content-type": "application/json" },
+            method: "POST",
+          },
+          policy: { allowPrivateNetwork: true },
+          auditContext: "qa-lab-crabline-transport-test",
+        });
+        await release();
+        expect(response.ok).toBe(true);
+
+        await expect(
+          transport.state.waitFor({
+            direction: "outbound",
+            kind: "message-text",
+            textIncludes: "assistant via fake telegram",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: {
+            id: "alice",
+            kind: "direct",
+          },
+          direction: "outbound",
+          text: "assistant via fake telegram",
+        });
+
+        await transport.state.reset();
+        const delivery = transport.buildAgentDelivery({ target: "dm:qa-operator" });
+        const { response: directResponse, release: directRelease } = await fetchWithSsrFGuard({
+          url: `${telegram?.apiRoot}/bot${telegram?.botToken}/sendMessage`,
+          init: {
+            body: JSON.stringify({
+              chat_id: delivery.to,
+              text: "assistant after reset",
+            }),
+            headers: { "content-type": "application/json" },
+            method: "POST",
+          },
+          policy: { allowPrivateNetwork: true },
+          auditContext: "qa-lab-crabline-transport-reset-test",
+        });
+        await directRelease();
+        expect(directResponse.ok).toBe(true);
+
+        await expect(
+          transport.state.waitFor({
+            direction: "outbound",
+            kind: "message-text",
+            textIncludes: "assistant after reset",
+            timeoutMs: 1_000,
+          }),
+        ).resolves.toMatchObject({
+          conversation: {
+            id: "qa-operator",
+            kind: "direct",
+          },
+          direction: "outbound",
+          text: "assistant after reset",
+        });
+      } finally {
+        await transport.cleanup?.();
+      }
+    });
+  });
+});

extensions/qa-lab/src/crabline-transport.ts

@@ -0,0 +1,389 @@
+// Qa Lab plugin module implements Crabline local-provider transport behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+import {
+  OPENCLAW_CRABLINE_MANIFEST_PATH,
+  startOpenClawCrablineAdapter,
+  type OpenClawCrablineChannelDriverSelection,
+  type OpenClawCrablineInbound,
+  type StartedOpenClawCrablineAdapter,
+} from "@openclaw/crabline";
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
+import {
+  isRecord,
+  normalizeStringifiedOptionalString,
+  readStringValue,
+} from "openclaw/plugin-sdk/string-coerce-runtime";
+import { createQaBusState, type QaBusState } from "./bus-state.js";
+import { QaSuiteInfraError } from "./errors.js";
+import {
+  QaStateBackedTransportAdapter,
+  waitForQaTransportOutboundSequence,
+} from "./qa-transport.js";
+import type {
+  QaTransportActionName,
+  QaTransportGatewayClient,
+  QaTransportGatewayConfig,
+  QaTransportNativeCommandInput,
+  QaTransportOutboundEvent,
+  QaTransportOutboundSequenceMatch,
+  QaTransportReportParams,
+  QaTransportState,
+} from "./qa-transport.js";
+import type {
+  QaBusInboundMessageInput,
+  QaBusMessage,
+  QaBusOutboundMessageInput,
+} from "./runtime-api.js";
+
+const CRABLINE_TRANSPORT_ID = "crabline";
+
+type QaCrablineTransportState = QaTransportState & {
+  cleanup: () => Promise<void>;
+  getOutboundEvents: () => Promise<readonly QaTransportOutboundEvent[]>;
+  observeEvent: (event: unknown) => void;
+  rememberProviderTarget: (providerTargetKey: string, qaTarget: string) => void;
+};
+
+const TELEGRAM_LIFECYCLE_METHOD_RE = /\/(sendMessage|editMessageText|deleteMessage)$/u;
+
+function readTelegramLifecycleEvent(params: {
+  cursor: number;
+  event: unknown;
+  messageByProviderId: Map<string, QaBusMessage>;
+  pendingByChat: Map<string, QaBusMessage[]>;
+}): QaTransportOutboundEvent | null {
+  if (!isRecord(params.event) || params.event.type !== "api") {
+    return null;
+  }
+  const pathValue = readStringValue(params.event.path);
+  const method = pathValue ? TELEGRAM_LIFECYCLE_METHOD_RE.exec(pathValue)?.[1] : undefined;
+  if (!method || !isRecord(params.event.body)) {
+    return null;
+  }
+  const chatId = normalizeStringifiedOptionalString(params.event.body.chat_id);
+  if (!chatId) {
+    return null;
+  }
+  const providerMessageId = normalizeStringifiedOptionalString(params.event.body.message_id);
+  const providerKey = providerMessageId ? `${chatId}:${providerMessageId}` : null;
+  let previous = providerKey ? params.messageByProviderId.get(providerKey) : undefined;
+  if (!previous && providerKey && providerMessageId) {
+    const pending = params.pendingByChat.get(chatId) ?? [];
+    if (pending.length === 1) {
+      previous = pending[0];
+      previous.id = providerMessageId;
+      params.messageByProviderId.set(providerKey, previous);
+      params.pendingByChat.delete(chatId);
+    }
+  }
+  const text = readStringValue(params.event.body.text) ?? previous?.text ?? "";
+  if (!text && method !== "deleteMessage") {
+    return null;
+  }
+  const threadId =
+    normalizeStringifiedOptionalString(params.event.body.message_thread_id) ?? previous?.threadId;
+  const message: QaBusMessage = {
+    id: providerMessageId ?? previous?.id ?? `crabline-${params.cursor}`,
+    accountId: "default",
+    direction: "outbound",
+    conversation: {
+      id: chatId,
+      kind: chatId.startsWith("-") ? "group" : "direct",
+    },
+    senderId: "openclaw",
+    senderName: "OpenClaw QA",
+    text,
+    timestamp: Date.now(),
+    ...(threadId ? { threadId } : {}),
+    ...(method === "deleteMessage" ? { deleted: true } : {}),
+    ...(method === "editMessageText" ? { editedAt: Date.now() } : {}),
+    reactions: [],
+  };
+  if (method === "sendMessage") {
+    const pending = params.pendingByChat.get(chatId) ?? [];
+    pending.push(message);
+    params.pendingByChat.set(chatId, pending);
+  } else if (providerKey) {
+    params.messageByProviderId.set(providerKey, message);
+  }
+  return {
+    cursor: params.cursor,
+    kind: method === "sendMessage" ? "sent" : method === "editMessageText" ? "edited" : "deleted",
+    message,
+  };
+}
+
+async function waitForCrablineReady(params: {
+  accountId: string;
+  channel: string;
+  gateway: QaTransportGatewayClient;
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+}) {
+  const timeoutMs = params.timeoutMs ?? 45_000;
+  const pollIntervalMs = params.pollIntervalMs ?? 500;
+  const startedAt = Date.now();
+  let lastAccountStatus = `no ${params.channel} accounts reported`;
+  let lastProbeError: string | null = null;
+
+  while (Date.now() - startedAt < timeoutMs) {
+    try {
+      const payload = (await params.gateway.call(
+        "channels.status",
+        { probe: false, timeoutMs: 2_000 },
+        { timeoutMs: 5_000 },
+      )) as {
+        channelAccounts?: Record<
+          string,
+          Array<{
+            accountId?: string;
+            running?: boolean;
+            restartPending?: boolean;
+          }>
+        >;
+      };
+      const accounts = payload.channelAccounts?.[params.channel] ?? [];
+      const account = accounts.find((entry) => entry.accountId === params.accountId) ?? accounts[0];
+      lastProbeError = null;
+      lastAccountStatus = account
+        ? JSON.stringify({
+            accountId: account.accountId ?? null,
+            running: account.running ?? null,
+            restartPending: account.restartPending ?? null,
+          })
+        : `no ${params.channel} accounts reported`;
+      if (account?.running && account.restartPending !== true) {
+        return;
+      }
+    } catch (error) {
+      lastProbeError = formatErrorMessage(error);
+    }
+    await sleep(pollIntervalMs);
+  }
+
+  throw new QaSuiteInfraError(
+    "transport_ready_timeout",
+    [
+      `timed out after ${timeoutMs}ms waiting for ${params.channel} ready`,
+      `last status: ${lastAccountStatus}`,
+      ...(lastProbeError ? [`last probe error: ${lastProbeError}`] : []),
+    ].join("; "),
+  );
+}
+
+async function postCrablineInbound(params: {
+  adapter: StartedOpenClawCrablineAdapter;
+  providerInbound: OpenClawCrablineInbound;
+}) {
+  const { response, release } = await fetchWithSsrFGuard({
+    url: params.adapter.manifest.endpoints.adminInboundUrl,
+    init: {
+      body: JSON.stringify(params.providerInbound.providerBody),
+      headers: {
+        "content-type": "application/json",
+        "x-crabline-admin-token": params.adapter.manifest.adminToken,
+      },
+      method: "POST",
+    },
+    policy: { allowPrivateNetwork: true },
+    auditContext: `qa-lab-crabline-${params.adapter.channel}-inbound`,
+  });
+  try {
+    if (!response.ok) {
+      throw new Error(
+        `Crabline ${params.adapter.channel} inbound injection failed with HTTP ${response.status}.`,
+      );
+    }
+  } finally {
+    await release();
+  }
+}
+
+function createCrablineState(params: {
+  adapter: StartedOpenClawCrablineAdapter;
+  state: QaBusState;
+}): QaCrablineTransportState {
+  const baseState = params.state;
+  const targetByProviderTarget = new Map<string, string>();
+  const telegramMessageByProviderId = new Map<string, QaBusMessage>();
+  const pendingTelegramMessagesByChat = new Map<string, QaBusMessage[]>();
+  const outboundEvents: QaTransportOutboundEvent[] = [];
+
+  return {
+    reset() {
+      baseState.reset();
+      targetByProviderTarget.clear();
+      telegramMessageByProviderId.clear();
+      pendingTelegramMessagesByChat.clear();
+      outboundEvents.length = 0;
+    },
+    getSnapshot: baseState.getSnapshot.bind(baseState),
+    async getOutboundEvents() {
+      return outboundEvents;
+    },
+    observeEvent(event) {
+      if (params.adapter.channel === "telegram") {
+        const lifecycle = readTelegramLifecycleEvent({
+          cursor: outboundEvents.length + 1,
+          event,
+          messageByProviderId: telegramMessageByProviderId,
+          pendingByChat: pendingTelegramMessagesByChat,
+        });
+        if (lifecycle) {
+          outboundEvents.push(lifecycle);
+        }
+      }
+      const outbound = params.adapter.createOutboundFromRecorderEvent({
+        event,
+        targetByProviderTarget,
+      }) as QaBusOutboundMessageInput | null;
+      if (outbound) {
+        baseState.addOutboundMessage(outbound);
+      }
+    },
+    async addInboundMessage(input: QaBusInboundMessageInput) {
+      const providerInbound = params.adapter.createInbound({ input });
+      targetByProviderTarget.set(providerInbound.providerTargetKey, providerInbound.qaTarget);
+      const message = baseState.addInboundMessage({
+        ...input,
+        conversation: providerInbound.stateConversation,
+        ...(providerInbound.threadId ? { threadId: providerInbound.threadId } : {}),
+      });
+      await postCrablineInbound({
+        adapter: params.adapter,
+        providerInbound,
+      });
+      return message;
+    },
+    rememberProviderTarget(providerTargetKey, qaTarget) {
+      targetByProviderTarget.set(providerTargetKey, qaTarget);
+    },
+    addOutboundMessage: baseState.addOutboundMessage.bind(baseState),
+    readMessage: baseState.readMessage.bind(baseState),
+    searchMessages: baseState.searchMessages.bind(baseState),
+    waitFor: baseState.waitFor.bind(baseState),
+    async cleanup() {
+      await params.adapter.close();
+    },
+  };
+}
+
+class QaCrablineTransport extends QaStateBackedTransportAdapter {
+  readonly #adapter: StartedOpenClawCrablineAdapter;
+  readonly #selection: OpenClawCrablineChannelDriverSelection;
+  readonly #state: QaCrablineTransportState;
+  readonly sendNativeCommand?: (input: QaTransportNativeCommandInput) => Promise<void>;
+  readonly waitForOutboundSequence?: (input: QaTransportOutboundSequenceMatch) => Promise<{
+    events: QaTransportOutboundEvent[];
+    final: QaBusMessage;
+  }>;
+
+  constructor(params: {
+    adapter: StartedOpenClawCrablineAdapter;
+    selection: OpenClawCrablineChannelDriverSelection;
+    state: QaCrablineTransportState;
+  }) {
+    super({
+      id: CRABLINE_TRANSPORT_ID,
+      label: `crabline local ${params.selection.channel}`,
+      accountId: params.adapter.accountId,
+      requiredPluginIds: params.adapter.requiredPluginIds,
+      state: params.state,
+    });
+    this.#adapter = params.adapter;
+    this.#selection = params.selection;
+    this.#state = params.state;
+    if (params.selection.channel === "telegram") {
+      this.sendNativeCommand = async (input) => {
+        const { command, ...message } = input;
+        await this.sendInbound({
+          ...message,
+          text: `/${command}`,
+          nativeCommand: { name: command },
+        });
+      };
+      this.waitForOutboundSequence = async (input) =>
+        await waitForQaTransportOutboundSequence({
+          input,
+          readEvents: () => this.#state.getOutboundEvents(),
+        });
+    }
+  }
+
+  createGatewayConfig = (params: { baseUrl: string }): QaTransportGatewayConfig =>
+    this.#adapter.createGatewayConfig(params) as QaTransportGatewayConfig;
+
+  waitReady = (params: {
+    gateway: QaTransportGatewayClient;
+    timeoutMs?: number;
+    pollIntervalMs?: number;
+  }) =>
+    waitForCrablineReady({
+      ...params,
+      accountId: this.#adapter.accountId,
+      channel: this.#adapter.channel,
+    });
+
+  buildAgentDelivery = ({ target }: { target: string }) => {
+    const delivery = this.#adapter.createAgentDelivery({ target });
+    this.#state.rememberProviderTarget(delivery.to ?? delivery.replyTo, target);
+    return delivery;
+  };
+
+  createRuntimeEnvPatch = () => this.#adapter.createChannelDriverSmokeEnv({});
+
+  handleAction = async (_params: {
+    action: QaTransportActionName;
+    args: Record<string, unknown>;
+    cfg: OpenClawConfig;
+    accountId?: string | null;
+  }) => {
+    throw new Error(`Crabline local-provider transport does not support ${_params.action} yet.`);
+  };
+
+  createReportNotes = (_params: QaTransportReportParams) => [
+    `Runs OpenClaw's ${this.#selection.channel} channel plugin against a Crabline local provider server.`,
+    "No live channel service or external credential lease is required.",
+  ];
+
+  async cleanup() {
+    await this.#state.cleanup();
+  }
+}
+
+export async function createQaCrablineTransportAdapter(params: {
+  outputDir: string;
+  selection: OpenClawCrablineChannelDriverSelection;
+  state?: QaBusState;
+}) {
+  const recorderPath = path.join(
+    params.outputDir,
+    "artifacts",
+    "crabline",
+    `${params.selection.channel}-fake-provider.jsonl`,
+  );
+  await fs.mkdir(path.dirname(recorderPath), { recursive: true });
+  let observeEvent = (_event: unknown) => {};
+  const adapter = await startOpenClawCrablineAdapter({
+    channel: params.selection.channel,
+    onEvent: (event) => observeEvent(event),
+    openclawConfig: {},
+    recorderPath,
+  });
+  await fs.writeFile(
+    path.join(params.outputDir, OPENCLAW_CRABLINE_MANIFEST_PATH),
+    `${JSON.stringify(adapter.manifest, null, 2)}\n`,
+    "utf8",
+  );
+
+  const state = createCrablineState({
+    adapter,
+    state: params.state ?? createQaBusState(),
+  });
+  observeEvent = state.observeEvent;
+  return new QaCrablineTransport({ adapter, selection: params.selection, state });
+}

extensions/qa-lab/src/cron-run-wait.test.ts

@@ -0,0 +1,79 @@
+// Qa Lab tests cover cron run wait plugin behavior.
+import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
+import { describe, expect, it, vi } from "vitest";
+import { resolveCronRunPollIntervalMs, waitForCronRunCompletion } from "./cron-run-wait.js";
+
+describe("waitForCronRunCompletion", () => {
+  it("ignores older entries and returns the newly finished run", async () => {
+    const callGateway = vi
+      .fn<
+        (method: string, rpcParams?: unknown, opts?: { timeoutMs?: number }) => Promise<unknown>
+      >()
+      .mockResolvedValueOnce({
+        entries: [{ ts: 100, status: "ok", summary: "older run" }],
+      })
+      .mockResolvedValueOnce({
+        entries: [{ ts: 180, status: "ok", summary: "new run" }],
+      });
+
+    const result = await waitForCronRunCompletion({
+      callGateway,
+      jobId: "dreaming-job",
+      afterTs: 150,
+      timeoutMs: 100,
+      intervalMs: 0,
+    });
+
+    expect(result).toEqual({ ts: 180, status: "ok", summary: "new run" });
+    expect(callGateway).toHaveBeenNthCalledWith(
+      1,
+      "cron.runs",
+      { id: "dreaming-job", limit: 20, sortDir: "desc" },
+      { timeoutMs: 100 },
+    );
+  });
+
+  it("surfaces recent run history on timeout", async () => {
+    const callGateway = vi
+      .fn<
+        (method: string, rpcParams?: unknown, opts?: { timeoutMs?: number }) => Promise<unknown>
+      >()
+      .mockResolvedValue({
+        entries: [{ ts: 100, status: "ok", summary: "older run" }],
+      });
+
+    await expect(
+      waitForCronRunCompletion({
+        callGateway,
+        jobId: "dreaming-job",
+        afterTs: 150,
+        timeoutMs: 5,
+        intervalMs: 0,
+      }),
+    ).rejects.toThrow(/timed out waiting for cron run completion/);
+  });
+
+  it("clamps oversized poll intervals before sleeping", () => {
+    expect(resolveCronRunPollIntervalMs(Number.MAX_SAFE_INTEGER)).toBe(MAX_TIMER_TIMEOUT_MS);
+  });
+
+  it("keeps oversized poll intervals within the overall timeout", async () => {
+    const callGateway = vi
+      .fn<
+        (method: string, rpcParams?: unknown, opts?: { timeoutMs?: number }) => Promise<unknown>
+      >()
+      .mockResolvedValue({
+        entries: [{ ts: 100, status: "ok", summary: "older run" }],
+      });
+
+    await expect(
+      waitForCronRunCompletion({
+        callGateway,
+        jobId: "dreaming-job",
+        afterTs: 150,
+        timeoutMs: 5,
+        intervalMs: Number.MAX_SAFE_INTEGER,
+      }),
+    ).rejects.toThrow(/timed out waiting for cron run completion/);
+  });
+});

extensions/qa-lab/src/cron-run-wait.ts

@@ -0,0 +1,67 @@
+// Qa Lab plugin module implements cron run wait behavior.
+import { setTimeout as sleep } from "node:timers/promises";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
+
+type QaCronRunLogEntry = {
+  ts?: number;
+  status?: "ok" | "error" | "skipped";
+  summary?: string;
+  error?: string;
+  deliveryStatus?: "delivered" | "not-delivered" | "unknown" | "not-requested";
+};
+
+type QaCronRunsPage = {
+  entries?: QaCronRunLogEntry[];
+};
+
+export function resolveCronRunPollIntervalMs(intervalMs: number | undefined): number {
+  return resolveTimerTimeoutMs(intervalMs ?? 1_000, 1_000, 0);
+}
+
+export async function waitForCronRunCompletion(params: {
+  callGateway: (
+    method: string,
+    rpcParams?: unknown,
+    opts?: { timeoutMs?: number },
+  ) => Promise<unknown>;
+  jobId: string;
+  afterTs: number;
+  timeoutMs?: number;
+  intervalMs?: number;
+}) {
+  const timeoutMs = params.timeoutMs ?? 90_000;
+  const intervalMs = resolveCronRunPollIntervalMs(params.intervalMs);
+  const startedAt = Date.now();
+  let lastEntries: QaCronRunLogEntry[] = [];
+  while (Date.now() - startedAt < timeoutMs) {
+    const page = (await params.callGateway(
+      "cron.runs",
+      {
+        id: params.jobId,
+        limit: 20,
+        sortDir: "desc",
+      },
+      { timeoutMs: Math.min(timeoutMs, 30_000) },
+    )) as QaCronRunsPage;
+    const entries = Array.isArray(page.entries) ? page.entries : [];
+    lastEntries = entries;
+    const completed = entries.find(
+      (entry) =>
+        typeof entry.ts === "number" &&
+        entry.ts >= params.afterTs &&
+        (entry.status === "ok" || entry.status === "error" || entry.status === "skipped"),
+    );
+    if (completed) {
+      return completed;
+    }
+    const remainingMs = timeoutMs - (Date.now() - startedAt);
+    if (remainingMs <= 0) {
+      break;
+    }
+    await sleep(Math.min(intervalMs, remainingMs));
+  }
+  throw new Error(
+    `timed out waiting for cron run completion for ${params.jobId}: ${formatErrorMessage(lastEntries)}`,
+  );
+}

extensions/qa-lab/src/discovery-eval.test.ts

@@ -0,0 +1,102 @@
+// Qa Lab tests cover discovery eval plugin behavior.
+import { describe, expect, it } from "vitest";
+import {
+  hasDiscoveryLabels,
+  reportsDiscoveryScopeLeak,
+  reportsMissingDiscoveryFiles,
+} from "./discovery-eval.js";
+
+describe("qa discovery evaluation", () => {
+  it("accepts rich discovery reports that explicitly confirm all required files were read", () => {
+    const report = `
+Worked
+- Read all three requested files: repo/qa/scenarios/index.yaml, repo/extensions/qa-lab/src/suite.ts, and repo/docs/help/testing.md.
+Failed
+- None.
+Blocked
+- Runtime execution not attempted here.
+Follow-up
+- Run the live suite next.
+
+The helper text mentions banned phrases like "not present", "missing files", "blocked by missing", and "could not inspect", but only as quoted examples.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("accepts numeric 'all 4 required files read' confirmations", () => {
+    const report = `
+Worked
+- Source: repo/qa/scenarios/index.yaml, repo/extensions/qa-lab/src/suite.ts, repo/docs/help/testing.md
+- all 3 required files read.
+Failed
+- None.
+Blocked
+- No runtime execution in this pass.
+Follow-up
+- Run the live suite next.
+
+The report may quote phrases like "not present" while describing the evaluator, but the files were read.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("accepts claude-style 'all four files retrieved' discovery summaries", () => {
+    const report = `
+Worked
+- All three files retrieved. Now let me compile the protocol report.
+- All three mandated files read successfully: repo/qa/scenarios/index.yaml, repo/extensions/qa-lab/src/suite.ts, repo/docs/help/testing.md.
+Failed
+- None.
+Blocked
+- Runtime execution not attempted here.
+Follow-up
+- Run the live suite next.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("still flags genuine file-miss language when the report never confirms the required reads", () => {
+    const report = `
+Worked
+- Read some of the requested files.
+Failed
+- repo/docs/help/testing.md was not present.
+Blocked
+- Could not inspect the remaining refs.
+Follow-up
+- Fix the workspace mount.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(true);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(false);
+  });
+
+  it("flags discovery replies that drift into unrelated suite wrap-up claims", () => {
+    const report = `
+Worked
+- All three requested files were read: repo/qa/scenarios/index.yaml, repo/extensions/qa-lab/src/suite.ts, repo/docs/help/testing.md.
+Failed
+- None.
+Blocked
+- Runtime execution not attempted here.
+Follow-up
+- Run the live suite next.
+
+Final QA tally update: all mandatory scenarios resolved. QA run complete.
+`.trim();
+
+    expect(hasDiscoveryLabels(report)).toBe(true);
+    expect(reportsMissingDiscoveryFiles(report)).toBe(false);
+    expect(reportsDiscoveryScopeLeak(report)).toBe(true);
+  });
+});

extensions/qa-lab/src/discovery-eval.ts

@@ -0,0 +1,73 @@
+// Qa Lab plugin module implements discovery eval behavior.
+import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
+import { readQaScenarioExecutionConfig } from "./scenario-catalog.js";
+
+function readRequiredDiscoveryRefs() {
+  const config = readQaScenarioExecutionConfig("source-docs-discovery-report") as
+    | { requiredFiles?: string[] }
+    | undefined;
+  return (
+    config?.requiredFiles ?? [
+      "repo/qa/scenarios/index.yaml",
+      "repo/extensions/qa-lab/src/suite.ts",
+      "repo/docs/help/testing.md",
+    ]
+  );
+}
+
+const REQUIRED_DISCOVERY_REFS = readRequiredDiscoveryRefs();
+
+const REQUIRED_DISCOVERY_REFS_LOWER = REQUIRED_DISCOVERY_REFS.map(normalizeLowercaseStringOrEmpty);
+
+const DISCOVERY_SCOPE_LEAK_PHRASES = [
+  "all mandatory scenarios",
+  "final qa tally",
+  "final qa tally update",
+  "qa run complete",
+  "scenario: `subagent-handoff`",
+  "scenario: subagent-handoff",
+] as const;
+
+function confirmsDiscoveryFileRead(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  const mentionsAllRefs = REQUIRED_DISCOVERY_REFS_LOWER.every((ref) => lower.includes(ref));
+  const mentionsReadVerb = /(?:read|retrieved|inspected|loaded|accessed|digested)/.test(lower);
+  const requiredCountPattern = "(?:three|3|four|4)";
+  const confirmsRead =
+    new RegExp(
+      `(?:read|retrieved|inspected|loaded|accessed|digested)\\s+all\\s+${requiredCountPattern}\\s+(?:(?:requested|required|mandated|seeded)\\s+)?files`,
+    ).test(lower) ||
+    new RegExp(
+      `all\\s+${requiredCountPattern}\\s+(?:(?:requested|required|mandated|seeded)\\s+)?files\\s+(?:were\\s+)?(?:read|retrieved|inspected|loaded|accessed|digested)(?:\\s+\\w+)?`,
+    ).test(lower) ||
+    new RegExp(`all\\s+${requiredCountPattern}\\s+seeded files readable`).test(lower);
+  return mentionsAllRefs && (confirmsRead || mentionsReadVerb);
+}
+
+export function hasDiscoveryLabels(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  return (
+    lower.includes("worked") &&
+    lower.includes("failed") &&
+    lower.includes("blocked") &&
+    (lower.includes("follow-up") || lower.includes("follow up"))
+  );
+}
+
+export function reportsMissingDiscoveryFiles(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  if (confirmsDiscoveryFileRead(text)) {
+    return false;
+  }
+  return (
+    lower.includes("not present") ||
+    lower.includes("missing files") ||
+    lower.includes("blocked by missing") ||
+    lower.includes("could not inspect")
+  );
+}
+
+export function reportsDiscoveryScopeLeak(text: string) {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  return DISCOVERY_SCOPE_LEAK_PHRASES.some((phrase) => lower.includes(phrase));
+}

extensions/qa-lab/src/docker-harness.test.ts

@@ -0,0 +1,188 @@
+// Qa Lab tests cover docker harness plugin behavior.
+import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import YAML from "yaml";
+import { buildQaDockerHarnessImage, writeQaDockerHarnessFiles } from "./docker-harness.js";
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+function parseComposeServices(compose: string) {
+  const parsed = YAML.parse(compose) as {
+    services?: Record<
+      string,
+      {
+        build?: { context?: string };
+        environment?: Record<string, string>;
+        volumes?: string[];
+      }
+    >;
+  };
+  return parsed.services ?? {};
+}
+
+describe("qa docker harness", () => {
+  it("writes compose, env, config, and workspace scaffold files", async () => {
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-test-"));
+    cleanups.push(async () => {
+      await rm(outputDir, { recursive: true, force: true });
+    });
+
+    const result = await writeQaDockerHarnessFiles({
+      outputDir,
+      gatewayPort: 18889,
+      qaLabPort: 43124,
+      gatewayToken: "qa-token",
+      providerBaseUrl: "http://host.docker.internal:45123/v1",
+      repoRoot: "/repo/openclaw",
+      usePrebuiltImage: true,
+      bindUiDist: true,
+    });
+
+    for (const expectedFile of [
+      path.join(outputDir, ".env.example"),
+      path.join(outputDir, "README.md"),
+      path.join(outputDir, "docker-compose.qa.yml"),
+      path.join(outputDir, "state", "openclaw.json"),
+      path.join(outputDir, "state", "seed-workspace", "QA_KICKOFF_TASK.md"),
+      path.join(outputDir, "state", "seed-workspace", "QA_SCENARIO_PLAN.md"),
+      path.join(outputDir, "state", "seed-workspace", "QA_SCENARIOS.yaml"),
+      path.join(outputDir, "state", "seed-workspace", "IDENTITY.md"),
+    ]) {
+      expect(result.files).toContain(expectedFile);
+    }
+
+    const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8");
+    const services = parseComposeServices(compose);
+    expect(compose).toContain("image: openclaw:qa-local-prebaked");
+    expect(compose).toContain("qa-mock-openai:");
+    expect(services["qa-mock-openai"]?.environment).toMatchObject({
+      OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1",
+      OPENCLAW_PROFILE: "",
+    });
+    expect(services["qa-mock-openai"]?.environment).not.toHaveProperty("OPENCLAW_CONFIG_PATH");
+    expect(services["qa-mock-openai"]?.volumes).toBeUndefined();
+    expect(services["qa-lab"]?.environment).toMatchObject({
+      OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1",
+      OPENCLAW_CONFIG_PATH: "/opt/openclaw-scaffold/openclaw.json",
+      OPENCLAW_STATE_DIR: "/tmp/openclaw/state",
+    });
+    expect(services["qa-lab"]?.volumes).toContain("./state:/opt/openclaw-scaffold:ro");
+    expect(compose).toContain('      - "127.0.0.1:18889:18789"');
+    expect(compose).toContain('      - "127.0.0.1:43124:43123"');
+    expect(compose).toContain(":/opt/openclaw-qa-lab-ui:ro");
+    expect(compose).toContain("      - sh");
+    expect(compose).toContain("      - -lc");
+    expect(compose).toContain(
+      '        - fetch("http://127.0.0.1:18789/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))',
+    );
+    expect(compose).toContain("--control-ui-proxy-target http://openclaw-qa-gateway:18789/");
+    expect(compose).not.toContain("--control-ui-token");
+    expect(compose).not.toContain("qa-token");
+    expect(compose).toContain("--send-kickoff-on-start");
+    expect(compose).toContain("--ui-dist-dir /opt/openclaw-qa-lab-ui");
+    expect(compose).toContain(":/opt/openclaw-repo:ro");
+    expect(compose).toContain("./state:/opt/openclaw-scaffold:ro");
+    expect(compose).toContain(
+      "cp -R /opt/openclaw-scaffold/seed-workspace/. /tmp/openclaw/workspace/ && rm -rf /tmp/openclaw/workspace/repo && ln -s /opt/openclaw-repo /tmp/openclaw/workspace/repo",
+    );
+    expect(compose).toContain("OPENCLAW_CONFIG_PATH: /tmp/openclaw/openclaw.json");
+    expect(compose).toContain("OPENCLAW_STATE_DIR: /tmp/openclaw/state");
+    expect(compose).toContain('OPENCLAW_NO_RESPAWN: "1"');
+
+    const envExample = await readFile(path.join(outputDir, ".env.example"), "utf8");
+    expect(envExample).toContain("OPENCLAW_GATEWAY_TOKEN=qa-token");
+    expect(envExample).toContain("QA_BUS_BASE_URL=http://qa-lab:43123");
+    expect(envExample).toContain("QA_PROVIDER_BASE_URL=http://host.docker.internal:45123/v1");
+    expect(envExample).toContain("QA_LAB_URL=http://127.0.0.1:43124");
+
+    const configText = await readFile(path.join(outputDir, "state", "openclaw.json"), "utf8");
+    const config = JSON.parse(configText) as {
+      plugins?: {
+        allow?: string[];
+        entries?: Record<string, { enabled?: boolean }>;
+      };
+    };
+    expect(configText).toContain('"allowInsecureAuth": true');
+    expect(configText).toContain('"pluginToolsMcpBridge": true');
+    expect(configText).toContain('"openClawToolsMcpBridge": true');
+    expect(configText).toContain("/app/dist/control-ui");
+    expect(configText).toContain("C-3PO QA");
+    expect(configText).toContain('"/tmp/openclaw/workspace"');
+    expect(config.plugins?.allow).toContain("qa-lab");
+    expect(config.plugins?.entries?.["qa-lab"]?.enabled).toBe(true);
+
+    const kickoff = await readFile(
+      path.join(outputDir, "state", "seed-workspace", "QA_KICKOFF_TASK.md"),
+      "utf8",
+    );
+    expect(kickoff).toContain("Lobster Invaders");
+
+    const scenarios = await readFile(
+      path.join(outputDir, "state", "seed-workspace", "QA_SCENARIOS.yaml"),
+      "utf8",
+    );
+    expect(scenarios).toContain("pack:");
+    expect(scenarios).toContain("subagent-fanout-synthesis");
+
+    const readme = await readFile(path.join(outputDir, "README.md"), "utf8");
+    expect(readme).toContain("in-process restarts inside Docker");
+    expect(readme).toContain("pnpm qa:lab:watch");
+  });
+
+  it("builds the reusable QA image with bundled QA extensions", async () => {
+    const calls: string[] = [];
+    const result = await buildQaDockerHarnessImage(
+      {
+        repoRoot: "/repo/openclaw",
+        imageName: "openclaw:qa-local-prebaked",
+      },
+      {
+        async runCommand(command, args, cwd) {
+          calls.push([command, ...args, `@${cwd}`].join(" "));
+          return { stdout: "", stderr: "" };
+        },
+      },
+    );
+
+    expect(result.imageName).toBe("openclaw:qa-local-prebaked");
+    expect(calls).toEqual([
+      "docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=qa-channel qa-lab -f Dockerfile . @/repo/openclaw",
+    ]);
+  });
+
+  it("quotes generated compose paths so shell-sensitive repo paths survive YAML parsing", async () => {
+    const tempRoot = await mkdtemp(path.join(os.tmpdir(), "qa-docker-paths-"));
+    const outputDir = path.join(tempRoot, "scaffold");
+    const repoRoot = path.join(tempRoot, "repo #hash");
+    cleanups.push(async () => {
+      await rm(tempRoot, { recursive: true, force: true });
+    });
+    await mkdir(repoRoot, { recursive: true });
+
+    await writeQaDockerHarnessFiles({
+      outputDir,
+      repoRoot,
+      gatewayToken: "qa-token",
+      usePrebuiltImage: false,
+      bindUiDist: true,
+    });
+
+    const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8");
+    const services = parseComposeServices(compose);
+    expect(services["qa-mock-openai"]?.build?.context).toBe("../repo #hash");
+    expect(services["qa-lab"]?.volumes).toContain(
+      "../repo #hash/extensions/qa-lab/web/dist:/opt/openclaw-qa-lab-ui:ro",
+    );
+    expect(services["openclaw-qa-gateway"]?.volumes).toContain(
+      "../repo #hash:/opt/openclaw-repo:ro",
+    );
+  });
+});

extensions/qa-lab/src/docker-harness.ts

@@ -0,0 +1,375 @@
+// Qa Lab plugin module implements docker harness behavior.
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { toQaErrorObject } from "./errors.js";
+import { seedQaAgentWorkspace } from "./qa-agent-workspace.js";
+import {
+  createQaChannelGatewayConfig,
+  QA_CHANNEL_REQUIRED_PLUGIN_IDS,
+} from "./qa-channel-transport.js";
+import { buildQaGatewayConfig } from "./qa-gateway-config.js";
+
+const QA_LAB_INTERNAL_PORT = 43123;
+const QA_LAB_UI_OVERLAY_DIR = "/opt/openclaw-qa-lab-ui";
+
+function toPosixRelative(fromDir: string, toPath: string): string {
+  return path.relative(fromDir, toPath).split(path.sep).join("/");
+}
+
+function yamlDoubleQuoted(value: string) {
+  return JSON.stringify(value);
+}
+
+function renderImageBlock(params: {
+  outputDir: string;
+  repoRoot: string;
+  imageName: string;
+  usePrebuiltImage: boolean;
+}) {
+  if (params.usePrebuiltImage) {
+    return `    image: ${params.imageName}\n`;
+  }
+  const context = toPosixRelative(params.outputDir, params.repoRoot) || ".";
+  return `    build:\n      context: ${yamlDoubleQuoted(context)}\n      dockerfile: Dockerfile\n      args:\n        OPENCLAW_EXTENSIONS: "qa-channel qa-lab"\n`;
+}
+
+function renderCompose(params: {
+  outputDir: string;
+  repoRoot: string;
+  imageName: string;
+  usePrebuiltImage: boolean;
+  bindUiDist: boolean;
+  gatewayPort: number;
+  qaLabPort: number;
+  includeQaLabUi: boolean;
+}) {
+  const imageBlock = renderImageBlock(params);
+  const repoMount = toPosixRelative(params.outputDir, params.repoRoot) || ".";
+  const qaLabUiMount = toPosixRelative(
+    params.outputDir,
+    path.join(params.repoRoot, "extensions", "qa-lab", "web", "dist"),
+  );
+
+  return `services:
+  qa-mock-openai:
+${imageBlock}    pull_policy: never
+    healthcheck:
+      test:
+        - CMD
+        - node
+        - -e
+        - fetch("http://127.0.0.1:44080/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 3s
+    environment:
+      OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
+      OPENCLAW_PROFILE: ""
+    command:
+      - node
+      - dist/index.js
+      - qa
+      - mock-openai
+      - --host
+      - "0.0.0.0"
+      - --port
+      - "44080"
+${
+  params.includeQaLabUi
+    ? `  qa-lab:
+${imageBlock}    pull_policy: never
+    ports:
+      - "127.0.0.1:${params.qaLabPort}:${QA_LAB_INTERNAL_PORT}"
+    volumes:
+      - ./state:/opt/openclaw-scaffold:ro
+${params.bindUiDist ? `      - ${yamlDoubleQuoted(`${qaLabUiMount}:${QA_LAB_UI_OVERLAY_DIR}:ro`)}\n` : ""}    healthcheck:
+      test:
+        - CMD
+        - node
+        - -e
+        - fetch("http://127.0.0.1:${QA_LAB_INTERNAL_PORT}/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 5s
+    environment:
+      OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
+      OPENCLAW_CONFIG_PATH: /opt/openclaw-scaffold/openclaw.json
+      OPENCLAW_STATE_DIR: /tmp/openclaw/state
+      OPENCLAW_SKIP_GMAIL_WATCHER: "1"
+      OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1"
+      OPENCLAW_SKIP_CANVAS_HOST: "1"
+      OPENCLAW_PROFILE: ""
+    command:
+      - sh
+      - -lc
+      - OPENCLAW_QA_CONTROL_UI_PROXY_TOKEN="$(node -e 'const fs=require("node:fs");const cfg=JSON.parse(fs.readFileSync("/opt/openclaw-scaffold/openclaw.json","utf8"));process.stdout.write(cfg.gateway?.auth?.token ?? "")')" exec node dist/index.js qa ui --host 0.0.0.0 --port ${QA_LAB_INTERNAL_PORT} --advertise-host 127.0.0.1 --advertise-port ${params.qaLabPort} --control-ui-url http://127.0.0.1:${params.gatewayPort}/ --control-ui-proxy-target http://openclaw-qa-gateway:18789/${params.bindUiDist ? ` --ui-dist-dir ${QA_LAB_UI_OVERLAY_DIR}` : ""} --auto-kickoff-target direct --send-kickoff-on-start --embedded-gateway disabled
+    depends_on:
+      qa-mock-openai:
+        condition: service_healthy
+`
+    : ""
+}  openclaw-qa-gateway:
+${imageBlock}    pull_policy: never
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    ports:
+      - "127.0.0.1:${params.gatewayPort}:18789"
+    environment:
+      OPENCLAW_CONFIG_PATH: /tmp/openclaw/openclaw.json
+      OPENCLAW_STATE_DIR: /tmp/openclaw/state
+      OPENCLAW_NO_RESPAWN: "1"
+      OPENCLAW_SKIP_GMAIL_WATCHER: "1"
+      OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1"
+      OPENCLAW_SKIP_CANVAS_HOST: "1"
+      OPENCLAW_PROFILE: ""
+    volumes:
+      - ./state:/opt/openclaw-scaffold:ro
+      - ${yamlDoubleQuoted(`${repoMount}:/opt/openclaw-repo:ro`)}
+    healthcheck:
+      test:
+        - CMD
+        - node
+        - -e
+        - fetch("http://127.0.0.1:18789/healthz").then((r)=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      start_period: 15s
+    depends_on:
+${
+  params.includeQaLabUi
+    ? `      qa-lab:
+        condition: service_healthy
+`
+    : ""
+}      qa-mock-openai:
+        condition: service_healthy
+    command:
+      - sh
+      - -lc
+      - mkdir -p /tmp/openclaw/workspace /tmp/openclaw/state && cp /opt/openclaw-scaffold/openclaw.json /tmp/openclaw/openclaw.json && cp -R /opt/openclaw-scaffold/seed-workspace/. /tmp/openclaw/workspace/ && rm -rf /tmp/openclaw/workspace/repo && ln -s /opt/openclaw-repo /tmp/openclaw/workspace/repo && exec node dist/index.js gateway run --port 18789 --bind lan --allow-unconfigured
+`;
+}
+
+function renderEnvExample(params: {
+  gatewayPort: number;
+  qaLabPort: number;
+  gatewayToken: string;
+  providerBaseUrl: string;
+  qaBusBaseUrl: string;
+  includeQaLabUi: boolean;
+}) {
+  return `# QA Docker harness example env
+OPENCLAW_GATEWAY_TOKEN=${params.gatewayToken}
+QA_GATEWAY_PORT=${params.gatewayPort}
+QA_BUS_BASE_URL=${params.qaBusBaseUrl}
+QA_PROVIDER_BASE_URL=${params.providerBaseUrl}
+${params.includeQaLabUi ? `QA_LAB_URL=http://127.0.0.1:${params.qaLabPort}\n` : ""}`;
+}
+
+function renderReadme(params: {
+  gatewayPort: number;
+  qaLabPort: number;
+  usePrebuiltImage: boolean;
+  bindUiDist: boolean;
+  includeQaLabUi: boolean;
+}) {
+  return `# QA Docker Harness
+
+Generated scaffold for the Docker-backed QA lane.
+
+Files:
+
+- \`docker-compose.qa.yml\`
+- \`.env.example\`
+- \`state/openclaw.json\`
+
+Suggested flow:
+
+1. Build the prebaked image once:
+   - \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="qa-channel qa-lab" -f Dockerfile .\`
+2. Start the stack:
+   - \`docker compose -f docker-compose.qa.yml up${params.usePrebuiltImage ? "" : " --build"} -d\`
+3. Open the QA dashboard:
+   - \`${params.includeQaLabUi ? `http://127.0.0.1:${params.qaLabPort}` : "not published in this scaffold"}\`
+4. The single QA site embeds both panes:
+   - left: Control UI
+   - right: Slack-ish QA lab
+5. The repo-backed kickoff task auto-injects on startup.
+
+Fast UI refresh:
+
+- Start once with a prebuilt image + bind-mounted QA Lab assets:
+  - \`pnpm qa:lab:up --use-prebuilt-image --bind-ui-dist --skip-ui-build\`
+- In another shell, rebuild the QA Lab bundle on change:
+  - \`pnpm qa:lab:watch\`
+- The browser auto-reloads when the QA Lab asset hash changes.
+
+Gateway:
+
+- health: \`http://127.0.0.1:${params.gatewayPort}/healthz\`
+- Control UI: \`http://127.0.0.1:${params.gatewayPort}/\`
+- Mock OpenAI: internal \`http://qa-mock-openai:44080/v1\`
+
+This scaffold uses localhost Control UI insecure-auth compatibility for QA only.
+The gateway runs with in-process restarts inside Docker so restart actions do not
+kill the container by detaching a replacement child.
+`;
+}
+
+export async function writeQaDockerHarnessFiles(params: {
+  outputDir: string;
+  repoRoot: string;
+  gatewayPort?: number;
+  qaLabPort?: number;
+  gatewayToken?: string;
+  providerBaseUrl?: string;
+  qaBusBaseUrl?: string;
+  imageName?: string;
+  usePrebuiltImage?: boolean;
+  bindUiDist?: boolean;
+  includeQaLabUi?: boolean;
+}) {
+  const gatewayPort = params.gatewayPort ?? 18789;
+  const qaLabPort = params.qaLabPort ?? 43124;
+  const gatewayToken = params.gatewayToken ?? `qa-token-${randomUUID()}`;
+  const providerBaseUrl = params.providerBaseUrl ?? "http://qa-mock-openai:44080/v1";
+  const qaBusBaseUrl = params.qaBusBaseUrl ?? "http://qa-lab:43123";
+  const imageName = params.imageName ?? "openclaw:qa-local-prebaked";
+  const usePrebuiltImage = params.usePrebuiltImage ?? false;
+  const bindUiDist = params.bindUiDist ?? false;
+  const includeQaLabUi = params.includeQaLabUi ?? true;
+
+  await fs.mkdir(path.join(params.outputDir, "state", "seed-workspace"), { recursive: true });
+  await seedQaAgentWorkspace({
+    workspaceDir: path.join(params.outputDir, "state", "seed-workspace"),
+    repoRoot: params.repoRoot,
+  });
+
+  const config = buildQaGatewayConfig({
+    bind: "lan",
+    gatewayPort: 18789,
+    gatewayToken,
+    providerBaseUrl,
+    workspaceDir: "/tmp/openclaw/workspace",
+    controlUiRoot: "/app/dist/control-ui",
+    transportPluginIds: QA_CHANNEL_REQUIRED_PLUGIN_IDS,
+    transportConfig: createQaChannelGatewayConfig({
+      baseUrl: qaBusBaseUrl,
+    }),
+  });
+
+  const files = [
+    path.join(params.outputDir, "docker-compose.qa.yml"),
+    path.join(params.outputDir, ".env.example"),
+    path.join(params.outputDir, "README.md"),
+    path.join(params.outputDir, "state", "openclaw.json"),
+  ];
+
+  await Promise.all([
+    fs.writeFile(
+      path.join(params.outputDir, "docker-compose.qa.yml"),
+      renderCompose({
+        outputDir: params.outputDir,
+        repoRoot: params.repoRoot,
+        imageName,
+        usePrebuiltImage,
+        bindUiDist,
+        gatewayPort,
+        qaLabPort,
+        includeQaLabUi,
+      }),
+      "utf8",
+    ),
+    fs.writeFile(
+      path.join(params.outputDir, ".env.example"),
+      renderEnvExample({
+        gatewayPort,
+        qaLabPort,
+        gatewayToken,
+        providerBaseUrl,
+        qaBusBaseUrl,
+        includeQaLabUi,
+      }),
+      "utf8",
+    ),
+    fs.writeFile(
+      path.join(params.outputDir, "README.md"),
+      renderReadme({
+        gatewayPort,
+        qaLabPort,
+        usePrebuiltImage,
+        bindUiDist,
+        includeQaLabUi,
+      }),
+      "utf8",
+    ),
+    fs.writeFile(
+      path.join(params.outputDir, "state", "openclaw.json"),
+      `${JSON.stringify(config, null, 2)}\n`,
+      "utf8",
+    ),
+  ]);
+
+  return {
+    outputDir: params.outputDir,
+    imageName,
+    files: [
+      ...files,
+      path.join(params.outputDir, "state", "seed-workspace", "IDENTITY.md"),
+      path.join(params.outputDir, "state", "seed-workspace", "QA_KICKOFF_TASK.md"),
+      path.join(params.outputDir, "state", "seed-workspace", "QA_SCENARIO_PLAN.md"),
+      path.join(params.outputDir, "state", "seed-workspace", "QA_SCENARIOS.yaml"),
+    ],
+  };
+}
+
+export async function buildQaDockerHarnessImage(
+  params: {
+    repoRoot: string;
+    imageName?: string;
+  },
+  deps?: {
+    runCommand?: (
+      command: string,
+      args: string[],
+      cwd: string,
+    ) => Promise<{ stdout: string; stderr: string }>;
+  },
+) {
+  const imageName = params.imageName ?? "openclaw:qa-local-prebaked";
+  const runCommand =
+    deps?.runCommand ??
+    (async (command: string, args: string[], cwd: string) => {
+      return await new Promise<{ stdout: string; stderr: string }>((resolve, reject) => {
+        execFile(command, args, { cwd }, (error, stdout, stderr) => {
+          if (error) {
+            reject(toQaErrorObject(error, "Non-Error rejection"));
+            return;
+          }
+          resolve({ stdout, stderr });
+        });
+      });
+    });
+
+  await runCommand(
+    "docker",
+    [
+      "build",
+      "-t",
+      imageName,
+      "--build-arg",
+      "OPENCLAW_EXTENSIONS=qa-channel qa-lab",
+      "-f",
+      "Dockerfile",
+      ".",
+    ],
+    params.repoRoot,
+  );
+
+  return { imageName };
+}

extensions/qa-lab/src/docker-runtime.ts

@@ -0,0 +1,22 @@
+// Qa Lab plugin module implements docker runtime behavior.
+import {
+  createQaDockerRuntime,
+  type QaDockerFetchLike as FetchLike,
+  type QaDockerRunCommand as RunCommand,
+} from "openclaw/plugin-sdk/qa-runtime";
+
+export type { FetchLike, RunCommand };
+
+const dockerRuntime = createQaDockerRuntime({
+  auditContext: "qa-lab-docker-health-check",
+  commandTimeoutMs: null,
+});
+
+export const {
+  execCommand,
+  fetchHealthUrl,
+  resolveComposeServiceUrl,
+  resolveHostPort,
+  waitForDockerServiceHealth,
+  waitForHealth,
+} = dockerRuntime;

extensions/qa-lab/src/docker-up.runtime.test.ts

@@ -0,0 +1,405 @@
+// Qa Lab tests cover docker up plugin behavior.
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it, vi } from "vitest";
+import { runQaDockerUp } from "./docker-up.runtime.js";
+import { shellQuote } from "./shell-quote.js";
+
+type QaDockerUpDeps = NonNullable<Parameters<typeof runQaDockerUp>[1]>;
+
+function createHealthyDockerDeps(calls: string[]): QaDockerUpDeps {
+  return {
+    async runCommand(command, args, cwd) {
+      calls.push([command, ...args, `@${cwd}`].join(" "));
+      if (args.join(" ").includes("ps --format json openclaw-qa-gateway")) {
+        return { stdout: '{"Health":"healthy","State":"running"}\n', stderr: "" };
+      }
+      return { stdout: "", stderr: "" };
+    },
+    fetchImpl: vi.fn(async () => ({ ok: true })),
+    sleepImpl: vi.fn(async () => {}),
+  };
+}
+
+describe("runQaDockerUp", () => {
+  it("builds the QA UI, writes the harness, starts compose, and waits for health", async () => {
+    const calls: string[] = [];
+    const fetchCalls: string[] = [];
+    const responseQueue = [false, true, true];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const repoRoot = path.resolve("/repo/openclaw");
+    const composeFile = path.join(outputDir, "docker-compose.qa.yml");
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot,
+          outputDir,
+          gatewayPort: 18889,
+          qaLabPort: 43124,
+        },
+        {
+          async runCommand(command, args, cwd) {
+            calls.push([command, ...args, `@${cwd}`].join(" "));
+            if (args.join(" ").includes("ps --format json openclaw-qa-gateway")) {
+              return { stdout: '[{"Health":"healthy","State":"running"}]\n', stderr: "" };
+            }
+            return { stdout: "", stderr: "" };
+          },
+          fetchImpl: vi.fn(async (input: string) => {
+            fetchCalls.push(input);
+            return { ok: responseQueue.shift() ?? true };
+          }),
+          sleepImpl: vi.fn(async () => {}),
+        },
+      );
+
+      expect(calls).toEqual([
+        `pnpm qa:lab:build @${repoRoot}`,
+        `docker compose -f ${composeFile} down --remove-orphans @${repoRoot}`,
+        `docker compose -f ${composeFile} up --build -d @${repoRoot}`,
+        `docker compose -f ${composeFile} ps --format json openclaw-qa-gateway @${repoRoot}`,
+      ]);
+      expect(fetchCalls).toEqual([
+        "http://127.0.0.1:43124/healthz",
+        "http://127.0.0.1:43124/healthz",
+        "http://127.0.0.1:18889/healthz",
+      ]);
+      expect(result.qaLabUrl).toBe("http://127.0.0.1:43124");
+      expect(result.gatewayUrl).toBe("http://127.0.0.1:18889/");
+      expect(result.composeFile).toBe(composeFile);
+      expect(result.stopCommand).toBe(`docker compose -f ${shellQuote(composeFile)} down`);
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("quotes the printed stop command when the compose path is shell-sensitive", async () => {
+    const calls: string[] = [];
+    const tempRoot = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const outputDir = path.join(tempRoot, "mac path's qa lab");
+    const repoRoot = path.resolve("/repo/openclaw");
+    const composeFile = path.join(outputDir, "docker-compose.qa.yml");
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot,
+          outputDir,
+          usePrebuiltImage: true,
+          skipUiBuild: true,
+        },
+        createHealthyDockerDeps(calls),
+      );
+
+      expect(result.stopCommand).toBe(`docker compose -f ${shellQuote(composeFile)} down`);
+      expect(calls).toContain(
+        `docker compose -f ${composeFile} down --remove-orphans @${repoRoot}`,
+      );
+    } finally {
+      await rm(tempRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("skips UI build and compose --build for prebuilt images", async () => {
+    const calls: string[] = [];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const repoRoot = path.resolve("/repo/openclaw");
+    const composeFile = path.join(outputDir, "docker-compose.qa.yml");
+
+    try {
+      await runQaDockerUp(
+        {
+          repoRoot,
+          outputDir,
+          usePrebuiltImage: true,
+          bindUiDist: true,
+          skipUiBuild: true,
+        },
+        createHealthyDockerDeps(calls),
+      );
+
+      expect(calls).toEqual([
+        `docker compose -f ${composeFile} down --remove-orphans @${repoRoot}`,
+        `docker compose -f ${composeFile} up -d @${repoRoot}`,
+        `docker compose -f ${composeFile} ps --format json openclaw-qa-gateway @${repoRoot}`,
+      ]);
+      const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8");
+      expect(compose).toContain(":/opt/openclaw-qa-lab-ui:ro");
+      expect(compose).toContain("--ui-dist-dir /opt/openclaw-qa-lab-ui");
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("falls back to Corepack for the QA UI build when pnpm is unavailable", async () => {
+    const calls: string[] = [];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const repoRoot = path.resolve("/repo/openclaw");
+    const composeFile = path.join(outputDir, "docker-compose.qa.yml");
+
+    try {
+      await runQaDockerUp(
+        {
+          repoRoot,
+          outputDir,
+          usePrebuiltImage: true,
+        },
+        {
+          async runCommand(command, args, cwd) {
+            calls.push([command, ...args, `@${cwd}`].join(" "));
+            if (command === "pnpm") {
+              throw Object.assign(new Error("spawn pnpm ENOENT"), { code: "ENOENT" });
+            }
+            if (args.join(" ").includes("ps --format json openclaw-qa-gateway")) {
+              return { stdout: '{"Health":"healthy","State":"running"}\n', stderr: "" };
+            }
+            return { stdout: "", stderr: "" };
+          },
+          fetchImpl: vi.fn(async () => ({ ok: true })),
+          sleepImpl: vi.fn(async () => {}),
+        },
+      );
+
+      expect(calls).toEqual([
+        `pnpm qa:lab:build @${repoRoot}`,
+        `corepack pnpm qa:lab:build @${repoRoot}`,
+        `docker compose -f ${composeFile} down --remove-orphans @${repoRoot}`,
+        `docker compose -f ${composeFile} up -d @${repoRoot}`,
+        `docker compose -f ${composeFile} ps --format json openclaw-qa-gateway @${repoRoot}`,
+      ]);
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("does not hide real QA UI build failures behind the Corepack fallback", async () => {
+    const calls: string[] = [];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const repoRoot = path.resolve("/repo/openclaw");
+
+    try {
+      await expect(
+        runQaDockerUp(
+          {
+            repoRoot,
+            outputDir,
+            usePrebuiltImage: true,
+          },
+          {
+            async runCommand(command, args, cwd) {
+              calls.push([command, ...args, `@${cwd}`].join(" "));
+              throw Object.assign(new Error("qa lab build failed"), { code: 1 });
+            },
+            fetchImpl: vi.fn(async () => ({ ok: true })),
+            sleepImpl: vi.fn(async () => {}),
+          },
+        ),
+      ).rejects.toThrow("qa lab build failed");
+
+      expect(calls).toEqual([`pnpm qa:lab:build @${repoRoot}`]);
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("uses a repo-root-relative default output dir when none is provided", async () => {
+    const calls: string[] = [];
+    const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-docker-root-"));
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot,
+          usePrebuiltImage: true,
+          skipUiBuild: true,
+        },
+        createHealthyDockerDeps(calls),
+      );
+
+      expect(result.outputDir).toBe(path.join(repoRoot, ".artifacts/qa-docker"));
+      expect(result.composeFile).toBe(
+        path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml"),
+      );
+      expect(calls).toEqual([
+        `docker compose -f ${path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml")} down --remove-orphans @${repoRoot}`,
+        `docker compose -f ${path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml")} up -d @${repoRoot}`,
+        `docker compose -f ${path.join(repoRoot, ".artifacts/qa-docker/docker-compose.qa.yml")} ps --format json openclaw-qa-gateway @${repoRoot}`,
+      ]);
+    } finally {
+      await rm(repoRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("falls back to free host ports when defaults are already occupied", async () => {
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const gatewayPort = 18789;
+    const qaLabPort = 43124;
+    const resolveHostPort = vi.fn(async (preferredPort: number, pinned: boolean) => {
+      expect(pinned).toBe(false);
+      if (preferredPort === gatewayPort) {
+        return 28001;
+      }
+      if (preferredPort === qaLabPort) {
+        return 28002;
+      }
+      return preferredPort;
+    });
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot: "/repo/openclaw",
+          outputDir,
+          skipUiBuild: true,
+          usePrebuiltImage: true,
+        },
+        {
+          async runCommand() {
+            return {
+              stdout: '{"Health":"healthy","State":"running"}\n',
+              stderr: "",
+            };
+          },
+          fetchImpl: vi.fn(async () => ({ ok: true })),
+          sleepImpl: vi.fn(async () => {}),
+          resolveHostPortImpl: resolveHostPort,
+        },
+      );
+
+      expect(result.gatewayUrl).not.toBe(`http://127.0.0.1:${gatewayPort}/`);
+      expect(result.qaLabUrl).not.toBe(`http://127.0.0.1:${qaLabPort}`);
+      expect(result.gatewayUrl).toBe("http://127.0.0.1:28001/");
+      expect(result.qaLabUrl).toBe("http://127.0.0.1:28002");
+      expect(resolveHostPort).toHaveBeenCalledWith(gatewayPort, false);
+      expect(resolveHostPort).toHaveBeenCalledWith(qaLabPort, false);
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("rejects explicit host port collisions before touching Docker", async () => {
+    const calls: string[] = [];
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+
+    try {
+      await expect(
+        runQaDockerUp(
+          {
+            repoRoot: "/repo/openclaw",
+            outputDir,
+            gatewayPort: 43124,
+            qaLabPort: 43124,
+            skipUiBuild: true,
+            usePrebuiltImage: true,
+          },
+          createHealthyDockerDeps(calls),
+        ),
+      ).rejects.toThrow(
+        "QA Lab gateway and UI host ports must be different. Both resolved to 43124.",
+      );
+
+      expect(calls).toEqual([]);
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("rejects resolved host port collisions before writing the harness", async () => {
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const resolveHostPort = vi.fn(async () => 28001);
+
+    try {
+      await expect(
+        runQaDockerUp(
+          {
+            repoRoot: "/repo/openclaw",
+            outputDir,
+            skipUiBuild: true,
+            usePrebuiltImage: true,
+          },
+          {
+            ...createHealthyDockerDeps([]),
+            resolveHostPortImpl: resolveHostPort,
+          },
+        ),
+      ).rejects.toThrow(
+        "QA Lab gateway and UI host ports must be different. Both resolved to 28001.",
+      );
+
+      await expect(readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8")).rejects.toThrow(
+        "ENOENT",
+      );
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("falls back to the container IP when the host gateway port is unreachable", async () => {
+    const calls: string[] = [];
+    const fetchCalls: string[] = [];
+    const hostGatewayCancel = vi.fn(async () => {});
+    const outputDir = await mkdtemp(path.join(os.tmpdir(), "qa-docker-up-"));
+    const repoRoot = path.resolve("/repo/openclaw");
+    const composeFile = path.join(outputDir, "docker-compose.qa.yml");
+
+    try {
+      const result = await runQaDockerUp(
+        {
+          repoRoot,
+          outputDir,
+          gatewayPort: 18889,
+          qaLabPort: 43124,
+          skipUiBuild: true,
+          usePrebuiltImage: true,
+        },
+        {
+          async runCommand(command, args, cwd) {
+            calls.push([command, ...args, `@${cwd}`].join(" "));
+            const joined = args.join(" ");
+            if (joined.includes("ps --format json openclaw-qa-gateway")) {
+              return { stdout: '{"Health":"healthy","State":"running"}\n', stderr: "" };
+            }
+            if (joined.includes("ps -q openclaw-qa-gateway")) {
+              return { stdout: "gateway-container\n", stderr: "" };
+            }
+            if (command === "docker" && args[0] === "inspect") {
+              return { stdout: "192.168.165.4\n", stderr: "" };
+            }
+            return { stdout: "", stderr: "" };
+          },
+          fetchImpl: vi.fn(async (input: string) => {
+            fetchCalls.push(input);
+            if (input === "http://127.0.0.1:18889/healthz") {
+              return { ok: false, body: { cancel: hostGatewayCancel } };
+            }
+            return {
+              ok:
+                input === "http://127.0.0.1:43124/healthz" ||
+                input === "http://192.168.165.4:18789/healthz",
+            };
+          }),
+          sleepImpl: vi.fn(async () => {}),
+        },
+      );
+
+      expect(calls).toEqual([
+        `docker compose -f ${composeFile} down --remove-orphans @${repoRoot}`,
+        `docker compose -f ${composeFile} up -d @${repoRoot}`,
+        `docker compose -f ${composeFile} ps --format json openclaw-qa-gateway @${repoRoot}`,
+        `docker compose -f ${composeFile} ps -q openclaw-qa-gateway @${repoRoot}`,
+        `docker inspect --format {{range .NetworkSettings.Networks}}{{println .IPAddress}}{{end}} gateway-container @${repoRoot}`,
+      ]);
+      expect(fetchCalls).toEqual([
+        "http://127.0.0.1:43124/healthz",
+        "http://127.0.0.1:18889/healthz",
+        "http://192.168.165.4:18789/healthz",
+      ]);
+      expect(result.gatewayUrl).toBe("http://192.168.165.4:18789/");
+      expect(hostGatewayCancel).toHaveBeenCalledTimes(1);
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+});

extensions/qa-lab/src/docker-up.runtime.ts

@@ -0,0 +1,194 @@
+// Qa Lab plugin module implements docker up behavior.
+import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { writeQaDockerHarnessFiles } from "./docker-harness.js";
+import {
+  execCommand,
+  fetchHealthUrl,
+  resolveComposeServiceUrl,
+  resolveHostPort,
+  waitForDockerServiceHealth,
+  waitForHealth,
+  type FetchLike,
+  type RunCommand,
+} from "./docker-runtime.js";
+import { shellQuote } from "./shell-quote.js";
+
+type QaDockerUpResult = {
+  outputDir: string;
+  composeFile: string;
+  qaLabUrl: string;
+  gatewayUrl: string;
+  stopCommand: string;
+};
+
+function resolveDefaultQaDockerDir(repoRoot: string) {
+  return path.resolve(repoRoot, ".artifacts/qa-docker");
+}
+
+async function isQaLabDockerHealthReachable(url: string, fetchImpl: FetchLike) {
+  let response: Awaited<ReturnType<FetchLike>> | undefined;
+  try {
+    response = await fetchImpl(url);
+    return response.ok;
+  } catch {
+    return false;
+  } finally {
+    try {
+      await response?.body?.cancel?.();
+    } catch {}
+  }
+}
+
+function isMissingCommandError(
+  error: unknown,
+  command: string,
+  seen = new Set<unknown>(),
+): boolean {
+  if (!error || seen.has(error)) {
+    return false;
+  }
+  seen.add(error);
+  if (typeof error !== "object") {
+    return formatErrorMessage(error).includes(`spawn ${command} ENOENT`);
+  }
+  const candidate = error as { cause?: unknown; code?: unknown; message?: unknown };
+  const message = typeof candidate.message === "string" ? candidate.message : "";
+  if (
+    candidate.code === "ENOENT" ||
+    message.includes(`spawn ${command} ENOENT`) ||
+    message.includes(`${command}: command not found`)
+  ) {
+    return true;
+  }
+  return isMissingCommandError(candidate.cause, command, seen);
+}
+
+async function runQaLabBuild(repoRoot: string, runCommand: RunCommand) {
+  try {
+    await runCommand("pnpm", ["qa:lab:build"], repoRoot);
+  } catch (error) {
+    if (!isMissingCommandError(error, "pnpm")) {
+      throw error;
+    }
+    await runCommand("corepack", ["pnpm", "qa:lab:build"], repoRoot);
+  }
+}
+
+export async function runQaDockerUp(
+  params: {
+    repoRoot?: string;
+    outputDir?: string;
+    gatewayPort?: number;
+    qaLabPort?: number;
+    providerBaseUrl?: string;
+    image?: string;
+    usePrebuiltImage?: boolean;
+    bindUiDist?: boolean;
+    skipUiBuild?: boolean;
+  },
+  deps?: {
+    runCommand?: RunCommand;
+    fetchImpl?: FetchLike;
+    sleepImpl?: (ms: number) => Promise<unknown>;
+    resolveHostPortImpl?: typeof resolveHostPort;
+  },
+): Promise<QaDockerUpResult> {
+  const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
+  const resolveHostPortImpl = deps?.resolveHostPortImpl ?? resolveHostPort;
+  const outputDir = path.resolve(params.outputDir ?? resolveDefaultQaDockerDir(repoRoot));
+  const gatewayPort = await resolveHostPortImpl(
+    params.gatewayPort ?? 18789,
+    params.gatewayPort != null,
+  );
+  const qaLabPort = await resolveHostPortImpl(params.qaLabPort ?? 43124, params.qaLabPort != null);
+  if (gatewayPort === qaLabPort) {
+    throw new Error(
+      `QA Lab gateway and UI host ports must be different. Both resolved to ${gatewayPort}.`,
+    );
+  }
+  const runCommand = deps?.runCommand ?? execCommand;
+  const fetchImpl = deps?.fetchImpl ?? fetchHealthUrl;
+  const sleepImpl = deps?.sleepImpl ?? sleep;
+
+  if (!params.skipUiBuild) {
+    await runQaLabBuild(repoRoot, runCommand);
+  }
+
+  await writeQaDockerHarnessFiles({
+    outputDir,
+    repoRoot,
+    gatewayPort,
+    qaLabPort,
+    providerBaseUrl: params.providerBaseUrl,
+    imageName: params.image,
+    usePrebuiltImage: params.usePrebuiltImage,
+    bindUiDist: params.bindUiDist,
+    includeQaLabUi: true,
+  });
+
+  const composeFile = path.join(outputDir, "docker-compose.qa.yml");
+
+  // Tear down any previous stack from this compose file so ports are freed
+  // and we get a clean restart every time.
+  try {
+    await runCommand(
+      "docker",
+      ["compose", "-f", composeFile, "down", "--remove-orphans"],
+      repoRoot,
+    );
+  } catch {
+    // First run or already stopped — ignore.
+  }
+
+  const composeArgs = ["compose", "-f", composeFile, "up"];
+  if (!params.usePrebuiltImage) {
+    composeArgs.push("--build");
+  }
+  composeArgs.push("-d");
+
+  await runCommand("docker", composeArgs, repoRoot);
+
+  // Brief settle delay so Docker Desktop finishes port-forwarding setup.
+  await sleepImpl(3_000);
+
+  const qaLabUrl = `http://127.0.0.1:${qaLabPort}`;
+  const hostGatewayUrl = `http://127.0.0.1:${gatewayPort}/`;
+
+  await waitForHealth(`${qaLabUrl}/healthz`, {
+    label: "QA Lab",
+    fetchImpl,
+    sleepImpl,
+    composeFile,
+  });
+  await waitForDockerServiceHealth(
+    "openclaw-qa-gateway",
+    composeFile,
+    repoRoot,
+    runCommand,
+    sleepImpl,
+  );
+  let gatewayUrl = hostGatewayUrl;
+  if (!(await isQaLabDockerHealthReachable(`${hostGatewayUrl}healthz`, fetchImpl))) {
+    const containerGatewayUrl = await resolveComposeServiceUrl(
+      "openclaw-qa-gateway",
+      18789,
+      composeFile,
+      repoRoot,
+      runCommand,
+      fetchImpl,
+    );
+    if (containerGatewayUrl) {
+      gatewayUrl = containerGatewayUrl;
+    }
+  }
+
+  return {
+    outputDir,
+    composeFile,
+    qaLabUrl,
+    gatewayUrl,
+    stopCommand: `docker compose -f ${shellQuote(composeFile)} down`,
+  };
+}

extensions/qa-lab/src/errors.ts

@@ -0,0 +1,50 @@
+// Qa Lab plugin module defines shared suite errors.
+export type QaSuiteArtifactErrorCode =
+  | "evidence_missing"
+  | "report_missing"
+  | "summary_missing"
+  | "summary_read_failed"
+  | "summary_parse_failed"
+  | "summary_failure_count_missing"
+  | "summary_blocking_count_missing";
+
+export class QaSuiteArtifactError extends Error {
+  readonly code: QaSuiteArtifactErrorCode;
+
+  constructor(code: QaSuiteArtifactErrorCode, message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.name = "QaSuiteArtifactError";
+    this.code = code;
+  }
+}
+
+export type QaSuiteInfraErrorCode =
+  | "agent_wait_failed"
+  | "gateway_startup_unhealthy"
+  | "gateway_ready_timeout"
+  | "qa_cli_timeout"
+  | "transport_ready_timeout";
+
+export class QaSuiteInfraError extends Error {
+  readonly code: QaSuiteInfraErrorCode;
+
+  constructor(code: QaSuiteInfraErrorCode, message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.name = "QaSuiteInfraError";
+    this.code = code;
+  }
+}
+
+export function toQaErrorObject(value: unknown, fallbackMessage: string): Error {
+  if (value instanceof Error) {
+    return value;
+  }
+  if (typeof value === "string") {
+    return new Error(value);
+  }
+  const error = new Error(fallbackMessage, { cause: value });
+  if ((typeof value === "object" && value !== null) || typeof value === "function") {
+    Object.assign(error, value);
+  }
+  return error;
+}

extensions/qa-lab/src/evidence-gallery.test.ts

@@ -0,0 +1,700 @@
+// Qa Lab tests cover generic QA evidence gallery behavior.
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import {
+  buildQaEvidenceGalleryModel,
+  resolveQaEvidenceArtifactFileByIndex,
+  resolveQaEvidenceArtifactFile,
+  resolveQaEvidenceProducerFile,
+  resolveQaEvidenceFile,
+} from "./evidence-gallery.js";
+import {
+  QA_EVIDENCE_FILENAME,
+  buildVitestEvidenceSummary,
+  type QaEvidenceSummaryJson,
+} from "./evidence-summary.js";
+
+async function createTempRepo() {
+  return fs.mkdtemp(path.join(os.tmpdir(), "qa-evidence-gallery-"));
+}
+
+async function writeJson(filePath: string, value: unknown) {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+}
+
+function producerRootLeakSegments(repoRoot: string) {
+  if (process.platform !== "win32") {
+    return [`nested${repoRoot}`];
+  }
+  return [
+    "nested",
+    ...repoRoot
+      .split(/[\\/]+/u)
+      .filter(Boolean)
+      .map((part) => part.replace(/[^A-Za-z0-9._-]/gu, "_")),
+  ];
+}
+
+function repoRelativePath(repoRoot: string, filePath: string) {
+  return path.relative(repoRoot, filePath).split(path.sep).join("/");
+}
+
+function vitestArtifactEvidence(params: {
+  id: string;
+  title: string;
+  artifact: { kind: string; path: string };
+}): QaEvidenceSummaryJson {
+  return {
+    kind: "openclaw.qa.evidence-summary",
+    schemaVersion: 2,
+    generatedAt: "2026-06-17T12:00:00.000Z",
+    evidenceMode: "full",
+    entries: [
+      {
+        test: { kind: "vitest-test", id: params.id, title: params.title },
+        coverage: [{ id: "qa.artifact", role: "primary" }],
+        execution: {
+          runner: "vitest",
+          environment: { ref: "gallery-test", os: "darwin", nodeVersion: "v24.0.0" },
+          provider: {
+            id: "mock-openai",
+            live: false,
+            model: { name: "mock-openai/gpt-5.5", ref: "mock-openai/gpt-5.5" },
+          },
+          packageSource: { kind: "source-checkout" },
+          artifacts: [{ ...params.artifact, source: "vitest" }],
+        },
+        result: { status: "pass" },
+      },
+    ],
+  };
+}
+
+describe("evidence gallery", () => {
+  it("builds a generic gallery model for non-UX QA Lab evidence", async () => {
+    const repoRoot = await createTempRepo();
+    const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "vitest");
+    await fs.mkdir(path.join(outputDir, "runner"), { recursive: true });
+    await fs.writeFile(path.join(outputDir, "runner", "result.json"), '{"ok":true}\n', "utf8");
+    await fs.writeFile(path.join(outputDir, "runner", "output.log"), "vitest pass\n", "utf8");
+
+    const evidence: QaEvidenceSummaryJson = buildVitestEvidenceSummary({
+      artifactPaths: [
+        { kind: "runner-result", path: "runner/result.json" },
+        { kind: "log", path: "runner/output.log" },
+      ],
+      env: {
+        OPENCLAW_QA_REF: "gallery-test",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-17T12:00:00.000Z",
+      primaryModel: "mock-openai/gpt-5.5",
+      providerMode: "mock-openai",
+      targets: [
+        {
+          id: "qa-lab.generic-vitest",
+          title: "Generic Vitest evidence",
+          sourcePath: "extensions/qa-lab/src/generic.test.ts",
+          primaryCoverageIds: ["qa.generic"],
+        },
+        {
+          id: "qa-lab.no-artifacts",
+          title: "Generic entry without artifacts",
+          sourcePath: "extensions/qa-lab/src/no-artifacts.test.ts",
+          primaryCoverageIds: ["qa.empty"],
+        },
+      ],
+      results: [
+        {
+          id: "qa-lab.generic-vitest",
+          status: "pass",
+          durationMs: 42,
+        },
+        {
+          id: "qa-lab.no-artifacts",
+          status: "skipped",
+          durationMs: 1,
+        },
+      ],
+    });
+    evidence.entries[1] = {
+      ...evidence.entries[1],
+      execution: {
+        ...evidence.entries[1].execution!,
+        artifacts: [],
+      },
+    };
+    const evidencePath = path.join(outputDir, QA_EVIDENCE_FILENAME);
+    await writeJson(evidencePath, evidence);
+
+    const model = await buildQaEvidenceGalleryModel({
+      evidencePath: outputDir,
+      repoRoot,
+    });
+
+    expect(model.counts).toMatchObject({ pass: 1, skipped: 1, fail: 0, blocked: 0 });
+    expect(model.evidencePath).toBe(".artifacts/qa-e2e/vitest/qa-evidence.json");
+    expect(model.producerContext).toBeNull();
+    expect(model.entries).toHaveLength(2);
+    expect(model.entries[0]).toMatchObject({
+      id: "qa-lab.generic-vitest",
+      kind: "vitest-test",
+      artifacts: [
+        expect.objectContaining({
+          exists: true,
+          kind: "runner-result",
+          href: "/api/evidence/artifact?evidencePath=.artifacts%2Fqa-e2e%2Fvitest%2Fqa-evidence.json&entryIndex=0&artifactIndex=0",
+          mediaKind: "json",
+          preview: '{\n  "ok": true\n}',
+        }),
+        expect.objectContaining({
+          exists: true,
+          kind: "log",
+          mediaKind: "text",
+          preview: "vitest pass\n",
+        }),
+      ],
+    });
+    expect(model.entries[1]).toMatchObject({
+      id: "qa-lab.no-artifacts",
+      artifacts: [],
+    });
+  });
+
+  it("sanitizes local roots from gallery failure reasons", async () => {
+    const repoRoot = await createTempRepo();
+    const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "vitest");
+    await fs.mkdir(outputDir, { recursive: true });
+    const evidence: QaEvidenceSummaryJson = vitestArtifactEvidence({
+      id: "qa-lab.failure-path",
+      title: "Failure path evidence",
+      artifact: { kind: "log", path: "missing.log" },
+    });
+    evidence.entries[0] = {
+      ...evidence.entries[0],
+      result: {
+        status: "blocked",
+        failure: {
+          class: "blocked",
+          reason: `Command failed at ${repoRoot}/openclaw.mjs and file://${repoRoot}/trace.log`,
+        },
+      },
+    };
+    await writeJson(path.join(outputDir, QA_EVIDENCE_FILENAME), evidence);
+
+    const model = await buildQaEvidenceGalleryModel({
+      evidencePath: outputDir,
+      repoRoot,
+    });
+
+    expect(model.entries[0].failureReason).toBe(
+      "Command failed at <repo-root>/openclaw.mjs and file://<repo-root>/trace.log",
+    );
+    expect(JSON.stringify(model)).not.toContain(repoRoot);
+  });
+
+  it("normalizes absolute source and declared artifact paths for gallery links", async () => {
+    const repoRoot = await createTempRepo();
+    const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "vitest");
+    const artifactPath = path.join(outputDir, "absolute.log");
+    await fs.mkdir(outputDir, { recursive: true });
+    await fs.writeFile(
+      artifactPath,
+      `absolute artifact ${repoRoot}\nfile://${repoRoot}/trace.log\n`,
+      "utf8",
+    );
+    const relativeLeakArtifactPath = `nested${repoRoot}/relative.log`;
+    const relativeLeakFile = path.resolve(outputDir, relativeLeakArtifactPath);
+    await fs.mkdir(path.dirname(relativeLeakFile), { recursive: true });
+    await fs.writeFile(relativeLeakFile, "relative artifact\n", "utf8");
+    const evidence: QaEvidenceSummaryJson = vitestArtifactEvidence({
+      id: "qa-lab.absolute-artifact-path",
+      title: "Absolute artifact path",
+      artifact: { kind: "log", path: artifactPath },
+    });
+    evidence.profile = `${repoRoot}/qa-profile`;
+    evidence.entries[0] = {
+      ...evidence.entries[0],
+      coverage: [{ id: `${repoRoot}/coverage`, role: `${repoRoot}/role` }],
+      execution: {
+        ...evidence.entries[0].execution!,
+        artifacts: [
+          {
+            ...evidence.entries[0].execution!.artifacts[0],
+            kind: `${repoRoot}/log`,
+            source: `${repoRoot}/vitest`,
+          },
+          {
+            kind: "log",
+            path: relativeLeakArtifactPath,
+            source: "vitest",
+          },
+        ],
+      },
+      test: {
+        ...evidence.entries[0].test,
+        id: `${repoRoot}/qa-lab.absolute-artifact-path`,
+        kind: `${repoRoot}/vitest-test`,
+        source: { path: path.join(repoRoot, "extensions/qa-lab/src/absolute.test.ts") },
+        title: `Absolute artifact path at ${repoRoot}`,
+      },
+    };
+    await writeJson(path.join(outputDir, QA_EVIDENCE_FILENAME), evidence);
+
+    const model = await buildQaEvidenceGalleryModel({
+      evidencePath: outputDir,
+      repoRoot,
+    });
+
+    const artifact = model.entries[0]?.artifacts[0];
+    expect(artifact).toMatchObject({
+      exists: true,
+      kind: "<repo-root>/log",
+      path: ".artifacts/qa-e2e/vitest/absolute.log",
+      preview: "absolute artifact <repo-root>\nfile://<repo-root>/trace.log\n",
+      source: "<repo-root>/vitest",
+    });
+    expect(artifact?.href).toContain("entryIndex=0&artifactIndex=0");
+    const relativeArtifact = model.entries[0]?.artifacts[1];
+    expect(relativeArtifact).toMatchObject({
+      exists: true,
+      path: expect.stringContaining(".artifacts/qa-e2e/vitest/nested"),
+      preview: "relative artifact\n",
+    });
+    expect(decodeURIComponent(relativeArtifact?.href ?? "")).not.toContain(repoRoot);
+    expect(relativeArtifact?.href).toContain("entryIndex=0&artifactIndex=1");
+    expect(model.entries[0]?.sourcePath).toBe("extensions/qa-lab/src/absolute.test.ts");
+    expect(model.entries[0]).toMatchObject({
+      coverage: [{ id: "<repo-root>/coverage", role: "<repo-root>/role" }],
+      id: "<repo-root>/qa-lab.absolute-artifact-path",
+      kind: "<repo-root>/vitest-test",
+      title: "Absolute artifact path at <repo-root>",
+    });
+    expect(model.profile).toBe("<repo-root>/qa-profile");
+    expect(JSON.stringify(model)).not.toContain(repoRoot);
+    await expect(
+      resolveQaEvidenceArtifactFile({
+        artifactPath: "<repo-root>/.artifacts/qa-e2e/vitest/absolute.log",
+        evidencePath: outputDir,
+        repoRoot,
+      }),
+    ).resolves.toBe(await fs.realpath(artifactPath));
+    await expect(
+      resolveQaEvidenceArtifactFileByIndex({
+        artifactIndex: 1,
+        entryIndex: 0,
+        evidencePath: outputDir,
+        repoRoot,
+      }),
+    ).resolves.toBe(await fs.realpath(relativeLeakFile));
+  });
+
+  it("detects UX Matrix producer context from suite-level evidence artifacts", async () => {
+    const repoRoot = await createTempRepo();
+    const suiteDir = path.join(repoRoot, ".artifacts", "qa-e2e", "suite");
+    const runDir = path.join(
+      suiteDir,
+      "script",
+      ...producerRootLeakSegments(repoRoot),
+      "ux-matrix-evidence-dashboard",
+      "run-1",
+    );
+    const expectedWebScreenshotNeedle =
+      process.platform === "win32"
+        ? ".artifacts/qa-e2e/suite/script/nested"
+        : ".artifacts/qa-e2e/suite/script/nested<repo-root>/ux-matrix-evidence-dashboard/run-1/surfaces/web-ui/stages/first-run/screenshot.png";
+    const expectedCliLogNeedle =
+      process.platform === "win32"
+        ? ".artifacts/qa-e2e/suite/script/nested"
+        : ".artifacts/qa-e2e/suite/script/nested<repo-root>/ux-matrix-evidence-dashboard/run-1/surfaces/cli/stages/error-state/logs.txt";
+    await fs.mkdir(path.join(runDir, "surfaces", "web-ui", "stages", "first-run"), {
+      recursive: true,
+    });
+    await fs.mkdir(path.join(runDir, "surfaces", "cli", "stages", "error-state"), {
+      recursive: true,
+    });
+    await fs.writeFile(
+      path.join(runDir, "surfaces", "web-ui", "stages", "first-run", "screenshot.png"),
+      "png",
+    );
+    await fs.writeFile(
+      path.join(runDir, "surfaces", "cli", "stages", "error-state", "logs.txt"),
+      "cli blocked\n",
+      "utf8",
+    );
+    await writeJson(path.join(runDir, "manifest.json"), {
+      run: {
+        runId: "run-1",
+        status: "pass",
+      },
+    });
+    await writeJson(path.join(runDir, "matrix.json"), {
+      counts: {
+        pass: 1,
+        blocked: 1,
+        "proof-gap": 1,
+      },
+      stages: [
+        { id: `${repoRoot}/diagnostics`, label: "Diagnostics" },
+        { id: "first-run", label: "First run" },
+        { id: "error-state", label: "Error state" },
+      ],
+      surfaces: [
+        { id: `${repoRoot}/native`, label: "Native" },
+        { id: "web-ui", label: "Web UI" },
+        { id: "cli", label: "CLI" },
+      ],
+      cells: [
+        null,
+        {
+          coverageIds: [`${repoRoot}/ui.control`],
+          runner: {
+            availability: "local",
+            command: `${repoRoot}/openclaw.mjs qa suite --scenario ux-matrix-evidence-dashboard`,
+            lane: "web-ui-playwright",
+            workflow: `${repoRoot}/.github/workflows/ux-matrix-qa.yml#ux-matrix-local`,
+          },
+          stage: "first-run",
+          status: "pass",
+          surface: "web-ui",
+        },
+        {
+          coverageIds: ["cli.entrypoint"],
+          runner: {
+            availability: "local",
+            command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard",
+            lane: "cli-status",
+            workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local",
+          },
+          stage: "first-run",
+          status: "proof-gap",
+          surface: "cli",
+        },
+        { stage: "error-state", status: "blocked", surface: "cli" },
+      ],
+    });
+    await writeJson(path.join(runDir, "release-ledger.json"), {
+      counts: {
+        pass: 1,
+        blocked: 1,
+        "proof-gap": 1,
+      },
+    });
+    await fs.writeFile(path.join(runDir, "scorecard.md"), "# UX Matrix\n\n- pass: 1\n", "utf8");
+    await fs.writeFile(path.join(runDir, "commands.txt"), "node ux matrix\n", "utf8");
+    await fs.mkdir(path.join(runDir, "preflight"), { recursive: true });
+    await fs.writeFile(path.join(runDir, "preflight", "memory.txt"), "memory ok\n", "utf8");
+    await fs.writeFile(
+      path.join(runDir, "preflight", "adb-devices.txt"),
+      "List of devices\n",
+      "utf8",
+    );
+
+    await writeJson(path.join(suiteDir, QA_EVIDENCE_FILENAME), {
+      kind: "openclaw.qa.evidence-summary",
+      schemaVersion: 2,
+      generatedAt: "2026-06-17T12:00:00.000Z",
+      evidenceMode: "full",
+      entries: [
+        {
+          test: {
+            kind: "ux-matrix-cell",
+            id: "ux-matrix.web-ui.first-run",
+            title: `UX Matrix: web-ui / first-run at ${repoRoot}`,
+            source: { path: "scripts/ux-matrix/dashboard.ts" },
+          },
+          coverage: [{ id: "ui.control", role: "primary" }],
+          execution: {
+            runner: "ux-matrix-dashboard",
+            environment: {
+              ref: "gallery-test",
+              os: "darwin",
+              nodeVersion: "v24.0.0",
+            },
+            provider: {
+              id: "ux-matrix",
+              live: false,
+              model: { name: null, ref: null },
+              fixture: "mocked-control-ui-and-isolated-cli",
+            },
+            packageSource: { kind: "source-checkout", sha: "abc123" },
+            artifacts: [
+              {
+                kind: "screenshot",
+                path: path.join(
+                  runDir,
+                  "surfaces",
+                  "web-ui",
+                  "stages",
+                  "first-run",
+                  "screenshot.png",
+                ),
+                source: "ux-matrix:web-ui:first-run",
+              },
+            ],
+          },
+          result: { status: "pass", timing: { wallMs: 1 } },
+        },
+        {
+          test: {
+            kind: "ux-matrix-cell",
+            id: "qa-lab.wrapper-cli-error",
+            title: "UX Matrix: cli / error-state",
+            source: { path: "scripts/ux-matrix/dashboard.ts" },
+          },
+          coverage: [{ id: "cli.status-snapshots", role: "primary" }],
+          execution: {
+            runner: "ux-matrix-dashboard",
+            environment: {
+              ref: "gallery-test",
+              os: "darwin",
+              nodeVersion: "v24.0.0",
+            },
+            provider: {
+              id: "ux-matrix",
+              live: false,
+              model: { name: null, ref: null },
+              fixture: "mocked-control-ui-and-isolated-cli",
+            },
+            packageSource: { kind: "source-checkout", sha: "abc123" },
+            artifacts: [
+              {
+                kind: "log",
+                path: repoRelativePath(
+                  repoRoot,
+                  path.join(runDir, "surfaces", "cli", "stages", "error-state", "logs.txt"),
+                ),
+                source: "ux-matrix:cli:error-state",
+              },
+            ],
+          },
+          result: {
+            status: "blocked",
+            failure: {
+              class: "blocked",
+              reason: "CLI error-state proof captured a blocked result.",
+            },
+            timing: { wallMs: 2 },
+          },
+        },
+      ],
+    });
+
+    const model = await buildQaEvidenceGalleryModel({
+      evidencePath: suiteDir,
+      repoRoot,
+    });
+
+    expect(model.producerContext).toMatchObject({
+      kind: "ux-matrix",
+      manifest: {
+        runId: "run-1",
+        runStatus: "pass",
+      },
+      matrix: {
+        counts: {
+          pass: 1,
+          blocked: 1,
+          "proof-gap": 1,
+        },
+        stages: ["<repo-root>/diagnostics", "first-run", "error-state"],
+        surfaces: ["<repo-root>/native", "web-ui", "cli"],
+      },
+      releaseLedger: {
+        counts: {
+          pass: 1,
+          blocked: 1,
+          "proof-gap": 1,
+        },
+      },
+    });
+    expect(model.producerContext?.matrix?.cells).toEqual([
+      {
+        artifactKinds: ["screenshot"],
+        artifactPaths: [expect.stringContaining(expectedWebScreenshotNeedle)],
+        coverageIds: ["<repo-root>/ui.control"],
+        runner: {
+          availability: "local",
+          command: "<repo-root>/openclaw.mjs qa suite --scenario ux-matrix-evidence-dashboard",
+          lane: "web-ui-playwright",
+          workflow: "<repo-root>/.github/workflows/ux-matrix-qa.yml#ux-matrix-local",
+        },
+        stage: "first-run",
+        status: "pass",
+        surface: "web-ui",
+        testId: "ux-matrix.web-ui.first-run",
+        title: "UX Matrix: web-ui / first-run at <repo-root>",
+      },
+      {
+        artifactKinds: [],
+        artifactPaths: [],
+        coverageIds: ["cli.entrypoint"],
+        runner: {
+          availability: "local",
+          command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard",
+          lane: "cli-status",
+          workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local",
+        },
+        stage: "first-run",
+        status: "proof-gap",
+        surface: "cli",
+        testId: null,
+        title: null,
+      },
+      {
+        artifactKinds: ["log"],
+        artifactPaths: [expect.stringContaining(expectedCliLogNeedle)],
+        coverageIds: [],
+        runner: null,
+        stage: "error-state",
+        status: "blocked",
+        surface: "cli",
+        testId: "qa-lab.wrapper-cli-error",
+        title: "UX Matrix: cli / error-state",
+      },
+    ]);
+    expect(model.producerContext?.scorecard?.preview).toContain("# UX Matrix");
+    expect(model.producerContext?.scorecard?.href).toContain("/api/evidence/artifact?");
+    expect(decodeURIComponent(model.producerContext?.scorecard?.href ?? "")).not.toContain(
+      repoRoot,
+    );
+    expect(model.producerContext?.commands?.preview).toBe("node ux matrix\n");
+    expect(model.producerContext?.commands?.path).toContain("commands.txt");
+    expect(decodeURIComponent(model.producerContext?.commands?.href ?? "")).not.toContain(repoRoot);
+    expect(model.producerContext?.manifest?.preview).toContain('"runId": "run-1"');
+    expect(model.producerContext?.releaseLedger?.preview).toContain('"proof-gap": 1');
+    expect(model.producerContext?.preflight.memory?.path).toContain("preflight/memory.txt");
+    expect(model.producerContext?.preflight.memory?.preview).toBe("memory ok\n");
+    expect(model.producerContext?.preflight.adbDevices?.path).toContain(
+      "preflight/adb-devices.txt",
+    );
+    expect(model.producerContext?.preflight.adbDevices?.preview).toBe("List of devices\n");
+    expect(model.evidencePath).toBe(".artifacts/qa-e2e/suite/qa-evidence.json");
+    expect(JSON.stringify(model)).not.toContain(repoRoot);
+    await expect(
+      resolveQaEvidenceProducerFile({
+        evidencePath: suiteDir,
+        producerFile: "scorecard",
+        repoRoot,
+      }),
+    ).resolves.toBe(await fs.realpath(path.join(runDir, "scorecard.md")));
+    const outsideDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-evidence-outside-"));
+    const outsideCommands = path.join(outsideDir, "commands.txt");
+    await fs.writeFile(outsideCommands, "outside secret\n", "utf8");
+    await fs.unlink(path.join(runDir, "commands.txt"));
+    await fs.symlink(outsideCommands, path.join(runDir, "commands.txt"));
+    const symlinkModel = await buildQaEvidenceGalleryModel({
+      evidencePath: suiteDir,
+      repoRoot,
+    });
+    expect(symlinkModel.producerContext?.commands).toBeNull();
+    expect(JSON.stringify(symlinkModel)).not.toContain("outside secret");
+    await expect(
+      resolveQaEvidenceArtifactFile({
+        artifactPath: path.relative(repoRoot, path.join(runDir, "scorecard.md")),
+        evidencePath: suiteDir,
+        repoRoot,
+      }),
+    ).resolves.toBe(await fs.realpath(path.join(runDir, "scorecard.md")));
+  });
+
+  it("resolves evidence and declared artifacts inside the repo root only", async () => {
+    const repoRoot = await createTempRepo();
+    const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "suite");
+    const evidencePath = path.join(outputDir, QA_EVIDENCE_FILENAME);
+    await fs.writeFile(path.join(repoRoot, "package.json"), '{"private":true}\n', "utf8");
+    await fs.mkdir(outputDir, { recursive: true });
+    await fs.writeFile(path.join(outputDir, "artifact.log"), "ok\n", "utf8");
+    await writeJson(
+      evidencePath,
+      vitestArtifactEvidence({
+        id: "qa-lab.declared-artifact",
+        title: "Declared artifact",
+        artifact: { kind: "log", path: "artifact.log" },
+      }),
+    );
+
+    await expect(resolveQaEvidenceFile({ inputPath: outputDir, repoRoot })).resolves.toBe(
+      await fs.realpath(evidencePath),
+    );
+    await expect(
+      resolveQaEvidenceArtifactFile({
+        artifactPath: "artifact.log",
+        evidencePath,
+        repoRoot,
+      }),
+    ).resolves.toBe(await fs.realpath(path.join(outputDir, "artifact.log")));
+    await fs.mkdir(path.join(repoRoot, "runner"), { recursive: true });
+    await fs.mkdir(path.join(outputDir, "runner"), { recursive: true });
+    await fs.writeFile(path.join(repoRoot, "runner", "result.json"), '{"from":"repo"}\n', "utf8");
+    await fs.writeFile(
+      path.join(outputDir, "runner", "result.json"),
+      '{"from":"evidence"}\n',
+      "utf8",
+    );
+    const collisionEvidence = vitestArtifactEvidence({
+      id: "qa-lab.colliding-artifact",
+      title: "Colliding artifact",
+      artifact: { kind: "runner-result", path: "runner/result.json" },
+    });
+    await writeJson(evidencePath, collisionEvidence);
+    await expect(
+      resolveQaEvidenceArtifactFile({
+        artifactPath: "runner/result.json",
+        evidencePath,
+        repoRoot,
+      }),
+    ).resolves.toBe(await fs.realpath(path.join(outputDir, "runner", "result.json")));
+    await fs.rm(path.join(outputDir, "runner", "result.json"));
+    const missingBundleModel = await buildQaEvidenceGalleryModel({ evidencePath, repoRoot });
+    expect(missingBundleModel.entries[0].artifacts[0]).toMatchObject({
+      exists: false,
+      error: "Evidence artifact not found.",
+      preview: null,
+    });
+    expect(JSON.stringify(missingBundleModel)).not.toContain('"from":"repo"');
+    await expect(
+      resolveQaEvidenceArtifactFile({
+        artifactPath: "package.json",
+        evidencePath,
+        repoRoot,
+      }),
+    ).rejects.toThrow("Evidence artifact not found.");
+    await fs.writeFile(path.join(outputDir, "undeclared.log"), "undeclared\n", "utf8");
+    await expect(
+      resolveQaEvidenceArtifactFile({
+        artifactPath: "undeclared.log",
+        evidencePath,
+        repoRoot,
+      }),
+    ).rejects.toThrow("Evidence artifact is not declared by this evidence summary.");
+    const outsideDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-evidence-outside-artifact-"));
+    const outsideArtifact = path.join(outsideDir, "artifact.log");
+    await fs.writeFile(outsideArtifact, "outside secret\n", "utf8");
+    await fs.symlink(outsideArtifact, path.join(outputDir, "escape.log"));
+    await writeJson(evidencePath, {
+      ...collisionEvidence,
+      entries: [
+        {
+          ...collisionEvidence.entries[0],
+          execution: {
+            ...collisionEvidence.entries[0].execution,
+            artifacts: [{ kind: "log", path: "escape.log", source: "vitest" }],
+          },
+        },
+      ],
+    });
+    await expect(
+      resolveQaEvidenceArtifactFile({
+        artifactPath: "escape.log",
+        evidencePath,
+        repoRoot,
+      }),
+    ).rejects.toThrow("Evidence artifact not found.");
+    await expect(
+      resolveQaEvidenceFile({ inputPath: "/tmp/not-openclaw-evidence.json", repoRoot }),
+    ).rejects.toThrow("Evidence path not found.");
+  });
+});

extensions/qa-lab/src/evidence-gallery.ts

@@ -0,0 +1,972 @@
+// Qa Lab plugin module implements generic QA evidence gallery data.
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import type {
+  QaEvidenceArtifactView,
+  QaEvidenceGalleryEntryView,
+  QaEvidenceGalleryModel,
+  QaEvidenceMatrixCellView,
+  QaEvidenceProducerContext,
+  QaEvidenceProducerContextFile,
+} from "../shared/evidence-gallery-types.js";
+import { toRepoPath, toRepoRelativePath } from "./cli-paths.js";
+import {
+  QA_EVIDENCE_FILENAME,
+  validateQaEvidenceSummaryJson,
+  type QaEvidenceStatus,
+  type QaEvidenceSummaryEntry,
+} from "./evidence-summary.js";
+
+export type {
+  QaEvidenceArtifactView,
+  QaEvidenceGalleryEntryView,
+  QaEvidenceGalleryModel,
+  QaEvidenceMatrixCellView,
+  QaEvidenceProducerContext,
+  QaEvidenceProducerContextFile,
+} from "../shared/evidence-gallery-types.js";
+
+const TEXT_PREVIEW_BYTES = 12 * 1024;
+const ARTIFACT_VIEW_CONCURRENCY = 8;
+const REPO_ROOT_ARTIFACT_PATH_PREFIX = "<repo-root>/";
+
+const UX_MATRIX_PRODUCER_FILES = [
+  { key: "commands", path: "commands.txt", previewKind: "text" },
+  { key: "manifest", path: "manifest.json", previewKind: "json" },
+  { key: "matrix", path: "matrix.json", previewKind: "json" },
+  { key: "releaseLedger", path: "release-ledger.json", previewKind: "json" },
+  { key: "scorecard", path: "scorecard.md", previewKind: "text" },
+  { key: "memory", path: path.join("preflight", "memory.txt"), previewKind: "text" },
+  { key: "adbDevices", path: path.join("preflight", "adb-devices.txt"), previewKind: "text" },
+] as const;
+
+type UxMatrixProducerFileKey = (typeof UX_MATRIX_PRODUCER_FILES)[number]["key"];
+type QaEvidenceArtifact = NonNullable<QaEvidenceSummaryEntry["execution"]>["artifacts"][number];
+
+export class QaEvidenceGalleryError extends Error {
+  readonly statusCode: number;
+
+  constructor(message: string, statusCode: number) {
+    super(message);
+    this.name = "QaEvidenceGalleryError";
+    this.statusCode = statusCode;
+  }
+}
+
+function evidenceError(message: string, statusCode: number): QaEvidenceGalleryError {
+  return new QaEvidenceGalleryError(message, statusCode);
+}
+
+function isInside(root: string, candidate: string) {
+  const relative = path.relative(root, candidate);
+  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+}
+
+function sanitizeGalleryText(
+  value: string,
+  params: {
+    extraRoots?: readonly string[];
+    repoRoot: string;
+  },
+) {
+  const localRoots = [...new Set([params.repoRoot, ...(params.extraRoots ?? [])])];
+  const roots = [
+    ...localRoots.flatMap((root) => [
+      { from: path.resolve(root), to: "<repo-root>" },
+      { from: pathToFileURL(path.resolve(root)).href, to: "file://<repo-root>" },
+    ]),
+    { from: os.homedir(), to: "<home>" },
+    { from: pathToFileURL(os.homedir()).href, to: "file://<home>" },
+  ].filter((entry) => entry.from && entry.from !== path.parse(entry.from).root);
+  return roots
+    .toSorted((a, b) => b.from.length - a.from.length)
+    .reduce((text, entry) => text.replaceAll(entry.from, entry.to), value);
+}
+
+function displayGalleryPath(
+  value: string,
+  params: {
+    extraRoots?: readonly string[];
+    repoRoot: string;
+  },
+) {
+  if (path.isAbsolute(value)) {
+    const absolute = path.resolve(value);
+    for (const root of [params.repoRoot, ...(params.extraRoots ?? [])]) {
+      const resolvedRoot = path.resolve(root);
+      if (isInside(resolvedRoot, absolute)) {
+        return sanitizeGalleryText(toRepoPath(path.relative(resolvedRoot, absolute)), params);
+      }
+    }
+  }
+  return sanitizeGalleryText(value, params);
+}
+
+function sanitizeGalleryPreview(
+  value: string | null,
+  params: {
+    extraRoots?: readonly string[];
+    repoRoot: string;
+  },
+) {
+  return value === null ? null : sanitizeGalleryText(value, params);
+}
+
+function sanitizeGalleryStringArray(
+  values: Iterable<unknown>,
+  params: {
+    extraRoots?: readonly string[];
+    repoRoot: string;
+  },
+) {
+  return readOrderedStringArray(
+    Array.from(values)
+      .filter((value): value is string => typeof value === "string")
+      .map((value) => sanitizeGalleryText(value, params)),
+  );
+}
+
+async function realpathIfExists(filePath: string): Promise<string | null> {
+  return fs.realpath(filePath).catch(() => null);
+}
+
+async function resolveContainedFileIfExists(
+  filePath: string,
+  allowedRoots: readonly string[],
+): Promise<string | null> {
+  const realFile = await realpathIfExists(filePath);
+  if (!realFile) {
+    return null;
+  }
+  if (!allowedRoots.some((root) => isInside(root, realFile))) {
+    return null;
+  }
+  const stats = await fs.stat(realFile).catch(() => null);
+  return stats?.isFile() ? realFile : null;
+}
+
+export async function resolveQaEvidenceFile(params: {
+  inputPath: string;
+  repoRoot: string;
+}): Promise<string> {
+  const repoRoot = await fs.realpath(path.resolve(params.repoRoot));
+  const raw = params.inputPath.trim();
+  if (!raw) {
+    throw evidenceError("Evidence path is required.", 400);
+  }
+  const candidate = path.resolve(repoRoot, raw);
+  const realCandidate = await realpathIfExists(candidate);
+  if (!realCandidate) {
+    throw evidenceError("Evidence path not found.", 404);
+  }
+  if (!isInside(repoRoot, realCandidate)) {
+    throw evidenceError("Evidence path must stay inside the repo root.", 403);
+  }
+  const stats = await fs.stat(realCandidate);
+  const evidencePath = stats.isDirectory()
+    ? path.join(realCandidate, QA_EVIDENCE_FILENAME)
+    : realCandidate;
+  const realEvidencePath = await realpathIfExists(evidencePath);
+  if (!realEvidencePath) {
+    throw evidenceError("qa-evidence.json not found.", 404);
+  }
+  if (!isInside(repoRoot, realEvidencePath)) {
+    throw evidenceError("qa-evidence.json must stay inside the repo root.", 403);
+  }
+  return realEvidencePath;
+}
+
+export async function resolveQaEvidenceArtifactFile(params: {
+  artifactPath: string;
+  evidencePath: string;
+  repoRoot: string;
+}): Promise<string> {
+  const repoRoot = await fs.realpath(path.resolve(params.repoRoot));
+  const evidencePath = await resolveQaEvidenceFile({ inputPath: params.evidencePath, repoRoot });
+  if (!params.artifactPath.trim()) {
+    throw evidenceError("Artifact path is required.", 400);
+  }
+  const summary = validateQaEvidenceSummaryJson(
+    JSON.parse(await fs.readFile(evidencePath, "utf8")) as unknown,
+  );
+  const artifactFile = await resolveArtifactFileWithinRoots({
+    artifactPath: params.artifactPath,
+    evidenceDir: path.dirname(evidencePath),
+    repoRoot,
+  });
+  if (!artifactFile) {
+    throw evidenceError("Evidence artifact not found.", 404);
+  }
+  const allowedArtifactFiles = await collectDeclaredQaEvidenceArtifactFiles({
+    evidencePath,
+    repoRoot,
+    summaryEntries: summary.entries,
+  });
+  if (allowedArtifactFiles.has(artifactFile)) {
+    return artifactFile;
+  }
+  throw evidenceError("Evidence artifact is not declared by this evidence summary.", 403);
+}
+
+export async function resolveQaEvidenceArtifactFileByIndex(params: {
+  artifactIndex: number;
+  entryIndex: number;
+  evidencePath: string;
+  repoRoot: string;
+}): Promise<string> {
+  const repoRoot = await fs.realpath(path.resolve(params.repoRoot));
+  const evidencePath = await resolveQaEvidenceFile({ inputPath: params.evidencePath, repoRoot });
+  if (
+    !Number.isSafeInteger(params.entryIndex) ||
+    params.entryIndex < 0 ||
+    !Number.isSafeInteger(params.artifactIndex) ||
+    params.artifactIndex < 0
+  ) {
+    throw evidenceError("Evidence artifact index is invalid.", 400);
+  }
+  const summary = validateQaEvidenceSummaryJson(
+    JSON.parse(await fs.readFile(evidencePath, "utf8")) as unknown,
+  );
+  const artifact = summary.entries[params.entryIndex]?.execution?.artifacts[params.artifactIndex];
+  if (!artifact) {
+    throw evidenceError("Evidence artifact not found.", 404);
+  }
+  const artifactFile = await resolveArtifactFileWithinRoots({
+    artifactPath: artifact.path,
+    evidenceDir: path.dirname(evidencePath),
+    repoRoot,
+  });
+  if (!artifactFile) {
+    throw evidenceError("Evidence artifact not found.", 404);
+  }
+  return artifactFile;
+}
+
+export async function resolveQaEvidenceProducerFile(params: {
+  evidencePath: string;
+  producerFile: string;
+  repoRoot: string;
+}): Promise<string> {
+  const repoRoot = await fs.realpath(path.resolve(params.repoRoot));
+  const evidencePath = await resolveQaEvidenceFile({ inputPath: params.evidencePath, repoRoot });
+  const producerFile = UX_MATRIX_PRODUCER_FILES.find((file) => file.key === params.producerFile);
+  if (!producerFile) {
+    throw evidenceError("Evidence producer file is unknown.", 400);
+  }
+  const summary = validateQaEvidenceSummaryJson(
+    JSON.parse(await fs.readFile(evidencePath, "utf8")) as unknown,
+  );
+  const producerRoot = await findUxMatrixProducerRoot({
+    evidencePath,
+    repoRoot,
+    summaryEntries: summary.entries,
+  });
+  if (!producerRoot) {
+    throw evidenceError("Evidence producer context not found.", 404);
+  }
+  const evidenceDir = path.dirname(evidencePath);
+  const producerPath = path.join(producerRoot, producerFile.path);
+  const realProducerFile = await resolveContainedFileIfExists(producerPath, [
+    repoRoot,
+    evidenceDir,
+  ]);
+  if (!realProducerFile) {
+    throw evidenceError("Evidence producer file not found.", 404);
+  }
+  return realProducerFile;
+}
+
+function isExplicitRepoRootArtifactPath(raw: string): boolean {
+  const normalized = raw.split(/[\\/]+/u).join("/");
+  return normalized.startsWith(".artifacts/");
+}
+
+function repoRootTokenArtifactPath(raw: string): string | null {
+  const normalized = raw.split(/[\\/]+/u).join("/");
+  return normalized.startsWith(REPO_ROOT_ARTIFACT_PATH_PREFIX)
+    ? normalized.slice(REPO_ROOT_ARTIFACT_PATH_PREFIX.length)
+    : null;
+}
+
+// Resolve an artifact path against pre-resolved roots without re-reading the evidence file.
+// Returns null when the path is missing or escapes both roots; callers map that to an error.
+async function resolveArtifactFileWithinRoots(params: {
+  artifactPath: string;
+  evidenceDir: string;
+  repoRoot: string;
+}): Promise<string | null> {
+  const raw = params.artifactPath.trim();
+  if (!raw) {
+    return null;
+  }
+  const tokenPath = repoRootTokenArtifactPath(raw);
+  const candidates = tokenPath
+    ? [path.resolve(params.repoRoot, tokenPath)]
+    : path.isAbsolute(raw)
+      ? [raw]
+      : [path.resolve(params.evidenceDir, raw)];
+  if (!tokenPath && !path.isAbsolute(raw) && isExplicitRepoRootArtifactPath(raw)) {
+    candidates.push(path.resolve(params.repoRoot, raw));
+  }
+  for (const candidate of candidates) {
+    const realCandidate = await realpathIfExists(candidate);
+    if (!realCandidate) {
+      continue;
+    }
+    if (!isInside(params.repoRoot, realCandidate) && !isInside(params.evidenceDir, realCandidate)) {
+      continue;
+    }
+    const stats = await fs.stat(realCandidate).catch(() => null);
+    if (stats?.isFile()) {
+      return realCandidate;
+    }
+  }
+  return null;
+}
+
+async function collectDeclaredQaEvidenceArtifactFiles(params: {
+  evidencePath: string;
+  repoRoot: string;
+  summaryEntries: readonly QaEvidenceSummaryEntry[];
+}): Promise<Set<string>> {
+  const repoRoot = await fs.realpath(path.resolve(params.repoRoot));
+  const evidenceDir = path.dirname(params.evidencePath);
+  const allowed = new Set<string>();
+  for (const entry of params.summaryEntries) {
+    for (const artifact of entry.execution?.artifacts ?? []) {
+      const artifactPath = await resolveArtifactFileWithinRoots({
+        artifactPath: artifact.path,
+        evidenceDir,
+        repoRoot,
+      });
+      if (artifactPath) {
+        allowed.add(artifactPath);
+      }
+    }
+  }
+  const producerRoot = await findUxMatrixProducerRoot({
+    evidencePath: params.evidencePath,
+    repoRoot: params.repoRoot,
+    summaryEntries: params.summaryEntries,
+  });
+  if (producerRoot) {
+    const producerFiles = [
+      ...UX_MATRIX_PRODUCER_FILES.map((file) => file.path),
+      QA_EVIDENCE_FILENAME,
+    ];
+    for (const producerFile of producerFiles) {
+      const realProducerFile = await realpathIfExists(path.join(producerRoot, producerFile));
+      if (realProducerFile) {
+        allowed.add(realProducerFile);
+      }
+    }
+  }
+  return allowed;
+}
+
+function classifyArtifact(kind: string, filePath: string): QaEvidenceArtifactView["mediaKind"] {
+  const normalizedKind = kind.toLowerCase();
+  const ext = path.extname(filePath).toLowerCase();
+  if (
+    normalizedKind.includes("screenshot") ||
+    normalizedKind.includes("gif") ||
+    [".png", ".jpg", ".jpeg", ".gif", ".webp"].includes(ext)
+  ) {
+    return "image";
+  }
+  if (normalizedKind.includes("video") || [".webm", ".mp4", ".mov"].includes(ext)) {
+    return "video";
+  }
+  if (
+    normalizedKind.includes("validation") ||
+    normalizedKind.includes("json") ||
+    ext === ".json" ||
+    ext === ".jsonl"
+  ) {
+    return "json";
+  }
+  if (
+    normalizedKind.includes("log") ||
+    normalizedKind.includes("report") ||
+    [".log", ".md", ".txt"].includes(ext)
+  ) {
+    return "text";
+  }
+  return "file";
+}
+
+async function readPreview(filePath: string, mediaKind: QaEvidenceArtifactView["mediaKind"]) {
+  if (mediaKind !== "json" && mediaKind !== "text") {
+    return null;
+  }
+  const handle = await fs.open(filePath, "r");
+  try {
+    const buffer = Buffer.alloc(TEXT_PREVIEW_BYTES);
+    const { bytesRead } = await handle.read(buffer, 0, TEXT_PREVIEW_BYTES, 0);
+    const text = buffer.subarray(0, bytesRead).toString("utf8");
+    if (mediaKind !== "json") {
+      return text;
+    }
+    try {
+      return JSON.stringify(JSON.parse(text), null, 2);
+    } catch {
+      return text;
+    }
+  } finally {
+    await handle.close();
+  }
+}
+
+async function readJsonIfExists(
+  filePath: string,
+  allowedRoots: readonly string[],
+): Promise<Record<string, unknown> | null> {
+  const realFile = await resolveContainedFileIfExists(filePath, allowedRoots);
+  if (!realFile) {
+    return null;
+  }
+  try {
+    const value = JSON.parse(await fs.readFile(realFile, "utf8")) as unknown;
+    return value && typeof value === "object" && !Array.isArray(value)
+      ? (value as Record<string, unknown>)
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+function artifactHref(
+  evidencePath: string,
+  artifact:
+    | {
+        artifactPath: string;
+      }
+    | {
+        artifactIndex: number;
+        entryIndex: number;
+      }
+    | {
+        producerFile: UxMatrixProducerFileKey;
+      },
+) {
+  const params = new URLSearchParams({ evidencePath });
+  if ("artifactPath" in artifact) {
+    params.set("artifactPath", artifact.artifactPath);
+  } else if ("producerFile" in artifact) {
+    params.set("producerFile", artifact.producerFile);
+  } else {
+    params.set("entryIndex", String(artifact.entryIndex));
+    params.set("artifactIndex", String(artifact.artifactIndex));
+  }
+  return `/api/evidence/artifact?${params.toString()}`;
+}
+
+async function buildProducerContextFile(params: {
+  allowedRoots: readonly string[];
+  extraRoots: readonly string[];
+  filePath: string;
+  hrefEvidencePath: string;
+  previewKind: "json" | "text";
+  producerFile: UxMatrixProducerFileKey;
+  repoRoot: string;
+}): Promise<QaEvidenceProducerContextFile | null> {
+  const realFile = await resolveContainedFileIfExists(params.filePath, params.allowedRoots);
+  if (!realFile) {
+    return null;
+  }
+  return {
+    href: artifactHref(params.hrefEvidencePath, { producerFile: params.producerFile }),
+    path: displayGalleryPath(params.filePath, params),
+    preview: await readPreview(realFile, params.previewKind)
+      .then((preview) =>
+        sanitizeGalleryPreview(preview, {
+          extraRoots: params.extraRoots,
+          repoRoot: params.repoRoot,
+        }),
+      )
+      .catch(() => null),
+  };
+}
+
+async function buildArtifactView(params: {
+  allowedArtifactFiles: ReadonlySet<string>;
+  artifactIndex: number;
+  artifact: QaEvidenceArtifact;
+  evidenceDir: string;
+  entryIndex: number;
+  extraRoots: readonly string[];
+  hrefEvidencePath: string;
+  repoRoot: string;
+}): Promise<QaEvidenceArtifactView> {
+  const mediaKind = classifyArtifact(params.artifact.kind, params.artifact.path);
+  const realFile = await resolveArtifactFileWithinRoots({
+    artifactPath: params.artifact.path,
+    evidenceDir: params.evidenceDir,
+    repoRoot: params.repoRoot,
+  }).catch(() => null);
+  const realFileRepoPath =
+    realFile && isInside(params.repoRoot, realFile)
+      ? toRepoRelativePath(params.repoRoot, realFile)
+      : null;
+  const displayPath =
+    (realFileRepoPath ? sanitizeGalleryText(realFileRepoPath, params) : null) ??
+    sanitizeGalleryText(params.artifact.path, {
+      extraRoots: params.extraRoots,
+      repoRoot: params.repoRoot,
+    });
+  if (!realFile || !params.allowedArtifactFiles.has(realFile)) {
+    return {
+      exists: false,
+      error: realFile
+        ? "Evidence artifact is not declared by this evidence summary."
+        : "Evidence artifact not found.",
+      href: null,
+      kind: sanitizeGalleryText(params.artifact.kind, params),
+      mediaKind,
+      path: displayPath,
+      preview: null,
+      source: sanitizeGalleryText(params.artifact.source, params),
+    };
+  }
+  return {
+    exists: true,
+    error: null,
+    href: artifactHref(params.hrefEvidencePath, {
+      artifactIndex: params.artifactIndex,
+      entryIndex: params.entryIndex,
+    }),
+    kind: sanitizeGalleryText(params.artifact.kind, params),
+    mediaKind,
+    path: displayPath,
+    preview: await readPreview(realFile, mediaKind)
+      .then((preview) =>
+        sanitizeGalleryPreview(preview, {
+          extraRoots: params.extraRoots,
+          repoRoot: params.repoRoot,
+        }),
+      )
+      .catch((error: unknown) =>
+        sanitizeGalleryText(`Preview unavailable: ${formatErrorMessage(error)}`, {
+          extraRoots: params.extraRoots,
+          repoRoot: params.repoRoot,
+        }),
+      ),
+    source: sanitizeGalleryText(params.artifact.source, params),
+  };
+}
+
+function readString(value: unknown): string | null {
+  return typeof value === "string" ? value : null;
+}
+
+function readRecord(value: unknown): Record<string, unknown> | null {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+function readCountRecord(value: unknown): Record<string, number> {
+  const record = readRecord(value);
+  if (!record) {
+    return {};
+  }
+  return Object.fromEntries(
+    Object.entries(record).filter(
+      (entry): entry is [string, number] => typeof entry[1] === "number",
+    ),
+  );
+}
+
+function readOrderedStringArray(values: Iterable<unknown>) {
+  return Array.from(
+    new Set(Array.from(values).filter((value): value is string => typeof value === "string")),
+  );
+}
+
+function readStringArray(values: Iterable<unknown>) {
+  return readOrderedStringArray(values).toSorted();
+}
+
+function readMatrixDimensionIds(params: {
+  extraRoots: readonly string[];
+  fallback: readonly string[];
+  repoRoot: string;
+  value: unknown;
+}): string[] {
+  if (!Array.isArray(params.value)) {
+    return sanitizeGalleryStringArray(params.fallback, params);
+  }
+  const ids = sanitizeGalleryStringArray(
+    params.value.map((entry) => {
+      if (typeof entry === "string") {
+        return entry;
+      }
+      return readString(readRecord(entry)?.id);
+    }),
+    params,
+  );
+  for (const rawFallbackId of params.fallback) {
+    const fallbackId = sanitizeGalleryText(rawFallbackId, params);
+    if (!ids.includes(fallbackId)) {
+      ids.push(fallbackId);
+    }
+  }
+  return ids;
+}
+
+function uxMatrixEntryKey(
+  entry: QaEvidenceSummaryEntry,
+): { stage: string; surface: string } | null {
+  const idMatch = /^ux-matrix\.([a-z0-9-]+)\.([a-z0-9-]+)$/u.exec(entry.test.id);
+  if (idMatch) {
+    return { surface: idMatch[1], stage: idMatch[2] };
+  }
+  for (const artifact of entry.execution?.artifacts ?? []) {
+    const sourceMatch = /^ux-matrix:([a-z0-9-]+):([a-z0-9-]+)$/u.exec(artifact.source);
+    if (sourceMatch) {
+      return { surface: sourceMatch[1], stage: sourceMatch[2] };
+    }
+  }
+  return null;
+}
+
+function buildUxMatrixEvidenceEntryIndex(entries: readonly QaEvidenceSummaryEntry[]) {
+  const indexed = new Map<string, QaEvidenceSummaryEntry>();
+  for (const entry of entries) {
+    const key = uxMatrixEntryKey(entry);
+    if (key) {
+      indexed.set(`${key.surface}:${key.stage}`, entry);
+    }
+  }
+  return indexed;
+}
+
+function readMatrixCells(params: {
+  extraRoots: readonly string[];
+  matrix: Record<string, unknown> | null;
+  repoRoot: string;
+  summaryEntries: readonly QaEvidenceSummaryEntry[];
+}): QaEvidenceMatrixCellView[] {
+  const rawCells = Array.isArray(params.matrix?.cells)
+    ? params.matrix.cells
+        .map(readRecord)
+        .filter((cell): cell is Record<string, unknown> => Boolean(cell))
+    : [];
+  const entriesByCell = buildUxMatrixEvidenceEntryIndex(params.summaryEntries);
+  return rawCells.flatMap((cell): QaEvidenceMatrixCellView[] => {
+    const rawSurface = readString(cell.surface);
+    const rawStage = readString(cell.stage);
+    const rawStatus = readString(cell.status) ?? "proof-gap";
+    if (!rawSurface || !rawStage) {
+      return [];
+    }
+    const entry =
+      rawStatus === "proof-gap" ? null : (entriesByCell.get(`${rawSurface}:${rawStage}`) ?? null);
+    const artifacts = entry?.execution?.artifacts ?? [];
+    const runner = readRecord(cell.runner);
+    const sanitizeCellString = (value: string) =>
+      sanitizeGalleryText(value, {
+        extraRoots: params.extraRoots,
+        repoRoot: params.repoRoot,
+      });
+    const readRunnerString = (value: unknown) => {
+      const text = readString(value);
+      return text ? sanitizeCellString(text) : null;
+    };
+    return [
+      {
+        artifactKinds: readStringArray(
+          artifacts.map((artifact) => sanitizeCellString(artifact.kind)),
+        ),
+        artifactPaths: artifacts.map((artifact) =>
+          displayGalleryPath(artifact.path, {
+            extraRoots: params.extraRoots,
+            repoRoot: params.repoRoot,
+          }),
+        ),
+        coverageIds: readStringArray(
+          (Array.isArray(cell.coverageIds) ? cell.coverageIds : []).map((coverageId) =>
+            typeof coverageId === "string" ? sanitizeCellString(coverageId) : coverageId,
+          ),
+        ),
+        runner: runner
+          ? {
+              availability: readRunnerString(runner.availability),
+              command: readRunnerString(runner.command),
+              lane: readRunnerString(runner.lane),
+              workflow: readRunnerString(runner.workflow),
+            }
+          : null,
+        stage: sanitizeCellString(rawStage),
+        status: sanitizeCellString(rawStatus),
+        surface: sanitizeCellString(rawSurface),
+        testId: entry?.test.id ? sanitizeCellString(entry.test.id) : null,
+        title: entry?.test.title ? sanitizeCellString(entry.test.title) : null,
+      },
+    ];
+  });
+}
+
+async function candidateProducerRoots(params: {
+  evidencePath: string;
+  repoRoot: string;
+  summaryEntries: readonly QaEvidenceSummaryEntry[];
+}) {
+  const repoRoot = await fs.realpath(path.resolve(params.repoRoot));
+  const evidenceDir = path.dirname(params.evidencePath);
+  const roots = new Set<string>([evidenceDir]);
+  for (const entry of params.summaryEntries) {
+    for (const artifact of entry.execution?.artifacts ?? []) {
+      const artifactPath = await resolveArtifactFileWithinRoots({
+        artifactPath: artifact.path,
+        evidenceDir,
+        repoRoot,
+      });
+      if (!artifactPath) {
+        continue;
+      }
+      let current = path.dirname(artifactPath);
+      while (isInside(repoRoot, current)) {
+        roots.add(current);
+        const parent = path.dirname(current);
+        if (parent === current) {
+          break;
+        }
+        current = parent;
+      }
+    }
+  }
+  return Array.from(roots);
+}
+
+async function findUxMatrixProducerRoot(params: {
+  evidencePath: string;
+  repoRoot: string;
+  summaryEntries: readonly QaEvidenceSummaryEntry[];
+}) {
+  for (const candidate of await candidateProducerRoots(params)) {
+    const [manifest, matrix] = await Promise.all([
+      realpathIfExists(path.join(candidate, "manifest.json")),
+      realpathIfExists(path.join(candidate, "matrix.json")),
+    ]);
+    if (manifest && matrix) {
+      return candidate;
+    }
+  }
+  return null;
+}
+
+async function buildProducerContext(params: {
+  evidencePath: string;
+  extraRoots: readonly string[];
+  hrefEvidencePath: string;
+  repoRoot: string;
+  summaryEntries: readonly QaEvidenceSummaryEntry[];
+}): Promise<QaEvidenceProducerContext | null> {
+  const rootPath = await findUxMatrixProducerRoot(params);
+  if (!rootPath) {
+    return null;
+  }
+  const repoRoot = await fs.realpath(path.resolve(params.repoRoot));
+  const evidenceDir = path.dirname(
+    await resolveQaEvidenceFile({ inputPath: params.evidencePath, repoRoot }),
+  );
+  const allowedRoots = [repoRoot, evidenceDir];
+  const producerPaths = Object.fromEntries(
+    UX_MATRIX_PRODUCER_FILES.map((file) => [file.key, path.join(rootPath, file.path)]),
+  ) as Record<(typeof UX_MATRIX_PRODUCER_FILES)[number]["key"], string>;
+  const manifestPath = producerPaths.manifest;
+  const matrixPath = producerPaths.matrix;
+  const releaseLedgerPath = producerPaths.releaseLedger;
+  const manifest = await readJsonIfExists(manifestPath, allowedRoots);
+  const matrix = await readJsonIfExists(matrixPath, allowedRoots);
+  const releaseLedger = await readJsonIfExists(releaseLedgerPath, allowedRoots);
+  const run = readRecord(manifest?.run);
+  const runId = readString(run?.runId);
+  const runStatus = readString(run?.status);
+  const producerFiles = Object.fromEntries(
+    await Promise.all(
+      UX_MATRIX_PRODUCER_FILES.map(async (file) => [
+        file.key,
+        await buildProducerContextFile({
+          allowedRoots,
+          extraRoots: params.extraRoots,
+          filePath: producerPaths[file.key],
+          hrefEvidencePath: params.hrefEvidencePath,
+          previewKind: file.previewKind,
+          producerFile: file.key,
+          repoRoot,
+        }),
+      ]),
+    ),
+  ) as Record<
+    (typeof UX_MATRIX_PRODUCER_FILES)[number]["key"],
+    QaEvidenceProducerContextFile | null
+  >;
+  const matrixCells = readMatrixCells({
+    extraRoots: params.extraRoots,
+    matrix,
+    repoRoot,
+    summaryEntries: params.summaryEntries,
+  });
+  return {
+    commands: producerFiles.commands,
+    kind: "ux-matrix",
+    manifest:
+      manifest && producerFiles.manifest
+        ? {
+            ...producerFiles.manifest,
+            runId: runId ? sanitizeGalleryText(runId, params) : null,
+            runStatus: runStatus ? sanitizeGalleryText(runStatus, params) : null,
+          }
+        : null,
+    matrix: matrix
+      ? {
+          cells: matrixCells,
+          counts: readCountRecord(matrix.counts),
+          path: displayGalleryPath(matrixPath, { extraRoots: params.extraRoots, repoRoot }),
+          stages: readMatrixDimensionIds({
+            extraRoots: params.extraRoots,
+            fallback: matrixCells.map((cell) => cell.stage),
+            repoRoot,
+            value: matrix.stages,
+          }),
+          surfaces: readMatrixDimensionIds({
+            extraRoots: params.extraRoots,
+            fallback: matrixCells.map((cell) => cell.surface),
+            repoRoot,
+            value: matrix.surfaces,
+          }),
+        }
+      : null,
+    preflight: {
+      adbDevices: producerFiles.adbDevices,
+      memory: producerFiles.memory,
+    },
+    releaseLedger:
+      releaseLedger && producerFiles.releaseLedger
+        ? {
+            ...producerFiles.releaseLedger,
+            counts: readCountRecord(releaseLedger.counts),
+          }
+        : null,
+    rootPath: displayGalleryPath(rootPath, { extraRoots: params.extraRoots, repoRoot }),
+    scorecard: producerFiles.scorecard,
+  };
+}
+
+function createConcurrencyLimit(limit: number) {
+  let active = 0;
+  const queue: Array<() => void> = [];
+  return async function runLimited<T>(task: () => Promise<T>): Promise<T> {
+    if (active >= limit) {
+      await new Promise<void>((resolve) => {
+        queue.push(resolve);
+      });
+    }
+    active += 1;
+    try {
+      return await task();
+    } finally {
+      active -= 1;
+      queue.shift()?.();
+    }
+  };
+}
+
+export async function buildQaEvidenceGalleryModel(params: {
+  evidencePath: string;
+  repoRoot: string;
+}): Promise<QaEvidenceGalleryModel> {
+  const requestedRepoRoot = path.resolve(params.repoRoot);
+  const repoRoot = await fs.realpath(requestedRepoRoot);
+  const evidencePath = await resolveQaEvidenceFile({
+    inputPath: params.evidencePath,
+    repoRoot,
+  });
+  const hrefEvidencePath = toRepoRelativePath(repoRoot, evidencePath);
+  const summary = validateQaEvidenceSummaryJson(
+    JSON.parse(await fs.readFile(evidencePath, "utf8")) as unknown,
+  );
+  const counts: Record<QaEvidenceStatus, number> = {
+    pass: 0,
+    fail: 0,
+    blocked: 0,
+    skipped: 0,
+  };
+  // Resolve the declared-artifact allowlist once; buildArtifactView then only checks membership
+  // instead of re-reading the evidence file and re-collecting the allowlist per artifact.
+  const evidenceDir = path.dirname(evidencePath);
+  const allowedArtifactFiles = await collectDeclaredQaEvidenceArtifactFiles({
+    evidencePath,
+    repoRoot,
+    summaryEntries: summary.entries,
+  });
+  const limitArtifactView = createConcurrencyLimit(ARTIFACT_VIEW_CONCURRENCY);
+  const entries = await Promise.all(
+    summary.entries.map(async (entry, entryIndex): Promise<QaEvidenceGalleryEntryView> => {
+      counts[entry.result.status] += 1;
+      const sanitizeEntryText = (value: string) =>
+        sanitizeGalleryText(value, {
+          extraRoots: [requestedRepoRoot],
+          repoRoot,
+        });
+      return {
+        artifacts: await Promise.all(
+          (entry.execution?.artifacts ?? []).map((artifact, artifactIndex) =>
+            limitArtifactView(() =>
+              buildArtifactView({
+                allowedArtifactFiles,
+                artifact,
+                artifactIndex,
+                evidenceDir,
+                entryIndex,
+                extraRoots: [requestedRepoRoot],
+                hrefEvidencePath,
+                repoRoot,
+              }),
+            ),
+          ),
+        ),
+        coverage: entry.coverage.map((coverage) => ({
+          id: sanitizeEntryText(coverage.id),
+          role: sanitizeEntryText(coverage.role),
+        })),
+        failureReason: entry.result.failure?.reason
+          ? sanitizeEntryText(entry.result.failure.reason)
+          : null,
+        id: sanitizeEntryText(entry.test.id),
+        kind: sanitizeEntryText(entry.test.kind),
+        sourcePath: entry.test.source?.path
+          ? displayGalleryPath(entry.test.source.path, {
+              extraRoots: [requestedRepoRoot],
+              repoRoot,
+            })
+          : null,
+        status: entry.result.status,
+        title: sanitizeEntryText(entry.test.title),
+      };
+    }),
+  );
+  return {
+    counts,
+    entries,
+    evidenceMode: summary.evidenceMode,
+    evidencePath: hrefEvidencePath,
+    generatedAt: summary.generatedAt,
+    profile: summary.profile
+      ? sanitizeGalleryText(summary.profile, { extraRoots: [requestedRepoRoot], repoRoot })
+      : null,
+    producerContext: await buildProducerContext({
+      evidencePath,
+      extraRoots: [requestedRepoRoot],
+      hrefEvidencePath,
+      repoRoot,
+      summaryEntries: summary.entries,
+    }),
+    schemaVersion: summary.schemaVersion,
+  };
+}

extensions/qa-lab/src/evidence-summary.test.ts

@@ -0,0 +1,668 @@
+// Qa Lab tests cover QA evidence summary behavior.
+import { execFileSync } from "node:child_process";
+import { describe, expect, it } from "vitest";
+import {
+  QA_EVIDENCE_SUMMARY_KIND,
+  QA_EVIDENCE_FILENAME,
+  QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
+  buildLiveTransportEvidenceSummary,
+  buildPlaywrightEvidenceSummary,
+  buildQaSuiteEvidenceSummary,
+  buildVitestEvidenceSummary,
+  validateQaEvidenceSummaryJson,
+} from "./evidence-summary.js";
+
+describe("evidence summary", () => {
+  it("builds QA suite evidence entries from catalog metadata", () => {
+    const evidence = buildQaSuiteEvidenceSummary({
+      artifactPaths: [
+        { kind: "summary", path: "qa-suite-summary.json" },
+        { kind: "report", path: "qa-suite-report.md" },
+      ],
+      scenarioDefinitions: [
+        {
+          id: "dm-chat-baseline",
+          title: "DM baseline conversation",
+          sourcePath: "qa/scenarios/channels/dm-chat-baseline.yaml",
+          surface: "dm",
+          coverage: {
+            primary: ["channels.dm"],
+            secondary: ["channels.qa-channel"],
+          },
+          runtimeParityTier: "standard",
+          docsRefs: ["docs/channels/qa-channel.md"],
+          codeRefs: ["extensions/qa-channel/src/gateway.ts"],
+        },
+      ],
+      channelId: "qa-channel",
+      env: {
+        OPENCLAW_QA_CHANNEL_DRIVER: "local-shim",
+        OPENCLAW_QA_REF: "abc123",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-07T12:00:00.000Z",
+      primaryModel: "mock-openai/gpt-5.5",
+      providerMode: "mock-openai",
+      scenarioResults: [{ name: "DM baseline conversation", status: "pass" }],
+    });
+
+    expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
+    expect(evidence.kind).toBe(QA_EVIDENCE_SUMMARY_KIND);
+    expect(evidence.schemaVersion).toBe(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION);
+    expect(evidence.evidenceMode).toBe("full");
+    expect(evidence.profile).toBeUndefined();
+    expect(evidence.entries).toHaveLength(1);
+    expect(evidence.entries[0]).toMatchObject({
+      test: {
+        kind: "qa-scenario",
+        id: "dm-chat-baseline",
+        title: "DM baseline conversation",
+        source: {
+          path: "qa/scenarios/channels/dm-chat-baseline.yaml",
+        },
+      },
+      coverage: [
+        {
+          id: "channels.dm",
+          role: "primary",
+        },
+        {
+          id: "channels.qa-channel",
+          role: "secondary",
+        },
+      ],
+      refs: [
+        {
+          kind: "docs",
+          path: "docs/channels/qa-channel.md",
+        },
+        {
+          kind: "code",
+          path: "extensions/qa-channel/src/gateway.ts",
+        },
+      ],
+      runtimeParityTier: "standard",
+      execution: {
+        runner: "host",
+        provider: {
+          id: "openai",
+          live: false,
+          model: {
+            name: "gpt-5.5",
+            ref: "mock-openai/gpt-5.5",
+          },
+          fixture: "mock-openai",
+        },
+        channel: {
+          id: "qa-channel",
+          live: false,
+          driver: "local-shim",
+        },
+        packageSource: {
+          kind: "source-checkout",
+        },
+        environment: {
+          ref: "abc123",
+          os: process.platform,
+          nodeVersion: process.version,
+        },
+        artifacts: [
+          {
+            kind: "summary",
+            path: "qa-suite-summary.json",
+            source: "qa-suite",
+          },
+          {
+            kind: "report",
+            path: "qa-suite-report.md",
+            source: "qa-suite",
+          },
+        ],
+      },
+      result: {
+        status: "pass",
+      },
+    });
+  });
+
+  it("prefers the checked-out ref over an inherited GitHub event SHA", () => {
+    const repoRoot = process.cwd();
+    const checkedOutRef = execFileSync("git", ["rev-parse", "--verify", "HEAD"], {
+      cwd: repoRoot,
+      encoding: "utf8",
+    }).trim();
+    const evidence = buildQaSuiteEvidenceSummary({
+      artifactPaths: [],
+      channelId: "qa-channel",
+      env: {
+        GITHUB_SHA: "bd479958c04a1eadbda8b6105e0722588d71e9ad",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-24T12:00:00.000Z",
+      primaryModel: "mock-openai/gpt-5.5",
+      providerMode: "mock-openai",
+      repoRoot,
+      scenarioDefinitions: [{ id: "ref-probe", title: "Ref probe" }],
+      scenarioResults: [{ name: "Ref probe", status: "pass" }],
+    });
+
+    expect(evidence.entries[0]?.execution?.environment.ref).toBe(checkedOutRef);
+  });
+
+  it("builds Telegram live transport evidence entries", () => {
+    const evidence = buildLiveTransportEvidenceSummary({
+      artifactPaths: [
+        { kind: "summary", path: QA_EVIDENCE_FILENAME },
+        { kind: "report", path: "telegram-qa-report.md" },
+      ],
+      env: {
+        OPENCLAW_QA_RUNNER: "crabbox",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-07T12:05:00.000Z",
+      primaryModel: "openai/gpt-5.5",
+      providerMode: "live-frontier",
+      checks: [
+        {
+          id: "telegram-canary",
+          coverageIds: ["channels.telegram.canary"],
+          title: "Telegram canary",
+          status: "fail",
+          details: "timed out waiting for SUT reply",
+          posture: "user-path",
+          rttMs: 4321,
+        },
+      ],
+      transportId: "telegram",
+    });
+
+    expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
+    expect(evidence.profile).toBeUndefined();
+    expect(evidence.entries).toEqual([
+      expect.objectContaining({
+        test: {
+          kind: "live-transport-check",
+          id: "telegram-canary",
+          title: "Telegram canary",
+        },
+        coverage: [
+          {
+            id: "channels.telegram.live",
+            role: "live-transport",
+          },
+          {
+            id: "channels.telegram.canary",
+            role: "live-transport-coverage",
+          },
+        ],
+        posture: "user-path",
+        execution: expect.objectContaining({
+          runner: "crabbox",
+          provider: {
+            id: "openai",
+            live: true,
+            model: {
+              name: "gpt-5.5",
+              ref: "openai/gpt-5.5",
+            },
+            auth: "live-frontier",
+          },
+          channel: {
+            id: "telegram",
+            live: true,
+            driver: "native",
+          },
+          artifacts: [
+            {
+              kind: "summary",
+              path: QA_EVIDENCE_FILENAME,
+              source: "telegram-live-transport",
+            },
+            {
+              kind: "report",
+              path: "telegram-qa-report.md",
+              source: "telegram-live-transport",
+            },
+          ],
+        }),
+        result: {
+          status: "fail",
+          failure: {
+            reason: "timed out waiting for SUT reply",
+          },
+          timing: {
+            rttMs: 4321,
+          },
+        },
+      }),
+    ]);
+  });
+
+  it("preserves aggregate live transport timing", () => {
+    const evidence = buildLiveTransportEvidenceSummary({
+      artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
+      generatedAt: "2026-06-07T12:05:00.000Z",
+      primaryModel: "openai/gpt-5.5",
+      providerMode: "live-frontier",
+      checks: [
+        {
+          id: "telegram-mentioned-message-reply",
+          coverageIds: ["channels.telegram.mention-gating"],
+          title: "Telegram mentioned message gets a reply",
+          status: "pass",
+          details: "5 samples collected.",
+          rttMs: 2000,
+          timing: {
+            rttMs: 1200,
+            avgMs: 1300,
+            p50Ms: 1200,
+            p95Ms: 1800,
+            maxMs: 2200,
+            samples: 5,
+            failedSamples: 1,
+          },
+        },
+      ],
+      transportId: "telegram",
+    });
+
+    expect(evidence.entries[0]?.result.timing).toEqual({
+      rttMs: 1200,
+      avgMs: 1300,
+      p50Ms: 1200,
+      p95Ms: 1800,
+      maxMs: 2200,
+      samples: 5,
+      failedSamples: 1,
+    });
+  });
+
+  it("builds Vitest runner evidence entries", () => {
+    const evidence = buildVitestEvidenceSummary({
+      artifactPaths: [
+        { kind: "runner-result", path: "vitest-results/runtime-boundary.vitest.json" },
+      ],
+      env: {
+        OPENCLAW_QA_REF: "abc123",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-07T12:06:00.000Z",
+      primaryModel: "mock-openai/gpt-5.5",
+      providerMode: "mock-openai",
+      targets: [
+        {
+          id: "runtime.agent-runner-boundary",
+          title: "Agent runner boundary integration tests",
+          sourcePath: "src/agents/agent-runner.e2e.test.ts",
+          primaryCoverageIds: ["runtime.agent-runner", "runtime.delivery"],
+          codeRefs: ["src/agents/agent-runner.ts"],
+        },
+      ],
+      results: [
+        {
+          id: "runtime.agent-runner-boundary",
+          status: "pass",
+          durationMs: 1234,
+        },
+      ],
+    });
+
+    expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
+    expect(evidence.profile).toBeUndefined();
+    expect(evidence.entries).toEqual([
+      expect.objectContaining({
+        test: {
+          kind: "vitest-test",
+          id: "runtime.agent-runner-boundary",
+          title: "Agent runner boundary integration tests",
+          source: {
+            path: "src/agents/agent-runner.e2e.test.ts",
+          },
+        },
+        coverage: [
+          {
+            id: "runtime.agent-runner",
+            role: "primary",
+          },
+          {
+            id: "runtime.delivery",
+            role: "primary",
+          },
+        ],
+        refs: [
+          {
+            kind: "code",
+            path: "src/agents/agent-runner.ts",
+          },
+        ],
+        execution: expect.objectContaining({
+          runner: "vitest",
+          provider: expect.objectContaining({
+            live: false,
+            fixture: "mock-openai",
+          }),
+          artifacts: [
+            {
+              kind: "runner-result",
+              path: "vitest-results/runtime-boundary.vitest.json",
+              source: "vitest",
+            },
+          ],
+        }),
+        result: {
+          status: "pass",
+          timing: {
+            wallMs: 1234,
+          },
+        },
+      }),
+    ]);
+  });
+
+  it("builds Playwright runner evidence entries", () => {
+    const evidence = buildPlaywrightEvidenceSummary({
+      artifactPaths: [
+        { kind: "runner-result", path: "playwright-results/control-ui.json" },
+        { kind: "report", path: "playwright-report/index.html" },
+      ],
+      env: {
+        GITHUB_SHA: "def456",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-07T12:07:00.000Z",
+      primaryModel: "mock-openai/gpt-5.5",
+      providerMode: "mock-openai",
+      targets: [
+        {
+          id: "control-ui.browser-run",
+          title: "Control UI browser workflow",
+          sourcePath: "ui/control-ui.e2e.test.ts",
+          primaryCoverageIds: ["ui.control"],
+          docsRefs: ["docs/concepts/qa-e2e-automation.md"],
+          codeRefs: ["ui/"],
+        },
+      ],
+      results: [
+        {
+          id: "control-ui.browser-run",
+          status: "fail",
+          durationMs: 2300,
+          failureMessage: "locator timed out",
+        },
+      ],
+    });
+
+    expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
+    expect(evidence.profile).toBeUndefined();
+    expect(evidence.entries[0]).toMatchObject({
+      test: {
+        kind: "playwright-test",
+        id: "control-ui.browser-run",
+        title: "Control UI browser workflow",
+        source: {
+          path: "ui/control-ui.e2e.test.ts",
+        },
+      },
+      coverage: [
+        {
+          id: "ui.control",
+          role: "primary",
+        },
+      ],
+      refs: [
+        {
+          kind: "docs",
+          path: "docs/concepts/qa-e2e-automation.md",
+        },
+        {
+          kind: "code",
+          path: "ui/",
+        },
+      ],
+      execution: {
+        runner: "playwright",
+        artifacts: [
+          {
+            kind: "runner-result",
+            path: "playwright-results/control-ui.json",
+            source: "playwright",
+          },
+          {
+            kind: "report",
+            path: "playwright-report/index.html",
+            source: "playwright",
+          },
+        ],
+      },
+      result: {
+        status: "fail",
+        failure: {
+          reason: "locator timed out",
+        },
+        timing: {
+          wallMs: 2300,
+        },
+      },
+    });
+  });
+
+  it("carries profile env values without hardcoding taxonomy coverage ids", () => {
+    const evidence = buildQaSuiteEvidenceSummary({
+      artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }],
+      scenarioDefinitions: [
+        {
+          id: "dm-chat-baseline",
+          title: "DM baseline conversation",
+          surface: "dm",
+          coverage: {
+            primary: ["channels.dm"],
+          },
+        },
+      ],
+      channelId: "qa-channel",
+      env: {
+        OPENCLAW_QA_PROFILE: "experimental-profile",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-07T12:09:00.000Z",
+      primaryModel: "mock-openai/gpt-5.5",
+      providerMode: "mock-openai",
+      scenarioResults: [{ name: "DM baseline conversation", status: "pass" }],
+    });
+
+    expect(evidence.profile).toBe("experimental-profile");
+  });
+
+  it.each([
+    { evidenceMode: undefined, expectedMode: "slim", hasExecution: false },
+    { evidenceMode: "full" as const, expectedMode: "full", hasExecution: true },
+  ])(
+    "resolves profile evidence mode $expectedMode",
+    ({ evidenceMode, expectedMode, hasExecution }) => {
+      const evidence = buildQaSuiteEvidenceSummary({
+        artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }],
+        ...(evidenceMode ? { evidenceMode } : {}),
+        profile: "smoke-ci",
+        scenarioDefinitions: [
+          {
+            id: "dm-chat-baseline",
+            title: "DM baseline conversation",
+            coverage: {
+              primary: ["channels.dm"],
+            },
+          },
+        ],
+        channelId: "qa-channel",
+        generatedAt: "2026-06-07T12:09:00.000Z",
+        primaryModel: "mock-openai/gpt-5.5",
+        providerMode: "mock-openai",
+        scenarioResults: [{ name: "DM baseline conversation", status: "pass" }],
+      });
+
+      expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
+      expect(evidence.evidenceMode).toBe(expectedMode);
+      expect("execution" in evidence.entries[0]).toBe(hasExecution);
+    },
+  );
+
+  it("keeps mock non-OpenAI model refs attributed to their model provider", () => {
+    const evidence = buildQaSuiteEvidenceSummary({
+      artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }],
+      scenarioDefinitions: [
+        {
+          id: "anthropic-parity",
+          title: "Anthropic parity",
+          surface: "runtime",
+          coverage: {
+            primary: ["providers.anthropic"],
+          },
+        },
+      ],
+      channelId: "qa-channel",
+      generatedAt: "2026-06-07T12:10:00.000Z",
+      primaryModel: "anthropic/claude-opus-4-8",
+      providerMode: "mock-openai",
+      scenarioResults: [{ name: "Anthropic parity", status: "pass" }],
+    });
+
+    expect(evidence.entries[0]?.execution).toMatchObject({
+      provider: {
+        id: "anthropic",
+        model: {
+          name: "claude-opus-4-8",
+          ref: "anthropic/claude-opus-4-8",
+        },
+      },
+    });
+    expect(evidence.entries[0]).toMatchObject({
+      execution: {
+        provider: {
+          live: false,
+          fixture: "mock-openai",
+        },
+      },
+    });
+  });
+
+  it("uses explicit package provenance from package runners", () => {
+    const evidence = buildLiveTransportEvidenceSummary({
+      artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
+      generatedAt: "2026-06-07T12:15:00.000Z",
+      packageSource: {
+        kind: "packed-tarball",
+        spec: "/tmp/openclaw.tgz",
+        sha: "abc123",
+      },
+      primaryModel: "openai/gpt-5.5",
+      providerMode: "live-frontier",
+      checks: [
+        {
+          id: "telegram-canary",
+          title: "Telegram canary",
+          details: "Canary passed.",
+          coverageIds: ["channels.telegram.canary"],
+          status: "pass",
+        },
+      ],
+      transportId: "telegram",
+    });
+
+    expect(evidence.entries[0]?.execution?.packageSource).toEqual({
+      kind: "packed-tarball",
+      spec: "/tmp/openclaw.tgz",
+      sha: "abc123",
+    });
+  });
+
+  it("derives package provenance from generic QA evidence env", () => {
+    const evidence = buildLiveTransportEvidenceSummary({
+      artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
+      env: {
+        OPENCLAW_QA_PACKAGE_SOURCE: "openclaw@beta",
+        OPENCLAW_QA_PACKAGE_SOURCE_KIND: "npm-package",
+        OPENCLAW_QA_PACKAGE_SOURCE_SHA: "def456",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-07T12:15:00.000Z",
+      primaryModel: "openai/gpt-5.5",
+      providerMode: "live-frontier",
+      checks: [
+        {
+          id: "telegram-canary",
+          title: "Telegram canary",
+          details: "Canary passed.",
+          coverageIds: ["channels.telegram.canary"],
+          status: "pass",
+        },
+      ],
+      transportId: "telegram",
+    });
+
+    expect(evidence.entries[0]?.execution?.packageSource).toEqual({
+      kind: "npm-package",
+      spec: "openclaw@beta",
+      sha: "def456",
+    });
+  });
+
+  it("does not infer package provenance from runner-specific env", () => {
+    const evidence = buildLiveTransportEvidenceSummary({
+      artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
+      env: {
+        OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE: "openclaw@beta",
+      } as NodeJS.ProcessEnv,
+      generatedAt: "2026-06-07T12:16:00.000Z",
+      primaryModel: "openai/gpt-5.5",
+      providerMode: "live-frontier",
+      checks: [
+        {
+          id: "telegram-canary",
+          title: "Telegram canary",
+          details: "Canary passed.",
+          coverageIds: ["channels.telegram.canary"],
+          status: "pass",
+        },
+      ],
+      transportId: "telegram",
+    });
+
+    expect(evidence.entries[0]?.execution?.packageSource).toEqual({
+      kind: "source-checkout",
+      spec: undefined,
+      sha: undefined,
+    });
+  });
+
+  it("keeps live transport check artifacts on the owning entry", () => {
+    const evidence = buildLiveTransportEvidenceSummary({
+      artifactPaths: [
+        { kind: "summary", path: QA_EVIDENCE_FILENAME },
+        { kind: "report", path: "discord-qa-report.md" },
+      ],
+      generatedAt: "2026-06-07T12:20:00.000Z",
+      primaryModel: "openai/gpt-5.5",
+      providerMode: "live-frontier",
+      checks: [
+        {
+          artifactPaths: {
+            screenshot: ".artifacts/discord/status.png",
+            video: ".artifacts/discord/status.mp4",
+          },
+          id: "discord-status-reactions-tool-only",
+          title: "Discord status reactions",
+          details: "Status reaction observed.",
+          status: "pass",
+        },
+      ],
+      transportId: "discord",
+    });
+
+    expect(evidence.entries[0]?.execution?.artifacts).toEqual(
+      expect.arrayContaining([
+        {
+          kind: "screenshot",
+          path: ".artifacts/discord/status.png",
+          source: "discord-live-transport:discord-status-reactions-tool-only",
+        },
+        {
+          kind: "video",
+          path: ".artifacts/discord/status.mp4",
+          source: "discord-live-transport:discord-status-reactions-tool-only",
+        },
+      ]),
+    );
+  });
+});

extensions/qa-lab/src/evidence-summary.ts

@@ -0,0 +1,832 @@
+// Qa Lab plugin module implements QA evidence summary behavior.
+import { execFileSync } from "node:child_process";
+import { z } from "zod";
+import { splitQaModelRef } from "./model-selection.js";
+import { getQaProvider, type QaProviderMode } from "./providers/index.js";
+import {
+  qaScorecardEvidenceModeSchema,
+  readQaScorecardProfileOptions,
+  type QaScorecardEvidenceMode,
+} from "./scorecard-taxonomy.js";
+
+export const QA_EVIDENCE_SUMMARY_KIND = "openclaw.qa.evidence-summary";
+export const QA_EVIDENCE_FILENAME = "qa-evidence.json";
+// v2 was introduced on this PR series and has no stable external readers yet.
+// Keep the version while the pre-release evidence shape settles.
+export const QA_EVIDENCE_SUMMARY_SCHEMA_VERSION = 2;
+
+const qaEvidenceStatusSchema = z.enum(["pass", "fail", "blocked", "skipped"]);
+const nonEmptyStringSchema = z.string().trim().min(1);
+const nullableStringSchema = nonEmptyStringSchema.nullable();
+const qaEvidenceProfileIdSchema = nonEmptyStringSchema;
+const qaEvidenceIdSchema = z.object({ id: nonEmptyStringSchema });
+
+const qaEvidenceProviderSchema = z
+  .object({
+    id: nonEmptyStringSchema,
+    live: z.boolean(),
+    model: z
+      .object({
+        name: nullableStringSchema,
+        ref: nullableStringSchema,
+      })
+      .strict(),
+    fixture: nonEmptyStringSchema.optional(),
+    auth: nonEmptyStringSchema.optional(),
+  })
+  .strict();
+
+const qaEvidenceChannelSchema = z
+  .object({
+    id: nonEmptyStringSchema,
+    live: z.boolean(),
+    driver: nonEmptyStringSchema.optional(),
+  })
+  .strict();
+
+const qaEvidenceEnvironmentSchema = z
+  .object({
+    ref: nullableStringSchema,
+    os: nonEmptyStringSchema,
+    nodeVersion: nonEmptyStringSchema,
+  })
+  .strict();
+
+const qaEvidencePackageSourceSchema = z
+  .object({
+    kind: nonEmptyStringSchema,
+    spec: nonEmptyStringSchema.optional(),
+    sha: nonEmptyStringSchema.optional(),
+  })
+  .strict();
+
+const qaEvidenceFailureSchema = z
+  .object({
+    class: nonEmptyStringSchema.optional(),
+    reason: nonEmptyStringSchema,
+  })
+  .strict();
+
+const qaEvidenceTimingSchema = z
+  .object({
+    wallMs: z.number().finite().positive().optional(),
+    rttMs: z.number().finite().positive().optional(),
+    avgMs: z.number().finite().positive().optional(),
+    p50Ms: z.number().finite().positive().optional(),
+    p95Ms: z.number().finite().positive().optional(),
+    maxMs: z.number().finite().positive().optional(),
+    samples: z.number().int().positive().optional(),
+    failedSamples: z.number().int().nonnegative().optional(),
+  })
+  .strict();
+
+const qaEvidenceTestSchema = z
+  .object({
+    kind: nonEmptyStringSchema,
+    id: nonEmptyStringSchema,
+    title: nonEmptyStringSchema,
+    source: z
+      .object({
+        path: nonEmptyStringSchema,
+      })
+      .strict()
+      .optional(),
+  })
+  .strict();
+
+const qaEvidenceRefSchema = z
+  .object({
+    kind: nonEmptyStringSchema,
+    path: nonEmptyStringSchema,
+  })
+  .strict();
+
+const qaEvidenceCoverageSchema = qaEvidenceIdSchema
+  .extend({
+    role: nonEmptyStringSchema,
+  })
+  .strict();
+
+const qaEvidenceScorecardCountSchema = z
+  .object({
+    total: z.number().int().nonnegative(),
+    fulfilled: z.number().int().nonnegative(),
+    partial: z.number().int().nonnegative().optional(),
+    missing: z.number().int().nonnegative(),
+    fulfillmentPercent: z.number().finite().nonnegative(),
+  })
+  .strict();
+
+const qaEvidenceScorecardCoverageCountSchema = qaEvidenceScorecardCountSchema.extend({
+  secondaryOnly: z.number().int().nonnegative(),
+});
+
+const qaEvidenceScorecardCategorySchema = z
+  .object({
+    id: nonEmptyStringSchema,
+    surfaceId: nonEmptyStringSchema,
+    name: nonEmptyStringSchema,
+    status: z.enum(["fulfilled", "partial", "missing"]),
+    features: qaEvidenceScorecardCountSchema,
+    coverageIds: qaEvidenceScorecardCoverageCountSchema,
+    missingCoverageIds: z.array(nonEmptyStringSchema),
+  })
+  .strict();
+
+const qaEvidenceScorecardSchema = z
+  .object({
+    filters: z
+      .object({
+        surface: nullableStringSchema,
+        category: nullableStringSchema,
+      })
+      .strict(),
+    run: z
+      .object({
+        evidenceEntryCount: z.number().int().nonnegative(),
+      })
+      .strict(),
+    categories: qaEvidenceScorecardCountSchema,
+    features: qaEvidenceScorecardCountSchema,
+    coverageIds: qaEvidenceScorecardCountSchema,
+    categoryReports: z.array(qaEvidenceScorecardCategorySchema),
+  })
+  .strict();
+
+const qaEvidenceArtifactSchema = z
+  .object({
+    kind: nonEmptyStringSchema,
+    path: nonEmptyStringSchema,
+    source: nonEmptyStringSchema,
+  })
+  .strict();
+
+const qaEvidenceExecutionSchema = z
+  .object({
+    runner: nonEmptyStringSchema,
+    environment: qaEvidenceEnvironmentSchema,
+    provider: qaEvidenceProviderSchema,
+    channel: qaEvidenceChannelSchema.optional(),
+    packageSource: qaEvidencePackageSourceSchema,
+    artifacts: z.array(qaEvidenceArtifactSchema),
+  })
+  .strict();
+
+const qaEvidenceResultSchema = z
+  .object({
+    status: qaEvidenceStatusSchema,
+    failure: qaEvidenceFailureSchema.optional(),
+    timing: qaEvidenceTimingSchema.optional(),
+  })
+  .strict();
+
+const qaEvidencePostureSchema = z.enum(["direct-gateway", "native-approval", "user-path"]);
+
+export const qaEvidenceSummaryEntrySchema = z
+  .object({
+    test: qaEvidenceTestSchema,
+    coverage: z.array(qaEvidenceCoverageSchema),
+    posture: qaEvidencePostureSchema.optional(),
+    refs: z.array(qaEvidenceRefSchema).optional(),
+    runtimeParityTier: nonEmptyStringSchema.optional(),
+    execution: qaEvidenceExecutionSchema.optional(),
+    result: qaEvidenceResultSchema,
+  })
+  .strict();
+
+export const qaEvidenceSummarySchema = z
+  .object({
+    kind: z.literal(QA_EVIDENCE_SUMMARY_KIND),
+    schemaVersion: z.literal(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION),
+    generatedAt: nonEmptyStringSchema,
+    evidenceMode: qaScorecardEvidenceModeSchema,
+    entries: z.array(qaEvidenceSummaryEntrySchema),
+    profile: qaEvidenceProfileIdSchema.optional(),
+    scorecard: qaEvidenceScorecardSchema.optional(),
+  })
+  .strict();
+
+export type QaEvidenceProfile = z.infer<typeof qaEvidenceProfileIdSchema>;
+export type QaEvidenceStatus = z.infer<typeof qaEvidenceStatusSchema>;
+export type QaEvidenceTiming = z.infer<typeof qaEvidenceTimingSchema>;
+export type QaEvidencePackageSource = z.infer<typeof qaEvidencePackageSourceSchema>;
+export type QaEvidenceScorecardJson = z.infer<typeof qaEvidenceScorecardSchema>;
+export type QaEvidenceSummaryEntry = z.infer<typeof qaEvidenceSummaryEntrySchema>;
+export type QaEvidenceSummaryJson = z.infer<typeof qaEvidenceSummarySchema>;
+
+type QaEvidenceStatusInput = QaEvidenceStatus | "skip";
+
+type QaEvidenceScenarioDefinitionInput = {
+  id: string;
+  title: string;
+  sourcePath?: string;
+  surface?: string;
+  surfaces?: readonly string[];
+  category?: string;
+  coverage?: {
+    primary?: readonly string[];
+    secondary?: readonly string[];
+  };
+  runtimeParityTier?: string;
+  docsRefs?: readonly string[];
+  codeRefs?: readonly string[];
+};
+
+type QaEvidenceScenarioResultInput = {
+  name: string;
+  status: QaEvidenceStatusInput;
+  details?: string;
+  timing?: QaEvidenceTiming;
+  rttMs?: number;
+  rttMeasurement?: {
+    finalMatchedReplyRttMs?: number;
+  };
+};
+
+type QaEvidenceLiveTransportCheckInput = {
+  id: string;
+  title: string;
+  status: QaEvidenceStatusInput;
+  details: string;
+  posture?: z.infer<typeof qaEvidencePostureSchema>;
+  timing?: QaEvidenceTiming;
+  rttMs?: number;
+  rttMeasurement?: {
+    finalMatchedReplyRttMs?: number;
+  };
+  coverageIds?: readonly string[];
+  artifactPaths?: Readonly<Record<string, string>>;
+};
+
+type QaEvidenceRttInput = Pick<
+  QaEvidenceScenarioResultInput,
+  "rttMeasurement" | "rttMs" | "timing"
+>;
+
+type QaEvidenceTestTargetInput = {
+  id: string;
+  title: string;
+  sourcePath: string;
+  primaryCoverageIds?: readonly string[];
+  secondaryCoverageIds?: readonly string[];
+  docsRefs?: readonly string[];
+  codeRefs?: readonly string[];
+};
+
+type QaEvidenceTestResultInput = {
+  id?: string;
+  title?: string;
+  sourcePath?: string;
+  status: QaEvidenceStatusInput;
+  durationMs?: number;
+  failureMessage?: string;
+};
+
+type QaEvidenceArtifactInput = {
+  kind: string;
+  path: string;
+};
+
+type QaEvidenceBuildBase = {
+  artifactPaths: readonly QaEvidenceArtifactInput[];
+  evidenceMode?: QaScorecardEvidenceMode;
+  env?: NodeJS.ProcessEnv;
+  generatedAt: string;
+  primaryModel: string;
+  providerMode: QaProviderMode;
+  channelDriver?: string;
+  packageSource?: QaEvidencePackageSource;
+  profile?: QaEvidenceProfile;
+  repoRoot?: string;
+  runner?: string;
+};
+
+function buildQaEvidenceRefs(params: {
+  docsRefs?: readonly string[];
+  codeRefs?: readonly string[];
+}) {
+  const buildRef = (kind: "docs" | "code", refPath: string) => {
+    const ref = {
+      kind,
+      path: refPath,
+    };
+    return ref;
+  };
+  const refs = [
+    ...(params.docsRefs ?? []).map((path) => buildRef("docs", path)),
+    ...(params.codeRefs ?? []).map((path) => buildRef("code", path)),
+  ];
+  return [...new Map(refs.map((ref) => [`${ref.kind}:${ref.path}`, ref])).values()];
+}
+
+function buildQaEvidenceCoverage(params: {
+  primaryCoverageIds?: readonly string[];
+  secondaryCoverageIds?: readonly string[];
+}) {
+  const buildCoverage = (id: string, role: "primary" | "secondary") => ({
+    id,
+    role,
+  });
+  return [
+    ...uniqueSortedStrings(params.primaryCoverageIds ?? []).map((id) =>
+      buildCoverage(id, "primary"),
+    ),
+    ...uniqueSortedStrings(params.secondaryCoverageIds ?? []).map((id) =>
+      buildCoverage(id, "secondary"),
+    ),
+  ];
+}
+
+function buildQaEvidenceArtifacts(paths: readonly QaEvidenceArtifactInput[], source: string) {
+  return paths.map((artifact) => ({
+    kind: artifact.kind,
+    path: artifact.path,
+    source,
+  }));
+}
+
+function buildQaEvidenceNamedArtifacts(paths: Readonly<Record<string, string>>, source: string) {
+  return Object.entries(paths).map(([kind, artifactPath]) => ({
+    kind,
+    path: artifactPath,
+    source,
+  }));
+}
+
+function uniqueSortedStrings(values: readonly (string | undefined)[]) {
+  return [...new Set(values.map((value) => value?.trim()).filter(Boolean) as string[])].toSorted(
+    (left, right) => left.localeCompare(right),
+  );
+}
+
+export function resolveQaEvidenceProfile(params: {
+  env?: NodeJS.ProcessEnv;
+  explicit?: QaEvidenceProfile;
+}) {
+  if (params.explicit) {
+    const explicit = params.explicit.trim();
+    if (!explicit) {
+      throw new Error("evidence profile must be a non-empty string.");
+    }
+    return explicit;
+  }
+
+  const envProfiles = [
+    ["OPENCLAW_E2E_PROFILE", params.env?.OPENCLAW_E2E_PROFILE],
+    ["OPENCLAW_QA_PROFILE", params.env?.OPENCLAW_QA_PROFILE],
+  ] as const;
+  for (const [, value] of envProfiles) {
+    const normalized = value?.trim();
+    if (!normalized) {
+      continue;
+    }
+    return normalized;
+  }
+
+  return undefined;
+}
+
+function resolveQaEvidenceRunner(params: { env?: NodeJS.ProcessEnv; fallback?: string }) {
+  return params.env?.OPENCLAW_QA_RUNNER?.trim() || params.fallback || "host";
+}
+
+function resolveQaEvidenceChannelDriver(params: { env?: NodeJS.ProcessEnv; fallback?: string }) {
+  const id =
+    params.fallback?.trim() ||
+    params.env?.OPENCLAW_QA_CHANNEL_DRIVER?.trim() ||
+    params.env?.OPENCLAW_E2E_CHANNEL_DRIVER?.trim();
+  return id ? { id } : undefined;
+}
+
+function resolveQaEvidenceCheckoutRef(repoRoot?: string) {
+  try {
+    const ref = execFileSync("git", ["rev-parse", "--verify", "HEAD"], {
+      cwd: repoRoot ?? process.cwd(),
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    }).trim();
+    return ref || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export function resolveQaEvidenceEnvironment(params: {
+  env?: NodeJS.ProcessEnv;
+  repoRoot?: string;
+}) {
+  return {
+    // GitHub's GITHUB_SHA describes the workflow event, not necessarily the
+    // checked-out ref selected by a manual or remote QA run.
+    ref:
+      params.env?.OPENCLAW_QA_REF?.trim() ||
+      resolveQaEvidenceCheckoutRef(params.repoRoot) ||
+      params.env?.GITHUB_SHA?.trim() ||
+      null,
+    os: process.platform,
+    nodeVersion: process.version,
+  };
+}
+
+function resolveQaEvidencePackageSource(env: NodeJS.ProcessEnv | undefined) {
+  const spec = env?.OPENCLAW_QA_PACKAGE_SOURCE?.trim() || undefined;
+  const sha = env?.OPENCLAW_QA_PACKAGE_SOURCE_SHA?.trim() || undefined;
+  const explicitKind = env?.OPENCLAW_QA_PACKAGE_SOURCE_KIND?.trim();
+  const kind =
+    explicitKind ||
+    (spec && spec.endsWith(".tgz") ? "packed-tarball" : spec ? "npm-package" : "source-checkout");
+  return {
+    kind,
+    spec,
+    sha,
+  };
+}
+
+function resolveQaEvidenceBuildPackageSource(params: QaEvidenceBuildBase) {
+  return params.packageSource ?? resolveQaEvidencePackageSource(params.env);
+}
+
+function buildQaEvidenceProvider(params: { providerMode: QaProviderMode; primaryModel: string }) {
+  const provider = getQaProvider(params.providerMode);
+  const split = splitQaModelRef(params.primaryModel);
+  const providerShape = {
+    id: split?.provider ?? params.providerMode,
+    model: {
+      name: split?.model ?? null,
+      ref: params.primaryModel || null,
+    },
+  };
+  if (provider.kind === "live") {
+    return {
+      ...providerShape,
+      live: true,
+      auth: params.providerMode,
+    };
+  }
+  const mockProviderId =
+    split?.provider && split.provider !== params.providerMode
+      ? split.provider
+      : params.providerMode === "mock-openai"
+        ? "openai"
+        : (split?.provider ?? params.providerMode);
+  return {
+    ...providerShape,
+    id: mockProviderId,
+    live: false,
+    fixture: params.providerMode,
+  };
+}
+
+function normalizeQaEvidenceStatus(status: QaEvidenceStatusInput): QaEvidenceStatus {
+  return status === "skip" ? "skipped" : status;
+}
+
+function failureForResult(result: {
+  details?: string;
+  failureMessage?: string;
+  status: QaEvidenceStatusInput;
+}) {
+  const status = normalizeQaEvidenceStatus(result.status);
+  if (status === "pass") {
+    return undefined;
+  }
+  return {
+    reason: result.details?.trim() || result.failureMessage?.trim() || `${status} test`,
+  };
+}
+
+function timingForRttResult(check: QaEvidenceRttInput) {
+  const timing: QaEvidenceTiming = { ...check.timing };
+  const rttMs = check.rttMeasurement?.finalMatchedReplyRttMs ?? check.rttMs;
+  if (
+    timing.rttMs === undefined &&
+    typeof rttMs === "number" &&
+    Number.isFinite(rttMs) &&
+    rttMs > 0
+  ) {
+    timing.rttMs = rttMs;
+  }
+  return Object.keys(timing).length > 0 ? timing : undefined;
+}
+
+function timingForTestResult(result: QaEvidenceTestResultInput) {
+  return typeof result.durationMs === "number" &&
+    Number.isFinite(result.durationMs) &&
+    result.durationMs > 0
+    ? { wallMs: result.durationMs }
+    : undefined;
+}
+
+function resultForEvidence(
+  result: { details?: string; failureMessage?: string; status: QaEvidenceStatusInput },
+  timing?: QaEvidenceTiming,
+) {
+  return {
+    status: normalizeQaEvidenceStatus(result.status),
+    failure: failureForResult(result),
+    timing,
+  };
+}
+
+function buildQaEvidenceSummary(params: {
+  entries: QaEvidenceSummaryEntry[];
+  evidenceMode?: QaScorecardEvidenceMode;
+  generatedAt: string;
+  profile?: QaEvidenceProfile;
+  scorecard?: QaEvidenceScorecardJson;
+}): QaEvidenceSummaryJson {
+  const profileOptions = readQaScorecardProfileOptions(params.profile);
+  const evidenceMode = params.evidenceMode ?? profileOptions.evidenceMode;
+  const entries =
+    evidenceMode === "slim"
+      ? params.entries.map((entry) => {
+          const { execution: _execution, ...withoutExecution } = entry;
+          return withoutExecution;
+        })
+      : params.entries;
+  return qaEvidenceSummarySchema.parse({
+    kind: QA_EVIDENCE_SUMMARY_KIND,
+    schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
+    generatedAt: params.generatedAt,
+    evidenceMode,
+    entries,
+    profile: params.profile,
+    scorecard: params.scorecard,
+  });
+}
+
+export function validateQaEvidenceSummaryJson(summary: unknown): QaEvidenceSummaryJson {
+  return qaEvidenceSummarySchema.parse(summary);
+}
+
+export function attachQaEvidenceScorecard(params: {
+  evidenceMode?: QaScorecardEvidenceMode;
+  summary: QaEvidenceSummaryJson;
+  profile: QaEvidenceProfile;
+  scorecard: QaEvidenceScorecardJson;
+}): QaEvidenceSummaryJson {
+  return buildQaEvidenceSummary({
+    entries: params.summary.entries,
+    evidenceMode: params.evidenceMode,
+    generatedAt: params.summary.generatedAt,
+    profile: params.profile,
+    scorecard: params.scorecard,
+  });
+}
+
+export function buildQaSuiteEvidenceSummary(
+  params: QaEvidenceBuildBase & {
+    channelId: string;
+    scenarioDefinitions: readonly QaEvidenceScenarioDefinitionInput[];
+    scenarioResults: readonly QaEvidenceScenarioResultInput[];
+  },
+): QaEvidenceSummaryJson {
+  const provider = buildQaEvidenceProvider(params);
+  const environment = resolveQaEvidenceEnvironment({
+    env: params.env,
+    repoRoot: params.repoRoot,
+  });
+  const packageSource = resolveQaEvidenceBuildPackageSource(params);
+  const runner = resolveQaEvidenceRunner({ env: params.env, fallback: params.runner });
+  const profile = resolveQaEvidenceProfile({
+    env: params.env,
+    explicit: params.profile,
+  });
+  const channelDriver = resolveQaEvidenceChannelDriver({
+    env: params.env,
+    fallback: params.channelDriver,
+  });
+  const entries = params.scenarioResults.map((result, index): QaEvidenceSummaryEntry => {
+    const scenario = params.scenarioDefinitions[index];
+    const primaryCoverageIds = uniqueSortedStrings(scenario?.coverage?.primary ?? []);
+    const coverageIds = uniqueSortedStrings([
+      ...(scenario?.coverage?.primary ?? []),
+      ...(scenario?.coverage?.secondary ?? []),
+    ]);
+    const runtimeParityTier = scenario?.runtimeParityTier;
+    const testId = scenario?.id ?? `scenario-${index + 1}`;
+    const refs = buildQaEvidenceRefs({
+      docsRefs: scenario?.docsRefs,
+      codeRefs: scenario?.codeRefs,
+    });
+    const timing = timingForRttResult(result);
+    return {
+      test: {
+        kind: "qa-scenario",
+        id: testId,
+        title: scenario?.title ?? result.name,
+        source: scenario?.sourcePath ? { path: scenario.sourcePath } : undefined,
+      },
+      coverage: buildQaEvidenceCoverage({
+        primaryCoverageIds,
+        secondaryCoverageIds: coverageIds.filter(
+          (coverageId) => !primaryCoverageIds.includes(coverageId),
+        ),
+      }),
+      refs: refs.length > 0 ? refs : undefined,
+      runtimeParityTier,
+      execution: {
+        runner,
+        environment,
+        provider,
+        channel: {
+          id: params.channelId,
+          live: false,
+          driver: channelDriver?.id,
+        },
+        packageSource,
+        artifacts: buildQaEvidenceArtifacts(params.artifactPaths, "qa-suite"),
+      },
+      result: resultForEvidence(result, timing),
+    };
+  });
+  return buildQaEvidenceSummary({
+    entries,
+    evidenceMode: params.evidenceMode,
+    generatedAt: params.generatedAt,
+    profile,
+  });
+}
+
+function buildTestRunnerEvidenceSummary(
+  params: QaEvidenceBuildBase & {
+    defaultRunner: string;
+    testKind: string;
+    targets: readonly QaEvidenceTestTargetInput[];
+    results: readonly QaEvidenceTestResultInput[];
+  },
+): QaEvidenceSummaryJson {
+  const provider = buildQaEvidenceProvider(params);
+  const environment = resolveQaEvidenceEnvironment({
+    env: params.env,
+    repoRoot: params.repoRoot,
+  });
+  const packageSource = resolveQaEvidenceBuildPackageSource(params);
+  const runner = resolveQaEvidenceRunner({
+    env: params.env,
+    fallback: params.runner ?? params.defaultRunner,
+  });
+  const profile = resolveQaEvidenceProfile({
+    env: params.env,
+    explicit: params.profile,
+  });
+  const targetById = new Map(params.targets.map((target) => [target.id, target]));
+  const targetByPath = new Map(params.targets.map((target) => [target.sourcePath, target]));
+  const entries = params.results.map((result, index): QaEvidenceSummaryEntry => {
+    const target = result.id
+      ? targetById.get(result.id)
+      : result.sourcePath
+        ? targetByPath.get(result.sourcePath)
+        : undefined;
+    const fallbackId = result.id ?? result.sourcePath ?? `test-${index + 1}`;
+    const sourcePath = target?.sourcePath ?? result.sourcePath;
+    const refs = buildQaEvidenceRefs({
+      docsRefs: target?.docsRefs,
+      codeRefs: target?.codeRefs,
+    });
+    const timing = timingForTestResult(result);
+    return {
+      test: {
+        kind: params.testKind,
+        id: target?.id ?? fallbackId,
+        title: target?.title ?? result.title ?? fallbackId,
+        source: sourcePath ? { path: sourcePath } : undefined,
+      },
+      coverage: buildQaEvidenceCoverage({
+        primaryCoverageIds: target?.primaryCoverageIds ?? [],
+        secondaryCoverageIds: target?.secondaryCoverageIds ?? [],
+      }),
+      refs: refs.length > 0 ? refs : undefined,
+      execution: {
+        runner,
+        environment,
+        provider,
+        packageSource,
+        artifacts: buildQaEvidenceArtifacts(params.artifactPaths, runner),
+      },
+      result: resultForEvidence(result, timing),
+    };
+  });
+  return buildQaEvidenceSummary({
+    entries,
+    evidenceMode: params.evidenceMode,
+    generatedAt: params.generatedAt,
+    profile,
+  });
+}
+
+export function buildVitestEvidenceSummary(
+  params: QaEvidenceBuildBase & {
+    targets: readonly QaEvidenceTestTargetInput[];
+    results: readonly QaEvidenceTestResultInput[];
+  },
+): QaEvidenceSummaryJson {
+  return buildTestRunnerEvidenceSummary({
+    ...params,
+    defaultRunner: "vitest",
+    testKind: "vitest-test",
+    runner: params.runner ?? "vitest",
+  });
+}
+
+export function buildPlaywrightEvidenceSummary(
+  params: QaEvidenceBuildBase & {
+    targets: readonly QaEvidenceTestTargetInput[];
+    results: readonly QaEvidenceTestResultInput[];
+  },
+): QaEvidenceSummaryJson {
+  return buildTestRunnerEvidenceSummary({
+    ...params,
+    defaultRunner: "playwright",
+    testKind: "playwright-test",
+    runner: params.runner ?? "playwright",
+  });
+}
+
+export function buildScriptEvidenceSummary(
+  params: QaEvidenceBuildBase & {
+    targets: readonly QaEvidenceTestTargetInput[];
+    results: readonly QaEvidenceTestResultInput[];
+  },
+): QaEvidenceSummaryJson {
+  return buildTestRunnerEvidenceSummary({
+    ...params,
+    defaultRunner: "script",
+    testKind: "script-test",
+    runner: params.runner ?? "script",
+  });
+}
+
+export function buildLiveTransportEvidenceSummary(
+  params: QaEvidenceBuildBase & {
+    checks: readonly QaEvidenceLiveTransportCheckInput[];
+    transportId: string;
+  },
+): QaEvidenceSummaryJson {
+  const provider = buildQaEvidenceProvider(params);
+  const environment = resolveQaEvidenceEnvironment({
+    env: params.env,
+    repoRoot: params.repoRoot,
+  });
+  const packageSource = resolveQaEvidenceBuildPackageSource(params);
+  const runner = resolveQaEvidenceRunner({ env: params.env, fallback: params.runner });
+  const profile = resolveQaEvidenceProfile({
+    env: params.env,
+    explicit: params.profile,
+  });
+  const channelDriver = resolveQaEvidenceChannelDriver({
+    env: params.env,
+    fallback: params.channelDriver ?? "native",
+  }) ?? { id: "native" };
+  const entries = params.checks.map((check): QaEvidenceSummaryEntry => {
+    const testId = check.id;
+    const liveCoverageId = `channels.${params.transportId}.live`;
+    const coverage = [
+      {
+        id: liveCoverageId,
+        role: "live-transport",
+      },
+      ...uniqueSortedStrings(check.coverageIds ?? [])
+        .filter((coverageId) => coverageId !== liveCoverageId)
+        .map((coverageId) => ({
+          id: coverageId,
+          role: "live-transport-coverage",
+        })),
+    ];
+    const timing = timingForRttResult(check);
+    return {
+      test: {
+        kind: "live-transport-check",
+        id: testId,
+        title: check.title,
+      },
+      coverage,
+      posture: check.posture,
+      execution: {
+        runner,
+        environment,
+        provider,
+        channel: {
+          id: params.transportId,
+          live: true,
+          driver: channelDriver.id,
+        },
+        packageSource,
+        artifacts: [
+          ...buildQaEvidenceArtifacts(params.artifactPaths, `${params.transportId}-live-transport`),
+          ...buildQaEvidenceNamedArtifacts(
+            check.artifactPaths ?? {},
+            `${params.transportId}-live-transport:${testId}`,
+          ),
+        ],
+      },
+      result: resultForEvidence(check, timing),
+    };
+  });
+  return buildQaEvidenceSummary({
+    entries,
+    evidenceMode: params.evidenceMode,
+    generatedAt: params.generatedAt,
+    profile,
+  });
+}

extensions/qa-lab/src/extract-tool-payload.ts

@@ -0,0 +1,2 @@
+// Qa Lab plugin module implements extract tool payload behavior.
+export { extractToolPayload as extractQaToolPayload } from "openclaw/plugin-sdk/tool-payload";

extensions/qa-lab/src/fixture-utils.ts

@@ -0,0 +1,321 @@
+// Qa Lab plugin module provides reusable fixture utilities.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { clearTimeout as clearNodeTimeout, setTimeout as setNodeTimeout } from "node:timers";
+
+export type QaFixtureFetchJsonOptions = {
+  fetchImpl?: (url: string, init: RequestInit) => Promise<Response>;
+  maxBodyBytes?: number;
+  timeoutMs?: number;
+};
+
+const DEFAULT_FETCH_BODY_MAX_BYTES = 1024 * 1024;
+const DEFAULT_FETCH_TIMEOUT_MS = 5_000;
+
+export function readPositiveIntEnv(name: string, fallback: number, env: NodeJS.ProcessEnv) {
+  const raw = env[name] ?? fallback;
+  const text = raw == null ? "unset" : String(raw).trim();
+  if (!/^\d+$/u.test(text)) {
+    throw new Error(`invalid ${name}: ${text}`);
+  }
+  const value = Number(text);
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`invalid ${name}: ${text}`);
+  }
+  return value;
+}
+
+function timeoutError(message: string) {
+  return Object.assign(new Error(message), { code: "ETIMEDOUT" });
+}
+
+function bodyTooLargeErrorMessage(url: string, byteLimit: number) {
+  return `HTTP response from ${url} exceeded ${byteLimit} bytes`;
+}
+
+function cancelReaderSoon(reader: ReadableStreamDefaultReader<Uint8Array>) {
+  void Promise.resolve()
+    .then(() => reader.cancel())
+    .catch(() => undefined);
+}
+
+async function readBoundedResponseText(params: {
+  response: Response;
+  url: string;
+  maxBytes: number;
+  timeoutPromise: Promise<never>;
+  signal: AbortSignal;
+}) {
+  const tooLargeError = () =>
+    Object.assign(new Error(bodyTooLargeErrorMessage(params.url, params.maxBytes)), {
+      code: "ETOOBIG",
+    });
+  const contentLength = params.response.headers.get("content-length");
+  if (contentLength && /^\d+$/u.test(contentLength) && Number(contentLength) > params.maxBytes) {
+    await params.response.body?.cancel().catch(() => undefined);
+    throw tooLargeError();
+  }
+  if (!params.response.body) {
+    return "";
+  }
+
+  const reader = params.response.body.getReader();
+  const decoder = new TextDecoder();
+  const chunks: string[] = [];
+  let totalBytes = 0;
+  let canceled = false;
+  try {
+    for (;;) {
+      const readPromise = reader.read();
+      let removeAbortListener: (() => void) | undefined;
+      const abortPromise = new Promise<never>((_resolve, reject) => {
+        const onAbort = () => {
+          canceled = true;
+          cancelReaderSoon(reader);
+          reject(
+            params.signal.reason instanceof Error
+              ? params.signal.reason
+              : new Error(`HTTP request to ${params.url} aborted`),
+          );
+        };
+        params.signal.addEventListener("abort", onAbort, { once: true });
+        removeAbortListener = () => params.signal.removeEventListener("abort", onAbort);
+      });
+      const { done, value } = await Promise.race([
+        readPromise,
+        abortPromise,
+        params.timeoutPromise,
+      ]).finally(() => removeAbortListener?.());
+      if (done) {
+        const tail = decoder.decode();
+        if (tail) {
+          chunks.push(tail);
+        }
+        break;
+      }
+      totalBytes += value.byteLength;
+      if (totalBytes > params.maxBytes) {
+        canceled = true;
+        await reader.cancel().catch(() => undefined);
+        throw tooLargeError();
+      }
+      chunks.push(decoder.decode(value, { stream: true }));
+    }
+  } finally {
+    if (!canceled) {
+      reader.releaseLock();
+    }
+  }
+  return chunks.join("");
+}
+
+export async function fetchQaFixtureJson(
+  url: string,
+  init: RequestInit = {},
+  options: QaFixtureFetchJsonOptions = {},
+): Promise<unknown> {
+  const timeoutMs = Math.max(1, options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS);
+  const maxBodyBytes = Math.max(1, options.maxBodyBytes ?? DEFAULT_FETCH_BODY_MAX_BYTES);
+  const controller = new AbortController();
+  const error = timeoutError(`HTTP request to ${url} timed out after ${timeoutMs}ms`);
+  let timeout: ReturnType<typeof setNodeTimeout> | undefined;
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeout = setNodeTimeout(() => {
+      controller.abort(error);
+      reject(error);
+    }, timeoutMs);
+  });
+
+  let response: Response;
+  let text: string;
+  try {
+    response = await Promise.race([
+      (options.fetchImpl ?? fetch)(url, {
+        ...init,
+        signal: controller.signal,
+      }),
+      timeoutPromise,
+    ]);
+    text = await readBoundedResponseText({
+      response,
+      url,
+      maxBytes: maxBodyBytes,
+      timeoutPromise,
+      signal: controller.signal,
+    });
+  } finally {
+    if (timeout) {
+      clearNodeTimeout(timeout);
+    }
+  }
+  let parsed: unknown;
+  try {
+    parsed = text ? JSON.parse(text) : {};
+  } catch {
+    parsed = text;
+  }
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} from ${url}: ${text}`);
+  }
+  return parsed;
+}
+
+export function outputToolNames(response: unknown): string[] {
+  const output = (response as { output?: Array<{ type?: unknown; name?: unknown }> }).output;
+  if (!Array.isArray(output)) {
+    return [];
+  }
+  return output
+    .filter((item) => item.type === "function_call" && typeof item.name === "string")
+    .map((item) => item.name as string);
+}
+
+export function outputText(response: unknown): string {
+  const output = (response as { output?: Array<{ type?: unknown; content?: unknown }> }).output;
+  if (!Array.isArray(output)) {
+    return "";
+  }
+  return output
+    .flatMap((item) => {
+      if (item.type !== "message" || !Array.isArray(item.content)) {
+        return [];
+      }
+      return item.content.flatMap((piece) => {
+        if (!piece || typeof piece !== "object") {
+          return [];
+        }
+        const record = piece as { text?: unknown };
+        return typeof record.text === "string" ? [record.text] : [];
+      });
+    })
+    .join("\n");
+}
+
+function readContentText(content: unknown): string {
+  if (typeof content === "string") {
+    return content;
+  }
+  if (!Array.isArray(content)) {
+    return "";
+  }
+  return content
+    .map((item) => {
+      if (!item || typeof item !== "object") {
+        return "";
+      }
+      const record = item as { type?: unknown; text?: unknown };
+      return typeof record.text === "string" ? record.text : "";
+    })
+    .join("\n");
+}
+
+export function countSystemPromptChars(body: unknown): number {
+  if (!body || typeof body !== "object") {
+    return 0;
+  }
+  const record = body as { instructions?: unknown; input?: unknown };
+  let total = typeof record.instructions === "string" ? record.instructions.length : 0;
+  if (Array.isArray(record.input)) {
+    for (const item of record.input) {
+      if (!item || typeof item !== "object") {
+        continue;
+      }
+      const inputRecord = item as { role?: unknown; content?: unknown };
+      if (inputRecord.role === "system" || inputRecord.role === "developer") {
+        total += readContentText(inputRecord.content).length;
+      }
+    }
+  }
+  return total;
+}
+
+function countOccurrences(haystack: string, needle: string): number {
+  if (!needle) {
+    return 0;
+  }
+  let count = 0;
+  let offset = 0;
+  for (;;) {
+    const next = haystack.indexOf(needle, offset);
+    if (next < 0) {
+      return count;
+    }
+    count += 1;
+    offset = next + needle.length;
+  }
+}
+
+function createCounts(needles: Record<string, string>): Record<string, number> {
+  return Object.fromEntries(Object.keys(needles).map((key) => [key, 0]));
+}
+
+function recordRole(record: unknown): string | undefined {
+  if (!record || typeof record !== "object") {
+    return undefined;
+  }
+  const candidate = record as { message?: unknown; role?: unknown };
+  if (typeof candidate.role === "string") {
+    return candidate.role;
+  }
+  if (!candidate.message || typeof candidate.message !== "object") {
+    return undefined;
+  }
+  const message = candidate.message as { role?: unknown };
+  return typeof message.role === "string" ? message.role : undefined;
+}
+
+function shouldScanSessionLogLine(line: string): boolean {
+  const trimmed = line.trim();
+  if (!trimmed) {
+    return false;
+  }
+  try {
+    return recordRole(JSON.parse(trimmed)) !== "user";
+  } catch {
+    return true;
+  }
+}
+
+async function countNeedlesInFile(filePath: string, needles: Record<string, string>) {
+  const text = await fs.readFile(filePath, "utf8").catch(() => "");
+  const counts = createCounts(needles);
+  for (const line of text.split(/\r?\n/u)) {
+    if (!shouldScanSessionLogLine(line)) {
+      continue;
+    }
+    for (const [key, needle] of Object.entries(needles)) {
+      counts[key] += countOccurrences(line, needle);
+    }
+  }
+  return counts;
+}
+
+export async function countSessionLogMentions(params: {
+  sessionsDir: string;
+  needles: Record<string, string>;
+}): Promise<Record<string, number>> {
+  const counts = createCounts(params.needles);
+  const files = await fs.readdir(params.sessionsDir, { recursive: true }).catch(() => []);
+  for (const file of files) {
+    if (typeof file !== "string" || !file.endsWith(".jsonl")) {
+      continue;
+    }
+    const fileCounts = await countNeedlesInFile(
+      path.join(params.sessionsDir, file),
+      params.needles,
+    );
+    for (const [key, count] of Object.entries(fileCounts)) {
+      counts[key] = (counts[key] ?? 0) + count;
+    }
+  }
+  return counts;
+}
+
+export function subtractMentionCounts(
+  after: Record<string, number>,
+  before: Record<string, number>,
+): Record<string, number> {
+  return Object.fromEntries(
+    Object.entries(after).map(([key, count]) => [key, count - (before[key] ?? 0)]),
+  );
+}

extensions/qa-lab/src/gateway-log-redaction.ts

@@ -0,0 +1,109 @@
+// Qa Lab plugin module implements gateway log redaction behavior.
+import { escapeRegExp } from "openclaw/plugin-sdk/text-utility-runtime";
+import {
+  QA_PROVIDER_SECRET_ENV_KEY_PATTERNS,
+  QA_PROVIDER_SECRET_ENV_VARS,
+} from "./providers/env.js";
+
+const QA_GATEWAY_DEBUG_SECRET_ENV_VARS = Object.freeze([
+  ...QA_PROVIDER_SECRET_ENV_VARS,
+  "OPENCLAW_GATEWAY_TOKEN",
+]);
+const QA_GATEWAY_DEBUG_SECRET_VALUE_KEYS = Object.freeze([
+  "accessToken",
+  "access_token",
+  "apiKey",
+  "api_key",
+  "botToken",
+  "clientSecret",
+  "client_secret",
+  "cookie",
+  "driverToken",
+  "sutToken",
+  "leaseToken",
+  "refreshToken",
+  "refresh_token",
+  "set-cookie",
+  "x-api-key",
+]);
+const QA_GATEWAY_DEBUG_SECRET_QUERY_KEYS = Object.freeze([
+  "access_token",
+  "api_key",
+  "apiKey",
+  "auth",
+  "deviceToken",
+  "id_token",
+  "key",
+  "password",
+  "refresh_token",
+  "token",
+]);
+const QA_GATEWAY_DEBUG_SECRET_HEADER_KEYS = Object.freeze(["cookie", "set-cookie", "x-api-key"]);
+
+function redactSecretEnvKeyPattern(text: string, pattern: RegExp) {
+  const source = pattern.source.replace(/^\^/u, "").replace(/\$$/u, "");
+  return text
+    .replace(
+      new RegExp(`\\b(${source})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "g"),
+      `$1$2<redacted>`,
+    )
+    .replace(new RegExp(`"(${source})"\\s*:\\s*"[^"]*"`, "g"), `"$1":"<redacted>"`);
+}
+
+function redactSecretValueKey(text: string, key: string) {
+  const escapedKey = escapeRegExp(key);
+  return text
+    .replace(new RegExp(`([?#&]${escapedKey}=)[^&\\s]+`, "gi"), "$1<redacted>")
+    .replace(
+      new RegExp(`(^|\\s)(--${escapedKey})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "gi"),
+      `$1$2$3<redacted>`,
+    )
+    .replace(
+      new RegExp(`(^|[^\\w?#&-])(${escapedKey})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "gi"),
+      `$1$2$3<redacted>`,
+    )
+    .replace(new RegExp(`("${escapedKey}"\\s*:\\s*)"[^"]*"`, "gi"), `$1"<redacted>"`);
+}
+
+export function redactQaGatewayDebugText(text: string) {
+  let redacted = text;
+  for (const key of QA_GATEWAY_DEBUG_SECRET_HEADER_KEYS) {
+    const escapedKey = escapeRegExp(key);
+    redacted = redacted.replace(
+      new RegExp(`^(\\s*${escapedKey}\\s*:\\s*).+$`, "gim"),
+      "$1<redacted>",
+    );
+  }
+  for (const envVar of QA_GATEWAY_DEBUG_SECRET_ENV_VARS) {
+    const escapedEnvVar = escapeRegExp(envVar);
+    redacted = redacted.replace(
+      new RegExp(`\\b(${escapedEnvVar})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "g"),
+      `$1$2<redacted>`,
+    );
+    redacted = redacted.replace(
+      new RegExp(`("${escapedEnvVar}"\\s*:\\s*)"[^"]*"`, "g"),
+      `$1"<redacted>"`,
+    );
+  }
+  for (const pattern of QA_PROVIDER_SECRET_ENV_KEY_PATTERNS) {
+    redacted = redactSecretEnvKeyPattern(redacted, pattern);
+  }
+  for (const key of QA_GATEWAY_DEBUG_SECRET_VALUE_KEYS) {
+    redacted = redactSecretValueKey(redacted, key);
+  }
+  return redacted
+    .replaceAll(/\bsk-ant-oat01-[A-Za-z0-9_-]+\b/g, "<redacted>")
+    .replaceAll(/\bBearer\s+[^\s"'<>]{8,}/gi, "Bearer <redacted>")
+    .replaceAll(
+      new RegExp(
+        `([?#&](?:${QA_GATEWAY_DEBUG_SECRET_QUERY_KEYS.map(escapeRegExp).join("|")})=)[^&\\s]+`,
+        "gi",
+      ),
+      "$1<redacted>",
+    );
+}
+
+export function formatQaGatewayLogsForError(logs: string) {
+  const sanitized = redactQaGatewayDebugText(logs).trim();
+  return sanitized.length > 0 ? `\nGateway logs:\n${sanitized}` : "";
+}

extensions/qa-lab/src/gateway-log-sentinel.test.ts

@@ -0,0 +1,126 @@
+// Qa Lab tests cover gateway log sentinel plugin behavior.
+import { describe, expect, it } from "vitest";
+import {
+  assertNoGatewayLogSentinels,
+  formatGatewayLogSentinelSummary,
+  scanDirectReplyTranscriptSentinels,
+  scanGatewayLogSentinels,
+} from "./gateway-log-sentinel.js";
+
+describe("gateway log sentinels", () => {
+  it("classifies May 13 beta.5 operational failure signatures", () => {
+    const findings = scanGatewayLogSentinels(
+      [
+        "2026-05-13T00:00:01Z plugin before_prompt_build hook failed: TypeError: boom",
+        "2026-05-13T00:00:02Z plugin before_tool_call crashed while evaluating policy",
+        "2026-05-13T00:00:03Z plugin manifest invalid: missing contracts.tools registration",
+        "[plugins] plugin must declare contracts.tools for: runtime_tool",
+        "2026-05-13T00:00:04Z codex app-server attempt timed out after 180000ms",
+        "2026-05-13T00:00:05Z codex_app_server progress stalled for run abc123",
+        "2026-05-13T00:00:06Z cron payload model openai/gpt-5.5 is not in model allowlist",
+        "2026-05-13T00:00:07Z OpenAI quota exceeded for live-frontier request",
+      ].join("\n"),
+    );
+
+    expect(findings.map((finding) => finding.kind)).toEqual([
+      "plugin-hook-failure",
+      "plugin-hook-failure",
+      "plugin-contract-error",
+      "plugin-contract-error",
+      "codex-app-server-timeout",
+      "stalled-agent-run",
+      "cron-model-allowlist",
+      "live-quota-or-subscription",
+    ]);
+    expect(findings.find((finding) => finding.kind === "plugin-hook-failure")).toMatchObject({
+      verdict: "qa-harness-bug",
+      owner: "plugin",
+      productImpact: "P1",
+    });
+    expect(findings.find((finding) => finding.kind === "live-quota-or-subscription")).toMatchObject(
+      {
+        verdict: "environment-blocked",
+        owner: "environment",
+        productImpact: "P4",
+      },
+    );
+  });
+
+  it("honors log cursors while preserving absolute line numbers", () => {
+    const prefix = "safe line\n";
+    const findings = scanGatewayLogSentinels(`${prefix}codex app-server attempt timed out`, {
+      since: prefix.length,
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]).toMatchObject({
+      kind: "codex-app-server-timeout",
+      line: 2,
+    });
+  });
+
+  it("throws actionable summaries unless only environment blockers are allowed", () => {
+    expect(() => assertNoGatewayLogSentinels("codex_app_server progress stalled")).toThrow(
+      "stalled-agent-run",
+    );
+    expect(() =>
+      assertNoGatewayLogSentinels("OpenAI quota exceeded", { allowEnvironmentBlocked: true }),
+    ).not.toThrow();
+    expect(formatGatewayLogSentinelSummary(scanGatewayLogSentinels("OpenAI quota exceeded"))).toBe(
+      "live-quota-or-subscription@1 environment-blocked owner=environment: OpenAI quota exceeded",
+    );
+  });
+
+  it("detects direct reply self-message transcripts separately from gateway logs", () => {
+    const findings = scanDirectReplyTranscriptSentinels(
+      [
+        JSON.stringify({
+          message: {
+            role: "assistant",
+            content: [
+              {
+                type: "tool_use",
+                name: "message",
+                input: { action: "send", conversationId: "qa-operator", text: "hello" },
+              },
+            ],
+          },
+        }),
+        JSON.stringify({ message: { role: "assistant", content: "Sent." } }),
+      ].join("\n"),
+    );
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]).toMatchObject({
+      kind: "direct-reply-self-message",
+      verdict: "product-bug",
+      owner: "openclaw-routing",
+    });
+  });
+
+  it("detects OpenAI function_call-shaped direct reply transcripts", () => {
+    const findings = scanDirectReplyTranscriptSentinels(
+      [
+        JSON.stringify({
+          message: {
+            role: "assistant",
+            content: [
+              {
+                type: "function_call",
+                name: "message",
+                arguments: JSON.stringify({
+                  action: "send",
+                  target: "current",
+                  text: "hello",
+                }),
+              },
+            ],
+          },
+        }),
+        JSON.stringify({ message: { role: "assistant", content: "Sent." } }),
+      ].join("\n"),
+    );
+
+    expect(findings.map((finding) => finding.kind)).toEqual(["direct-reply-self-message"]);
+  });
+});

extensions/qa-lab/src/gateway-log-sentinel.ts

@@ -0,0 +1,384 @@
+// Qa Lab plugin module implements gateway log sentinel behavior.
+import {
+  isRecord,
+  normalizeOptionalString as readNonEmptyString,
+} from "openclaw/plugin-sdk/string-coerce-runtime";
+
+export type GatewayLogSentinelKind =
+  | "plugin-hook-failure"
+  | "plugin-contract-error"
+  | "direct-reply-self-message"
+  | "codex-app-server-timeout"
+  | "stalled-agent-run"
+  | "cron-model-allowlist"
+  | "live-quota-or-subscription";
+
+export type GatewayLogSentinelVerdict =
+  | "product-bug"
+  | "qa-harness-bug"
+  | "fixture-bug"
+  | "environment-blocked";
+
+export type GatewayLogSentinelOwner =
+  | "plugin"
+  | "openclaw-routing"
+  | "codex-runtime"
+  | "openclaw-cron"
+  | "environment";
+
+export type GatewayLogSentinelFinding = {
+  kind: GatewayLogSentinelKind;
+  verdict: GatewayLogSentinelVerdict;
+  owner: GatewayLogSentinelOwner;
+  productImpact: "P0" | "P1" | "P2" | "P3" | "P4";
+  qaImpact: "P0" | "P1" | "P2" | "P3" | "P4";
+  line: number;
+  text: string;
+};
+
+export type GatewayLogSentinelScanOptions = {
+  since?: number;
+  kinds?: readonly GatewayLogSentinelKind[];
+  ignoreKinds?: readonly GatewayLogSentinelKind[];
+};
+
+export type GatewayLogSentinelAssertOptions = GatewayLogSentinelScanOptions & {
+  allowEnvironmentBlocked?: boolean;
+};
+
+type GatewayLogSentinelRule = Omit<GatewayLogSentinelFinding, "line" | "text"> & {
+  test: (line: string) => boolean;
+};
+
+type GatewayLogSentinelToolCall = {
+  name: string;
+  args: unknown;
+};
+
+const GATEWAY_LOG_SENTINEL_RULES: GatewayLogSentinelRule[] = [
+  {
+    kind: "plugin-hook-failure",
+    verdict: "qa-harness-bug",
+    owner: "plugin",
+    productImpact: "P1",
+    qaImpact: "P0",
+    test: (line) =>
+      /\bbefore_(?:prompt_build|tool_call)\b/iu.test(line) &&
+      /\b(?:crash(?:ed)?|exception|failed|failure|error)\b/iu.test(line),
+  },
+  {
+    kind: "plugin-contract-error",
+    verdict: "qa-harness-bug",
+    owner: "plugin",
+    productImpact: "P1",
+    qaImpact: "P0",
+    test: (line) =>
+      /\bcontracts\.tools\b/iu.test(line) &&
+      /\b(?:missing|invalid|registration|register|manifest|contract|schema|declare|error)\b/iu.test(
+        line,
+      ),
+  },
+  {
+    kind: "codex-app-server-timeout",
+    verdict: "product-bug",
+    owner: "codex-runtime",
+    productImpact: "P1",
+    qaImpact: "P0",
+    test: (line) =>
+      /\bcodex app-server\b.*\btimed out\b|\btimed out\b.*\bcodex app-server\b/iu.test(line),
+  },
+  {
+    kind: "stalled-agent-run",
+    verdict: "product-bug",
+    owner: "codex-runtime",
+    productImpact: "P1",
+    qaImpact: "P0",
+    test: (line) =>
+      /\bcodex_app_server\b.*\b(?:stalled|no progress|progress stalled)\b|\b(?:stalled|no progress|progress stalled)\b.*\bcodex_app_server\b/iu.test(
+        line,
+      ),
+  },
+  {
+    kind: "cron-model-allowlist",
+    verdict: "product-bug",
+    owner: "openclaw-cron",
+    productImpact: "P2",
+    qaImpact: "P0",
+    test: (line) =>
+      /\bcron\b/iu.test(line) &&
+      (/\bmodel allowlist\b/iu.test(line) ||
+        /\ballowlist\b.*\bmodel\b/iu.test(line) ||
+        /\bmodel\b.*\b(?:not in|outside|blocked by)\b.*\ballowlist\b/iu.test(line)),
+  },
+  {
+    kind: "live-quota-or-subscription",
+    verdict: "environment-blocked",
+    owner: "environment",
+    productImpact: "P4",
+    qaImpact: "P0",
+    test: (line) =>
+      /\b(?:quota exceeded|insufficient_quota|subscription exhausted|no active subscription|billing hard limit|usage limit)\b/iu.test(
+        line,
+      ),
+  },
+];
+
+function filterGatewayLogSentinelFindings(
+  findings: GatewayLogSentinelFinding[],
+  options: GatewayLogSentinelScanOptions | undefined,
+) {
+  const kinds = new Set(options?.kinds ?? []);
+  const ignoreKinds = new Set(options?.ignoreKinds ?? []);
+  return findings.filter((finding) => {
+    if (kinds.size > 0 && !kinds.has(finding.kind)) {
+      return false;
+    }
+    return !ignoreKinds.has(finding.kind);
+  });
+}
+
+function lineNumberForOffset(logs: string, offset: number) {
+  if (offset <= 0) {
+    return 1;
+  }
+  return logs.slice(0, offset).split(/\r?\n/u).length;
+}
+
+export function extractGatewayMessageText(message: Record<string, unknown>) {
+  const rawContent = message.content;
+  if (typeof rawContent === "string") {
+    return rawContent.trim();
+  }
+  if (!Array.isArray(rawContent)) {
+    return "";
+  }
+  const parts: string[] = [];
+  for (const block of rawContent) {
+    if (typeof block === "string") {
+      if (block.trim()) {
+        parts.push(block.trim());
+      }
+      continue;
+    }
+    if (!isRecord(block)) {
+      continue;
+    }
+    const text = readNonEmptyString(block.text);
+    if (text) {
+      parts.push(text);
+      continue;
+    }
+    const nestedText = readNonEmptyString(block.content);
+    if (
+      nestedText &&
+      (block.type === "output_text" || block.type === "text" || block.type === "message")
+    ) {
+      parts.push(nestedText);
+    }
+  }
+  return parts.join("\n").trim();
+}
+
+function parseJsonArguments(value: unknown): unknown {
+  if (typeof value !== "string") {
+    return value;
+  }
+  try {
+    return JSON.parse(value) as unknown;
+  } catch {
+    return value;
+  }
+}
+
+function extractAssistantToolCalls(message: Record<string, unknown>): GatewayLogSentinelToolCall[] {
+  const calls: GatewayLogSentinelToolCall[] = [];
+  const rawContent = message.content;
+  if (Array.isArray(rawContent)) {
+    for (const block of rawContent) {
+      if (!isRecord(block)) {
+        continue;
+      }
+      const type = readNonEmptyString(block.type)?.toLowerCase();
+      if (
+        type !== "tool_use" &&
+        type !== "toolcall" &&
+        type !== "tool_call" &&
+        type !== "function_call"
+      ) {
+        continue;
+      }
+      calls.push({
+        name: readNonEmptyString(block.name) ?? "unknown",
+        args: parseJsonArguments(block.input ?? block.arguments ?? block.args ?? null),
+      });
+    }
+  }
+
+  const rawToolCalls =
+    message.tool_calls ?? message.toolCalls ?? message.function_call ?? message.functionCall;
+  const toolCalls = Array.isArray(rawToolCalls) ? rawToolCalls : rawToolCalls ? [rawToolCalls] : [];
+  for (const call of toolCalls) {
+    if (!isRecord(call)) {
+      continue;
+    }
+    const functionRecord = isRecord(call.function) ? call.function : undefined;
+    calls.push({
+      name: readNonEmptyString(call.name) ?? readNonEmptyString(functionRecord?.name) ?? "unknown",
+      args: parseJsonArguments(
+        call.arguments ?? functionRecord?.arguments ?? call.input ?? functionRecord?.input ?? null,
+      ),
+    });
+  }
+  return calls;
+}
+
+function isCurrentChatMessageSend(call: GatewayLogSentinelToolCall) {
+  if (call.name !== "message") {
+    return false;
+  }
+  if (!isRecord(call.args) || readNonEmptyString(call.args.action)?.toLowerCase() !== "send") {
+    return false;
+  }
+  const explicitTarget =
+    readNonEmptyString(call.args.conversationId) ??
+    readNonEmptyString(call.args.conversation) ??
+    readNonEmptyString(call.args.to) ??
+    readNonEmptyString(call.args.target);
+  if (!explicitTarget) {
+    return true;
+  }
+  return /\b(?:current|same-chat|qa-operator|dm:qa-operator)\b/iu.test(explicitTarget);
+}
+
+function normalizeTranscriptText(text: string) {
+  return text.replace(/\s+/gu, " ").trim();
+}
+
+function createDirectReplyFinding(): GatewayLogSentinelFinding {
+  return {
+    kind: "direct-reply-self-message",
+    verdict: "product-bug",
+    owner: "openclaw-routing",
+    productImpact: "P1",
+    qaImpact: "P0",
+    line: 1,
+    text: "assistant called message(action=send) and then produced final text Sent.",
+  };
+}
+
+export function createDirectReplyTranscriptSentinelScanner() {
+  let lastAssistantText = "";
+  const toolCalls: GatewayLogSentinelToolCall[] = [];
+  return {
+    recordMessage(message: Record<string, unknown>) {
+      if (message.role !== "assistant") {
+        return;
+      }
+      const text = extractGatewayMessageText(message);
+      if (text) {
+        lastAssistantText = text;
+      }
+      toolCalls.push(...extractAssistantToolCalls(message));
+    },
+    findings(): GatewayLogSentinelFinding[] {
+      const hasDirectReply =
+        toolCalls.some(isCurrentChatMessageSend) &&
+        normalizeTranscriptText(lastAssistantText).toLowerCase() === "sent.";
+      return hasDirectReply ? [createDirectReplyFinding()] : [];
+    },
+  };
+}
+
+function transcriptHasDirectReplySelfMessage(transcriptBytes: string) {
+  const scanner = createDirectReplyTranscriptSentinelScanner();
+  for (const line of transcriptBytes.split(/\r?\n/u)) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      continue;
+    }
+    try {
+      const parsed = JSON.parse(trimmed) as unknown;
+      const message = isRecord(parsed) && isRecord(parsed.message) ? parsed.message : undefined;
+      if (!message || message.role !== "assistant") {
+        continue;
+      }
+      scanner.recordMessage(message);
+    } catch {
+      // Ignore malformed QA transcript rows and keep sentinel scans deterministic.
+    }
+  }
+  return scanner.findings().length > 0;
+}
+
+export function scanGatewayLogSentinels(
+  logs: string | undefined,
+  options?: GatewayLogSentinelScanOptions,
+): GatewayLogSentinelFinding[] {
+  if (!logs) {
+    return [];
+  }
+  const startOffset = Math.max(0, Math.min(logs.length, Math.floor(options?.since ?? 0)));
+  const lineOffset = lineNumberForOffset(logs, startOffset) - 1;
+  const findings: GatewayLogSentinelFinding[] = [];
+  for (const [index, rawLine] of logs.slice(startOffset).split(/\r?\n/u).entries()) {
+    const text = rawLine.trim();
+    if (!text) {
+      continue;
+    }
+    for (const rule of GATEWAY_LOG_SENTINEL_RULES) {
+      if (!rule.test(text)) {
+        continue;
+      }
+      findings.push({
+        kind: rule.kind,
+        verdict: rule.verdict,
+        owner: rule.owner,
+        productImpact: rule.productImpact,
+        qaImpact: rule.qaImpact,
+        line: lineOffset + index + 1,
+        text,
+      });
+    }
+  }
+  return filterGatewayLogSentinelFindings(findings, options);
+}
+
+export function scanDirectReplyTranscriptSentinels(
+  transcriptBytes: string,
+): GatewayLogSentinelFinding[] {
+  if (!transcriptHasDirectReplySelfMessage(transcriptBytes)) {
+    return [];
+  }
+  return [createDirectReplyFinding()];
+}
+
+export function formatGatewayLogSentinelSummary(findings: readonly GatewayLogSentinelFinding[]) {
+  if (findings.length === 0) {
+    return "no gateway log sentinels";
+  }
+  return findings
+    .map(
+      (finding) =>
+        `${finding.kind}@${finding.line} ${finding.verdict} owner=${finding.owner}: ${finding.text}`,
+    )
+    .join("\n");
+}
+
+export function assertNoGatewayLogSentinels(
+  logs: string | undefined,
+  options?: GatewayLogSentinelAssertOptions,
+) {
+  const findings = scanGatewayLogSentinels(logs, options);
+  if (findings.length === 0) {
+    return findings;
+  }
+  if (
+    options?.allowEnvironmentBlocked === true &&
+    findings.every((finding) => finding.verdict === "environment-blocked")
+  ) {
+    return findings;
+  }
+  throw new Error(
+    `Gateway log sentinel(s) detected:\n${formatGatewayLogSentinelSummary(findings)}`,
+  );
+}

extensions/qa-lab/src/gateway-rpc-client.test.ts

@@ -0,0 +1,229 @@
+// Qa Lab tests cover gateway rpc client plugin behavior.
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const gatewayRpcMock = vi.hoisted(() => {
+  const callGatewayFromCli = vi.fn(async () => ({ ok: true }));
+  return {
+    callGatewayFromCli,
+    reset() {
+      callGatewayFromCli.mockReset().mockResolvedValue({ ok: true });
+    },
+  };
+});
+
+vi.mock("openclaw/plugin-sdk/gateway-runtime", () => ({
+  callGatewayFromCli: gatewayRpcMock.callGatewayFromCli,
+}));
+
+import { startQaGatewayRpcClient } from "./gateway-rpc-client.js";
+
+function expectRequestResolver(
+  callback: ((value: { ok: boolean }) => void) | null,
+): (value: { ok: boolean }) => void {
+  if (callback === null) {
+    throw new Error("Expected first request resolver callback to be captured");
+  }
+  return callback;
+}
+
+function expectReleaseCallback(callback: (() => void) | null): () => void {
+  if (callback === null) {
+    throw new Error("Expected first request release callback to be captured");
+  }
+  return callback;
+}
+
+describe("startQaGatewayRpcClient", () => {
+  beforeEach(() => {
+    gatewayRpcMock.reset();
+  });
+
+  it("calls the in-process gateway cli helper without mutating process.env", async () => {
+    const originalHome = process.env.OPENCLAW_HOME;
+    delete process.env.OPENCLAW_HOME;
+
+    try {
+      gatewayRpcMock.callGatewayFromCli.mockImplementationOnce(async () => {
+        expect(process.env.OPENCLAW_HOME).toBeUndefined();
+        return { ok: true };
+      });
+
+      const client = await startQaGatewayRpcClient({
+        wsUrl: "ws://127.0.0.1:18789",
+        token: "qa-token",
+        logs: () => "qa logs",
+      });
+
+      await expect(
+        client.request("agent.run", { prompt: "hi" }, { expectFinal: true, timeoutMs: 45_000 }),
+      ).resolves.toEqual({ ok: true });
+
+      expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenCalledWith(
+        "agent.run",
+        {
+          url: "ws://127.0.0.1:18789",
+          token: "qa-token",
+          timeout: "45000",
+          expectFinal: true,
+          json: true,
+        },
+        { prompt: "hi" },
+        {
+          clientName: "gateway-client",
+          deviceIdentity: null,
+          expectFinal: true,
+          mode: "backend",
+          progress: false,
+          scopes: ["operator.admin"],
+        },
+      );
+    } finally {
+      if (originalHome === undefined) {
+        delete process.env.OPENCLAW_HOME;
+      } else {
+        process.env.OPENCLAW_HOME = originalHome;
+      }
+    }
+
+    expect(process.env.OPENCLAW_HOME).toBe(originalHome);
+  });
+
+  it("wraps request failures with gateway logs", async () => {
+    gatewayRpcMock.callGatewayFromCli.mockRejectedValueOnce(new Error("gateway not connected"));
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "OPENCLAW_GATEWAY_TOKEN=secret-token\nAuthorization: Bearer secret+/token=123456",
+    });
+
+    await expect(client.request("health")).rejects.toThrow(
+      "gateway not connected\nGateway logs:\nOPENCLAW_GATEWAY_TOKEN=<redacted>\nAuthorization: Bearer <redacted>",
+    );
+  });
+
+  it("rejects new requests after stop", async () => {
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "url=http://127.0.0.1:18789/#token=abc123",
+    });
+
+    await client.stop();
+
+    await expect(client.request("health")).rejects.toThrow(
+      "gateway rpc client already stopped\nGateway logs:\nurl=http://127.0.0.1:18789/#token=<redacted>",
+    );
+  });
+
+  it("does not serialize requests across different gateway clients", async () => {
+    let resolveFirst: ((value: { ok: boolean }) => void) | null = null;
+    gatewayRpcMock.callGatewayFromCli
+      .mockImplementationOnce(
+        async () =>
+          await new Promise<{ ok: boolean }>((resolve) => {
+            resolveFirst = resolve;
+          }),
+      )
+      .mockResolvedValueOnce({ ok: true });
+
+    const firstClient = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token-a",
+      logs: () => "qa logs a",
+    });
+    const secondClient = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:28789",
+      token: "qa-token-b",
+      logs: () => "qa logs b",
+    });
+
+    const firstRequest = firstClient.request("health");
+    await Promise.resolve();
+
+    await expect(secondClient.request("status")).resolves.toEqual({ ok: true });
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenNthCalledWith(
+      2,
+      "status",
+      {
+        url: "ws://127.0.0.1:28789",
+        token: "qa-token-b",
+        timeout: "20000",
+        expectFinal: undefined,
+        json: true,
+      },
+      {},
+      {
+        clientName: "gateway-client",
+        deviceIdentity: null,
+        expectFinal: undefined,
+        mode: "backend",
+        progress: false,
+        scopes: ["operator.admin"],
+      },
+    );
+
+    expectRequestResolver(resolveFirst)({ ok: true });
+    await expect(firstRequest).resolves.toEqual({ ok: true });
+  });
+
+  it("still serializes requests within the same gateway client", async () => {
+    let releaseFirst: (() => void) | null = null;
+    gatewayRpcMock.callGatewayFromCli
+      .mockImplementationOnce(
+        async () =>
+          await new Promise<{ ok: boolean }>((resolve) => {
+            releaseFirst = () => resolve({ ok: true });
+          }),
+      )
+      .mockResolvedValueOnce({ ok: true });
+
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "qa logs",
+    });
+
+    const firstRequest = client.request("health");
+    await Promise.resolve();
+    const secondRequest = client.request("status");
+    await Promise.resolve();
+
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenCalledTimes(1);
+
+    expectReleaseCallback(releaseFirst)();
+
+    await expect(firstRequest).resolves.toEqual({ ok: true });
+    await expect(secondRequest).resolves.toEqual({ ok: true });
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenCalledTimes(2);
+  });
+
+  it("rejects queued requests that have not started before stop", async () => {
+    let releaseFirst: (() => void) | null = null;
+    gatewayRpcMock.callGatewayFromCli.mockImplementationOnce(
+      async () =>
+        await new Promise<{ ok: boolean }>((resolve) => {
+          releaseFirst = () => resolve({ ok: true });
+        }),
+    );
+
+    const client = await startQaGatewayRpcClient({
+      wsUrl: "ws://127.0.0.1:18789",
+      token: "qa-token",
+      logs: () => "qa logs",
+    });
+
+    const firstRequest = client.request("health");
+    await Promise.resolve();
+    const secondRequest = client.request("status");
+    await Promise.resolve();
+
+    await client.stop();
+    expectReleaseCallback(releaseFirst)();
+
+    await expect(firstRequest).resolves.toEqual({ ok: true });
+    await expect(secondRequest).rejects.toThrow(
+      "gateway rpc client already stopped\nGateway logs:\nqa logs",
+    );
+    expect(gatewayRpcMock.callGatewayFromCli).toHaveBeenCalledTimes(1);
+  });
+});

extensions/qa-lab/src/gateway-rpc-client.ts

@@ -0,0 +1,84 @@
+// Qa Lab plugin module implements gateway rpc client behavior.
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { callGatewayFromCli } from "openclaw/plugin-sdk/gateway-runtime";
+import { formatQaGatewayLogsForError } from "./gateway-log-redaction.js";
+
+type QaGatewayRpcRequestOptions = {
+  expectFinal?: boolean;
+  timeoutMs?: number;
+};
+
+type QaGatewayRpcClient = {
+  request(method: string, rpcParams?: unknown, opts?: QaGatewayRpcRequestOptions): Promise<unknown>;
+  stop(): Promise<void>;
+};
+
+function formatQaGatewayRpcError(error: unknown, logs: () => string) {
+  const details = formatErrorMessage(error);
+  return new Error(`${details}${formatQaGatewayLogsForError(logs())}`);
+}
+
+function runQueuedQaGatewayRpc<T>(queue: Promise<void>, task: () => Promise<T>) {
+  const run = queue.then(task, task);
+  const nextQueue = run.then(
+    () => undefined,
+    () => undefined,
+  );
+  return { run, nextQueue };
+}
+
+export async function startQaGatewayRpcClient(params: {
+  wsUrl: string;
+  token: string;
+  logs: () => string;
+}): Promise<QaGatewayRpcClient> {
+  const wrapError = (error: unknown) => formatQaGatewayRpcError(error, params.logs);
+  let stopped = false;
+  let queue = Promise.resolve();
+  const assertNotStopped = () => {
+    if (stopped) {
+      throw new Error("gateway rpc client already stopped");
+    }
+  };
+
+  return {
+    async request(method, rpcParams, opts) {
+      try {
+        assertNotStopped();
+      } catch (error) {
+        throw wrapError(error);
+      }
+      try {
+        const { run, nextQueue } = runQueuedQaGatewayRpc(queue, async () => {
+          assertNotStopped();
+          return await callGatewayFromCli(
+            method,
+            {
+              url: params.wsUrl,
+              token: params.token,
+              timeout: String(opts?.timeoutMs ?? 20_000),
+              expectFinal: opts?.expectFinal,
+              json: true,
+            },
+            rpcParams ?? {},
+            {
+              clientName: "gateway-client",
+              deviceIdentity: null,
+              expectFinal: opts?.expectFinal,
+              mode: "backend",
+              progress: false,
+              scopes: ["operator.admin"],
+            },
+          );
+        });
+        queue = nextQueue;
+        return await run;
+      } catch (error) {
+        throw wrapError(error);
+      }
+    },
+    async stop() {
+      stopped = true;
+    },
+  };
+}

extensions/qa-lab/src/harness-parity.test.ts

@@ -0,0 +1,285 @@
+// Qa Lab tests cover harness parity plugin behavior.
+import { describe, expect, it } from "vitest";
+import {
+  buildHarnessParityCell,
+  buildHarnessParityResult,
+  type HarnessRuntimeParityCell,
+  type HarnessVariant,
+} from "./harness-parity.js";
+import type { RuntimeId } from "./runtime-parity.js";
+import type { RuntimeParityComparisonMode } from "./runtime-tool-metadata.js";
+
+const LEFT: HarnessVariant = { id: "left", label: "Left", runtime: "openclaw" };
+const RIGHT: HarnessVariant = { id: "right", label: "Right", runtime: "openclaw" };
+
+const BASE_PROMPT_REPORT = {
+  systemPrompt: {
+    chars: 100,
+    projectContextChars: 40,
+    nonProjectContextChars: 60,
+    hash: "system-a",
+  },
+  skills: {
+    promptChars: 12,
+    hash: "skills-a",
+  },
+  tools: {
+    schemaChars: 20,
+    entries: [
+      {
+        name: "read",
+        summaryChars: 8,
+        summaryHash: "summary-a",
+        schemaChars: 20,
+        schemaHash: "schema-a",
+        propertiesCount: 1,
+      },
+    ],
+  },
+};
+
+function makeCell(
+  runtime: RuntimeId,
+  overrides: Partial<HarnessRuntimeParityCell> = {},
+): HarnessRuntimeParityCell {
+  return {
+    runtime,
+    transcriptBytes: '{"message":{"role":"assistant","content":"same"}}\n',
+    toolCalls: [],
+    finalText: "same",
+    usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+    wallClockMs: 1,
+    bootStateLines: [],
+    systemPromptReport: BASE_PROMPT_REPORT,
+    ...overrides,
+  };
+}
+
+function classify(
+  left: Partial<HarnessRuntimeParityCell>,
+  right: Partial<HarnessRuntimeParityCell>,
+  comparisonMode?: RuntimeParityComparisonMode,
+) {
+  return buildHarnessParityResult({
+    scenarioId: "scenario",
+    left: buildHarnessParityCell({
+      variant: LEFT,
+      cell: makeCell("openclaw", left),
+      tokenUsageSource: "live-usage",
+    }),
+    right: buildHarnessParityCell({
+      variant: RIGHT,
+      cell: makeCell("openclaw", right),
+      tokenUsageSource: "live-usage",
+    }),
+    ...(comparisonMode ? { comparisonMode } : {}),
+  }).drift;
+}
+
+describe("harness parity", () => {
+  it("classifies prompt and tool surface drift before behavioral drift", () => {
+    expect(
+      classify(
+        {},
+        {
+          systemPromptReport: {
+            ...BASE_PROMPT_REPORT,
+            systemPrompt: { chars: 101, projectContextChars: 40, nonProjectContextChars: 61 },
+          },
+        },
+      ),
+    ).toBe("system-prompt");
+    expect(
+      classify(
+        {},
+        {
+          systemPromptReport: {
+            ...BASE_PROMPT_REPORT,
+            systemPrompt: {
+              chars: 100,
+              projectContextChars: 40,
+              nonProjectContextChars: 60,
+              hash: "system-b",
+            },
+          },
+        },
+      ),
+    ).toBe("system-prompt");
+    expect(
+      classify(
+        {},
+        {
+          systemPromptReport: {
+            ...BASE_PROMPT_REPORT,
+            skills: { promptChars: 12, hash: "skills-b" },
+          },
+        },
+      ),
+    ).toBe("system-prompt");
+    expect(
+      classify(
+        {},
+        {
+          systemPromptReport: {
+            ...BASE_PROMPT_REPORT,
+            tools: {
+              schemaChars: 20,
+              entries: [
+                {
+                  name: "read",
+                  summaryChars: 8,
+                  summaryHash: "summary-b",
+                  schemaChars: 20,
+                  schemaHash: "schema-a",
+                  propertiesCount: 1,
+                },
+              ],
+            },
+          },
+        },
+      ),
+    ).toBe("tool-description");
+    expect(
+      classify(
+        {},
+        {
+          systemPromptReport: {
+            ...BASE_PROMPT_REPORT,
+            tools: {
+              schemaChars: 20,
+              entries: [
+                {
+                  name: "read",
+                  summaryChars: 8,
+                  summaryHash: "summary-a",
+                  schemaChars: 20,
+                  schemaHash: "schema-b",
+                  propertiesCount: 1,
+                },
+              ],
+            },
+          },
+        },
+      ),
+    ).toBe("tool-schema");
+  });
+
+  it("classifies behavioral harness drift", () => {
+    expect(
+      classify(
+        { toolCalls: [{ tool: "read", argsHash: "a", resultHash: "r" }] },
+        { toolCalls: [{ tool: "read", argsHash: "b", resultHash: "r" }] },
+      ),
+    ).toBe("tool-call-shape");
+    expect(
+      classify(
+        { toolCalls: [{ tool: "read", argsHash: "a", resultHash: "r1" }] },
+        { toolCalls: [{ tool: "read", argsHash: "a", resultHash: "r2" }] },
+      ),
+    ).toBe("tool-result-shape");
+    expect(classify({ finalText: "same text" }, { finalText: "different text" })).toBe("text-only");
+    expect(
+      classify(
+        {
+          transcriptBytes:
+            '{"type":"model_change","modelId":"gpt-5.5"}\n' +
+            '{"type":"thinking_level_change","thinkingLevel":"off"}\n' +
+            '{"type":"custom","customType":"model-snapshot"}\n' +
+            '{"message":{"role":"assistant","content":"same"}}\n',
+        },
+        { transcriptBytes: '{"message":{"role":"assistant","content":"same"}}\n' },
+      ),
+    ).toBe("none");
+    expect(
+      classify(
+        { transcriptBytes: '{"message":{"role":"assistant"}}\n' },
+        { transcriptBytes: '{"message":{"role":"assistant"}}\n{"message":{"role":"tool"}}\n' },
+      ),
+    ).toBe("structural");
+    expect(
+      classify(
+        { transcriptBytes: '{"role":"assistant","content":"same"}\n' },
+        {
+          transcriptBytes:
+            '{"role":"assistant","content":"same"}\n{"role":"tool","content":"same"}\n',
+        },
+      ),
+    ).toBe("structural");
+    expect(classify({ runtimeErrorClass: "timeout" }, {})).toBe("failure-mode");
+  });
+
+  it("honors native workspace comparison mode for outcome-only harness proofs", () => {
+    expect(
+      classify(
+        {
+          transcriptBytes:
+            '{"message":{"role":"assistant","content":"same"}}\n' +
+            '{"message":{"role":"tool","content":"same result"}}\n',
+          toolCalls: [{ tool: "bash", argsHash: "sed-160", resultHash: "same-result" }],
+        },
+        {
+          transcriptBytes: '{"message":{"role":"assistant","content":"same"}}\n',
+          toolCalls: [{ tool: "bash", argsHash: "sed-200", resultHash: "same-result" }],
+        },
+        "codex-native-workspace",
+      ),
+    ).toBe("none");
+
+    expect(
+      classify(
+        { toolCalls: [{ tool: "bash", argsHash: "a", resultHash: "r1" }] },
+        { toolCalls: [{ tool: "bash", argsHash: "b", resultHash: "r2" }] },
+        "outcome-only",
+      ),
+    ).toBe("none");
+  });
+
+  it("keeps prompt and tool surface checks strict under native workspace comparison mode", () => {
+    expect(
+      classify(
+        {},
+        {
+          systemPromptReport: {
+            ...BASE_PROMPT_REPORT,
+            systemPrompt: { chars: 101, projectContextChars: 40, nonProjectContextChars: 61 },
+          },
+          toolCalls: [{ tool: "bash", argsHash: "changed", resultHash: "changed" }],
+        },
+        "codex-native-workspace",
+      ),
+    ).toBe("system-prompt");
+    expect(
+      classify(
+        {},
+        {
+          systemPromptReport: {
+            ...BASE_PROMPT_REPORT,
+            tools: {
+              schemaChars: 20,
+              entries: [{ name: "read", summaryChars: 9, schemaChars: 20, propertiesCount: 1 }],
+            },
+          },
+          toolCalls: [{ tool: "bash", argsHash: "changed", resultHash: "changed" }],
+        },
+        "outcome-only",
+      ),
+    ).toBe("tool-description");
+  });
+
+  it("labels mock token estimates separately from live usage", () => {
+    const sourceCell = makeCell("openclaw", {
+      usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+    });
+    const cell = buildHarnessParityCell({
+      variant: LEFT,
+      cell: sourceCell,
+      tokenUsageSource: "mock-estimate",
+    });
+    const inputChars = 100 + 12 + 8 + 20 + sourceCell.transcriptBytes.length;
+
+    expect(cell.tokenUsageSource).toBe("mock-estimate");
+    expect(cell.tokenUsage.totalTokens).toBeGreaterThan(0);
+    expect(cell.tokenUsage.inputTokens).toBe(Math.ceil(inputChars / 4));
+    expect(cell.promptStats.toolCount).toBe(1);
+  });
+});

extensions/qa-lab/src/harness-parity.ts

@@ -0,0 +1,404 @@
+import { compareToolCallShape, stableHash } from "./parity-shared.js";
+// Qa Lab plugin module implements harness parity behavior.
+import type {
+  RuntimeId,
+  RuntimeParityCell,
+  RuntimeParityDrift,
+  RuntimeParityToolCall,
+  RuntimeParityUsage,
+} from "./runtime-parity.js";
+import type { RuntimeParityComparisonMode } from "./runtime-tool-metadata.js";
+
+export type HarnessVariant = {
+  id: string;
+  label: string;
+  runtime?: RuntimeId;
+  model?: string;
+  configPatch?: Record<string, unknown>;
+  systemPromptOverlay?: string;
+  toolDescriptionOverlay?: Record<string, string>;
+};
+
+export type HarnessParityDrift =
+  | RuntimeParityDrift
+  | "system-prompt"
+  | "tool-description"
+  | "tool-schema";
+
+export type HarnessParityPromptStats = {
+  systemPromptChars: number;
+  projectContextChars: number;
+  nonProjectContextChars: number;
+  skillPromptChars: number;
+  toolSummaryChars: number;
+  toolSchemaChars: number;
+  toolCount: number;
+};
+
+export type RuntimeParitySystemPromptReport = {
+  systemPrompt?: {
+    chars?: number;
+    projectContextChars?: number;
+    nonProjectContextChars?: number;
+    text?: string;
+    hash?: string;
+    contentHash?: string;
+  };
+  skills?: {
+    promptChars?: number;
+    prompt?: string;
+    hash?: string;
+    contentHash?: string;
+  };
+  tools?: {
+    listChars?: number;
+    schemaChars?: number;
+    entries?: Array<{
+      name?: string;
+      summary?: string;
+      summaryHash?: string;
+      summaryChars?: number;
+      schema?: unknown;
+      schemaHash?: string;
+      schemaChars?: number;
+      propertiesCount?: number;
+    }>;
+  };
+};
+
+export type HarnessRuntimeParityCell = RuntimeParityCell & {
+  systemPromptReport?: RuntimeParitySystemPromptReport;
+};
+
+export type HarnessParityCell = HarnessRuntimeParityCell & {
+  variant: HarnessVariant;
+  promptStats: HarnessParityPromptStats;
+  systemPromptHash: string;
+  toolDescriptionHash: string;
+  toolSchemaHash: string;
+  tokenUsage: RuntimeParityUsage;
+  tokenUsageSource: "live-usage" | "mock-estimate";
+};
+
+export type HarnessParityResult = {
+  scenarioId: string;
+  left: HarnessParityCell;
+  right: HarnessParityCell;
+  drift: HarnessParityDrift;
+  driftDetails?: string;
+  promptDelta: {
+    systemPromptChars: number;
+    projectContextChars: number;
+    skillPromptChars: number;
+    toolSummaryChars: number;
+    toolSchemaChars: number;
+    toolCount: number;
+  };
+  tokenDeltaPercent: number;
+  firstDriftTurn?: number;
+};
+
+function countComparableTranscriptRecords(transcriptBytes: string) {
+  let count = 0;
+  for (const line of transcriptBytes.split(/\r?\n/u)) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      continue;
+    }
+    try {
+      const parsed = JSON.parse(trimmed) as {
+        message?: { role?: unknown };
+        role?: unknown;
+      };
+      if (
+        (parsed.message && typeof parsed.message.role === "string") ||
+        typeof parsed.role === "string"
+      ) {
+        count += 1;
+      }
+    } catch {
+      // Ignore malformed QA transcript rows and keep parity classification deterministic.
+    }
+  }
+  return count;
+}
+
+function readPositiveNumber(value: unknown) {
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
+}
+
+function buildPromptStats(report: RuntimeParitySystemPromptReport | undefined) {
+  const toolEntries = Array.isArray(report?.tools?.entries) ? report.tools.entries : [];
+  return {
+    systemPromptChars: readPositiveNumber(report?.systemPrompt?.chars),
+    projectContextChars: readPositiveNumber(report?.systemPrompt?.projectContextChars),
+    nonProjectContextChars: readPositiveNumber(report?.systemPrompt?.nonProjectContextChars),
+    skillPromptChars: readPositiveNumber(report?.skills?.promptChars),
+    toolSummaryChars: toolEntries.reduce(
+      (sum, entry) => sum + readPositiveNumber(entry.summaryChars),
+      0,
+    ),
+    toolSchemaChars: readPositiveNumber(report?.tools?.schemaChars),
+    toolCount: toolEntries.length,
+  };
+}
+
+function estimateUsage(
+  cell: RuntimeParityCell,
+  stats: HarnessParityPromptStats,
+): RuntimeParityUsage {
+  const inputChars =
+    stats.systemPromptChars +
+    stats.skillPromptChars +
+    stats.toolSummaryChars +
+    stats.toolSchemaChars +
+    cell.transcriptBytes.length;
+  const outputChars = cell.finalText.length + cell.toolCalls.length * 80;
+  const inputTokens = Math.ceil(inputChars / 4);
+  const outputTokens = Math.ceil(outputChars / 4);
+  return {
+    inputTokens,
+    outputTokens,
+    totalTokens: inputTokens + outputTokens,
+  };
+}
+
+function normalizeTextForParity(text: string) {
+  return text.replace(/\s+/gu, " ").trim();
+}
+
+function compareToolResultShape(left: RuntimeParityToolCall[], right: RuntimeParityToolCall[]) {
+  const total = Math.min(left.length, right.length);
+  for (let index = 0; index < total; index += 1) {
+    const leftCall = left[index];
+    const rightCall = right[index];
+    if (!leftCall || !rightCall) {
+      continue;
+    }
+    if (
+      leftCall.resultHash !== rightCall.resultHash ||
+      (leftCall.errorClass ?? "") !== (rightCall.errorClass ?? "")
+    ) {
+      return `tool result ${index + 1} differs (${leftCall.tool})`;
+    }
+  }
+  return undefined;
+}
+
+function firstDriftTurn(leftTranscript: string, rightTranscript: string): number | undefined {
+  const leftLines = leftTranscript.trim().length ? leftTranscript.trim().split(/\r?\n/u) : [];
+  const rightLines = rightTranscript.trim().length ? rightTranscript.trim().split(/\r?\n/u) : [];
+  const total = Math.max(leftLines.length, rightLines.length);
+  for (let index = 0; index < total; index += 1) {
+    if ((leftLines[index] ?? "") !== (rightLines[index] ?? "")) {
+      return index + 1;
+    }
+  }
+  return undefined;
+}
+
+export function buildHarnessParityCell(params: {
+  variant: HarnessVariant;
+  cell: HarnessRuntimeParityCell;
+  tokenUsageSource: HarnessParityCell["tokenUsageSource"];
+}): HarnessParityCell {
+  const report = params.cell.systemPromptReport;
+  const promptStats = buildPromptStats(report);
+  const toolEntries = report?.tools?.entries ?? [];
+  const tokenUsage =
+    params.tokenUsageSource === "live-usage"
+      ? params.cell.usage
+      : estimateUsage(params.cell, promptStats);
+  return {
+    ...params.cell,
+    variant: params.variant,
+    ...(report ? { systemPromptReport: report } : {}),
+    promptStats,
+    systemPromptHash: stableHash({
+      systemPrompt: report?.systemPrompt ?? null,
+      skills: report?.skills ?? null,
+    }),
+    toolDescriptionHash: stableHash(
+      toolEntries.map((entry) => {
+        return {
+          name: entry.name,
+          summary: entry.summary,
+          summaryHash: entry.summaryHash,
+          summaryChars: entry.summaryChars,
+        };
+      }),
+    ),
+    toolSchemaHash: stableHash({
+      listChars: report?.tools?.listChars,
+      schemaChars: report?.tools?.schemaChars,
+      entries: toolEntries.map((entry) => {
+        return {
+          name: entry.name,
+          schema: entry.schema,
+          schemaHash: entry.schemaHash,
+          schemaChars: entry.schemaChars,
+          propertiesCount: entry.propertiesCount,
+        };
+      }),
+    }),
+    tokenUsage,
+    tokenUsageSource: params.tokenUsageSource,
+  };
+}
+
+export function buildHarnessParityResult(params: {
+  scenarioId: string;
+  left: HarnessParityCell;
+  right: HarnessParityCell;
+  comparisonMode?: RuntimeParityComparisonMode;
+}): HarnessParityResult {
+  const promptDelta = {
+    systemPromptChars:
+      params.right.promptStats.systemPromptChars - params.left.promptStats.systemPromptChars,
+    projectContextChars:
+      params.right.promptStats.projectContextChars - params.left.promptStats.projectContextChars,
+    skillPromptChars:
+      params.right.promptStats.skillPromptChars - params.left.promptStats.skillPromptChars,
+    toolSummaryChars:
+      params.right.promptStats.toolSummaryChars - params.left.promptStats.toolSummaryChars,
+    toolSchemaChars:
+      params.right.promptStats.toolSchemaChars - params.left.promptStats.toolSchemaChars,
+    toolCount: params.right.promptStats.toolCount - params.left.promptStats.toolCount,
+  };
+  const tokenDeltaPercent =
+    params.left.tokenUsage.totalTokens === 0
+      ? params.right.tokenUsage.totalTokens === 0
+        ? 0
+        : 100
+      : ((params.right.tokenUsage.totalTokens - params.left.tokenUsage.totalTokens) /
+          params.left.tokenUsage.totalTokens) *
+        100;
+  const failDetails =
+    params.left.transportErrorClass || params.right.transportErrorClass
+      ? "at least one harness variant hit a transport failure"
+      : params.left.runtimeErrorClass || params.right.runtimeErrorClass
+        ? "at least one harness variant hit a runtime failure"
+        : undefined;
+  if (failDetails) {
+    return {
+      scenarioId: params.scenarioId,
+      left: params.left,
+      right: params.right,
+      drift: "failure-mode",
+      driftDetails: failDetails,
+      promptDelta,
+      tokenDeltaPercent,
+      firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+    };
+  }
+  if (params.left.systemPromptHash !== params.right.systemPromptHash) {
+    return {
+      scenarioId: params.scenarioId,
+      left: params.left,
+      right: params.right,
+      drift: "system-prompt",
+      driftDetails: "system prompt report differs",
+      promptDelta,
+      tokenDeltaPercent,
+      firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+    };
+  }
+  if (params.left.toolDescriptionHash !== params.right.toolDescriptionHash) {
+    return {
+      scenarioId: params.scenarioId,
+      left: params.left,
+      right: params.right,
+      drift: "tool-description",
+      driftDetails: "tool description summary shape differs",
+      promptDelta,
+      tokenDeltaPercent,
+      firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+    };
+  }
+  if (params.left.toolSchemaHash !== params.right.toolSchemaHash) {
+    return {
+      scenarioId: params.scenarioId,
+      left: params.left,
+      right: params.right,
+      drift: "tool-schema",
+      driftDetails: "tool schema shape differs",
+      promptDelta,
+      tokenDeltaPercent,
+      firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+    };
+  }
+  const compareToolShapes =
+    params.comparisonMode !== "codex-native-workspace" && params.comparisonMode !== "outcome-only";
+  const compareTranscriptStructure =
+    params.comparisonMode !== "codex-native-workspace" && params.comparisonMode !== "outcome-only";
+
+  if (compareToolShapes) {
+    const toolCallDrift = compareToolCallShape(params.left.toolCalls, params.right.toolCalls);
+    if (toolCallDrift) {
+      return {
+        scenarioId: params.scenarioId,
+        left: params.left,
+        right: params.right,
+        drift: "tool-call-shape",
+        driftDetails: toolCallDrift,
+        promptDelta,
+        tokenDeltaPercent,
+        firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+      };
+    }
+    const toolResultDrift = compareToolResultShape(params.left.toolCalls, params.right.toolCalls);
+    if (toolResultDrift) {
+      return {
+        scenarioId: params.scenarioId,
+        left: params.left,
+        right: params.right,
+        drift: "tool-result-shape",
+        driftDetails: toolResultDrift,
+        promptDelta,
+        tokenDeltaPercent,
+        firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+      };
+    }
+  }
+  const leftTranscriptRecords = countComparableTranscriptRecords(params.left.transcriptBytes);
+  const rightTranscriptRecords = countComparableTranscriptRecords(params.right.transcriptBytes);
+  if (
+    compareTranscriptStructure &&
+    (leftTranscriptRecords !== rightTranscriptRecords ||
+      (!params.left.finalText && Boolean(params.right.finalText)) ||
+      (Boolean(params.left.finalText) && !params.right.finalText))
+  ) {
+    return {
+      scenarioId: params.scenarioId,
+      left: params.left,
+      right: params.right,
+      drift: "structural",
+      driftDetails: `transcript/final-text structure differs (${leftTranscriptRecords} message records vs ${rightTranscriptRecords} message records)`,
+      promptDelta,
+      tokenDeltaPercent,
+      firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+    };
+  }
+  if (
+    normalizeTextForParity(params.left.finalText) !== normalizeTextForParity(params.right.finalText)
+  ) {
+    return {
+      scenarioId: params.scenarioId,
+      left: params.left,
+      right: params.right,
+      drift: "text-only",
+      driftDetails: "final text differs after whitespace normalization",
+      promptDelta,
+      tokenDeltaPercent,
+      firstDriftTurn: firstDriftTurn(params.left.transcriptBytes, params.right.transcriptBytes),
+    };
+  }
+  return {
+    scenarioId: params.scenarioId,
+    left: params.left,
+    right: params.right,
+    drift: "none",
+    promptDelta,
+    tokenDeltaPercent,
+  };
+}

extensions/qa-lab/src/harness-runtime.ts

@@ -0,0 +1,126 @@
+// Qa Lab plugin module implements harness runtime behavior.
+import {
+  buildMentionRegexes,
+  implicitMentionKindWhen,
+  matchesMentionPatterns,
+  matchesMentionWithExplicit,
+  resolveInboundMentionDecision,
+} from "openclaw/plugin-sdk/channel-inbound";
+import type { PluginRuntime } from "openclaw/plugin-sdk/runtime-store";
+
+type SessionRecord = {
+  sessionKey: string;
+  body: string;
+};
+
+export function createQaRunnerRuntime(): PluginRuntime {
+  const sessions = new Map<string, SessionRecord>();
+  return {
+    channel: {
+      routing: {
+        resolveAgentRoute({
+          accountId,
+          peer,
+        }: {
+          accountId?: string | null;
+          peer?: { kind?: string; id?: string } | null;
+        }) {
+          return {
+            agentId: "qa-agent",
+            accountId: accountId ?? "default",
+            sessionKey: `qa-agent:${peer?.kind ?? "direct"}:${peer?.id ?? "default"}`,
+            mainSessionKey: "qa-agent:main",
+            lastRoutePolicy: "session",
+            matchedBy: "default",
+            channel: "qa-channel",
+          };
+        },
+      },
+      session: {
+        resolveStorePath(_store: string | undefined, { agentId }: { agentId: string }) {
+          return agentId;
+        },
+        readSessionUpdatedAt({ sessionKey }: { sessionKey: string }) {
+          return sessions.has(sessionKey) ? Date.now() : undefined;
+        },
+        recordInboundSession({
+          sessionKey,
+          ctx,
+        }: {
+          sessionKey: string;
+          ctx: { BodyForAgent?: string; Body?: string };
+        }) {
+          sessions.set(sessionKey, {
+            sessionKey,
+            body: ctx.BodyForAgent ?? ctx.Body ?? "",
+          });
+        },
+      },
+      mentions: {
+        buildMentionRegexes,
+        matchesMentionPatterns,
+        matchesMentionWithExplicit,
+        implicitMentionKindWhen,
+        resolveInboundMentionDecision,
+      },
+      reply: {
+        resolveEnvelopeFormatOptions() {
+          return {};
+        },
+        formatAgentEnvelope({ body }: { body: string }) {
+          return body;
+        },
+        finalizeInboundContext(ctx: Record<string, unknown>) {
+          return ctx as typeof ctx & { CommandAuthorized: boolean };
+        },
+        async dispatchReplyWithBufferedBlockDispatcher({
+          ctx,
+          dispatcherOptions,
+        }: {
+          ctx: { BodyForAgent?: string; Body?: string };
+          dispatcherOptions: { deliver: (payload: { text: string }) => Promise<void> };
+        }) {
+          await dispatcherOptions.deliver({
+            text: `qa-echo: ${ctx.BodyForAgent ?? ctx.Body ?? ""}`,
+          });
+        },
+      },
+      inbound: {
+        async dispatchReply(
+          params: Parameters<PluginRuntime["channel"]["inbound"]["dispatchReply"]>[0],
+        ) {
+          const sessionKey =
+            typeof params.ctxPayload.SessionKey === "string"
+              ? params.ctxPayload.SessionKey
+              : params.routeSessionKey;
+          await params.recordInboundSession({
+            storePath: params.storePath,
+            sessionKey,
+            ctx: params.ctxPayload,
+            onRecordError: params.record?.onRecordError ?? (() => undefined),
+          });
+          const dispatchResult = await params.dispatchReplyWithBufferedBlockDispatcher({
+            ctx: params.ctxPayload,
+            cfg: params.cfg,
+            dispatcherOptions: {
+              ...params.dispatcherOptions,
+              deliver: async (payload, info) => {
+                await params.delivery.deliver(payload, info);
+              },
+              onError: params.delivery.onError,
+            },
+            replyOptions: params.replyOptions,
+            replyResolver: params.replyResolver,
+          });
+          return {
+            admission: params.admission ?? { kind: "dispatch" },
+            dispatched: true,
+            ctxPayload: params.ctxPayload,
+            routeSessionKey: params.routeSessionKey,
+            dispatchResult,
+          };
+        },
+      },
+    },
+  } as unknown as PluginRuntime;
+}

extensions/qa-lab/src/jsonl-replay.test.ts

@@ -0,0 +1,170 @@
+// Qa Lab tests cover jsonl replay plugin behavior.
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  createMockJsonlReplayCellRunner,
+  extractJsonlReplayUserTurns,
+  renderJsonlReplayMarkdownReport,
+  runJsonlReplay,
+  type JsonlReplayCellRunner,
+} from "./jsonl-replay.js";
+import type { RuntimeId, RuntimeParityCell, RuntimeParityToolCall } from "./runtime-parity.js";
+
+const tempRoots: string[] = [];
+
+function makeCell(
+  runtime: RuntimeId,
+  overrides: Partial<RuntimeParityCell> = {},
+): RuntimeParityCell {
+  return {
+    runtime,
+    transcriptBytes: `{"message":{"role":"assistant","content":"${runtime} reply"}}\n`,
+    toolCalls: [],
+    finalText: "same reply",
+    usage: {
+      inputTokens: 10,
+      outputTokens: 5,
+      totalTokens: 15,
+    },
+    wallClockMs: 12,
+    bootStateLines: [],
+    ...overrides,
+  };
+}
+
+function makeToolCall(overrides: Partial<RuntimeParityToolCall> = {}): RuntimeParityToolCall {
+  return {
+    tool: "read",
+    argsHash: "args-a",
+    resultHash: "result-a",
+    ...overrides,
+  };
+}
+
+async function makeTempDir() {
+  const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "jsonl-replay-"));
+  tempRoots.push(tempRoot);
+  return tempRoot;
+}
+
+afterEach(async () => {
+  await Promise.all(
+    tempRoots.splice(0).map((tempRoot) => fs.rm(tempRoot, { recursive: true, force: true })),
+  );
+});
+
+describe("jsonl replay", () => {
+  it("extracts user-turn boundaries while ignoring system, tool-only, empty, and malformed rows", () => {
+    const turns = extractJsonlReplayUserTurns(
+      [
+        `{"message":{"role":"system","content":"System setup"}}`,
+        `{"message":{"role":"tool","content":"tool-only prelude"}}`,
+        `{"message":{"role":"user","content":"   "}}`,
+        `{not-json`,
+        `{"message":{"role":"assistant","content":"Ready."}}`,
+        `{"message":{"role":"user","content":[{"type":"text","text":"Plan the release"},{"type":"tool_result","content":"ignored"}]}}`,
+        `{"role":"user","content":[{"type":"input_text","text":"Check the follow-up"}]}`,
+      ].join("\n"),
+    );
+
+    expect(turns).toEqual([
+      expect.objectContaining({
+        turn: 1,
+        lineNumber: 6,
+        userText: "Plan the release",
+      }),
+      expect.objectContaining({
+        turn: 2,
+        lineNumber: 7,
+        userText: "Check the follow-up",
+      }),
+    ]);
+    expect(turns[0]?.transcriptPrefix).toContain(`"role":"system"`);
+    expect(turns[0]?.transcriptPrefix).not.toContain("{not-json");
+  });
+
+  it("reports the earliest divergent turn using runtime parity drift classes", async () => {
+    const transcriptDir = await makeTempDir();
+    await fs.writeFile(
+      path.join(transcriptDir, "three-turns.jsonl"),
+      [
+        `{"message":{"role":"user","content":"Turn one"}}`,
+        `{"message":{"role":"assistant","content":"Ready"}}`,
+        `{"message":{"role":"user","content":"Turn two"}}`,
+        `{"message":{"role":"assistant","content":"Using a tool"}}`,
+        `{"message":{"role":"user","content":"Turn three"}}`,
+      ].join("\n"),
+      "utf8",
+    );
+
+    const runCell: JsonlReplayCellRunner = async ({ runtime, turn }) => {
+      if (turn.turn === 2) {
+        return {
+          scenarioStatus: "pass",
+          cell: makeCell(runtime, {
+            toolCalls: [makeToolCall(runtime === "openclaw" ? {} : { argsHash: "args-codex" })],
+          }),
+        };
+      }
+      if (turn.turn === 3) {
+        return {
+          scenarioStatus: "pass",
+          cell: makeCell(runtime, {
+            finalText: runtime === "openclaw" ? "openclaw wording" : "codex wording",
+          }),
+        };
+      }
+      return {
+        scenarioStatus: "pass",
+        cell: makeCell(runtime),
+      };
+    };
+
+    const result = await runJsonlReplay(
+      {
+        directory: transcriptDir,
+        runtimePair: ["openclaw", "codex"],
+        providerMode: "mock-openai",
+      },
+      { runCell },
+    );
+
+    expect(result.transcripts).toHaveLength(1);
+    expect(result.transcripts[0]).toEqual(
+      expect.objectContaining({
+        userTurnCount: 3,
+        drift: ["none", "tool-call-shape", "text-only"],
+        firstDriftAtTurn: 2,
+      }),
+    );
+    expect(result.transcripts[0]?.cells.openclaw).toHaveLength(3);
+    expect(result.transcripts[0]?.cells.codex).toHaveLength(3);
+  });
+
+  it("runs the curated replay fixture set in mock-openai mode", async () => {
+    const fixtureDir = path.resolve("qa/scenarios/jsonl-replay");
+
+    const result = await runJsonlReplay(
+      {
+        directory: fixtureDir,
+        runtimePair: ["openclaw", "codex"],
+        providerMode: "mock-openai",
+      },
+      { runCell: createMockJsonlReplayCellRunner() },
+    );
+
+    expect(result.transcripts).toHaveLength(7);
+    expect(result.transcripts.map((entry) => entry.userTurnCount)).toEqual([2, 2, 3, 2, 2, 2, 2]);
+    expect(result.transcripts.every((entry) => entry.firstDriftAtTurn === undefined)).toBe(true);
+    expect(
+      renderJsonlReplayMarkdownReport({
+        generatedAt: "2026-05-10T00:00:00.000Z",
+        providerMode: "mock-openai",
+        runtimePair: ["openclaw", "codex"],
+        transcripts: result.transcripts,
+      }),
+    ).toContain("| plan-mode-boundaries.jsonl | 3 |  | none, none, none |");
+  });
+});

extensions/qa-lab/src/jsonl-replay.ts

@@ -0,0 +1,268 @@
+// Qa Lab plugin module implements jsonl replay behavior.
+import fs from "node:fs/promises";
+import path from "node:path";
+import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
+import {
+  runRuntimeParityScenario,
+  type RuntimeId,
+  type RuntimeParityCell,
+  type RuntimeParityResult,
+  type RuntimeParityScenarioExecution,
+} from "./runtime-parity.js";
+
+export type JsonlReplayInput = {
+  directory: string;
+  runtimePair: ["openclaw", "codex"];
+  providerMode: "mock-openai" | "live-frontier";
+};
+
+export type JsonlReplayTurn = {
+  turn: number;
+  lineNumber: number;
+  userText: string;
+  transcriptPrefix: string;
+};
+
+export type JsonlReplayCellRunner = (params: {
+  runtime: RuntimeId;
+  transcriptPath: string;
+  turn: JsonlReplayTurn;
+  turns: readonly JsonlReplayTurn[];
+  providerMode: JsonlReplayInput["providerMode"];
+}) => Promise<RuntimeParityScenarioExecution>;
+
+export type JsonlReplayResult = {
+  transcripts: Array<{
+    transcriptPath: string;
+    userTurnCount: number;
+    cells: { openclaw: RuntimeParityCell[]; codex: RuntimeParityCell[] };
+    drift: Array<RuntimeParityResult["drift"]>;
+    firstDriftAtTurn?: number;
+  }>;
+};
+
+export type JsonlReplayOptions = {
+  runCell?: JsonlReplayCellRunner;
+};
+
+export type JsonlReplayMarkdownReport = {
+  generatedAt: string;
+  providerMode: JsonlReplayInput["providerMode"];
+  runtimePair: JsonlReplayInput["runtimePair"];
+  transcripts: JsonlReplayResult["transcripts"];
+};
+
+function readString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+}
+
+function readReplayMessage(record: Record<string, unknown>): Record<string, unknown> | undefined {
+  if (isRecord(record.message)) {
+    return record.message;
+  }
+  return readString(record.role) ? record : undefined;
+}
+
+function readRole(message: Record<string, unknown>) {
+  return readString(message.role)?.toLowerCase();
+}
+
+function isTextLikeContentBlock(block: Record<string, unknown>) {
+  const type = readString(block.type)?.toLowerCase();
+  return (
+    !type ||
+    type === "text" ||
+    type === "input_text" ||
+    type === "message" ||
+    type === "output_text" ||
+    type === "user_text"
+  );
+}
+
+function extractTextContent(content: unknown): string {
+  if (typeof content === "string") {
+    return content.trim();
+  }
+  if (!Array.isArray(content)) {
+    return "";
+  }
+  const parts: string[] = [];
+  for (const block of content) {
+    if (typeof block === "string") {
+      if (block.trim()) {
+        parts.push(block.trim());
+      }
+      continue;
+    }
+    if (!isRecord(block) || !isTextLikeContentBlock(block)) {
+      continue;
+    }
+    const text = readString(block.text) ?? readString(block.content);
+    if (text) {
+      parts.push(text);
+    }
+  }
+  return parts.join("\n").trim();
+}
+
+export function extractJsonlReplayUserTurns(transcriptBytes: string): JsonlReplayTurn[] {
+  const turns: JsonlReplayTurn[] = [];
+  const acceptedLines: string[] = [];
+  for (const [lineIndex, rawLine] of transcriptBytes.split(/\r?\n/u).entries()) {
+    const trimmed = rawLine.trim();
+    if (!trimmed) {
+      continue;
+    }
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(trimmed) as unknown;
+    } catch {
+      continue;
+    }
+    if (!isRecord(parsed)) {
+      continue;
+    }
+    acceptedLines.push(trimmed);
+    const message = readReplayMessage(parsed);
+    if (!message || readRole(message) !== "user") {
+      continue;
+    }
+    const userText = extractTextContent(message.content);
+    if (!userText) {
+      continue;
+    }
+    turns.push({
+      turn: turns.length + 1,
+      lineNumber: lineIndex + 1,
+      userText,
+      transcriptPrefix: `${acceptedLines.join("\n")}\n`,
+    });
+  }
+  return turns;
+}
+
+async function listJsonlFiles(directory: string): Promise<string[]> {
+  const entries = await fs.readdir(directory, { withFileTypes: true });
+  const files: string[] = [];
+  for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) {
+    const entryPath = path.join(directory, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...(await listJsonlFiles(entryPath)));
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith(".jsonl")) {
+      files.push(entryPath);
+    }
+  }
+  return files;
+}
+
+function defaultRunCell(): Promise<RuntimeParityScenarioExecution> {
+  throw new Error(
+    "jsonl replay requires a runtime cell runner; CLI/suite wiring should provide the Phase 1 runtime parity runner",
+  );
+}
+
+function assertSupportedRuntimePair(runtimePair: JsonlReplayInput["runtimePair"]) {
+  if (runtimePair[0] !== "openclaw" || runtimePair[1] !== "codex") {
+    throw new Error(`unsupported jsonl replay runtime pair: ${runtimePair.join(",")}`);
+  }
+}
+
+export function createMockJsonlReplayCellRunner(): JsonlReplayCellRunner {
+  return async ({ runtime, turn }) => ({
+    scenarioStatus: "pass",
+    cell: {
+      runtime,
+      transcriptBytes: turn.transcriptPrefix,
+      toolCalls: [],
+      finalText: `Replayed curated turn ${turn.turn}.`,
+      usage: {
+        inputTokens: Math.max(1, Math.ceil(turn.transcriptPrefix.length / 4)),
+        outputTokens: 8,
+        totalTokens: Math.max(1, Math.ceil(turn.transcriptPrefix.length / 4)) + 8,
+      },
+      wallClockMs: 1,
+      bootStateLines: [],
+    },
+  });
+}
+
+export async function runJsonlReplay(
+  input: JsonlReplayInput,
+  options: JsonlReplayOptions = {},
+): Promise<JsonlReplayResult> {
+  assertSupportedRuntimePair(input.runtimePair);
+  const directory = path.resolve(input.directory);
+  const transcriptPaths = await listJsonlFiles(directory);
+  const runCell = options.runCell ?? defaultRunCell;
+  const transcripts: JsonlReplayResult["transcripts"] = [];
+
+  for (const transcriptPath of transcriptPaths) {
+    const transcriptBytes = await fs.readFile(transcriptPath, "utf8");
+    const turns = extractJsonlReplayUserTurns(transcriptBytes);
+    const cells: { openclaw: RuntimeParityCell[]; codex: RuntimeParityCell[] } = {
+      openclaw: [],
+      codex: [],
+    };
+    const drift: Array<RuntimeParityResult["drift"]> = [];
+    let firstDriftAtTurn: number | undefined;
+
+    for (const turn of turns) {
+      const parity = await runRuntimeParityScenario({
+        scenarioId: `${path.basename(transcriptPath)}#turn-${turn.turn}`,
+        runCell: async (runtime) =>
+          runCell({
+            runtime,
+            transcriptPath,
+            turn,
+            turns,
+            providerMode: input.providerMode,
+          }),
+      });
+      cells.openclaw.push(parity.cells.openclaw);
+      cells.codex.push(parity.cells.codex);
+      drift.push(parity.drift);
+      if (firstDriftAtTurn === undefined && parity.drift !== "none") {
+        firstDriftAtTurn = turn.turn;
+      }
+    }
+
+    transcripts.push({
+      transcriptPath,
+      userTurnCount: turns.length,
+      cells,
+      drift,
+      ...(firstDriftAtTurn !== undefined ? { firstDriftAtTurn } : {}),
+    });
+  }
+
+  return { transcripts };
+}
+
+export function renderJsonlReplayMarkdownReport(report: JsonlReplayMarkdownReport): string {
+  const totalTurns = report.transcripts.reduce((sum, entry) => sum + entry.userTurnCount, 0);
+  const driftedTranscripts = report.transcripts.filter(
+    (entry) => entry.firstDriftAtTurn !== undefined,
+  );
+  const lines = [
+    `# OpenClaw JSONL Replay Report - ${report.runtimePair[0]} vs ${report.runtimePair[1]}`,
+    "",
+    `- Generated at: ${report.generatedAt}`,
+    `- Provider mode: ${report.providerMode}`,
+    `- Transcripts: ${report.transcripts.length}`,
+    `- User turns: ${totalTurns}`,
+    `- Drifted transcripts: ${driftedTranscripts.length}`,
+    "",
+    "| Transcript | User turns | First drift turn | Drift sequence |",
+    "| --- | ---: | ---: | --- |",
+  ];
+
+  for (const transcript of report.transcripts) {
+    lines.push(
+      `| ${path.basename(transcript.transcriptPath)} | ${transcript.userTurnCount} | ${transcript.firstDriftAtTurn ?? ""} | ${transcript.drift.join(", ")} |`,
+    );
+  }
+
+  return `${lines.join("\n").trimEnd()}\n`;
+}

extensions/qa-lab/src/lab-server-capture.test.ts

@@ -0,0 +1,60 @@
+// Qa Lab tests cover lab server capture plugin behavior.
+import { createServer } from "node:http";
+import { afterEach, describe, expect, it } from "vitest";
+import { mapCaptureEventForQa, probeTcpReachability } from "./lab-server-capture.js";
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+describe("qa-lab server capture helpers", () => {
+  it("maps capture rows into QA-friendly fields", () => {
+    const record = mapCaptureEventForQa({
+      flowId: "flow-1",
+      dataText: '{"hello":"world"}',
+      metaJson: JSON.stringify({
+        provider: "openai",
+        api: "responses",
+        model: "gpt-5.5",
+        captureOrigin: "shared-fetch",
+      }),
+    }) as ReturnType<typeof mapCaptureEventForQa> & { flowId?: string };
+    expect(record.flowId).toBe("flow-1");
+    expect(record.payloadPreview).toBe('{"hello":"world"}');
+    expect(record.provider).toBe("openai");
+    expect(record.api).toBe("responses");
+    expect(record.model).toBe("gpt-5.5");
+    expect(record.captureOrigin).toBe("shared-fetch");
+  });
+
+  it("probes tcp reachability for reachable and unreachable targets", async () => {
+    const server = createServer((_req, res) => {
+      res.writeHead(200);
+      res.end("ok");
+    });
+    await new Promise<void>((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(0, "127.0.0.1", () => resolve());
+    });
+    cleanups.push(
+      async () =>
+        await new Promise<void>((resolve, reject) => {
+          server.close((error) => (error ? reject(error) : resolve()));
+        }),
+    );
+
+    const address = server.address();
+    if (!address || typeof address === "string") {
+      throw new Error("expected tcp probe address");
+    }
+
+    const reachable = await probeTcpReachability(`http://127.0.0.1:${address.port}`);
+    expect(reachable.ok).toBe(true);
+    const unreachable = await probeTcpReachability("http://127.0.0.1:9", 50);
+    expect(unreachable.ok).toBe(false);
+  });
+});

extensions/qa-lab/src/lab-server-capture.ts

@@ -0,0 +1,128 @@
+// Qa Lab plugin module implements lab server capture behavior.
+import net from "node:net";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+
+const CAPTURE_QUERY_PRESETS = new Set([
+  "double-sends",
+  "retry-storms",
+  "cache-busting",
+  "ws-duplicate-frames",
+  "missing-ack",
+  "error-bursts",
+]);
+
+type QaStartupProbeStatus = {
+  label: string;
+  url: string;
+  ok: boolean;
+  error?: string;
+};
+
+export function isCaptureQueryPreset(
+  value: string,
+): value is Parameters<
+  ReturnType<
+    typeof import("openclaw/plugin-sdk/proxy-capture").getDebugProxyCaptureStore
+  >["queryPreset"]
+>[0] {
+  return CAPTURE_QUERY_PRESETS.has(value);
+}
+
+function parseCaptureMeta(metaJson: unknown): Record<string, unknown> | null {
+  if (typeof metaJson !== "string" || metaJson.trim().length === 0) {
+    return null;
+  }
+  try {
+    const parsed = JSON.parse(metaJson) as unknown;
+    return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : null;
+  } catch {
+    return null;
+  }
+}
+
+function readCaptureMetaString(
+  meta: Record<string, unknown> | null,
+  key: string,
+): string | undefined {
+  const value = meta?.[key];
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+}
+
+export function mapCaptureEventForQa(row: Record<string, unknown>) {
+  const meta = parseCaptureMeta(row.metaJson);
+  return {
+    ...row,
+    payloadPreview: typeof row.dataText === "string" ? row.dataText : undefined,
+    provider: readCaptureMetaString(meta, "provider"),
+    api: readCaptureMetaString(meta, "api"),
+    model: readCaptureMetaString(meta, "model"),
+    captureOrigin: readCaptureMetaString(meta, "captureOrigin"),
+  };
+}
+
+function defaultPortForProtocol(protocol: string): number {
+  if (protocol === "https:") {
+    return 443;
+  }
+  if (protocol === "http:") {
+    return 80;
+  }
+  return 0;
+}
+
+export async function probeTcpReachability(
+  rawUrl: string,
+  timeoutMs = 700,
+): Promise<QaStartupProbeStatus> {
+  let parsed: URL;
+  try {
+    parsed = new URL(rawUrl);
+  } catch {
+    return {
+      label: rawUrl,
+      url: rawUrl,
+      ok: false,
+      error: "invalid url",
+    };
+  }
+  const host = parsed.hostname;
+  const port = parsed.port ? Number(parsed.port) : defaultPortForProtocol(parsed.protocol);
+  if (!host || !Number.isFinite(port) || port <= 0) {
+    return {
+      label: parsed.origin,
+      url: parsed.toString(),
+      ok: false,
+      error: "missing host or port",
+    };
+  }
+  try {
+    await new Promise<void>((resolve, reject) => {
+      const socket = net.createConnection({ host, port });
+      const onError = (error: Error) => {
+        socket.destroy();
+        reject(error);
+      };
+      socket.setTimeout(timeoutMs, () => {
+        socket.destroy(new Error("timeout"));
+      });
+      socket.once("connect", () => {
+        socket.end();
+        resolve();
+      });
+      socket.once("error", onError);
+      socket.once("timeout", () => onError(new Error("timeout")));
+    });
+    return {
+      label: parsed.host,
+      url: parsed.toString(),
+      ok: true,
+    };
+  } catch (error) {
+    return {
+      label: parsed.host,
+      url: parsed.toString(),
+      ok: false,
+      error: formatErrorMessage(error),
+    };
+  }
+}

extensions/qa-lab/src/lab-server-ui.test.ts

@@ -0,0 +1,92 @@
+// Qa Lab tests cover lab server ui plugin behavior.
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  detectContentType,
+  missingUiHtml,
+  resolveUiAssetVersion,
+  tryResolveUiAsset,
+} from "./lab-server-ui.js";
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+describe("qa-lab server ui helpers", () => {
+  it("detects basic UI asset content types", () => {
+    expect(detectContentType("index.html")).toBe("text/html; charset=utf-8");
+    expect(detectContentType("styles.css")).toBe("text/css; charset=utf-8");
+    expect(detectContentType("main.js")).toBe("text/javascript; charset=utf-8");
+    expect(detectContentType("icon.svg")).toBe("image/svg+xml");
+  });
+
+  it("renders the missing-ui placeholder html", () => {
+    expect(missingUiHtml()).toContain("QA Lab UI not built");
+    expect(missingUiHtml()).toContain("pnpm qa:lab:build");
+  });
+
+  it("hashes built UI assets and changes when bundle contents change", async () => {
+    const uiDistDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-ui-dist-"));
+    cleanups.push(async () => {
+      await rm(uiDistDir, { recursive: true, force: true });
+    });
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><head><title>QA Lab</title></head><body><div id='app'></div></body></html>",
+      "utf8",
+    );
+
+    const version1 = resolveUiAssetVersion(uiDistDir);
+    expect(version1).toMatch(/^[0-9a-f]{12}$/);
+
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><head><title>QA Lab Updated</title></head><body><div id='app'></div></body></html>",
+      "utf8",
+    );
+
+    const version2 = resolveUiAssetVersion(uiDistDir);
+    expect(version2).toMatch(/^[0-9a-f]{12}$/);
+    expect(version2).not.toBe(version1);
+  });
+
+  it("never resolves sibling files outside the UI dist root", async () => {
+    const rootDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-ui-boundary-"));
+    cleanups.push(async () => {
+      await rm(rootDir, { recursive: true, force: true });
+    });
+    const uiDistDir = path.join(rootDir, "dist");
+    const siblingDir = path.join(rootDir, "dist-other");
+    await mkdir(uiDistDir, { recursive: true });
+    await mkdir(siblingDir, { recursive: true });
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><body>bundle-root</body></html>",
+      "utf8",
+    );
+    await writeFile(path.join(siblingDir, "secret.txt"), "sibling-secret", "utf8");
+
+    expect(tryResolveUiAsset("/", uiDistDir, rootDir)).toBe(path.join(uiDistDir, "index.html"));
+    expect(tryResolveUiAsset("/../dist-other/secret.txt", uiDistDir, rootDir)).toBeNull();
+  });
+
+  it("rejects malformed percent-encoded UI asset paths", async () => {
+    const uiDistDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-ui-malformed-"));
+    cleanups.push(async () => {
+      await rm(uiDistDir, { recursive: true, force: true });
+    });
+    await writeFile(
+      path.join(uiDistDir, "index.html"),
+      "<!doctype html><html><body>bundle-root</body></html>",
+      "utf8",
+    );
+
+    expect(tryResolveUiAsset("/%E0%A4", uiDistDir, uiDistDir)).toBeNull();
+  });
+});

extensions/qa-lab/src/lab-server-ui.ts

@@ -0,0 +1,302 @@
+// Qa Lab plugin module implements lab server ui behavior.
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import { request as httpRequest, type IncomingMessage, type ServerResponse } from "node:http";
+import { request as httpsRequest } from "node:https";
+import net from "node:net";
+import path from "node:path";
+import type { Duplex } from "node:stream";
+import tls from "node:tls";
+import { fileURLToPath } from "node:url";
+import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
+import { writeError } from "./bus-server.js";
+
+export function detectContentType(filePath: string): string {
+  if (filePath.endsWith(".css")) {
+    return "text/css; charset=utf-8";
+  }
+  if (filePath.endsWith(".js")) {
+    return "text/javascript; charset=utf-8";
+  }
+  if (filePath.endsWith(".json")) {
+    return "application/json; charset=utf-8";
+  }
+  if (filePath.endsWith(".svg")) {
+    return "image/svg+xml";
+  }
+  return "text/html; charset=utf-8";
+}
+
+export function missingUiHtml() {
+  return `<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>QA Lab UI Missing</title>
+    <style>
+      body { font-family: ui-sans-serif, system-ui, sans-serif; background: #0f1115; color: #f5f7fb; margin: 0; display: grid; place-items: center; min-height: 100vh; }
+      main { max-width: 42rem; padding: 2rem; background: #171b22; border: 1px solid #283140; border-radius: 18px; box-shadow: 0 30px 80px rgba(0,0,0,.35); }
+      code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; color: #9ee8d8; }
+      h1 { margin-top: 0; }
+    </style>
+  </head>
+  <body>
+    <main>
+      <h1>QA Lab UI not built</h1>
+      <p>Build the private debugger bundle, then reload this page.</p>
+      <p><code>pnpm qa:lab:build</code></p>
+    </main>
+  </body>
+</html>`;
+}
+
+function resolveUiDistDir(overrideDir?: string | null, repoRoot = process.cwd()) {
+  if (overrideDir?.trim()) {
+    return overrideDir;
+  }
+  const candidates = [
+    path.resolve(repoRoot, "extensions/qa-lab/web/dist"),
+    path.resolve(repoRoot, "dist/extensions/qa-lab/web/dist"),
+    fileURLToPath(new URL("../web/dist", import.meta.url)),
+  ];
+  return (
+    candidates.find((candidate) => {
+      if (!fs.existsSync(candidate)) {
+        return false;
+      }
+      const indexPath = path.join(candidate, "index.html");
+      return fs.existsSync(indexPath) && fs.statSync(indexPath).isFile();
+    }) ?? candidates[0]
+  );
+}
+
+function listUiAssetFiles(rootDir: string, currentDir = rootDir): string[] {
+  const entries = fs
+    .readdirSync(currentDir, { withFileTypes: true })
+    .toSorted((left, right) => left.name.localeCompare(right.name));
+  const files: string[] = [];
+  for (const entry of entries) {
+    const resolved = path.join(currentDir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...listUiAssetFiles(rootDir, resolved));
+      continue;
+    }
+    if (!entry.isFile()) {
+      continue;
+    }
+    files.push(path.relative(rootDir, resolved));
+  }
+  return files;
+}
+
+export function resolveUiAssetVersion(
+  overrideDir?: string | null,
+  repoRoot = process.cwd(),
+): string | null {
+  try {
+    const distDir = resolveUiDistDir(overrideDir, repoRoot);
+    const indexPath = path.join(distDir, "index.html");
+    if (!fs.existsSync(indexPath) || !fs.statSync(indexPath).isFile()) {
+      return null;
+    }
+    const hash = createHash("sha1");
+    for (const relativeFile of listUiAssetFiles(distDir)) {
+      hash.update(relativeFile);
+      hash.update("\0");
+      hash.update(fs.readFileSync(path.join(distDir, relativeFile)));
+      hash.update("\0");
+    }
+    return hash.digest("hex").slice(0, 12);
+  } catch {
+    return null;
+  }
+}
+
+export function resolveAdvertisedBaseUrl(params: {
+  bindHost?: string;
+  bindPort: number;
+  advertiseHost?: string;
+  advertisePort?: number;
+}) {
+  const advertisedHost =
+    params.advertiseHost?.trim() ||
+    (params.bindHost && params.bindHost !== "0.0.0.0" ? params.bindHost : "127.0.0.1");
+  const advertisedPort =
+    typeof params.advertisePort === "number" && Number.isFinite(params.advertisePort)
+      ? params.advertisePort
+      : params.bindPort;
+  return `http://${advertisedHost}:${advertisedPort}`;
+}
+
+export function isControlUiProxyPath(pathname: string) {
+  return pathname === "/control-ui" || pathname.startsWith("/control-ui/");
+}
+
+function rewriteControlUiProxyPath(pathname: string, search: string) {
+  const stripped = pathname === "/control-ui" ? "/" : pathname.slice("/control-ui".length) || "/";
+  return `${stripped}${search}`;
+}
+
+function rewriteEmbeddedControlUiHeaders(
+  headers: IncomingMessage["headers"],
+): Record<string, string | string[] | number | undefined> {
+  const rewritten: Record<string, string | string[] | number | undefined> = { ...headers };
+  delete rewritten["x-frame-options"];
+
+  const csp = headers["content-security-policy"];
+  if (typeof csp === "string") {
+    rewritten["content-security-policy"] = csp.includes("frame-ancestors")
+      ? csp.replace(/frame-ancestors\s+[^;]+/i, "frame-ancestors 'self'")
+      : `${csp}; frame-ancestors 'self'`;
+  }
+
+  return rewritten;
+}
+
+export async function proxyHttpRequest(params: {
+  req: IncomingMessage;
+  res: ServerResponse;
+  target: URL;
+  pathname: string;
+  search: string;
+  authorizationToken?: string | null;
+}) {
+  const client = params.target.protocol === "https:" ? httpsRequest : httpRequest;
+  const upstreamReq = client(
+    {
+      protocol: params.target.protocol,
+      hostname: params.target.hostname,
+      port: params.target.port || (params.target.protocol === "https:" ? 443 : 80),
+      method: params.req.method,
+      path: rewriteControlUiProxyPath(params.pathname, params.search),
+      headers: {
+        ...params.req.headers,
+        host: params.target.host,
+        ...(params.authorizationToken
+          ? { authorization: `Bearer ${params.authorizationToken}` }
+          : {}),
+      },
+    },
+    (upstreamRes) => {
+      params.res.writeHead(
+        upstreamRes.statusCode ?? 502,
+        rewriteEmbeddedControlUiHeaders(upstreamRes.headers),
+      );
+      upstreamRes.pipe(params.res);
+    },
+  );
+
+  upstreamReq.on("error", (error) => {
+    if (!params.res.headersSent) {
+      writeError(params.res, 502, error);
+      return;
+    }
+    params.res.destroy(error);
+  });
+
+  if (params.req.method === "GET" || params.req.method === "HEAD") {
+    upstreamReq.end();
+    return;
+  }
+  params.req.pipe(upstreamReq);
+}
+
+export function proxyUpgradeRequest(params: {
+  req: IncomingMessage;
+  socket: Duplex;
+  head: Buffer;
+  target: URL;
+  authorizationToken?: string | null;
+}) {
+  const requestUrl = new URL(params.req.url ?? "/", "http://127.0.0.1");
+  const port = Number(params.target.port || (params.target.protocol === "https:" ? 443 : 80));
+  const upstream =
+    params.target.protocol === "https:"
+      ? tls.connect({
+          host: params.target.hostname,
+          port,
+          servername: params.target.hostname,
+        })
+      : net.connect({
+          host: params.target.hostname,
+          port,
+        });
+
+  const headerLines: string[] = [];
+  for (let index = 0; index < params.req.rawHeaders.length; index += 2) {
+    const name = params.req.rawHeaders[index];
+    const value = params.req.rawHeaders[index + 1] ?? "";
+    const normalizedName = normalizeLowercaseStringOrEmpty(name);
+    if (
+      normalizedName === "host" ||
+      (params.authorizationToken && normalizedName === "authorization")
+    ) {
+      continue;
+    }
+    headerLines.push(`${name}: ${value}`);
+  }
+
+  upstream.once("connect", () => {
+    const requestText = [
+      `${params.req.method ?? "GET"} ${rewriteControlUiProxyPath(requestUrl.pathname, requestUrl.search)} HTTP/${params.req.httpVersion}`,
+      `Host: ${params.target.host}`,
+      ...(params.authorizationToken ? [`Authorization: Bearer ${params.authorizationToken}`] : []),
+      ...headerLines,
+      "",
+      "",
+    ].join("\r\n");
+    upstream.write(requestText);
+    if (params.head.length > 0) {
+      upstream.write(params.head);
+    }
+    upstream.pipe(params.socket);
+    params.socket.pipe(upstream);
+  });
+
+  const closeBoth = () => {
+    if (!params.socket.destroyed) {
+      params.socket.destroy();
+    }
+    if (!upstream.destroyed) {
+      upstream.destroy();
+    }
+  };
+
+  upstream.on("error", () => {
+    if (!params.socket.destroyed) {
+      params.socket.write("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n");
+    }
+    closeBoth();
+  });
+  params.socket.on("error", closeBoth);
+  params.socket.on("close", closeBoth);
+}
+
+export function tryResolveUiAsset(
+  pathname: string,
+  overrideDir?: string | null,
+  repoRoot = process.cwd(),
+): string | null {
+  const distDir = resolveUiDistDir(overrideDir, repoRoot);
+  if (!fs.existsSync(distDir)) {
+    return null;
+  }
+  const safePath = pathname === "/" ? "/index.html" : pathname;
+  let decoded: string;
+  try {
+    decoded = decodeURIComponent(safePath);
+  } catch {
+    return null;
+  }
+  const candidate = path.resolve(distDir, `.${decoded.startsWith("/") ? decoded : `/${decoded}`}`);
+  const relative = path.relative(distDir, candidate);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    return null;
+  }
+  if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) {
+    return candidate;
+  }
+  const fallback = path.join(distDir, "index.html");
+  return fs.existsSync(fallback) ? fallback : null;
+}

extensions/qa-lab/src/lab-server.ts

@@ -0,0 +1,886 @@
+// Qa Lab plugin module implements lab server behavior.
+import fs from "node:fs";
+import { createServer } from "node:http";
+import path from "node:path";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import {
+  acquireDebugProxyCaptureStore,
+  resolveDebugProxySettings,
+} from "openclaw/plugin-sdk/proxy-capture";
+import {
+  closeQaHttpServer,
+  handleQaBusRequest,
+  readQaJsonBody,
+  writeError,
+  writeJson,
+  writeQaRequestBodyLimitError,
+} from "./bus-server.js";
+import { createQaBusState, type QaBusState } from "./bus-state.js";
+import {
+  QaEvidenceGalleryError,
+  buildQaEvidenceGalleryModel,
+  resolveQaEvidenceArtifactFileByIndex,
+  resolveQaEvidenceArtifactFile,
+  resolveQaEvidenceProducerFile,
+} from "./evidence-gallery.js";
+import { createQaRunnerRuntime } from "./harness-runtime.js";
+import {
+  isCaptureQueryPreset,
+  mapCaptureEventForQa,
+  probeTcpReachability,
+} from "./lab-server-capture.js";
+import {
+  detectContentType,
+  isControlUiProxyPath,
+  missingUiHtml,
+  proxyHttpRequest,
+  proxyUpgradeRequest,
+  resolveAdvertisedBaseUrl,
+  resolveUiAssetVersion,
+  tryResolveUiAsset,
+} from "./lab-server-ui.js";
+import type {
+  QaLabLatestReport,
+  QaLabScenarioOutcome,
+  QaLabScenarioRun,
+  QaLabServerHandle,
+  QaLabServerStartParams,
+} from "./lab-server.types.js";
+import type { QaRunnerModelOption } from "./model-catalog.runtime.js";
+import { createQaChannelGatewayConfig } from "./qa-channel-transport.js";
+import {
+  createIdleQaRunnerSnapshot,
+  createQaRunOutputDir,
+  normalizeQaRunSelection,
+} from "./run-config.js";
+import { qaChannelPlugin, setQaChannelRuntime, type OpenClawConfig } from "./runtime-api.js";
+import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
+import { runQaSelfCheckAgainstState, type QaSelfCheckResult } from "./self-check.js";
+
+type QaLabBootstrapDefaults = {
+  conversationKind: "direct" | "channel";
+  conversationId: string;
+  senderId: string;
+  senderName: string;
+};
+
+export type {
+  QaLabLatestReport,
+  QaLabScenarioOutcome,
+  QaLabScenarioRun,
+  QaLabServerHandle,
+  QaLabServerStartParams,
+} from "./lab-server.types.js";
+
+export function writeQaLabServerError(res: Parameters<typeof writeError>[0], error: unknown): void {
+  if (writeQaRequestBodyLimitError(res, error)) {
+    return;
+  }
+  if (error instanceof QaEvidenceGalleryError) {
+    writeError(res, error.statusCode, error.message);
+    return;
+  }
+  writeError(res, 500, error);
+}
+
+function countQaLabScenarioRun(scenarios: QaLabScenarioOutcome[]) {
+  return {
+    total: scenarios.length,
+    pending: scenarios.filter((scenario) => scenario.status === "pending").length,
+    running: scenarios.filter((scenario) => scenario.status === "running").length,
+    passed: scenarios.filter((scenario) => scenario.status === "pass").length,
+    failed: scenarios.filter((scenario) => scenario.status === "fail").length,
+    skipped: scenarios.filter((scenario) => scenario.status === "skip").length,
+  };
+}
+
+function withQaLabRunCounts(run: Omit<QaLabScenarioRun, "counts">): QaLabScenarioRun {
+  return {
+    ...run,
+    counts: countQaLabScenarioRun(run.scenarios),
+  };
+}
+
+function parseQaEvidenceArtifactIndexText(value: string): number {
+  if (!/^(0|[1-9]\d*)$/.test(value)) {
+    throw new QaEvidenceGalleryError("Evidence artifact index is invalid.", 400);
+  }
+  const index = Number(value);
+  if (!Number.isSafeInteger(index) || String(index) !== value) {
+    throw new QaEvidenceGalleryError("Evidence artifact index is invalid.", 400);
+  }
+  return index;
+}
+
+function injectKickoffMessage(params: {
+  state: QaBusState;
+  defaults: QaLabBootstrapDefaults;
+  kickoffTask: string;
+}) {
+  return params.state.addInboundMessage({
+    conversation: {
+      id: params.defaults.conversationId,
+      kind: params.defaults.conversationKind,
+      ...(params.defaults.conversationKind === "channel"
+        ? { title: params.defaults.conversationId }
+        : {}),
+    },
+    senderId: params.defaults.senderId,
+    senderName: params.defaults.senderName,
+    text: params.kickoffTask,
+  });
+}
+
+function createBootstrapDefaults(autoKickoffTarget?: string): QaLabBootstrapDefaults {
+  if (autoKickoffTarget === "channel") {
+    return {
+      conversationKind: "channel",
+      conversationId: "qa-lab",
+      senderId: "qa-operator",
+      senderName: "QA Operator",
+    };
+  }
+  return {
+    conversationKind: "direct",
+    conversationId: "qa-operator",
+    senderId: "qa-operator",
+    senderName: "QA Operator",
+  };
+}
+
+const CONTROL_UI_CREDENTIAL_QUERY_KEYS = new Set([
+  "access_token",
+  "api_key",
+  "apikey",
+  "auth",
+  "devicetoken",
+  "id_token",
+  "password",
+  "refresh_token",
+  "token",
+]);
+const CONTROL_UI_CREDENTIAL_QUERY_PATTERN =
+  /([?&])(?:access_token|api_?key|auth|deviceToken|id_token|password|refresh_token|token)=[^&#\s]*&?/gi;
+
+function stripSensitiveQueryParamsFromText(rawUrl: string): string {
+  let sanitized = rawUrl;
+  for (;;) {
+    const next = sanitized
+      .replace(CONTROL_UI_CREDENTIAL_QUERY_PATTERN, (match: string, separator: string) =>
+        match.endsWith("&") ? separator : "",
+      )
+      .replace(/[?&]$/, "")
+      .replace("?&", "?");
+    if (next === sanitized) {
+      return next;
+    }
+    sanitized = next;
+  }
+}
+
+function stripSensitiveQueryParams(rawUrl: string): string {
+  try {
+    const url = new URL(rawUrl);
+    for (const key of Array.from(url.searchParams.keys())) {
+      if (CONTROL_UI_CREDENTIAL_QUERY_KEYS.has(key.toLowerCase())) {
+        url.searchParams.delete(key);
+      }
+    }
+    return url.toString();
+  } catch {
+    return stripSensitiveQueryParamsFromText(rawUrl);
+  }
+}
+
+function sanitizeControlUiPublicUrl(url: string | null): string | null {
+  if (!url) {
+    return null;
+  }
+  const fragmentIndex = url.indexOf("#");
+  const withoutFragment = fragmentIndex === -1 ? url : url.slice(0, fragmentIndex);
+  return stripSensitiveQueryParams(withoutFragment);
+}
+
+function createQaLabConfig(baseUrl: string): OpenClawConfig {
+  return createQaChannelGatewayConfig({ baseUrl });
+}
+
+function normalizeQaLabCleanupError(error: unknown): Error {
+  return error instanceof Error ? error : new Error(formatErrorMessage(error));
+}
+
+function detectQaEvidenceArtifactContentType(filePath: string): string {
+  const lower = filePath.toLowerCase();
+  if (lower.endsWith(".png")) {
+    return "image/png";
+  }
+  if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) {
+    return "image/jpeg";
+  }
+  if (lower.endsWith(".gif")) {
+    return "image/gif";
+  }
+  if (lower.endsWith(".webp")) {
+    return "image/webp";
+  }
+  if (lower.endsWith(".webm")) {
+    return "video/webm";
+  }
+  if (lower.endsWith(".mp4")) {
+    return "video/mp4";
+  }
+  if (lower.endsWith(".mov")) {
+    return "video/quicktime";
+  }
+  if (lower.endsWith(".json") || lower.endsWith(".jsonl")) {
+    return "application/json; charset=utf-8";
+  }
+  if (lower.endsWith(".md") || lower.endsWith(".txt") || lower.endsWith(".log")) {
+    return "text/plain; charset=utf-8";
+  }
+  return "application/octet-stream";
+}
+
+async function startQaGatewayLoop(params: { state: QaBusState; baseUrl: string }) {
+  const runtime = createQaRunnerRuntime();
+  setQaChannelRuntime(runtime);
+  const cfg = createQaLabConfig(params.baseUrl);
+  const account = qaChannelPlugin.config.resolveAccount(cfg, "default");
+  const abort = new AbortController();
+  const task = Promise.resolve().then(
+    async () =>
+      await qaChannelPlugin.gateway?.startAccount?.({
+        accountId: account.accountId,
+        account,
+        cfg,
+        runtime: {
+          log: () => undefined,
+          error: () => undefined,
+          exit: () => undefined,
+        },
+        abortSignal: abort.signal,
+        log: {
+          info: () => undefined,
+          warn: () => undefined,
+          error: () => undefined,
+          debug: () => undefined,
+        },
+        getStatus: () => ({
+          accountId: account.accountId,
+          configured: true,
+          enabled: true,
+          running: true,
+        }),
+        setStatus: () => undefined,
+      }),
+  );
+  return {
+    cfg,
+    async stop() {
+      abort.abort();
+      await task;
+    },
+  };
+}
+
+export async function startQaLabServer(
+  params?: QaLabServerStartParams,
+): Promise<QaLabServerHandle> {
+  const repoRoot = path.resolve(params?.repoRoot ?? process.cwd());
+  const captureSettings = resolveDebugProxySettings();
+  const captureStoreLease = acquireDebugProxyCaptureStore();
+  const captureStore = captureStoreLease.store;
+  const state = createQaBusState();
+  let latestReport: QaLabLatestReport | null = null;
+  let latestScenarioRun: QaLabScenarioRun | null = null;
+  const scenarioCatalog = readQaBootstrapScenarioCatalog();
+  const bootstrapDefaults = createBootstrapDefaults(params?.autoKickoffTarget);
+  let runnerModelOptions: QaRunnerModelOption[] = [];
+  let runnerModelCatalogStatus: "loading" | "ready" | "failed" = "loading";
+  let runnerSnapshot = createIdleQaRunnerSnapshot(scenarioCatalog.scenarios);
+  let activeSuiteRun: Promise<void> | null = null;
+  let controlUiProxyTarget = params?.controlUiProxyTarget?.trim()
+    ? new URL(params.controlUiProxyTarget)
+    : null;
+  let controlUiProxyToken = params?.controlUiProxyToken?.trim() || null;
+  let controlUiUrl = sanitizeControlUiPublicUrl(params?.controlUiUrl?.trim() || null);
+  let gateway:
+    | {
+        cfg: OpenClawConfig;
+        stop: () => Promise<void>;
+      }
+    | undefined;
+  const embeddedGatewayEnabled = params?.embeddedGateway !== "disabled";
+  let labHandle: QaLabServerHandle | null = null;
+  let captureStoreReleased = false;
+  let serverListening = false;
+
+  let listenUrl = "";
+  let publicBaseUrl = "";
+  let runnerModelCatalogPromise: Promise<void> | null = null;
+  let runnerModelCatalogAbort: AbortController | null = null;
+  const ensureRunnerModelCatalog = () => {
+    if (runnerModelCatalogPromise) {
+      return runnerModelCatalogPromise;
+    }
+    runnerModelCatalogAbort = new AbortController();
+    runnerModelCatalogPromise = (async () => {
+      try {
+        const { loadQaRunnerModelOptions } = await import("./model-catalog.runtime.js");
+        runnerModelOptions = await loadQaRunnerModelOptions({
+          repoRoot,
+          signal: runnerModelCatalogAbort?.signal,
+        });
+        runnerModelCatalogStatus = "ready";
+      } catch {
+        runnerModelOptions = [];
+        runnerModelCatalogStatus = "failed";
+      }
+    })().finally(() => {
+      runnerModelCatalogAbort = null;
+    });
+    return runnerModelCatalogPromise;
+  };
+
+  async function runSelfCheck(): Promise<QaSelfCheckResult> {
+    latestScenarioRun = withQaLabRunCounts({
+      kind: "self-check",
+      status: "running",
+      startedAt: new Date().toISOString(),
+      scenarios: [
+        {
+          id: "qa-self-check",
+          name: "Synthetic Slack-class roundtrip",
+          status: "running",
+        },
+      ],
+    });
+    const result = await runQaSelfCheckAgainstState({
+      state,
+      cfg: gateway?.cfg ?? createQaLabConfig(listenUrl),
+      transportId: "qa-channel",
+      outputPath: params?.outputPath,
+      repoRoot,
+      waitTimeoutMs: params?.selfCheckWaitTimeoutMs,
+    });
+    latestScenarioRun = withQaLabRunCounts({
+      kind: "self-check",
+      status: "completed",
+      startedAt: latestScenarioRun.startedAt,
+      finishedAt: new Date().toISOString(),
+      scenarios: [
+        {
+          id: "qa-self-check",
+          name: result.scenarioResult.name,
+          status: result.scenarioResult.status,
+          details: result.scenarioResult.details,
+          steps: result.scenarioResult.steps,
+        },
+      ],
+    });
+    latestReport = {
+      outputPath: result.outputPath,
+      markdown: result.report,
+      generatedAt: new Date().toISOString(),
+    };
+    return result;
+  }
+
+  const server = createServer((req, res) => {
+    void (async () => {
+      const url = new URL(req.url ?? "/", "http://127.0.0.1");
+
+      if (await handleQaBusRequest({ req, res, state })) {
+        return;
+      }
+
+      try {
+        if (controlUiProxyTarget && isControlUiProxyPath(url.pathname)) {
+          await proxyHttpRequest({
+            req,
+            res,
+            target: controlUiProxyTarget,
+            pathname: url.pathname,
+            search: url.search,
+            authorizationToken: controlUiProxyToken,
+          });
+          return;
+        }
+
+        if (req.method === "GET" && url.pathname === "/api/bootstrap") {
+          void ensureRunnerModelCatalog();
+          const resolvedControlUiUrl = controlUiProxyTarget
+            ? `${publicBaseUrl}/control-ui/`
+            : controlUiUrl;
+          const safeControlUiUrl = sanitizeControlUiPublicUrl(resolvedControlUiUrl);
+          writeJson(res, 200, {
+            baseUrl: publicBaseUrl,
+            latestReport,
+            controlUiUrl: safeControlUiUrl,
+            controlUiEmbeddedUrl: safeControlUiUrl,
+            kickoffTask: scenarioCatalog.kickoffTask,
+            scenarios: scenarioCatalog.scenarios,
+            defaults: bootstrapDefaults,
+            runner: runnerSnapshot,
+            runnerCatalog: {
+              status: runnerModelCatalogStatus,
+              real: runnerModelOptions,
+            },
+          });
+          return;
+        }
+        if (req.method === "GET" && (url.pathname === "/healthz" || url.pathname === "/readyz")) {
+          writeJson(res, 200, { ok: true, status: "live" });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/state") {
+          writeJson(res, 200, state.getSnapshot());
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/report") {
+          writeJson(res, 200, { report: latestReport });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/ui-version") {
+          res.writeHead(200, {
+            "content-type": "application/json; charset=utf-8",
+            "cache-control": "no-store",
+          });
+          res.end(JSON.stringify({ version: resolveUiAssetVersion(params?.uiDistDir, repoRoot) }));
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/outcomes") {
+          writeJson(res, 200, { run: latestScenarioRun });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/evidence") {
+          const evidencePath =
+            url.searchParams.get("path")?.trim() || runnerSnapshot.artifacts?.evidencePath;
+          if (!evidencePath) {
+            res.writeHead(200, {
+              "content-type": "application/json; charset=utf-8",
+              "cache-control": "no-store",
+            });
+            res.end(JSON.stringify({ evidence: null }));
+            return;
+          }
+          // Build the model before sending any headers so a thrown QaEvidenceGalleryError
+          // still routes through writeQaLabServerError (writing headers first would make the
+          // error response throw ERR_HTTP_HEADERS_SENT and reset the connection).
+          const evidence = await buildQaEvidenceGalleryModel({ evidencePath, repoRoot });
+          res.writeHead(200, {
+            "content-type": "application/json; charset=utf-8",
+            "cache-control": "no-store",
+          });
+          res.end(JSON.stringify({ evidence }));
+          return;
+        }
+        if (
+          (req.method === "GET" || req.method === "HEAD") &&
+          url.pathname === "/api/evidence/artifact"
+        ) {
+          const evidencePath = url.searchParams.get("evidencePath")?.trim();
+          const artifactPath = url.searchParams.get("artifactPath")?.trim();
+          const producerFile = url.searchParams.get("producerFile")?.trim();
+          const entryIndexText = url.searchParams.get("entryIndex");
+          const artifactIndexText = url.searchParams.get("artifactIndex");
+          if (
+            !evidencePath ||
+            (!artifactPath && !producerFile && (!entryIndexText || !artifactIndexText))
+          ) {
+            writeError(res, 400, "Missing evidencePath and artifact selector");
+            return;
+          }
+          const artifactFile = artifactPath
+            ? await resolveQaEvidenceArtifactFile({
+                artifactPath,
+                evidencePath,
+                repoRoot,
+              })
+            : producerFile
+              ? await resolveQaEvidenceProducerFile({
+                  evidencePath,
+                  producerFile,
+                  repoRoot,
+                })
+              : await resolveQaEvidenceArtifactFileByIndex({
+                  artifactIndex: parseQaEvidenceArtifactIndexText(artifactIndexText!),
+                  entryIndex: parseQaEvidenceArtifactIndexText(entryIndexText!),
+                  evidencePath,
+                  repoRoot,
+                });
+          const artifactStats = await fs.promises.stat(artifactFile);
+          res.writeHead(200, {
+            "content-type": detectQaEvidenceArtifactContentType(artifactFile),
+            "content-length": artifactStats.size,
+            "cache-control": "no-store",
+            "x-content-type-options": "nosniff",
+          });
+          if (req.method === "HEAD") {
+            res.end();
+            return;
+          }
+          fs.createReadStream(artifactFile)
+            .on("error", (error) => res.destroy(normalizeQaLabCleanupError(error)))
+            .pipe(res);
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/capture/sessions") {
+          writeJson(res, 200, {
+            sessions: captureStore.listSessions(50),
+          });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/capture/startup-status") {
+          const proxyUrl = captureSettings.proxyUrl || "http://127.0.0.1:7799";
+          const gatewayUrl = controlUiUrl || "http://127.0.0.1:18789/";
+          const [proxy, gatewayLocal] = await Promise.all([
+            probeTcpReachability(proxyUrl),
+            probeTcpReachability(gatewayUrl),
+          ]);
+          writeJson(res, 200, {
+            status: {
+              proxy: {
+                ...proxy,
+                label: "Proxy",
+              },
+              gateway: {
+                ...gatewayLocal,
+                label: "Gateway",
+              },
+              qaLab: {
+                label: "QA Lab",
+                url: publicBaseUrl,
+                ok: true,
+              },
+            },
+          });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/capture/events") {
+          const sessionId = url.searchParams.get("sessionId")?.trim();
+          writeJson(res, 200, {
+            events: sessionId
+              ? captureStore.getSessionEvents(sessionId, 200).map(mapCaptureEventForQa)
+              : [],
+          });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/capture/coverage") {
+          const sessionId = url.searchParams.get("sessionId")?.trim();
+          if (!sessionId) {
+            writeError(res, 400, "Missing sessionId");
+            return;
+          }
+          writeJson(res, 200, {
+            coverage: captureStore.summarizeSessionCoverage(sessionId),
+          });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/capture/query") {
+          const preset = url.searchParams.get("preset")?.trim();
+          const sessionId = url.searchParams.get("sessionId")?.trim() || undefined;
+          if (!preset) {
+            writeError(res, 400, "Missing preset");
+            return;
+          }
+          if (!isCaptureQueryPreset(preset)) {
+            writeError(res, 400, "Unknown preset");
+            return;
+          }
+          writeJson(res, 200, {
+            rows: captureStore.queryPreset(preset, sessionId),
+          });
+          return;
+        }
+        if (req.method === "GET" && url.pathname === "/api/capture/blob") {
+          const blobId = url.searchParams.get("id")?.trim();
+          if (!blobId) {
+            writeError(res, 400, "Missing blob id");
+            return;
+          }
+          const content = captureStore.readBlob(blobId);
+          if (content == null) {
+            writeError(res, 404, "Blob not found");
+            return;
+          }
+          writeJson(res, 200, { id: blobId, content });
+          return;
+        }
+        if (req.method === "POST" && url.pathname === "/api/capture/delete-sessions") {
+          const body = (await readQaJsonBody(req)) as { sessionIds?: unknown };
+          const sessionIds = Array.isArray(body.sessionIds)
+            ? body.sessionIds.filter((value): value is string => typeof value === "string")
+            : [];
+          writeJson(res, 200, {
+            result: captureStore.deleteSessions(sessionIds),
+          });
+          return;
+        }
+        if (req.method === "POST" && url.pathname === "/api/capture/purge") {
+          writeJson(res, 200, {
+            result: captureStore.purgeAll(),
+          });
+          return;
+        }
+        if (req.method === "POST" && url.pathname === "/api/reset") {
+          if (activeSuiteRun) {
+            writeError(res, 409, "QA suite run already in progress");
+            return;
+          }
+          state.reset();
+          latestReport = null;
+          latestScenarioRun = null;
+          runnerSnapshot = {
+            ...runnerSnapshot,
+            status: "idle",
+            artifacts: null,
+            error: null,
+            startedAt: undefined,
+            finishedAt: undefined,
+          };
+          writeJson(res, 200, { ok: true });
+          return;
+        }
+        if (req.method === "POST" && url.pathname === "/api/inbound/message") {
+          const body = await readQaJsonBody(req);
+          writeJson(res, 200, {
+            message: state.addInboundMessage(
+              body as Parameters<QaBusState["addInboundMessage"]>[0],
+            ),
+          });
+          return;
+        }
+        if (req.method === "POST" && url.pathname === "/api/kickoff") {
+          writeJson(res, 200, {
+            message: injectKickoffMessage({
+              state,
+              defaults: bootstrapDefaults,
+              kickoffTask: scenarioCatalog.kickoffTask,
+            }),
+          });
+          return;
+        }
+        if (req.method === "POST" && url.pathname === "/api/scenario/self-check") {
+          if (activeSuiteRun) {
+            writeError(res, 409, "QA suite run already in progress");
+            return;
+          }
+          const result = await runSelfCheck();
+          writeJson(res, 200, serializeSelfCheck(result));
+          return;
+        }
+        if (req.method === "POST" && url.pathname === "/api/scenario/suite") {
+          if (activeSuiteRun) {
+            writeError(res, 409, "QA suite run already in progress");
+            return;
+          }
+          const selection = normalizeQaRunSelection(
+            await readQaJsonBody(req),
+            scenarioCatalog.scenarios,
+          );
+          state.reset();
+          latestReport = null;
+          latestScenarioRun = null;
+          const startedAt = new Date().toISOString();
+          runnerSnapshot = {
+            status: "running",
+            selection,
+            startedAt,
+            finishedAt: undefined,
+            artifacts: null,
+            error: null,
+          };
+          activeSuiteRun = (async () => {
+            try {
+              const { runQaFlowSuite } = await import("./suite.js");
+              const result = await runQaFlowSuite({
+                lab: labHandle ?? undefined,
+                startLab: startQaLabServer,
+                outputDir: createQaRunOutputDir(repoRoot),
+                providerMode: selection.providerMode,
+                primaryModel: selection.primaryModel,
+                alternateModel: selection.alternateModel,
+                scenarioIds: selection.scenarioIds,
+              });
+              runnerSnapshot = {
+                status: "completed",
+                selection,
+                startedAt,
+                finishedAt: new Date().toISOString(),
+                artifacts: {
+                  outputDir: result.outputDir,
+                  evidencePath: result.evidencePath,
+                  reportPath: result.reportPath,
+                  summaryPath: result.summaryPath,
+                  watchUrl: result.watchUrl,
+                },
+                error: null,
+              };
+            } catch (error) {
+              runnerSnapshot = {
+                status: "failed",
+                selection,
+                startedAt,
+                finishedAt: new Date().toISOString(),
+                artifacts: null,
+                error: formatErrorMessage(error),
+              };
+            } finally {
+              activeSuiteRun = null;
+            }
+          })();
+          writeJson(res, 202, {
+            ok: true,
+            runner: runnerSnapshot,
+          });
+          return;
+        }
+
+        if (req.method !== "GET" && req.method !== "HEAD") {
+          writeError(res, 404, "not found");
+          return;
+        }
+
+        const asset = tryResolveUiAsset(url.pathname, params?.uiDistDir, repoRoot);
+        if (!asset) {
+          const html = missingUiHtml();
+          res.writeHead(200, {
+            "content-type": "text/html; charset=utf-8",
+            "content-length": Buffer.byteLength(html),
+          });
+          if (req.method === "HEAD") {
+            res.end();
+            return;
+          }
+          res.end(html);
+          return;
+        }
+
+        const body = fs.readFileSync(asset);
+        res.writeHead(200, {
+          "content-type": detectContentType(asset),
+          "content-length": body.byteLength,
+        });
+        if (req.method === "HEAD") {
+          res.end();
+          return;
+        }
+        res.end(body);
+      } catch (error) {
+        writeQaLabServerError(res, error);
+      }
+    })();
+  });
+
+  const releaseCaptureStore = () => {
+    if (captureStoreReleased) {
+      return;
+    }
+    captureStoreReleased = true;
+    captureStoreLease.release();
+  };
+
+  const stopLabServerResources = async (): Promise<Error | undefined> => {
+    runnerModelCatalogAbort?.abort();
+    await runnerModelCatalogPromise?.catch(() => undefined);
+    const results = await Promise.allSettled([
+      Promise.resolve().then(() => gateway?.stop()),
+      Promise.resolve().then(() => (serverListening ? closeQaHttpServer(server) : undefined)),
+      Promise.resolve().then(releaseCaptureStore),
+    ]);
+    const failed = results.find((result) => result.status === "rejected");
+    return failed ? normalizeQaLabCleanupError(failed.reason) : undefined;
+  };
+
+  try {
+    await new Promise<void>((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(params?.port ?? 0, params?.host ?? "127.0.0.1", () => resolve());
+    });
+    serverListening = true;
+    const address = server.address();
+    if (!address || typeof address === "string") {
+      throw new Error("qa-lab failed to bind");
+    }
+    listenUrl = resolveAdvertisedBaseUrl({
+      bindHost: params?.host ?? "127.0.0.1",
+      bindPort: address.port,
+    });
+    publicBaseUrl = resolveAdvertisedBaseUrl({
+      bindHost: params?.host ?? "127.0.0.1",
+      bindPort: address.port,
+      advertiseHost: params?.advertiseHost,
+      advertisePort: params?.advertisePort,
+    });
+    if (embeddedGatewayEnabled) {
+      gateway = await startQaGatewayLoop({ state, baseUrl: listenUrl });
+    }
+    if (params?.sendKickoffOnStart) {
+      injectKickoffMessage({
+        state,
+        defaults: bootstrapDefaults,
+        kickoffTask: scenarioCatalog.kickoffTask,
+      });
+    }
+
+    server.on("upgrade", (req, socket, head) => {
+      const url = new URL(req.url ?? "/", "http://127.0.0.1");
+      if (!controlUiProxyTarget || !isControlUiProxyPath(url.pathname)) {
+        socket.destroy();
+        return;
+      }
+      proxyUpgradeRequest({
+        req,
+        socket,
+        head,
+        target: controlUiProxyTarget,
+        authorizationToken: controlUiProxyToken,
+      });
+    });
+
+    const lab = {
+      baseUrl: publicBaseUrl,
+      listenUrl,
+      state,
+      setControlUi(next: {
+        controlUiUrl?: string | null;
+        controlUiProxyToken?: string | null;
+        controlUiProxyTarget?: string | null;
+      }) {
+        controlUiUrl = sanitizeControlUiPublicUrl(next.controlUiUrl?.trim() || null);
+        controlUiProxyToken = next.controlUiProxyToken?.trim() || null;
+        controlUiProxyTarget = next.controlUiProxyTarget?.trim()
+          ? new URL(next.controlUiProxyTarget)
+          : null;
+      },
+      setScenarioRun(next: Omit<QaLabScenarioRun, "counts"> | null) {
+        latestScenarioRun = next ? withQaLabRunCounts(next) : null;
+      },
+      setLatestReport(next: QaLabLatestReport | null) {
+        latestReport = next;
+      },
+      runSelfCheck,
+      async stop() {
+        const cleanupError = await stopLabServerResources();
+        if (cleanupError) {
+          throw cleanupError;
+        }
+      },
+    };
+    labHandle = lab;
+    return lab;
+  } catch (error) {
+    await stopLabServerResources().catch(() => undefined);
+    throw error;
+  }
+}
+
+function serializeSelfCheck(result: QaSelfCheckResult) {
+  return {
+    outputPath: result.outputPath,
+    report: result.report,
+    checks: result.checks,
+    scenario: result.scenarioResult,
+  };
+}

extensions/qa-lab/src/lab-server.types.ts

@@ -0,0 +1,75 @@
+// Qa Lab type declarations define plugin contracts.
+import type { QaBusState } from "./bus-state.js";
+import type { QaSelfCheckResult } from "./self-check.js";
+
+export type QaLabLatestReport = {
+  outputPath: string;
+  markdown: string;
+  generatedAt: string;
+};
+
+type QaLabRunStatus = "idle" | "running" | "completed";
+
+type QaLabScenarioStep = {
+  name: string;
+  status: "pass" | "fail" | "skip";
+  details?: string;
+};
+
+export type QaLabScenarioOutcome = {
+  id: string;
+  name: string;
+  status: "pending" | "running" | "pass" | "fail" | "skip";
+  details?: string;
+  steps?: QaLabScenarioStep[];
+  startedAt?: string;
+  finishedAt?: string;
+};
+
+export type QaLabScenarioRun = {
+  kind: "suite" | "self-check";
+  status: QaLabRunStatus;
+  startedAt?: string;
+  finishedAt?: string;
+  scenarios: QaLabScenarioOutcome[];
+  counts: {
+    total: number;
+    pending: number;
+    running: number;
+    passed: number;
+    failed: number;
+    skipped: number;
+  };
+};
+
+export type QaLabServerStartParams = {
+  repoRoot?: string;
+  host?: string;
+  port?: number;
+  outputPath?: string;
+  advertiseHost?: string;
+  advertisePort?: number;
+  controlUiUrl?: string;
+  controlUiProxyToken?: string;
+  controlUiProxyTarget?: string;
+  uiDistDir?: string;
+  autoKickoffTarget?: string;
+  embeddedGateway?: string;
+  sendKickoffOnStart?: boolean;
+  selfCheckWaitTimeoutMs?: number;
+};
+
+export type QaLabServerHandle = {
+  baseUrl: string;
+  listenUrl: string;
+  state: QaBusState;
+  setControlUi: (next: {
+    controlUiUrl?: string | null;
+    controlUiProxyToken?: string | null;
+    controlUiProxyTarget?: string | null;
+  }) => void;
+  setScenarioRun: (next: Omit<QaLabScenarioRun, "counts"> | null) => void;
+  setLatestReport: (next: QaLabLatestReport | null) => void;
+  runSelfCheck: () => Promise<QaSelfCheckResult>;
+  stop: () => Promise<void>;
+};

extensions/qa-lab/src/live-timeout.test.ts

@@ -0,0 +1,97 @@
+// Qa Lab tests cover live timeout plugin behavior.
+import { describe, expect, it } from "vitest";
+import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
+
+describe("qa live timeout policy", () => {
+  it("keeps mock lanes on the caller fallback", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "mock-openai",
+          primaryModel: "anthropic/claude-sonnet-4-6",
+          alternateModel: "anthropic/claude-opus-4-8",
+        },
+        30_000,
+      ),
+    ).toBe(30_000);
+  });
+
+  it("uses the higher gpt-5 live floor for openai heavy turns", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "openai/gpt-5.5",
+          alternateModel: "openai/gpt-5.5",
+        },
+        30_000,
+      ),
+    ).toBe(360_000);
+  });
+
+  it("keeps the standard live floor for other non-anthropic models", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "google/gemini-3-flash",
+          alternateModel: "google/gemini-3-flash",
+        },
+        30_000,
+      ),
+    ).toBe(120_000);
+  });
+
+  it("uses the anthropic floor for sonnet turns", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "anthropic/claude-sonnet-4-6",
+          alternateModel: "anthropic/claude-opus-4-8",
+        },
+        30_000,
+      ),
+    ).toBe(180_000);
+  });
+
+  it("uses the opus floor when the switched turn runs on claude opus", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "anthropic/claude-sonnet-4-6",
+          alternateModel: "anthropic/claude-opus-4-8",
+        },
+        30_000,
+        "anthropic/claude-opus-4-8",
+      ),
+    ).toBe(240_000);
+  });
+
+  it("uses the anthropic floor for claude-cli sonnet turns", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "claude-cli/claude-sonnet-4-6",
+          alternateModel: "claude-cli/claude-opus-4-8",
+        },
+        30_000,
+      ),
+    ).toBe(180_000);
+  });
+
+  it("uses the opus floor for claude-cli opus turns", () => {
+    expect(
+      resolveQaLiveTurnTimeoutMs(
+        {
+          providerMode: "live-frontier",
+          primaryModel: "claude-cli/claude-opus-4-8",
+          alternateModel: "claude-cli/claude-opus-4-8",
+        },
+        30_000,
+      ),
+    ).toBe(240_000);
+  });
+});

extensions/qa-lab/src/live-timeout.ts

@@ -0,0 +1,22 @@
+// Qa Lab plugin module implements live timeout behavior.
+import type { QaProviderMode } from "./model-selection.js";
+import { getQaProvider } from "./providers/index.js";
+
+type QaLiveTimeoutProfile = {
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+};
+
+export function resolveQaLiveTurnTimeoutMs(
+  profile: QaLiveTimeoutProfile,
+  fallbackMs: number,
+  modelRef = profile.primaryModel,
+) {
+  return getQaProvider(profile.providerMode).resolveTurnTimeoutMs({
+    primaryModel: profile.primaryModel,
+    alternateModel: profile.alternateModel,
+    modelRef,
+    fallbackMs,
+  });
+}

extensions/qa-lab/src/live-transports/cli.ts

@@ -0,0 +1,57 @@
+// Qa Lab plugin module implements cli behavior.
+import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime";
+import { discordQaCliRegistration } from "./discord/cli.js";
+import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js";
+import { slackQaCliRegistration } from "./slack/cli.js";
+import { telegramQaCliRegistration } from "./telegram/cli.js";
+import { whatsappQaCliRegistration } from "./whatsapp/cli.js";
+
+function createBlockedQaRunnerCliRegistration(params: {
+  commandName: string;
+  description?: string;
+  pluginId: string;
+}): LiveTransportQaCliRegistration {
+  return {
+    commandName: params.commandName,
+    register(qa) {
+      qa.command(params.commandName)
+        .description(params.description ?? `Run the ${params.commandName} live QA lane`)
+        .action(() => {
+          throw new Error(
+            `QA runner "${params.commandName}" is installed but not active. Enable or allow plugin "${params.pluginId}" in your OpenClaw config, then try again.`,
+          );
+        });
+    },
+  };
+}
+
+function createQaRunnerCliRegistration(
+  runner: ReturnType<typeof listQaRunnerCliContributions>[number],
+): LiveTransportQaCliRegistration {
+  if (runner.status === "available") {
+    return runner.registration;
+  }
+  return createBlockedQaRunnerCliRegistration({
+    commandName: runner.commandName,
+    description: runner.description,
+    pluginId: runner.pluginId,
+  });
+}
+
+const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
+  telegramQaCliRegistration,
+  discordQaCliRegistration,
+  slackQaCliRegistration,
+  whatsappQaCliRegistration,
+];
+
+export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] {
+  const liveRegistrations = [...LIVE_TRANSPORT_QA_CLI_REGISTRATIONS];
+  const discoveredRunners = listQaRunnerCliContributions();
+
+  for (const runner of discoveredRunners) {
+    liveRegistrations.push(createQaRunnerCliRegistration(runner));
+  }
+
+  return liveRegistrations;
+}

extensions/qa-lab/src/live-transports/discord/cli.runtime.ts

@@ -0,0 +1,24 @@
+import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js";
+// Qa Lab plugin module implements cli behavior.
+import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js";
+import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
+import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js";
+import { runDiscordQaLive } from "./discord-live.runtime.js";
+
+export async function runQaDiscordCommand(opts: LiveTransportQaCommandOptions) {
+  const runOptions = resolveLiveTransportQaRunOptions(opts);
+  const result = await runDiscordQaLive(runOptions);
+  printLiveTransportQaArtifacts("Discord QA", {
+    report: result.reportPath,
+    summary: result.summaryPath,
+    "observed messages": result.observedMessagesPath,
+    ...(result.reactionTimelinesPath ? { "reaction timelines": result.reactionTimelinesPath } : {}),
+    ...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}),
+  });
+  if (!runOptions.allowFailures) {
+    const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
+    if (failedScenarioCount > 0) {
+      process.exitCode = 1;
+    }
+  }
+}

extensions/qa-lab/src/live-transports/discord/cli.ts

@@ -0,0 +1,33 @@
+// Qa Lab plugin module implements cli behavior.
+import {
+  createLazyCliRuntimeLoader,
+  createLiveTransportQaCliRegistration,
+  type LiveTransportQaCliRegistration,
+  type LiveTransportQaCommandOptions,
+} from "../shared/live-transport-cli.js";
+
+type DiscordQaCliRuntime = typeof import("./cli.runtime.js");
+
+const loadDiscordQaCliRuntime = createLazyCliRuntimeLoader<DiscordQaCliRuntime>(
+  () => import("./cli.runtime.js"),
+);
+
+async function runQaDiscord(opts: LiveTransportQaCommandOptions) {
+  const runtime = await loadDiscordQaCliRuntime();
+  await runtime.runQaDiscordCommand(opts);
+}
+
+export const discordQaCliRegistration: LiveTransportQaCliRegistration =
+  createLiveTransportQaCliRegistration({
+    commandName: "discord",
+    credentialOptions: {
+      sourceDescription: "Credential source for Discord QA: env or convex (default: env)",
+      roleDescription:
+        "Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)",
+    },
+    description: "Run the Discord live QA lane against a private guild bot-to-bot harness",
+    outputDirHelp: "Discord QA artifact directory",
+    scenarioHelp: "Run only the named Discord QA scenario (repeatable)",
+    sutAccountHelp: "Temporary Discord account id inside the QA gateway config",
+    run: runQaDiscord,
+  });

extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts

@@ -0,0 +1,739 @@
+// Qa Lab tests cover discord live plugin behavior.
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+  findMissingLiveTransportStandardScenarios,
+} from "../shared/live-transport-scenarios.js";
+import { testing } from "./discord-live.runtime.js";
+
+describe("discord live qa runtime", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+  });
+
+  it("resolves required Discord QA env vars", () => {
+    expect(
+      testing.resolveDiscordQaRuntimeEnv({
+        OPENCLAW_QA_DISCORD_GUILD_ID: "123456789012345678",
+        OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678",
+        OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver",
+        OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut",
+        OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID: "323456789012345678",
+      }),
+    ).toEqual({
+      guildId: "123456789012345678",
+      channelId: "223456789012345678",
+      driverBotToken: "driver",
+      sutBotToken: "sut",
+      sutApplicationId: "323456789012345678",
+    });
+  });
+
+  it("resolves optional Discord QA voice channel env var", () => {
+    expect(
+      testing.resolveDiscordQaRuntimeEnv({
+        OPENCLAW_QA_DISCORD_GUILD_ID: "123456789012345678",
+        OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678",
+        OPENCLAW_QA_DISCORD_VOICE_CHANNEL_ID: "523456789012345678",
+        OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver",
+        OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut",
+        OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID: "323456789012345678",
+      }),
+    ).toEqual({
+      guildId: "123456789012345678",
+      channelId: "223456789012345678",
+      voiceChannelId: "523456789012345678",
+      driverBotToken: "driver",
+      sutBotToken: "sut",
+      sutApplicationId: "323456789012345678",
+    });
+  });
+
+  it("fails when a required Discord QA env var is missing", () => {
+    expect(() =>
+      testing.resolveDiscordQaRuntimeEnv({
+        OPENCLAW_QA_DISCORD_GUILD_ID: "123456789012345678",
+        OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678",
+        OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver",
+        OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut",
+      }),
+    ).toThrow("OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID");
+  });
+
+  it("fails when Discord IDs are not snowflakes", () => {
+    expect(() =>
+      testing.resolveDiscordQaRuntimeEnv({
+        OPENCLAW_QA_DISCORD_GUILD_ID: "qa-guild",
+        OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678",
+        OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver",
+        OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut",
+        OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID: "323456789012345678",
+      }),
+    ).toThrow("OPENCLAW_QA_DISCORD_GUILD_ID must be a Discord snowflake.");
+  });
+
+  it("parses Discord pooled credential payloads", () => {
+    expect(
+      testing.parseDiscordQaCredentialPayload({
+        guildId: "123456789012345678",
+        channelId: "223456789012345678",
+        voiceChannelId: "523456789012345678",
+        driverBotToken: "driver",
+        sutBotToken: "sut",
+        sutApplicationId: "323456789012345678",
+      }),
+    ).toEqual({
+      guildId: "123456789012345678",
+      channelId: "223456789012345678",
+      voiceChannelId: "523456789012345678",
+      driverBotToken: "driver",
+      sutBotToken: "sut",
+      sutApplicationId: "323456789012345678",
+    });
+  });
+
+  it("rejects Discord pooled credential payloads with bad snowflakes", () => {
+    expect(() =>
+      testing.parseDiscordQaCredentialPayload({
+        guildId: "123456789012345678",
+        channelId: "channel",
+        driverBotToken: "driver",
+        sutBotToken: "sut",
+        sutApplicationId: "323456789012345678",
+      }),
+    ).toThrow("Discord credential payload_CHANNEL_ID must be a Discord snowflake.");
+  });
+
+  it("injects a temporary Discord account into the QA gateway config", () => {
+    const baseCfg: OpenClawConfig = {
+      plugins: {
+        allow: ["memory-core", "qa-channel"],
+        entries: {
+          "memory-core": { enabled: true },
+          "qa-channel": { enabled: true },
+        },
+      },
+      channels: {
+        "qa-channel": {
+          enabled: true,
+          baseUrl: "http://127.0.0.1:43123",
+          botUserId: "openclaw",
+          botDisplayName: "OpenClaw QA",
+          allowFrom: ["*"],
+        },
+      },
+    };
+
+    const next = testing.buildDiscordQaConfig(baseCfg, {
+      guildId: "123456789012345678",
+      channelId: "223456789012345678",
+      driverBotId: "423456789012345678",
+      sutAccountId: "sut",
+      sutBotToken: "sut-token",
+    });
+
+    expect(next.plugins?.allow).toContain("discord");
+    expect(next.plugins?.entries?.discord).toEqual({ enabled: true });
+    expect(next.messages?.groupChat?.visibleReplies).toBe("automatic");
+    expect(next.channels?.discord).toEqual({
+      enabled: true,
+      defaultAccount: "sut",
+      accounts: {
+        sut: {
+          enabled: true,
+          token: "sut-token",
+          allowBots: "mentions",
+          groupPolicy: "allowlist",
+          guilds: {
+            "123456789012345678": {
+              requireMention: true,
+              users: ["423456789012345678"],
+              channels: {
+                "223456789012345678": {
+                  enabled: true,
+                  requireMention: true,
+                  users: ["423456789012345678"],
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+  });
+
+  it("injects Discord voice auto-join config for the voice smoke", () => {
+    const next = testing.buildDiscordQaConfig(
+      {},
+      {
+        guildId: "123456789012345678",
+        channelId: "223456789012345678",
+        driverBotId: "423456789012345678",
+        sutAccountId: "sut",
+        sutBotToken: "sut-token",
+      },
+      {
+        voiceAutoJoin: {
+          guildId: "123456789012345678",
+          channelId: "523456789012345678",
+        },
+      },
+    );
+
+    expect(next.channels?.discord?.voice).toEqual({
+      enabled: true,
+      autoJoin: [
+        {
+          guildId: "123456789012345678",
+          channelId: "523456789012345678",
+        },
+      ],
+    });
+  });
+
+  it("injects tool-only Discord status reaction config for the Mantis scenario", () => {
+    const next = testing.buildDiscordQaConfig(
+      {},
+      {
+        guildId: "123456789012345678",
+        channelId: "223456789012345678",
+        driverBotId: "423456789012345678",
+        sutAccountId: "sut",
+        sutBotToken: "sut-token",
+      },
+      { statusReactionsToolOnly: true },
+    );
+
+    expect(next.messages?.ackReaction).toBe("👀");
+    expect(next.messages?.ackReactionScope).toBe("all");
+    expect(next.messages?.groupChat?.visibleReplies).toBe("message_tool");
+    expect(next.messages?.statusReactions?.enabled).toBe(true);
+    expect(next.messages?.statusReactions?.timing?.debounceMs).toBe(0);
+    const discordAccount = next.channels?.discord?.accounts?.sut;
+    expect(discordAccount?.allowBots).toBe(true);
+    expect(discordAccount?.guilds?.["123456789012345678"]?.requireMention).toBe(false);
+    expect(
+      discordAccount?.guilds?.["123456789012345678"]?.channels?.["223456789012345678"]
+        ?.requireMention,
+    ).toBe(false);
+  });
+
+  it("normalizes observed Discord messages", () => {
+    expect(
+      testing.normalizeDiscordObservedMessage({
+        id: "523456789012345678",
+        channel_id: "223456789012345678",
+        guild_id: "123456789012345678",
+        content: "hello",
+        timestamp: "2026-04-22T12:00:00.000Z",
+        author: {
+          id: "423456789012345678",
+          username: "driver",
+          bot: true,
+        },
+        referenced_message: { id: "323456789012345678" },
+      }),
+    ).toEqual({
+      messageId: "523456789012345678",
+      channelId: "223456789012345678",
+      guildId: "123456789012345678",
+      senderId: "423456789012345678",
+      senderIsBot: true,
+      senderUsername: "driver",
+      text: "hello",
+      replyToMessageId: "323456789012345678",
+      timestamp: "2026-04-22T12:00:00.000Z",
+    });
+  });
+
+  it("matches Discord scenario replies by SUT id and marker", () => {
+    expect(
+      testing.matchesDiscordScenarioReply({
+        channelId: "223456789012345678",
+        sutBotId: "323456789012345678",
+        matchText: "DISCORD_QA_ECHO_TOKEN",
+        message: {
+          messageId: "523456789012345678",
+          channelId: "223456789012345678",
+          senderId: "323456789012345678",
+          senderIsBot: true,
+          text: "reply DISCORD_QA_ECHO_TOKEN",
+        },
+      }),
+    ).toBe(true);
+    expect(
+      testing.matchesDiscordScenarioReply({
+        channelId: "223456789012345678",
+        sutBotId: "323456789012345678",
+        matchText: "DISCORD_QA_ECHO_TOKEN",
+        message: {
+          messageId: "523456789012345679",
+          channelId: "223456789012345678",
+          senderId: "423456789012345678",
+          senderIsBot: true,
+          text: "reply DISCORD_QA_ECHO_TOKEN",
+        },
+      }),
+    ).toBe(false);
+  });
+
+  it("computes Discord RTT from trigger and reply timestamps", () => {
+    expect(
+      testing.computeDiscordRttMs("2026-04-22T11:59:59.125Z", "2026-04-22T12:00:00.875Z"),
+    ).toBe(1750);
+    expect(testing.computeDiscordRttMs("bad", "2026-04-22T12:00:00.875Z")).toBeUndefined();
+  });
+
+  it("includes the Discord live scenarios", () => {
+    expect(testing.findScenario().map((scenario) => scenario.id)).toEqual([
+      "discord-canary",
+      "discord-mention-gating",
+      "discord-native-help-command-registration",
+    ]);
+    expect(
+      testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id),
+    ).toEqual(["discord-status-reactions-tool-only"]);
+    expect(testing.findScenario(["discord-voice-autojoin"]).map((scenario) => scenario.id)).toEqual(
+      ["discord-voice-autojoin"],
+    );
+    expect(
+      testing
+        .findScenario(["discord-thread-reply-filepath-attachment"])
+        .map((scenario) => scenario.id),
+    ).toEqual(["discord-thread-reply-filepath-attachment"]);
+  });
+
+  it("collects the status reaction sequence across timeline snapshots", () => {
+    expect(
+      testing.collectSeenReactionSequence(
+        [
+          {
+            elapsedMs: 0,
+            observedAt: "2026-05-03T12:00:00.000Z",
+            reactions: [{ emoji: "👀", count: 1, me: true }],
+          },
+          {
+            elapsedMs: 250,
+            observedAt: "2026-05-03T12:00:00.250Z",
+            reactions: [
+              { emoji: "👀", count: 1, me: true },
+              { emoji: "🤔", count: 1, me: true },
+            ],
+          },
+          {
+            elapsedMs: 500,
+            observedAt: "2026-05-03T12:00:00.500Z",
+            reactions: [{ emoji: "👍", count: 1, me: true }],
+          },
+        ],
+        ["👀", "🤔", "👍"],
+      ),
+    ).toEqual(["👀", "🤔", "👍"]);
+  });
+
+  it("normalizes reaction snapshots from Discord messages", () => {
+    expect(
+      testing.normalizeDiscordReactionSnapshot({
+        startedAtMs: new Date("2026-05-03T12:00:00.000Z").getTime(),
+        observedAt: new Date("2026-05-03T12:00:01.000Z"),
+        message: {
+          id: "523456789012345678",
+          channel_id: "223456789012345678",
+          reactions: [
+            { count: 1, emoji: { name: "🤔" }, me: true },
+            { count: 2, emoji: { name: "👀" }, me: false },
+          ],
+        },
+      }),
+    ).toEqual({
+      elapsedMs: 1000,
+      observedAt: "2026-05-03T12:00:01.000Z",
+      reactions: [
+        { emoji: "👀", count: 2, me: false },
+        { emoji: "🤔", count: 1, me: true },
+      ],
+    });
+  });
+
+  it("renders a human-readable status reaction timeline artifact", () => {
+    const html = testing.renderDiscordStatusReactionHtml({
+      scenarioTitle: "Discord status reactions",
+      expectedSequence: ["👀", "🤔", "👍"],
+      seenSequence: ["👀", "🤔"],
+      snapshots: [
+        {
+          elapsedMs: 0,
+          observedAt: "2026-05-03T12:00:00.000Z",
+          reactions: [{ emoji: "👀", count: 1, me: true }],
+        },
+      ],
+    });
+
+    expect(html).toContain("Discord status reactions");
+    expect(html).toContain("Expected: 👀 → 🤔 → 👍");
+    expect(html).toContain("Seen: 👀 → 🤔");
+  });
+
+  it("renders a human-readable thread attachment artifact", () => {
+    const html = testing.renderDiscordThreadReplyAttachmentHtml({
+      attachmentFilenames: [],
+      expectedAttachmentFilename: "mantis-thread-report.md",
+      messageContent: "Mantis thread attachment reply",
+      scenarioTitle: "Discord thread reply preserves filePath attachment",
+      status: "fail",
+      threadName: "mantis-thread-filepath-1234",
+    });
+
+    expect(html).toContain("Attachment missing");
+    expect(html).toContain("No attachments on the SUT thread reply");
+    expect(html).toContain("mantis-thread-report.md");
+  });
+
+  it("builds Discord Web message URLs for logged-in Mantis capture", () => {
+    expect(
+      testing.buildDiscordWebMessageUrl({
+        guildId: "111111111111111111",
+        messageId: "333333333333333333",
+        threadId: "222222222222222222",
+      }),
+    ).toBe("https://discord.com/channels/111111111111111111/222222222222222222/333333333333333333");
+  });
+
+  it("waits for the Discord account to become connected, not just running", async () => {
+    vi.useFakeTimers();
+    try {
+      const gateway = {
+        call: vi
+          .fn()
+          .mockResolvedValueOnce({
+            channelAccounts: {
+              discord: [
+                { accountId: "sut", running: true, connected: false, restartPending: false },
+              ],
+            },
+          })
+          .mockResolvedValueOnce({
+            channelAccounts: {
+              discord: [
+                { accountId: "sut", running: true, connected: true, restartPending: false },
+              ],
+            },
+          }),
+      } as unknown as Parameters<typeof testing.waitForDiscordChannelRunning>[0];
+
+      const readyPromise = testing.waitForDiscordChannelRunning(gateway, "sut");
+      await vi.advanceTimersByTimeAsync(600);
+
+      await expect(readyPromise).resolves.toBeUndefined();
+      expect(gateway["call"]).toHaveBeenCalledTimes(2);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("reports the last Discord status when connection readiness times out", async () => {
+    vi.useFakeTimers();
+    try {
+      const gateway = {
+        call: vi.fn().mockResolvedValue({
+          channelAccounts: {
+            discord: [
+              {
+                accountId: "sut",
+                running: true,
+                connected: false,
+                restartPending: false,
+                lastError: null,
+                lastDisconnect: { error: "runtime-not-ready" },
+              },
+            ],
+          },
+        }),
+      } as unknown as Parameters<typeof testing.waitForDiscordChannelRunning>[0];
+
+      const readyPromise = testing.waitForDiscordChannelRunning(gateway, "sut");
+      const assertion = expect(readyPromise).rejects.toThrow(
+        'discord account "sut" did not become connected (last status: running=true connected=false',
+      );
+      await vi.advanceTimersByTimeAsync(45_500);
+      await assertion;
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("fails when any requested Discord scenario id is unknown", () => {
+    expect(() => testing.findScenario(["discord-canary", "typo-scenario"])).toThrow(
+      "unknown Discord QA scenario id(s): typo-scenario",
+    );
+  });
+
+  it("tracks Discord live coverage against the shared transport contract", () => {
+    expect(testing.DISCORD_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]);
+    expect(
+      findMissingLiveTransportStandardScenarios({
+        coveredStandardScenarioIds: testing.DISCORD_QA_STANDARD_SCENARIO_IDS,
+        expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+      }),
+    ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]);
+  });
+
+  it("lists Discord application commands through the REST API", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => {
+        expect(init?.headers).toBeInstanceOf(Headers);
+        expect((init!.headers as Headers).get("authorization")).toBe("Bot token");
+        return new Response(
+          JSON.stringify([
+            { id: "623456789012345678", name: "help" },
+            { id: "623456789012345679", name: "commands" },
+          ]),
+          {
+            status: 200,
+            headers: {
+              "content-type": "application/json",
+            },
+          },
+        );
+      }),
+    );
+
+    await expect(
+      testing.listApplicationCommands({
+        token: "token",
+        applicationId: "323456789012345678",
+      }),
+    ).resolves.toEqual([
+      { id: "623456789012345678", name: "help" },
+      { id: "623456789012345679", name: "commands" },
+    ]);
+  });
+
+  it("discovers the first visible Discord voice channel for the voice smoke", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(
+            JSON.stringify([
+              { id: "123456789012345678", name: "general", position: 0, type: 0 },
+              { id: "523456789012345678", name: "qa-voice", position: 1, type: 2 },
+              { id: "623456789012345678", name: "stage", position: 2, type: 13 },
+            ]),
+            {
+              status: 200,
+              headers: {
+                "content-type": "application/json",
+              },
+            },
+          ),
+      ),
+    );
+
+    const voiceChannel = await testing.resolveDiscordQaVoiceChannel({
+      token: "token",
+      guildId: "123456789012345678",
+    });
+    expect(voiceChannel.id).toBe("523456789012345678");
+    expect(voiceChannel.name).toBe("qa-voice");
+  });
+
+  it("normalizes missing current Discord voice state to null", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(JSON.stringify({ message: "Unknown Voice State" }), {
+            status: 404,
+            headers: {
+              "content-type": "application/json",
+            },
+          }),
+      ),
+    );
+
+    await expect(
+      testing.getCurrentDiscordVoiceState({
+        token: "token",
+        guildId: "123456789012345678",
+      }),
+    ).resolves.toBeNull();
+  });
+
+  it("waits for required Discord application commands to be registered", async () => {
+    vi.useFakeTimers();
+    try {
+      vi.stubGlobal(
+        "fetch",
+        vi
+          .fn()
+          .mockResolvedValueOnce(
+            new Response(JSON.stringify([{ id: "623456789012345679", name: "commands" }]), {
+              status: 200,
+              headers: {
+                "content-type": "application/json",
+              },
+            }),
+          )
+          .mockResolvedValueOnce(
+            new Response(
+              JSON.stringify([
+                { id: "623456789012345679", name: "commands" },
+                { id: "623456789012345678", name: "help" },
+              ]),
+              {
+                status: 200,
+                headers: {
+                  "content-type": "application/json",
+                },
+              },
+            ),
+          ),
+      );
+
+      const registeredPromise = testing.assertDiscordApplicationCommandsRegistered({
+        token: "token",
+        applicationId: "323456789012345678",
+        expectedCommandNames: ["help"],
+        timeoutMs: 5_000,
+      });
+      await vi.advanceTimersByTimeAsync(1_100);
+
+      await expect(registeredPromise).resolves.toEqual({
+        commandNames: ["commands", "help"],
+      });
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("uses the Discord API helper timeout for identity probes", async () => {
+    const controller = new AbortController();
+    const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal);
+    let signal: AbortSignal | undefined;
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => {
+        signal = init?.signal as AbortSignal | undefined;
+        return new Response(JSON.stringify({ id: "423456789012345678" }), {
+          status: 200,
+          headers: {
+            "content-type": "application/json",
+          },
+        });
+      }),
+    );
+
+    await expect(testing.getCurrentDiscordUser("token")).resolves.toEqual({
+      id: "423456789012345678",
+    });
+    expect(timeoutSpy).toHaveBeenCalledWith(15_000);
+    expect(signal).toBe(controller.signal);
+    expect(signal?.aborted).toBe(false);
+    controller.abort();
+    expect(signal?.aborted).toBe(true);
+  });
+
+  it("retries Discord REST requests after a 429 rate limit", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi
+        .fn()
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ message: "You are being rate limited.", retry_after: 0 }), {
+            status: 429,
+            headers: {
+              "content-type": "application/json",
+            },
+          }),
+        )
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ id: "423456789012345678" }), {
+            status: 200,
+            headers: {
+              "content-type": "application/json",
+            },
+          }),
+        ),
+    );
+
+    await expect(testing.getCurrentDiscordUser("token")).resolves.toEqual({
+      id: "423456789012345678",
+    });
+    expect(fetch).toHaveBeenCalledTimes(2);
+  });
+
+  it("redacts observed message content by default in artifacts", () => {
+    expect(
+      testing.buildObservedMessagesArtifact({
+        includeContent: false,
+        redactMetadata: false,
+        observedMessages: [
+          {
+            messageId: "523456789012345678",
+            channelId: "223456789012345678",
+            guildId: "123456789012345678",
+            senderId: "323456789012345678",
+            senderIsBot: true,
+            senderUsername: "sut",
+            text: "secret text",
+            triggerMessageId: "423456789012345678",
+            triggerTimestamp: "2026-04-22T11:59:59.000Z",
+            timestamp: "2026-04-22T12:00:00.000Z",
+          },
+        ],
+      }),
+    ).toEqual([
+      {
+        messageId: "523456789012345678",
+        channelId: "223456789012345678",
+        guildId: "123456789012345678",
+        senderId: "323456789012345678",
+        senderIsBot: true,
+        senderUsername: "sut",
+        triggerMessageId: "423456789012345678",
+        triggerTimestamp: "2026-04-22T11:59:59.000Z",
+        replyToMessageId: undefined,
+        timestamp: "2026-04-22T12:00:00.000Z",
+      },
+    ]);
+  });
+
+  it("preserves observed message timing when metadata is redacted", () => {
+    expect(
+      testing.buildObservedMessagesArtifact({
+        includeContent: false,
+        redactMetadata: true,
+        observedMessages: [
+          {
+            messageId: "523456789012345678",
+            channelId: "223456789012345678",
+            guildId: "123456789012345678",
+            senderId: "323456789012345678",
+            senderIsBot: true,
+            senderUsername: "sut",
+            scenarioId: "canary",
+            scenarioTitle: "Canary",
+            matchedScenario: true,
+            text: "secret text",
+            triggerMessageId: "423456789012345678",
+            triggerTimestamp: "2026-04-22T11:59:59.000Z",
+            timestamp: "2026-04-22T12:00:00.000Z",
+          },
+        ],
+      }),
+    ).toEqual([
+      {
+        senderIsBot: true,
+        scenarioId: "canary",
+        scenarioTitle: "Canary",
+        matchedScenario: true,
+        triggerTimestamp: "2026-04-22T11:59:59.000Z",
+        timestamp: "2026-04-22T12:00:00.000Z",
+      },
+    ]);
+  });
+});

extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.test.ts

@@ -0,0 +1,729 @@
+// Qa Lab tests cover credential lease plugin behavior.
+import { createServer } from "node:http";
+import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  acquireQaCredentialLease,
+  startQaCredentialLeaseHeartbeat,
+} from "./credential-lease.runtime.js";
+
+function jsonResponse(payload: unknown, status = 200) {
+  return new Response(JSON.stringify(payload), {
+    status,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+type FetchMock = { mock: { calls: Parameters<typeof fetch>[] } };
+
+function fetchCall(fetchImpl: FetchMock, index = 0): Parameters<typeof fetch> {
+  const call = fetchImpl.mock.calls[index];
+  if (!call) {
+    throw new Error(`expected fetch call ${index}`);
+  }
+  return call;
+}
+
+function fetchUrl(fetchImpl: FetchMock, index = 0): string {
+  const url = fetchCall(fetchImpl, index)[0];
+  if (typeof url !== "string") {
+    throw new Error(`expected fetch call ${index} URL`);
+  }
+  return url;
+}
+
+function fetchInit(fetchImpl: FetchMock, index = 0): RequestInit {
+  const init = fetchCall(fetchImpl, index)[1];
+  if (!init || typeof init !== "object") {
+    throw new Error(`expected fetch call ${index} init`);
+  }
+  return init;
+}
+
+async function startStreamingFailureBroker(params: {
+  chunkBytes?: number;
+  intervalMs?: number;
+  totalBytes?: number;
+}) {
+  const chunkBytes = params.chunkBytes ?? 64 * 1024;
+  const intervalMs = params.intervalMs ?? 1;
+  const totalBytes = params.totalBytes ?? 4 * 1024 * 1024;
+  let bytesWritten = 0;
+  let requestCount = 0;
+  let resolveClose: () => void = () => {};
+  const closePromise = new Promise<void>((resolve) => {
+    resolveClose = resolve;
+  });
+
+  const server = createServer((_req, res) => {
+    requestCount += 1;
+    res.writeHead(500, { "content-type": "text/plain" });
+    const interval = setInterval(() => {
+      if (bytesWritten >= totalBytes || res.destroyed) {
+        clearInterval(interval);
+        if (!res.destroyed) {
+          res.end();
+        }
+        return;
+      }
+      const nextBytes = Math.min(chunkBytes, totalBytes - bytesWritten);
+      bytesWritten += nextBytes;
+      res.write("x".repeat(nextBytes));
+    }, intervalMs);
+    res.on("close", () => {
+      clearInterval(interval);
+      resolveClose();
+    });
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      server.off("error", reject);
+      resolve();
+    });
+  });
+
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("expected streaming broker address");
+  }
+  return {
+    closePromise,
+    getBytesWritten: () => bytesWritten,
+    getRequestCount: () => requestCount,
+    totalBytes,
+    url: `http://127.0.0.1:${address.port}`,
+    stop: async () => {
+      await new Promise<void>((resolve, reject) => {
+        server.close((error) => {
+          if (error) {
+            reject(error);
+            return;
+          }
+          resolve();
+        });
+      });
+    },
+  };
+}
+
+describe("credential lease runtime", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  it("uses env credentials by default", async () => {
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      resolveEnvPayload: () => ({ groupId: "-100123", driverToken: "driver", sutToken: "sut" }),
+      parsePayload: () => {
+        throw new Error("should not parse convex payload in env mode");
+      },
+      env: {},
+    });
+
+    expect(lease.source).toBe("env");
+    expect(lease.payload).toEqual({
+      groupId: "-100123",
+      driverToken: "driver",
+      sutToken: "sut",
+    });
+  });
+
+  it("acquires, heartbeats, and releases convex credentials", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-1",
+          leaseToken: "lease-1",
+          payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+          leaseTtlMs: 1_200_000,
+          heartbeatIntervalMs: 30_000,
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }))
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }));
+
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      role: "maintainer",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(lease.source).toBe("convex");
+    expect(lease.credentialId).toBe("cred-1");
+    expect(lease.payload.groupId).toBe("-100123");
+
+    await lease.heartbeat();
+    await lease.release();
+
+    expect(fetchImpl).toHaveBeenCalledTimes(3);
+    expect(fetchUrl(fetchImpl)).toContain("/qa-credentials/v1/acquire");
+    const firstInit = fetchInit(fetchImpl);
+    const headers = firstInit?.headers as Record<string, string>;
+    expect(headers.authorization).toBe("Bearer maintainer-secret");
+  });
+
+  it("bounds oversized convex broker failure bodies before parsing", async () => {
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      new Response("x".repeat(1_048_577), {
+        status: 500,
+        headers: { "content-type": "text/plain" },
+      }),
+    );
+
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "maintainer",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        },
+        fetchImpl,
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("Convex credential broker: text response exceeds 1048576 bytes");
+
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+  });
+
+  it("cancels a streaming convex broker failure body after the response cap", async () => {
+    const broker = await startStreamingFailureBroker({});
+    try {
+      await expect(
+        acquireQaCredentialLease({
+          kind: "telegram",
+          source: "convex",
+          role: "maintainer",
+          env: {
+            OPENCLAW_QA_CONVEX_SITE_URL: broker.url,
+            OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+            OPENCLAW_QA_ALLOW_INSECURE_HTTP: "1",
+          },
+          resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+          parsePayload: (payload) =>
+            payload as { groupId: string; driverToken: string; sutToken: string },
+        }),
+      ).rejects.toThrow("Convex credential broker: text response exceeds 1048576 bytes");
+
+      await broker.closePromise;
+      expect(broker.getRequestCount()).toBe(1);
+      expect(broker.getBytesWritten()).toBeLessThan(broker.totalBytes);
+    } finally {
+      await broker.stop();
+    }
+  });
+
+  it("hydrates chunked convex credential payloads after acquire", async () => {
+    const serialized = JSON.stringify({
+      groupId: "-100123",
+      driverToken: "driver",
+      sutToken: "sut",
+    });
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-chunked",
+          leaseToken: "lease-chunked",
+          payload: {
+            __openclawQaCredentialPayloadChunksV1: true,
+            byteLength: serialized.length,
+            chunkCount: 2,
+          },
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok", data: serialized.slice(0, 20) }))
+      .mockResolvedValueOnce(jsonResponse({ status: "ok", data: serialized.slice(20) }));
+
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      role: "ci",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_CI: "ci-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(lease.payload).toEqual({
+      groupId: "-100123",
+      driverToken: "driver",
+      sutToken: "sut",
+    });
+    expect(fetchImpl).toHaveBeenCalledTimes(3);
+    expect(fetchUrl(fetchImpl, 1)).toBe(
+      "https://qa-cred.example.convex.site/qa-credentials/v1/payload-chunk",
+    );
+    const chunkRequestBody = fetchInit(fetchImpl, 1).body;
+    expect(chunkRequestBody).toBeTypeOf("string");
+    const chunkRequest = JSON.parse(chunkRequestBody as string) as {
+      credentialId?: string;
+      index?: number;
+      leaseToken?: string;
+    };
+    expect(chunkRequest.credentialId).toBe("cred-chunked");
+    expect(chunkRequest.index).toBe(0);
+    expect(chunkRequest.leaseToken).toBe("lease-chunked");
+  });
+
+  it("validates chunked convex payload length as utf8 bytes", async () => {
+    const serialized = JSON.stringify({
+      groupId: "-100123",
+      driverToken: "driv\u00e9r",
+      sutToken: "sut",
+    });
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-utf8",
+          leaseToken: "lease-utf8",
+          payload: {
+            __openclawQaCredentialPayloadChunksV1: true,
+            byteLength: Buffer.byteLength(serialized, "utf8"),
+            chunkCount: 1,
+          },
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok", data: serialized }));
+
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      role: "ci",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_CI: "ci-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(lease.payload.driverToken).toBe("driv\u00e9r");
+  });
+
+  it("rejects chunked convex payload markers above the configured chunk cap", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-many-chunks",
+          leaseToken: "lease-many-chunks",
+          payload: {
+            __openclawQaCredentialPayloadChunksV1: true,
+            byteLength: 1,
+            chunkCount: 3,
+          },
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }));
+
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "ci",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_CI: "ci-secret",
+          OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS: "2",
+        },
+        fetchImpl,
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("Chunked credential payload marker exceeds 2 chunks.");
+
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+    expect(fetchUrl(fetchImpl, 1)).toBe(
+      "https://qa-cred.example.convex.site/qa-credentials/v1/release",
+    );
+  });
+
+  it("rejects chunked convex payload markers above the configured byte cap", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-large-payload",
+          leaseToken: "lease-large-payload",
+          payload: {
+            __openclawQaCredentialPayloadChunksV1: true,
+            byteLength: 33,
+            chunkCount: 1,
+          },
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }));
+
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "ci",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_CI: "ci-secret",
+          OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES: "32",
+        },
+        fetchImpl,
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("Chunked credential payload marker exceeds 32 bytes.");
+
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+    expect(fetchUrl(fetchImpl, 1)).toBe(
+      "https://qa-cred.example.convex.site/qa-credentials/v1/release",
+    );
+  });
+
+  it("stops chunked convex payload hydration when chunk data exceeds the marker", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-overrun",
+          leaseToken: "lease-overrun",
+          payload: {
+            __openclawQaCredentialPayloadChunksV1: true,
+            byteLength: 2,
+            chunkCount: 2,
+          },
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok", data: "abc" }))
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }));
+
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "ci",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_CI: "ci-secret",
+        },
+        fetchImpl,
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("Chunked credential payload exceeded declared byteLength.");
+
+    expect(fetchImpl).toHaveBeenCalledTimes(3);
+    expect(fetchUrl(fetchImpl, 1)).toBe(
+      "https://qa-cred.example.convex.site/qa-credentials/v1/payload-chunk",
+    );
+    expect(fetchUrl(fetchImpl, 2)).toBe(
+      "https://qa-cred.example.convex.site/qa-credentials/v1/release",
+    );
+  });
+
+  it("defaults convex credential role to maintainer outside CI", async () => {
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      jsonResponse({
+        status: "ok",
+        credentialId: "cred-maintainer-default",
+        leaseToken: "lease-maintainer-default",
+        payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+      }),
+    );
+
+    await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    const firstInit = fetchInit(fetchImpl);
+    const headers = firstInit?.headers as Record<string, string>;
+    expect(headers.authorization).toBe("Bearer maintainer-secret");
+  });
+
+  it("defaults convex credential role to ci when CI=true", async () => {
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      jsonResponse({
+        status: "ok",
+        credentialId: "cred-ci-default",
+        leaseToken: "lease-ci-default",
+        payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+      }),
+    );
+
+    await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      env: {
+        CI: "true",
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_CI: "ci-secret",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    const firstInit = fetchInit(fetchImpl);
+    const headers = firstInit?.headers as Record<string, string>;
+    expect(headers.authorization).toBe("Bearer ci-secret");
+  });
+
+  it("retries convex acquire while the pool is exhausted", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "error",
+          code: "POOL_EXHAUSTED",
+          message: "wait",
+        }),
+      )
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "error",
+          code: "POOL_EXHAUSTED",
+          message: "wait",
+        }),
+      )
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-2",
+          leaseToken: "lease-2",
+          payload: { groupId: "-100456", driverToken: "driver-2", sutToken: "sut-2" },
+        }),
+      );
+
+    const sleeps: number[] = [];
+    let nowMs = 0;
+
+    const lease = await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "90000",
+      },
+      fetchImpl,
+      randomImpl: () => 0,
+      timeImpl: () => nowMs,
+      sleepImpl: async (ms) => {
+        sleeps.push(ms);
+        nowMs += ms;
+      },
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(lease.credentialId).toBe("cred-2");
+    expect(fetchImpl).toHaveBeenCalledTimes(3);
+    expect(sleeps.length).toBe(2);
+    expect(sleeps[0]).toBeGreaterThanOrEqual(100);
+    expect(sleeps[1]).toBeGreaterThan(sleeps[0] ?? 0);
+  });
+
+  it("rejects non-https convex site URLs unless local insecure opt-in is enabled", async () => {
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "http://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        },
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("must use https://");
+  });
+
+  it("allows loopback http URLs when OPENCLAW_QA_ALLOW_INSECURE_HTTP is enabled", async () => {
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      jsonResponse({
+        status: "ok",
+        credentialId: "cred-local",
+        leaseToken: "lease-local",
+        payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+      }),
+    );
+
+    await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      role: "maintainer",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "http://127.0.0.1:3210",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        OPENCLAW_QA_ALLOW_INSECURE_HTTP: "1",
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(fetchUrl(fetchImpl)).toBe("http://127.0.0.1:3210/qa-credentials/v1/acquire");
+  });
+
+  it("caps oversized convex HTTP timeouts before creating abort signals", async () => {
+    const timeoutController = new AbortController();
+    const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(timeoutController.signal);
+    const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
+      jsonResponse({
+        status: "ok",
+        credentialId: "cred-timeout",
+        leaseToken: "lease-timeout",
+        payload: { groupId: "-100123", driverToken: "driver", sutToken: "sut" },
+      }),
+    );
+
+    await acquireQaCredentialLease({
+      kind: "telegram",
+      source: "convex",
+      role: "maintainer",
+      env: {
+        OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: String(Number.MAX_SAFE_INTEGER),
+      },
+      fetchImpl,
+      resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+      parsePayload: (payload) =>
+        payload as { groupId: string; driverToken: string; sutToken: string },
+    });
+
+    expect(timeoutSpy).toHaveBeenCalledWith(MAX_TIMER_TIMEOUT_MS);
+    expect(fetchInit(fetchImpl).signal).toBe(timeoutController.signal);
+  });
+
+  it("rejects unsafe endpoint prefix overrides", async () => {
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+          OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX: "//evil.example",
+        },
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must be an absolute path");
+  });
+
+  it("releases acquired lease when payload parsing fails", async () => {
+    const fetchImpl = vi
+      .fn<typeof fetch>()
+      .mockResolvedValueOnce(
+        jsonResponse({
+          status: "ok",
+          credentialId: "cred-parse-fail",
+          leaseToken: "lease-parse-fail",
+          payload: { broken: true },
+        }),
+      )
+      .mockResolvedValueOnce(jsonResponse({ status: "ok" }));
+
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "maintainer",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+          OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
+        },
+        fetchImpl,
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: () => {
+          throw new Error("bad payload shape");
+        },
+      }),
+    ).rejects.toThrow("bad payload shape");
+
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+    expect(fetchUrl(fetchImpl, 1)).toBe(
+      "https://qa-cred.example.convex.site/qa-credentials/v1/release",
+    );
+  });
+
+  it("fails convex mode when auth secret is missing", async () => {
+    await expect(
+      acquireQaCredentialLease({
+        kind: "telegram",
+        source: "convex",
+        role: "maintainer",
+        env: {
+          OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-cred.example.convex.site",
+        },
+        resolveEnvPayload: () => ({ groupId: "-1", driverToken: "unused", sutToken: "unused" }),
+        parsePayload: (payload) =>
+          payload as { groupId: string; driverToken: string; sutToken: string },
+      }),
+    ).rejects.toThrow("OPENCLAW_QA_CONVEX_SECRET_MAINTAINER");
+  });
+
+  it("captures heartbeat failures for fail-fast checks", async () => {
+    vi.useFakeTimers();
+    const heartbeat = startQaCredentialLeaseHeartbeat(
+      {
+        source: "convex",
+        kind: "telegram",
+        heartbeatIntervalMs: 50,
+        heartbeat: async () => {
+          throw new Error("heartbeat-down");
+        },
+      },
+      { intervalMs: 50 },
+    );
+
+    await vi.advanceTimersByTimeAsync(55);
+    expect(heartbeat.getFailure()).toBeInstanceOf(Error);
+    expect(() => heartbeat.throwIfFailed()).toThrow("heartbeat-down");
+    await heartbeat.stop();
+  });
+});

extensions/qa-lab/src/live-transports/shared/credential-lease.runtime.ts

@@ -0,0 +1,649 @@
+// Qa Lab plugin module implements credential lease behavior.
+import { randomUUID } from "node:crypto";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
+import { readProviderTextResponse } from "openclaw/plugin-sdk/provider-http";
+import { z } from "zod";
+import {
+  isQaCredentialTruthyOptIn,
+  joinQaCredentialEndpoint,
+  normalizeQaCredentialConvexSiteUrl,
+  normalizeQaCredentialEndpointPrefix,
+  parseQaCredentialPositiveIntegerEnv,
+  QA_CREDENTIALS_DEFAULT_ENDPOINT_PREFIX,
+} from "../../qa-credentials-common.runtime.js";
+
+const DEFAULT_ACQUIRE_TIMEOUT_MS = 90_000;
+const DEFAULT_ENDPOINT_PREFIX = QA_CREDENTIALS_DEFAULT_ENDPOINT_PREFIX;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;
+const DEFAULT_HTTP_TIMEOUT_MS = 15_000;
+const DEFAULT_LEASE_TTL_MS = 20 * 60 * 1_000;
+const DEFAULT_CHUNKED_PAYLOAD_MAX_BYTES = 64 * 1024 * 1024;
+const DEFAULT_CHUNKED_PAYLOAD_MAX_CHUNKS = 4096;
+const CONVEX_BROKER_RESPONSE_MAX_BYTES = 1 * 1024 * 1024;
+const CHUNKED_PAYLOAD_MAX_BYTES_ENV = "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES";
+const CHUNKED_PAYLOAD_MAX_CHUNKS_ENV = "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS";
+const RETRY_BACKOFF_MS = [500, 1_000, 2_000, 4_000, 5_000] as const;
+const RETRYABLE_ACQUIRE_CODES = new Set(["POOL_EXHAUSTED", "NO_CREDENTIAL_AVAILABLE"]);
+const CHUNKED_PAYLOAD_MARKER = "__openclawQaCredentialPayloadChunksV1";
+
+const convexAcquireSuccessSchema = z.object({
+  status: z.literal("ok"),
+  credentialId: z.string().min(1),
+  leaseToken: z.string().min(1),
+  payload: z.unknown(),
+  leaseTtlMs: z.number().int().positive().optional(),
+  heartbeatIntervalMs: z.number().int().positive().optional(),
+});
+
+const convexErrorSchema = z.object({
+  status: z.literal("error"),
+  code: z.string().min(1),
+  message: z.string().optional(),
+  retryAfterMs: z.number().int().positive().optional(),
+});
+
+const convexOkSchema = z.object({
+  status: z.literal("ok"),
+});
+
+const convexPayloadChunkSuccessSchema = z.object({
+  status: z.literal("ok"),
+  data: z.string(),
+});
+
+type ConvexCredentialBrokerConfig = {
+  acquireTimeoutMs: number;
+  acquireUrl: string;
+  authToken: string;
+  heartbeatIntervalMs: number;
+  heartbeatUrl: string;
+  httpTimeoutMs: number;
+  leaseTtlMs: number;
+  ownerId: string;
+  payloadMaxBytes: number;
+  payloadMaxChunks: number;
+  payloadChunkUrl: string;
+  releaseUrl: string;
+  role: QaCredentialRole;
+};
+
+type QaCredentialLeaseHeartbeat = {
+  getFailure(): Error | null;
+  stop(): Promise<void>;
+  throwIfFailed(): void;
+};
+
+export type QaCredentialRole = "ci" | "maintainer";
+
+type QaCredentialLeaseSource = "convex" | "env";
+
+type QaCredentialLease<TPayload> = {
+  credentialId?: string;
+  heartbeat(): Promise<void>;
+  heartbeatIntervalMs: number;
+  kind: string;
+  leaseToken?: string;
+  leaseTtlMs: number;
+  ownerId?: string;
+  payload: TPayload;
+  release(): Promise<void>;
+  role?: QaCredentialRole;
+  source: QaCredentialLeaseSource;
+};
+
+type AcquireQaCredentialLeaseOptions<TPayload> = {
+  env?: NodeJS.ProcessEnv;
+  fetchImpl?: typeof fetch;
+  kind: string;
+  ownerId?: string;
+  parsePayload: (payload: unknown) => TPayload;
+  randomImpl?: () => number;
+  resolveEnvPayload: () => TPayload;
+  role?: string;
+  sleepImpl?: (ms: number) => Promise<unknown>;
+  source?: string;
+  timeImpl?: () => number;
+};
+
+class QaCredentialBrokerError extends Error {
+  code: string;
+  retryAfterMs?: number;
+
+  constructor(params: { code: string; message: string; retryAfterMs?: number }) {
+    super(params.message);
+    this.name = "QaCredentialBrokerError";
+    this.code = params.code;
+    this.retryAfterMs = params.retryAfterMs;
+  }
+}
+
+function parsePositiveIntegerEnv(env: NodeJS.ProcessEnv, key: string, fallback: number): number {
+  return parseQaCredentialPositiveIntegerEnv({ env, key, fallback });
+}
+
+function normalizeQaCredentialSource(value: string | undefined): QaCredentialLeaseSource {
+  const normalized = value?.trim().toLowerCase() || "env";
+  if (normalized === "env" || normalized === "convex") {
+    return normalized;
+  }
+  throw new Error(`Credential source must be one of env or convex, got "${value}".`);
+}
+
+function normalizeQaCredentialRole(
+  value: string | undefined,
+  env: NodeJS.ProcessEnv = process.env,
+): QaCredentialRole {
+  const defaultRole = isQaCredentialTruthyOptIn(env.CI) ? "ci" : "maintainer";
+  const normalized = value?.trim().toLowerCase() || defaultRole;
+  if (normalized === "maintainer" || normalized === "ci") {
+    return normalized;
+  }
+  throw new Error(`Credential role must be one of maintainer or ci, got "${value}".`);
+}
+
+function normalizeConvexSiteUrl(raw: string, env: NodeJS.ProcessEnv): string {
+  return normalizeQaCredentialConvexSiteUrl({ raw, env });
+}
+
+function normalizeEndpointPrefix(value: string | undefined): string {
+  return normalizeQaCredentialEndpointPrefix({
+    value,
+    fallback: DEFAULT_ENDPOINT_PREFIX,
+    invalidAbsoluteMessage:
+      "OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must be an absolute path like /qa-credentials/v1.",
+    invalidSegmentsMessage:
+      "OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must not contain backslashes or .. path segments.",
+  });
+}
+
+function resolveConvexAuthToken(env: NodeJS.ProcessEnv, role: QaCredentialRole): string {
+  const roleToken =
+    role === "ci"
+      ? env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim()
+      : env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim();
+  const token = roleToken;
+  if (token) {
+    return token;
+  }
+  if (role === "ci") {
+    throw new Error("Missing OPENCLAW_QA_CONVEX_SECRET_CI for CI credential access.");
+  }
+  throw new Error("Missing OPENCLAW_QA_CONVEX_SECRET_MAINTAINER for maintainer credential access.");
+}
+
+function resolveConvexCredentialBrokerConfig(params: {
+  env: NodeJS.ProcessEnv;
+  ownerId?: string;
+  role: QaCredentialRole;
+}): ConvexCredentialBrokerConfig {
+  const siteUrl = params.env.OPENCLAW_QA_CONVEX_SITE_URL?.trim();
+  if (!siteUrl) {
+    throw new Error("Missing OPENCLAW_QA_CONVEX_SITE_URL for --credential-source convex.");
+  }
+  const baseUrl = normalizeConvexSiteUrl(siteUrl, params.env);
+  const endpointPrefix = normalizeEndpointPrefix(params.env.OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX);
+  const ownerId =
+    params.ownerId?.trim() ||
+    params.env.OPENCLAW_QA_CREDENTIAL_OWNER_ID?.trim() ||
+    `qa-lab-${params.role}-${process.pid}-${randomUUID().slice(0, 8)}`;
+  return {
+    role: params.role,
+    ownerId,
+    authToken: resolveConvexAuthToken(params.env, params.role),
+    leaseTtlMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS",
+      DEFAULT_LEASE_TTL_MS,
+    ),
+    heartbeatIntervalMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS",
+      DEFAULT_HEARTBEAT_INTERVAL_MS,
+    ),
+    acquireTimeoutMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS",
+      DEFAULT_ACQUIRE_TIMEOUT_MS,
+    ),
+    httpTimeoutMs: parsePositiveIntegerEnv(
+      params.env,
+      "OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS",
+      DEFAULT_HTTP_TIMEOUT_MS,
+    ),
+    payloadMaxBytes: parsePositiveIntegerEnv(
+      params.env,
+      CHUNKED_PAYLOAD_MAX_BYTES_ENV,
+      DEFAULT_CHUNKED_PAYLOAD_MAX_BYTES,
+    ),
+    payloadMaxChunks: parsePositiveIntegerEnv(
+      params.env,
+      CHUNKED_PAYLOAD_MAX_CHUNKS_ENV,
+      DEFAULT_CHUNKED_PAYLOAD_MAX_CHUNKS,
+    ),
+    acquireUrl: joinQaCredentialEndpoint(baseUrl, endpointPrefix, "acquire"),
+    heartbeatUrl: joinQaCredentialEndpoint(baseUrl, endpointPrefix, "heartbeat"),
+    payloadChunkUrl: joinQaCredentialEndpoint(baseUrl, endpointPrefix, "payload-chunk"),
+    releaseUrl: joinQaCredentialEndpoint(baseUrl, endpointPrefix, "release"),
+  };
+}
+
+function parseChunkedPayloadMarker(
+  payload: unknown,
+  limits: { maxBytes: number; maxChunks: number },
+) {
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
+    return null;
+  }
+  const record = payload as Record<string, unknown>;
+  if (record[CHUNKED_PAYLOAD_MARKER] !== true) {
+    return null;
+  }
+  if (
+    typeof record.chunkCount !== "number" ||
+    !Number.isInteger(record.chunkCount) ||
+    record.chunkCount < 1
+  ) {
+    throw new Error("Chunked credential payload marker has an invalid chunkCount.");
+  }
+  if (record.chunkCount > limits.maxChunks) {
+    throw new Error(`Chunked credential payload marker exceeds ${limits.maxChunks} chunks.`);
+  }
+  if (
+    typeof record.byteLength !== "number" ||
+    !Number.isInteger(record.byteLength) ||
+    record.byteLength < 0
+  ) {
+    throw new Error("Chunked credential payload marker has an invalid byteLength.");
+  }
+  if (record.byteLength > limits.maxBytes) {
+    throw new Error(`Chunked credential payload marker exceeds ${limits.maxBytes} bytes.`);
+  }
+  return {
+    chunkCount: record.chunkCount,
+    byteLength: record.byteLength,
+  };
+}
+
+function toBrokerError(params: {
+  payload: unknown;
+  fallback: string;
+}): QaCredentialBrokerError | null {
+  const parsed = convexErrorSchema.safeParse(params.payload);
+  if (!parsed.success) {
+    return null;
+  }
+  return new QaCredentialBrokerError({
+    code: parsed.data.code,
+    message: parsed.data.message?.trim() || params.fallback,
+    retryAfterMs: parsed.data.retryAfterMs,
+  });
+}
+
+async function postConvexBroker(params: {
+  authToken: string;
+  body: Record<string, unknown>;
+  fetchImpl: typeof fetch;
+  maxBytes: number;
+  timeoutMs: number;
+  url: string;
+}): Promise<unknown> {
+  const timeoutMs = resolveTimerTimeoutMs(params.timeoutMs, DEFAULT_HTTP_TIMEOUT_MS);
+  const response = await params.fetchImpl(params.url, {
+    method: "POST",
+    headers: {
+      authorization: `Bearer ${params.authToken}`,
+      "content-type": "application/json",
+    },
+    body: JSON.stringify(params.body),
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+
+  // Keep ordinary broker responses small, while allowing chunk payloads to use
+  // the larger declared payload ceiling.
+  const text = await readProviderTextResponse(response, "Convex credential broker", {
+    maxBytes: params.maxBytes,
+  });
+  const payload: unknown = (() => {
+    if (!text.trim()) {
+      return undefined;
+    }
+    try {
+      return JSON.parse(text) as unknown;
+    } catch {
+      return text;
+    }
+  })();
+
+  const brokerError = toBrokerError({
+    payload,
+    fallback: `Convex credential broker request failed (${response.status}).`,
+  });
+  if (brokerError) {
+    throw brokerError;
+  }
+  if (!response.ok) {
+    throw new Error(
+      `Convex credential broker request to ${params.url} failed with HTTP ${response.status}.`,
+    );
+  }
+  return payload;
+}
+
+async function resolveConvexCredentialPayload(params: {
+  acquired: z.infer<typeof convexAcquireSuccessSchema>;
+  config: ConvexCredentialBrokerConfig;
+  fetchImpl: typeof fetch;
+  kind: string;
+}) {
+  const marker = parseChunkedPayloadMarker(params.acquired.payload, {
+    maxBytes: params.config.payloadMaxBytes,
+    maxChunks: params.config.payloadMaxChunks,
+  });
+  if (!marker) {
+    return params.acquired.payload;
+  }
+  const chunks: string[] = [];
+  let serializedBytes = 0;
+  for (let index = 0; index < marker.chunkCount; index += 1) {
+    const payload = await postConvexBroker({
+      fetchImpl: params.fetchImpl,
+      maxBytes: params.config.payloadMaxBytes,
+      timeoutMs: params.config.httpTimeoutMs,
+      authToken: params.config.authToken,
+      url: params.config.payloadChunkUrl,
+      body: {
+        kind: params.kind,
+        ownerId: params.config.ownerId,
+        actorRole: params.config.role,
+        credentialId: params.acquired.credentialId,
+        leaseToken: params.acquired.leaseToken,
+        index,
+      },
+    });
+    const parsed = convexPayloadChunkSuccessSchema.parse(payload);
+    serializedBytes += Buffer.byteLength(parsed.data, "utf8");
+    if (serializedBytes > marker.byteLength) {
+      throw new Error("Chunked credential payload exceeded declared byteLength.");
+    }
+    chunks.push(parsed.data);
+  }
+  const serialized = chunks.join("");
+  if (serializedBytes !== marker.byteLength) {
+    throw new Error("Chunked credential payload length mismatch.");
+  }
+  return JSON.parse(serialized) as unknown;
+}
+
+function computeAcquireBackoffMs(params: {
+  attempt: number;
+  randomImpl: () => number;
+  retryAfterMs?: number;
+}): number {
+  if (params.retryAfterMs && params.retryAfterMs > 0) {
+    return params.retryAfterMs;
+  }
+  const base = RETRY_BACKOFF_MS[Math.min(RETRY_BACKOFF_MS.length - 1, params.attempt - 1)];
+  const jitter = 0.75 + params.randomImpl() * 0.5;
+  return Math.max(100, Math.round(base * jitter));
+}
+
+function assertConvexOk(payload: unknown, actionLabel: string) {
+  if (payload === undefined) {
+    return;
+  }
+  if (convexOkSchema.safeParse(payload).success) {
+    return;
+  }
+  const brokerError = toBrokerError({
+    payload,
+    fallback: `Convex credential ${actionLabel} failed.`,
+  });
+  if (brokerError) {
+    throw brokerError;
+  }
+  throw new Error(`Convex credential ${actionLabel} failed with an invalid response payload.`);
+}
+
+export async function acquireQaCredentialLease<TPayload>(
+  opts: AcquireQaCredentialLeaseOptions<TPayload>,
+): Promise<QaCredentialLease<TPayload>> {
+  const env = opts.env ?? process.env;
+  const source = normalizeQaCredentialSource(opts.source ?? env.OPENCLAW_QA_CREDENTIAL_SOURCE);
+  if (source === "env") {
+    return {
+      source: "env",
+      kind: opts.kind,
+      payload: opts.resolveEnvPayload(),
+      heartbeatIntervalMs: 0,
+      leaseTtlMs: 0,
+      async heartbeat() {},
+      async release() {},
+    };
+  }
+
+  const role = normalizeQaCredentialRole(opts.role ?? env.OPENCLAW_QA_CREDENTIAL_ROLE, env);
+  const config = resolveConvexCredentialBrokerConfig({
+    env,
+    role,
+    ownerId: opts.ownerId,
+  });
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const sleepImpl =
+    opts.sleepImpl ??
+    ((ms: number) =>
+      new Promise((resolve) => {
+        setTimeout(resolve, ms);
+      }));
+  const timeImpl = opts.timeImpl ?? (() => Date.now());
+  const randomImpl = opts.randomImpl ?? (() => Math.random());
+  const startedAt = timeImpl();
+  let attempt = 0;
+
+  while (true) {
+    attempt += 1;
+    try {
+      const payload = await postConvexBroker({
+        fetchImpl,
+        maxBytes: CONVEX_BROKER_RESPONSE_MAX_BYTES,
+        timeoutMs: config.httpTimeoutMs,
+        authToken: config.authToken,
+        url: config.acquireUrl,
+        body: {
+          kind: opts.kind,
+          ownerId: config.ownerId,
+          actorRole: config.role,
+          leaseTtlMs: config.leaseTtlMs,
+          heartbeatIntervalMs: config.heartbeatIntervalMs,
+        },
+      });
+      const acquired = convexAcquireSuccessSchema.parse(payload);
+      const releaseLease = async () => {
+        const releasePayload = await postConvexBroker({
+          fetchImpl,
+          maxBytes: CONVEX_BROKER_RESPONSE_MAX_BYTES,
+          timeoutMs: config.httpTimeoutMs,
+          authToken: config.authToken,
+          url: config.releaseUrl,
+          body: {
+            kind: opts.kind,
+            ownerId: config.ownerId,
+            credentialId: acquired.credentialId,
+            leaseToken: acquired.leaseToken,
+            actorRole: config.role,
+          },
+        });
+        assertConvexOk(releasePayload, "release");
+      };
+      let parsedPayload: TPayload;
+      try {
+        const resolvedPayload = await resolveConvexCredentialPayload({
+          acquired,
+          config,
+          fetchImpl,
+          kind: opts.kind,
+        });
+        parsedPayload = opts.parsePayload(resolvedPayload);
+      } catch (error) {
+        try {
+          await releaseLease();
+        } catch (releaseError) {
+          throw new Error(
+            `Convex credential payload validation failed for kind "${opts.kind}" and cleanup release failed: ${formatErrorMessage(error)}; release failed: ${formatErrorMessage(releaseError)}`,
+            { cause: releaseError },
+          );
+        }
+        throw new Error(
+          `Convex credential payload validation failed for kind "${opts.kind}": ${formatErrorMessage(error)}`,
+          { cause: error },
+        );
+      }
+      const leaseTtlMs = acquired.leaseTtlMs ?? config.leaseTtlMs;
+      const heartbeatIntervalMs = acquired.heartbeatIntervalMs ?? config.heartbeatIntervalMs;
+      return {
+        source: "convex",
+        kind: opts.kind,
+        role,
+        ownerId: config.ownerId,
+        credentialId: acquired.credentialId,
+        leaseToken: acquired.leaseToken,
+        leaseTtlMs,
+        heartbeatIntervalMs,
+        payload: parsedPayload,
+        async heartbeat() {
+          const heartbeatPayload = await postConvexBroker({
+            fetchImpl,
+            maxBytes: CONVEX_BROKER_RESPONSE_MAX_BYTES,
+            timeoutMs: config.httpTimeoutMs,
+            authToken: config.authToken,
+            url: config.heartbeatUrl,
+            body: {
+              kind: opts.kind,
+              ownerId: config.ownerId,
+              credentialId: acquired.credentialId,
+              leaseToken: acquired.leaseToken,
+              actorRole: config.role,
+              leaseTtlMs,
+            },
+          });
+          assertConvexOk(heartbeatPayload, "heartbeat");
+        },
+        async release() {
+          await releaseLease();
+        },
+      };
+    } catch (error) {
+      if (error instanceof QaCredentialBrokerError && RETRYABLE_ACQUIRE_CODES.has(error.code)) {
+        const elapsed = timeImpl() - startedAt;
+        if (elapsed >= config.acquireTimeoutMs) {
+          throw new Error(
+            `Convex credential pool exhausted for kind "${opts.kind}" after ${config.acquireTimeoutMs}ms.`,
+            { cause: error },
+          );
+        }
+        const delayMs = Math.min(
+          computeAcquireBackoffMs({
+            attempt,
+            retryAfterMs: error.retryAfterMs,
+            randomImpl,
+          }),
+          Math.max(0, config.acquireTimeoutMs - elapsed),
+        );
+        if (delayMs > 0) {
+          await sleepImpl(delayMs);
+        }
+        continue;
+      }
+      if (error instanceof z.ZodError) {
+        throw new Error(
+          `Convex credential acquire response did not match the expected payload for kind "${opts.kind}": ${error.message}`,
+          { cause: error },
+        );
+      }
+      throw new Error(
+        `Convex credential acquire failed for kind "${opts.kind}": ${formatErrorMessage(error)}`,
+        { cause: error },
+      );
+    }
+  }
+}
+
+export function startQaCredentialLeaseHeartbeat(
+  lease: Pick<QaCredentialLease<unknown>, "heartbeat" | "heartbeatIntervalMs" | "kind" | "source">,
+  opts?: {
+    intervalMs?: number;
+    setTimeoutImpl?: typeof setTimeout;
+    clearTimeoutImpl?: typeof clearTimeout;
+  },
+): QaCredentialLeaseHeartbeat {
+  if (lease.source !== "convex") {
+    return {
+      getFailure: () => null,
+      async stop() {},
+      throwIfFailed() {},
+    };
+  }
+  const intervalMs = opts?.intervalMs ?? lease.heartbeatIntervalMs;
+  if (!Number.isFinite(intervalMs) || intervalMs < 1) {
+    return {
+      getFailure: () => null,
+      async stop() {},
+      throwIfFailed() {},
+    };
+  }
+
+  const setTimeoutImpl = opts?.setTimeoutImpl ?? setTimeout;
+  const clearTimeoutImpl = opts?.clearTimeoutImpl ?? clearTimeout;
+  let failure: Error | null = null;
+  let stopped = false;
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  let inFlight: Promise<void> | null = null;
+
+  const schedule = () => {
+    if (stopped || failure) {
+      return;
+    }
+    timer = setTimeoutImpl(() => {
+      timer = null;
+      if (stopped || failure) {
+        return;
+      }
+      inFlight = (async () => {
+        try {
+          await lease.heartbeat();
+        } catch (error) {
+          failure = new Error(
+            `Credential lease heartbeat failed for kind "${lease.kind}": ${formatErrorMessage(error)}`,
+          );
+          return;
+        } finally {
+          inFlight = null;
+        }
+        schedule();
+      })();
+    }, intervalMs);
+  };
+
+  schedule();
+
+  return {
+    getFailure() {
+      return failure;
+    },
+    throwIfFailed() {
+      if (failure) {
+        throw failure;
+      }
+    },
+    async stop() {
+      stopped = true;
+      if (timer) {
+        clearTimeoutImpl(timer);
+        timer = null;
+      }
+      if (inFlight) {
+        await inFlight.catch(() => {});
+      }
+    },
+  };
+}

extensions/qa-lab/src/live-transports/shared/live-approval-result.ts

@@ -0,0 +1,44 @@
+export function formatApprovalResultValue(value: unknown) {
+  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return String(value);
+  }
+  if (value == null) {
+    return "<missing>";
+  }
+  return JSON.stringify(value) ?? "<unserializable>";
+}
+
+export function readAcceptedApprovalRequest(result: unknown) {
+  const accepted =
+    typeof result === "object" && result !== null
+      ? (result as { id?: unknown; status?: unknown })
+      : null;
+  if (accepted?.status !== "accepted") {
+    throw new Error(
+      `approval request status was ${formatApprovalResultValue(
+        accepted?.status,
+      )} instead of accepted`,
+    );
+  }
+  return accepted;
+}
+
+export function readAcceptedApprovalRequestId(result: unknown) {
+  const id = readAcceptedApprovalRequest(result).id;
+  if (typeof id !== "string" || id.trim().length === 0) {
+    throw new Error(`approval request id was ${formatApprovalResultValue(id)}`);
+  }
+  return id;
+}
+
+export function assertApprovalDecisionResult(params: { decision: string; result: unknown }) {
+  const resultDecision =
+    typeof params.result === "object" && params.result !== null
+      ? (params.result as { decision?: unknown }).decision
+      : undefined;
+  if (resultDecision !== params.decision) {
+    throw new Error(
+      `approval decision was ${formatApprovalResultValue(resultDecision)} instead of ${params.decision}`,
+    );
+  }
+}

extensions/qa-lab/src/live-transports/shared/live-artifacts.test.ts

@@ -0,0 +1,36 @@
+// Qa Lab tests cover live artifacts plugin behavior.
+import { describe, expect, it } from "vitest";
+import { redactQaLiveLaneDetails, redactQaLiveLaneIssues } from "./live-artifacts.js";
+
+describe("live transport artifacts", () => {
+  it("uses a stable public metadata redaction marker", () => {
+    expect(redactQaLiveLaneDetails()).toBe(
+      "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)",
+    );
+  });
+
+  it("preserves cleanup phase labels while redacting details", () => {
+    expect(
+      redactQaLiveLaneIssues([
+        "credential lease release: broker rejected release for group -100123",
+        "live gateway cleanup: failed to stop pid 123",
+      ]),
+    ).toEqual([
+      "credential lease release: details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)",
+      "live gateway cleanup: details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)",
+    ]);
+  });
+
+  it("redacts multi-line artifact errors without preserving later section labels", () => {
+    expect(
+      redactQaLiveLaneIssues([
+        [
+          "WhatsApp QA failed before scenario completion.",
+          "raw startup error with +15550000002",
+          "Artifacts:",
+          "- gatewayDebug: /tmp/openclaw-whatsapp-qa/gateway-debug",
+        ].join("\n"),
+      ]),
+    ).toEqual(["details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)"]);
+  });
+});

extensions/qa-lab/src/live-transports/shared/live-artifacts.ts

@@ -0,0 +1,44 @@
+// Qa Lab plugin module implements live artifacts behavior.
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+
+const REDACTED_QA_LIVE_LANE_ISSUE_DETAILS =
+  "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)";
+
+export function redactQaLiveLaneDetails() {
+  return REDACTED_QA_LIVE_LANE_ISSUE_DETAILS;
+}
+
+export function appendQaLiveLaneIssue(issues: string[], label: string, error: unknown) {
+  issues.push(`${label}: ${formatErrorMessage(error)}`);
+}
+
+export function redactQaLiveLaneIssues(issues: readonly string[]) {
+  return issues.map((issue) => {
+    const firstLine = issue.split(/\r?\n/u, 1)[0] ?? "";
+    const separatorIndex = firstLine.indexOf(":");
+    const label = separatorIndex < 0 ? "" : firstLine.slice(0, separatorIndex).trim();
+    return label ? `${label}: ${redactQaLiveLaneDetails()}` : redactQaLiveLaneDetails();
+  });
+}
+
+export function buildQaLiveLaneArtifactsError(params: {
+  heading: string;
+  artifacts: Record<string, string>;
+  details?: string[];
+}) {
+  return [
+    params.heading,
+    ...(params.details ?? []),
+    "Artifacts:",
+    ...Object.entries(params.artifacts).map(([label, filePath]) => `- ${label}: ${filePath}`),
+  ].join("\n");
+}
+
+export function printLiveTransportQaArtifacts(
+  laneLabel: string,
+  artifacts: Record<string, string>,
+) {
+  for (const [label, filePath] of Object.entries(artifacts)) {
+    process.stdout.write(`${laneLabel} ${label}: ${filePath}\n`);
+  }
+}

extensions/qa-lab/src/live-transports/shared/live-credential-source.ts

@@ -0,0 +1,10 @@
+export type QaInferredCredentialSource = "convex" | "env";
+
+export function inferQaCredentialSource(
+  value: string | undefined,
+  env: NodeJS.ProcessEnv = process.env,
+): QaInferredCredentialSource {
+  const normalized =
+    value?.trim().toLowerCase() || env.OPENCLAW_QA_CREDENTIAL_SOURCE?.trim().toLowerCase();
+  return normalized === "convex" ? "convex" : "env";
+}

extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts

@@ -0,0 +1,271 @@
+// Qa Lab tests cover live gateway plugin behavior.
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { startQaGatewayChild, startQaProviderServer } = vi.hoisted(() => ({
+  startQaGatewayChild: vi.fn(),
+  startQaProviderServer: vi.fn(),
+}));
+
+vi.mock("../../gateway-child.js", () => ({
+  startQaGatewayChild,
+}));
+
+vi.mock("../../providers/server-runtime.js", () => ({
+  startQaProviderServer,
+}));
+
+import { startQaLiveLaneGateway } from "./live-gateway.runtime.js";
+
+type GatewayOptions = {
+  providerBaseUrl?: string;
+  providerMode?: string;
+  transportBaseUrl?: string;
+  mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
+};
+
+function createStubTransport(baseUrl = "http://127.0.0.1:43123") {
+  return {
+    requiredPluginIds: ["qa-channel"],
+    createGatewayConfig: () => ({
+      channels: {
+        "qa-channel": {
+          enabled: true,
+          baseUrl,
+          botUserId: "openclaw",
+          botDisplayName: "OpenClaw QA",
+          allowFrom: ["*"],
+          pollTimeoutMs: 250,
+        },
+      },
+      messages: {
+        groupChat: {
+          mentionPatterns: ["\\b@?openclaw\\b"],
+        },
+      },
+    }),
+  };
+}
+
+function firstGatewayOptions(): GatewayOptions | undefined {
+  return startQaGatewayChild.mock.calls[0]?.[0] as GatewayOptions | undefined;
+}
+
+describe("startQaLiveLaneGateway", () => {
+  const gatewayStop = vi.fn();
+  const gatewayCall = vi.fn();
+  const mockStop = vi.fn();
+
+  beforeEach(() => {
+    gatewayStop.mockReset();
+    gatewayCall.mockReset();
+    mockStop.mockReset();
+    startQaGatewayChild.mockReset();
+    startQaProviderServer.mockReset();
+
+    startQaGatewayChild.mockResolvedValue({
+      call: gatewayCall,
+      cfg: {},
+      stop: gatewayStop,
+    });
+    startQaProviderServer.mockImplementation(async (providerMode: string) =>
+      providerMode === "mock-openai"
+        ? {
+            baseUrl: "http://127.0.0.1:44080",
+            stop: mockStop,
+          }
+        : null,
+    );
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("threads the mock provider base url into the gateway child", async () => {
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.5",
+      alternateModel: "mock-openai/gpt-5.5-alt",
+      controlUiEnabled: false,
+    });
+
+    expect(startQaProviderServer).toHaveBeenCalledWith("mock-openai");
+    const gatewayOptions = firstGatewayOptions();
+    expect(gatewayOptions?.transportBaseUrl).toBe("http://127.0.0.1:43123");
+    expect(gatewayOptions?.providerBaseUrl).toBe("http://127.0.0.1:44080/v1");
+    expect(gatewayOptions?.providerMode).toBe("mock-openai");
+
+    await harness.stop();
+    expect(gatewayStop).toHaveBeenCalledTimes(1);
+    expect(mockStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("disables memory search for transport-only live lanes", async () => {
+    await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.5",
+      alternateModel: "mock-openai/gpt-5.5-alt",
+      controlUiEnabled: false,
+    });
+
+    const { mutateConfig } = firstGatewayOptions() ?? {};
+    if (!mutateConfig) {
+      throw new Error("expected gateway config mutator");
+    }
+    const cfg = mutateConfig({
+      plugins: {
+        allow: ["acpx", "memory-core", "qa-channel"],
+        entries: {
+          acpx: { enabled: true },
+          "memory-core": { enabled: true },
+          "qa-channel": { enabled: true },
+        },
+        slots: {
+          memory: "memory-core",
+          contextEngine: "qmd",
+        },
+      },
+      agents: {
+        defaults: {
+          memorySearch: {
+            enabled: true,
+            sync: {
+              onSearch: true,
+              onSessionStart: true,
+              watch: true,
+            },
+          },
+        },
+      },
+    });
+
+    expect(cfg?.plugins?.allow).toEqual(["acpx", "qa-channel"]);
+    expect(cfg?.plugins?.entries).not.toHaveProperty("memory-core");
+    expect(cfg?.plugins?.slots?.memory).toBe("none");
+    expect(cfg?.plugins?.slots?.contextEngine).toBe("qmd");
+    expect(cfg?.agents?.defaults?.memorySearch?.enabled).toBe(false);
+    expect(cfg?.agents?.defaults?.memorySearch?.sync?.onSearch).toBe(false);
+    expect(cfg?.agents?.defaults?.memorySearch?.sync?.onSessionStart).toBe(false);
+    expect(cfg?.agents?.defaults?.memorySearch?.sync?.watch).toBe(false);
+  });
+
+  it("forwards gateway stop options to the child harness", async () => {
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.5",
+      alternateModel: "mock-openai/gpt-5.5-alt",
+      controlUiEnabled: false,
+    });
+
+    await harness.stop({ preserveToDir: ".artifacts/qa-e2e/debug" });
+    expect(gatewayStop).toHaveBeenCalledWith({ preserveToDir: ".artifacts/qa-e2e/debug" });
+    expect(mockStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("skips mock bootstrap for live frontier runs", async () => {
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "live-frontier",
+      primaryModel: "openai/gpt-5.5",
+      alternateModel: "openai/gpt-5.5",
+      controlUiEnabled: false,
+    });
+
+    expect(startQaProviderServer).toHaveBeenCalledWith("live-frontier");
+    const gatewayOptions = firstGatewayOptions();
+    expect(gatewayOptions?.transportBaseUrl).toBe("http://127.0.0.1:43123");
+    expect(gatewayOptions?.providerBaseUrl).toBeUndefined();
+    expect(gatewayOptions?.providerMode).toBe("live-frontier");
+
+    await harness.stop();
+    expect(gatewayStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("stops the mock server when gateway startup fails", async () => {
+    startQaGatewayChild.mockRejectedValueOnce(new Error("gateway failed"));
+
+    await expect(
+      startQaLiveLaneGateway({
+        repoRoot: "/tmp/openclaw-repo",
+        transport: createStubTransport(),
+        transportBaseUrl: "http://127.0.0.1:43123",
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.5",
+        alternateModel: "mock-openai/gpt-5.5-alt",
+        controlUiEnabled: false,
+      }),
+    ).rejects.toThrow("gateway failed");
+
+    expect(mockStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("reports mock cleanup failures after gateway startup failures", async () => {
+    startQaGatewayChild.mockRejectedValueOnce(new Error("gateway failed"));
+    mockStop.mockRejectedValueOnce(new Error("mock stuck"));
+
+    await expect(
+      startQaLiveLaneGateway({
+        repoRoot: "/tmp/openclaw-repo",
+        transport: createStubTransport(),
+        transportBaseUrl: "http://127.0.0.1:43123",
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.5",
+        alternateModel: "mock-openai/gpt-5.5-alt",
+        controlUiEnabled: false,
+      }),
+    ).rejects.toThrow(
+      "failed to start QA live lane gateway:\ngateway startup failed: gateway failed\nmock provider stop failed: mock stuck",
+    );
+
+    expect(mockStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("still stops the mock server when gateway shutdown fails", async () => {
+    gatewayStop.mockRejectedValueOnce(new Error("gateway down"));
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.5",
+      alternateModel: "mock-openai/gpt-5.5-alt",
+      controlUiEnabled: false,
+    });
+
+    await expect(harness.stop()).rejects.toThrow(
+      "failed to stop QA live lane resources:\ngateway stop failed: gateway down",
+    );
+    expect(gatewayStop).toHaveBeenCalledTimes(1);
+    expect(mockStop).toHaveBeenCalledTimes(1);
+  });
+
+  it("reports both gateway and mock shutdown failures together", async () => {
+    gatewayStop.mockRejectedValueOnce(new Error("gateway down"));
+    mockStop.mockRejectedValueOnce(new Error("mock down"));
+    const harness = await startQaLiveLaneGateway({
+      repoRoot: "/tmp/openclaw-repo",
+      transport: createStubTransport(),
+      transportBaseUrl: "http://127.0.0.1:43123",
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.5",
+      alternateModel: "mock-openai/gpt-5.5-alt",
+      controlUiEnabled: false,
+    });
+
+    await expect(harness.stop()).rejects.toThrow(
+      "failed to stop QA live lane resources:\ngateway stop failed: gateway down\nmock provider stop failed: mock down",
+    );
+  });
+});

extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts

@@ -0,0 +1,147 @@
+// Qa Lab plugin module implements live gateway behavior.
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
+import {
+  startQaGatewayChild,
+  type QaCliBackendAuthMode,
+  type QaGatewayChildCommand,
+} from "../../gateway-child.js";
+import type { QaProviderMode } from "../../model-selection.js";
+import { startQaProviderServer } from "../../providers/server-runtime.js";
+import type { QaThinkingLevel } from "../../qa-gateway-config.js";
+import { appendQaLiveLaneIssue as appendLiveLaneIssue } from "./live-artifacts.js";
+
+async function stopQaLiveLaneResources(
+  resources: {
+    gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
+    mock: { baseUrl: string; stop(): Promise<void> } | null;
+  },
+  opts?: { keepTemp?: boolean; preserveToDir?: string },
+) {
+  const errors: string[] = [];
+  try {
+    await resources.gateway.stop(opts);
+  } catch (error) {
+    appendLiveLaneIssue(errors, "gateway stop failed", error);
+  }
+  if (resources.mock) {
+    try {
+      await resources.mock.stop();
+    } catch (error) {
+      appendLiveLaneIssue(errors, "mock provider stop failed", error);
+    }
+  }
+  if (errors.length > 0) {
+    throw new Error(`failed to stop QA live lane resources:\n${errors.join("\n")}`);
+  }
+}
+
+function omitMemoryCoreEntry<T extends Record<string, unknown> | undefined>(entries: T): T {
+  if (!entries || !Object.hasOwn(entries, "memory-core")) {
+    return entries;
+  }
+  const { "memory-core": _memoryCore, ...rest } = entries;
+  return rest as T;
+}
+
+function prepareLiveTransportGatewayConfig(cfg: OpenClawConfig): OpenClawConfig {
+  const defaults = cfg.agents?.defaults ?? {};
+  return {
+    ...cfg,
+    plugins: cfg.plugins
+      ? {
+          ...cfg.plugins,
+          allow: cfg.plugins.allow?.filter((pluginId) => pluginId !== "memory-core"),
+          entries: omitMemoryCoreEntry(cfg.plugins.entries),
+          slots: {
+            ...cfg.plugins.slots,
+            memory: "none",
+          },
+        }
+      : {
+          slots: {
+            memory: "none",
+          },
+        },
+    agents: {
+      ...cfg.agents,
+      defaults: {
+        ...defaults,
+        memorySearch: {
+          ...defaults.memorySearch,
+          enabled: false,
+          sync: {
+            ...defaults.memorySearch?.sync,
+            onSearch: false,
+            onSessionStart: false,
+            watch: false,
+          },
+        },
+      },
+    },
+  };
+}
+
+export async function startQaLiveLaneGateway(params: {
+  repoRoot: string;
+  command?: QaGatewayChildCommand;
+  transport: {
+    requiredPluginIds: readonly string[];
+    createGatewayConfig: (params: {
+      baseUrl: string;
+    }) => Pick<OpenClawConfig, "channels" | "messages">;
+  };
+  transportBaseUrl: string;
+  controlUiAllowedOrigins?: string[];
+  providerMode: QaProviderMode;
+  primaryModel: string;
+  alternateModel: string;
+  fastMode?: boolean;
+  thinkingDefault?: QaThinkingLevel;
+  claudeCliAuthMode?: QaCliBackendAuthMode;
+  controlUiEnabled?: boolean;
+  mockAuthAgentIds?: readonly string[];
+  mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
+}) {
+  const mock = await startQaProviderServer(params.providerMode);
+  try {
+    const gateway = await startQaGatewayChild({
+      repoRoot: params.repoRoot,
+      command: params.command,
+      providerBaseUrl: mock ? `${mock.baseUrl}/v1` : undefined,
+      transport: params.transport,
+      transportBaseUrl: params.transportBaseUrl,
+      controlUiAllowedOrigins: params.controlUiAllowedOrigins,
+      providerMode: params.providerMode,
+      primaryModel: params.primaryModel,
+      alternateModel: params.alternateModel,
+      fastMode: params.fastMode,
+      thinkingDefault: params.thinkingDefault,
+      claudeCliAuthMode: params.claudeCliAuthMode,
+      controlUiEnabled: params.controlUiEnabled,
+      mockAuthAgentIds: params.mockAuthAgentIds,
+      mutateConfig: (cfg) =>
+        prepareLiveTransportGatewayConfig(params.mutateConfig ? params.mutateConfig(cfg) : cfg),
+    });
+    return {
+      gateway,
+      mock,
+      async stop(opts?: { keepTemp?: boolean; preserveToDir?: string }) {
+        await stopQaLiveLaneResources({ gateway, mock }, opts);
+      },
+    };
+  } catch (error) {
+    if (mock) {
+      try {
+        await mock.stop();
+      } catch (cleanupError) {
+        const errors: string[] = [];
+        appendLiveLaneIssue(errors, "gateway startup failed", error);
+        appendLiveLaneIssue(errors, "mock provider stop failed", cleanupError);
+        throw new Error(`failed to start QA live lane gateway:\n${errors.join("\n")}`, {
+          cause: cleanupError,
+        });
+      }
+    }
+    throw error;
+  }
+}

extensions/qa-lab/src/live-transports/shared/live-scenario-reply.ts

@@ -0,0 +1,21 @@
+type LiveScenarioReplyMessage = {
+  messageId: string | number;
+  text: string;
+  [key: string]: unknown;
+};
+
+export function assertLiveScenarioReply(params: {
+  expectedTextIncludes?: string[];
+  message: LiveScenarioReplyMessage;
+}) {
+  if (!params.message.text.trim()) {
+    throw new Error(`reply message ${params.message.messageId} was empty`);
+  }
+  for (const expected of params.expectedTextIncludes ?? []) {
+    if (!params.message.text.includes(expected)) {
+      throw new Error(
+        `reply message ${params.message.messageId} missing expected text: ${expected}`,
+      );
+    }
+  }
+}

extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts

@@ -0,0 +1,21 @@
+// Qa Lab tests cover live transport cli plugin behavior.
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { resolveLiveTransportQaRunOptions } from "./live-transport-cli.runtime.js";
+
+describe("resolveLiveTransportQaRunOptions", () => {
+  it("drops blank model refs so live transports can use provider defaults", () => {
+    const options = resolveLiveTransportQaRunOptions({
+      repoRoot: "/tmp/openclaw-repo",
+      providerMode: "live-frontier",
+      primaryModel: " ",
+      alternateModel: "",
+      listScenarios: true,
+    });
+    expect(options.repoRoot).toBe(path.resolve("/tmp/openclaw-repo"));
+    expect(options.providerMode).toBe("live-frontier");
+    expect(options.primaryModel).toBeUndefined();
+    expect(options.alternateModel).toBeUndefined();
+    expect(options.listScenarios).toBe(true);
+  });
+});

extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts

@@ -0,0 +1,40 @@
+// Qa Lab plugin module implements live transport cli behavior.
+import path from "node:path";
+import { resolveRepoRelativeOutputDir } from "../../cli-paths.js";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
+import type { QaProviderMode } from "../../run-config.js";
+import { normalizeQaProviderMode } from "../../run-config.js";
+import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
+
+function normalizeLiveTransportModelRef(input: string | undefined) {
+  const model = input?.trim();
+  return model && model.length > 0 ? model : undefined;
+}
+
+export function resolveLiveTransportQaRunOptions(
+  opts: LiveTransportQaCommandOptions,
+): LiveTransportQaCommandOptions & {
+  repoRoot: string;
+  providerMode: QaProviderMode;
+} {
+  return {
+    repoRoot: path.resolve(opts.repoRoot ?? process.cwd()),
+    outputDir: resolveRepoRelativeOutputDir(
+      path.resolve(opts.repoRoot ?? process.cwd()),
+      opts.outputDir,
+    ),
+    providerMode:
+      opts.providerMode === undefined
+        ? DEFAULT_QA_LIVE_PROVIDER_MODE
+        : normalizeQaProviderMode(opts.providerMode),
+    primaryModel: normalizeLiveTransportModelRef(opts.primaryModel),
+    alternateModel: normalizeLiveTransportModelRef(opts.alternateModel),
+    fastMode: opts.fastMode,
+    allowFailures: opts.allowFailures,
+    scenarioIds: opts.scenarioIds,
+    listScenarios: opts.listScenarios,
+    sutAccountId: opts.sutAccountId,
+    credentialSource: opts.credentialSource?.trim(),
+    credentialRole: opts.credentialRole?.trim(),
+  };
+}

extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts

@@ -0,0 +1,28 @@
+// Qa Lab plugin module implements live transport cli behavior.
+import {
+  createLiveTransportQaCliRegistration as createSharedLiveTransportQaCliRegistration,
+  type LiveTransportQaCliRegistrationOptions,
+} from "openclaw/plugin-sdk/qa-runtime";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE, formatQaProviderModeHelp } from "../../providers/index.js";
+
+export {
+  createLazyCliRuntimeLoader,
+  type LiveTransportQaCliRegistration,
+  type LiveTransportQaCommandOptions,
+} from "openclaw/plugin-sdk/qa-runtime";
+
+type QaLabLiveTransportQaCliRegistrationOptions = Omit<
+  LiveTransportQaCliRegistrationOptions,
+  "allowFailuresHelp" | "defaultProviderMode" | "providerModeHelp"
+>;
+
+export function createLiveTransportQaCliRegistration(
+  params: QaLabLiveTransportQaCliRegistrationOptions,
+) {
+  return createSharedLiveTransportQaCliRegistration({
+    ...params,
+    allowFailuresHelp: "Write artifacts without setting a failing exit code when scenarios fail",
+    defaultProviderMode: DEFAULT_QA_LIVE_PROVIDER_MODE,
+    providerModeHelp: formatQaProviderModeHelp(),
+  });
+}

extensions/qa-lab/src/live-transports/shared/live-transport-result.ts

@@ -0,0 +1,24 @@
+// Qa Lab plugin module implements shared live-transport result shapes.
+import type { QaEvidenceTiming } from "../../evidence-summary.js";
+
+export type LiveTransportRttMeasurement = {
+  finalMatchedReplyRttMs: number;
+  requestStartedAt: string;
+  responseObservedAt: string;
+  source: "request-to-observed-message";
+};
+
+export type LiveTransportCheckResult = {
+  id: string;
+  title: string;
+  status: "pass" | "fail";
+  details: string;
+  coverageIds?: readonly string[];
+  timing?: QaEvidenceTiming;
+  rttMs?: number;
+  requestStartedAt?: string;
+  responseObservedAt?: string;
+  rttMeasurement?: LiveTransportRttMeasurement;
+  sentMessageId?: number;
+  responseMessageId?: number;
+};

extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts

@@ -0,0 +1,83 @@
+// Qa Lab plugin module implements shared live-transport RTT behavior.
+import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
+import type { QaEvidenceTiming } from "../../evidence-summary.js";
+
+export type LiveTransportRttOptions<CheckId extends string = string> = {
+  count: number;
+  timeoutMs: number;
+  maxFailures: number;
+  checkIds: Set<CheckId>;
+};
+
+export type LiveTransportRttSample = {
+  rttMs?: number;
+  status: "pass" | "fail";
+};
+
+function normalizePositiveRttInteger(value: number | undefined) {
+  if (
+    typeof value !== "number" ||
+    !Number.isSafeInteger(value) ||
+    value <= 0 ||
+    value > MAX_TIMER_TIMEOUT_MS
+  ) {
+    return undefined;
+  }
+  return value;
+}
+
+export function normalizeLiveTransportRttOptions<CheckId extends string>(params: {
+  count?: number;
+  defaultCheckIds: readonly CheckId[];
+  knownCheckIds: ReadonlySet<CheckId>;
+  maxFailures?: number;
+  rawCheckIds?: readonly string[];
+  timeoutMs?: number;
+  unknownCheckMessage: (checkId: string) => string;
+}): LiveTransportRttOptions<CheckId> | undefined {
+  const count = normalizePositiveRttInteger(params.count);
+  if (count === undefined) {
+    return undefined;
+  }
+  const rawCheckIds =
+    params.rawCheckIds && params.rawCheckIds.length > 0
+      ? params.rawCheckIds
+      : params.defaultCheckIds;
+  const checkIds = new Set<CheckId>();
+  for (const checkId of rawCheckIds) {
+    if (!params.knownCheckIds.has(checkId as CheckId)) {
+      throw new Error(params.unknownCheckMessage(checkId));
+    }
+    checkIds.add(checkId as CheckId);
+  }
+  return {
+    count,
+    maxFailures: normalizePositiveRttInteger(params.maxFailures) ?? count,
+    checkIds,
+    timeoutMs: normalizePositiveRttInteger(params.timeoutMs) ?? 30_000,
+  };
+}
+
+export function percentile(sortedValues: readonly number[], percentileValue: number) {
+  if (sortedValues.length === 0) {
+    return undefined;
+  }
+  const index = Math.ceil((percentileValue / 100) * sortedValues.length) - 1;
+  return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)];
+}
+
+export function summarizeLiveTransportRttSamples(samples: readonly LiveTransportRttSample[]) {
+  const passed = samples.filter((sample) => sample.status === "pass" && sample.rttMs !== undefined);
+  const sorted = passed.map((sample) => sample.rttMs as number).toSorted((a, b) => a - b);
+  const sum = sorted.reduce((total, value) => total + value, 0);
+  const timing: QaEvidenceTiming = {
+    rttMs: percentile(sorted, 50),
+    avgMs: sorted.length > 0 ? Math.round(sum / sorted.length) : undefined,
+    p50Ms: percentile(sorted, 50),
+    p95Ms: percentile(sorted, 95),
+    maxMs: sorted.at(-1),
+    samples: samples.length,
+    failedSamples: samples.length - passed.length,
+  };
+  return { passed: passed.length, failed: samples.length - passed.length, timing };
+}

extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts

@@ -0,0 +1,144 @@
+// Qa Lab tests cover live transport scenarios plugin behavior.
+import fs from "node:fs";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+import { __testing as discordTesting } from "../discord/discord-live.runtime.js";
+import { __testing as slackTesting } from "../slack/slack-live.runtime.js";
+import { __testing as telegramTesting } from "../telegram/telegram-live.runtime.js";
+import { __testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js";
+import {
+  LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+  buildLiveTransportCoverageLaneSummaries,
+  collectLiveTransportStandardScenarioCoverage,
+  findMissingLiveTransportStandardScenarios,
+  loadNonYamlScenarioRefs,
+  selectLiveTransportScenarios,
+} from "./live-transport-scenarios.js";
+
+describe("live transport scenario helpers", () => {
+  it("loads every non-YAML scenario id exactly once", async () => {
+    const refs = await loadNonYamlScenarioRefs();
+
+    expect(refs.length).toBeGreaterThan(0);
+    expect(new Set(refs.map((ref) => ref.id)).size).toBe(refs.length);
+  });
+
+  it("uses the public live transport scenario SDK seam", () => {
+    const source = fs.readFileSync(
+      fileURLToPath(new URL("./live-transport-scenarios.ts", import.meta.url)),
+      "utf8",
+    );
+
+    expect(source).toContain("openclaw/plugin-sdk/qa-live-transport-scenarios");
+    expect(source).not.toContain("openclaw/plugin-sdk/qa-runtime");
+  });
+
+  it("keeps the repo-wide baseline contract ordered", () => {
+    expect(LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS).toEqual([
+      "canary",
+      "mention-gating",
+      "allowlist-block",
+      "top-level-reply-shape",
+      "restart-resume",
+    ]);
+  });
+
+  it("selects requested scenarios and reports unknown ids with the lane label", () => {
+    const definitions = [
+      { id: "alpha", timeoutMs: 1_000, title: "alpha" },
+      { id: "beta", timeoutMs: 1_000, title: "beta" },
+    ] as const;
+
+    expect(
+      selectLiveTransportScenarios({
+        ids: ["beta"],
+        laneLabel: "Demo",
+        scenarios: definitions,
+      }),
+    ).toEqual([definitions[1]]);
+
+    expect(() =>
+      selectLiveTransportScenarios({
+        ids: ["alpha", "missing"],
+        laneLabel: "Demo",
+        scenarios: definitions,
+      }),
+    ).toThrow("unknown Demo QA scenario id(s): missing");
+  });
+
+  it("dedupes always-on and scenario-backed standard coverage", () => {
+    const covered = collectLiveTransportStandardScenarioCoverage({
+      alwaysOnStandardScenarioIds: ["canary"],
+      scenarios: [
+        {
+          id: "scenario-1",
+          standardId: "mention-gating",
+          timeoutMs: 1_000,
+          title: "mention",
+        },
+        {
+          id: "scenario-2",
+          standardId: "mention-gating",
+          timeoutMs: 1_000,
+          title: "mention again",
+        },
+        {
+          id: "scenario-3",
+          standardId: "restart-resume",
+          timeoutMs: 1_000,
+          title: "restart",
+        },
+      ],
+    });
+
+    expect(covered).toEqual(["canary", "mention-gating", "restart-resume"]);
+    expect(
+      findMissingLiveTransportStandardScenarios({
+        coveredStandardScenarioIds: covered,
+        expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+      }),
+    ).toEqual(["allowlist-block", "top-level-reply-shape"]);
+  });
+
+  it("summarizes live transport lane membership for coverage reports", () => {
+    const lanes = buildLiveTransportCoverageLaneSummaries();
+
+    expect(lanes.map((lane) => lane.transportId)).toEqual([
+      "discord",
+      "slack",
+      "telegram",
+      "whatsapp",
+    ]);
+    expect(lanes.find((lane) => lane.transportId === "telegram")?.members).toContainEqual({
+      standardId: "canary",
+    });
+    expect(lanes.find((lane) => lane.transportId === "slack")?.members).toContainEqual({
+      standardId: "thread-follow-up",
+      scenarioId: "slack-thread-follow-up",
+    });
+    expect(lanes.find((lane) => lane.transportId === "whatsapp")?.members).toContainEqual({
+      standardId: "allowlist-block",
+      scenarioId: "whatsapp-group-allowlist-block",
+    });
+    expect(
+      lanes.find((lane) => lane.transportId === "discord")?.baselineMissingStandardScenarioIds,
+    ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]);
+    expect(
+      lanes.find((lane) => lane.transportId === "whatsapp")?.baselineMissingStandardScenarioIds,
+    ).toEqual([]);
+  });
+
+  it("keeps coverage report lane summaries aligned with runtime lanes", () => {
+    const lanes = new Map(
+      buildLiveTransportCoverageLaneSummaries().map((lane) => [
+        lane.transportId,
+        lane.standardScenarioIds,
+      ]),
+    );
+
+    expect(lanes.get("discord")).toEqual(discordTesting.DISCORD_QA_STANDARD_SCENARIO_IDS);
+    expect(lanes.get("slack")).toEqual(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS);
+    expect(lanes.get("telegram")).toEqual(telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS);
+    expect(lanes.get("whatsapp")).toEqual(whatsAppTesting.WHATSAPP_QA_STANDARD_SCENARIO_IDS);
+  });
+});