Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
46
test/e2e/qa-lab/config/cli-channel-picker.test.ts
Normal file
46
test/e2e/qa-lab/config/cli-channel-picker.test.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
// CLI channel picker producer tests cover its unique config and redaction assertions.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { cliChannelPickerTestApi } from "./cli-channel-picker.js";
|
||||
|
||||
function validPickerConfig() {
|
||||
return {
|
||||
plugins: { entries: { telegram: { enabled: true } } },
|
||||
channels: {
|
||||
telegram: {
|
||||
enabled: true,
|
||||
botToken: cliChannelPickerTestApi.testBotToken,
|
||||
groups: { "*": { requireMention: true } },
|
||||
},
|
||||
},
|
||||
wizard: { lastRunCommand: "configure", lastRunMode: "local" },
|
||||
};
|
||||
}
|
||||
|
||||
describe("CLI channel picker producer", () => {
|
||||
it("accepts only the expected isolated Telegram configuration", () => {
|
||||
expect(cliChannelPickerTestApi.assertPickerConfig(validPickerConfig())).toMatchObject({
|
||||
channelEnabled: true,
|
||||
defaultGroupRequiresMention: true,
|
||||
pluginEnabled: true,
|
||||
selectedChannel: "telegram",
|
||||
wizardCommand: "configure",
|
||||
wizardMode: "local",
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
cliChannelPickerTestApi.assertPickerConfig({
|
||||
...validPickerConfig(),
|
||||
channels: { telegram: { enabled: true, botToken: "wrong" } },
|
||||
}),
|
||||
).toThrow("entered Telegram bot token");
|
||||
});
|
||||
|
||||
it("removes the synthetic token and ANSI control sequences from evidence", () => {
|
||||
const sanitized = cliChannelPickerTestApi.sanitizePickerTranscript(
|
||||
`\u001b[31m${cliChannelPickerTestApi.testBotToken}\u001b[0m`,
|
||||
);
|
||||
|
||||
expect(sanitized).toBe("<test-token>");
|
||||
expect(sanitized).not.toContain("123456");
|
||||
});
|
||||
});
|
||||
300
test/e2e/qa-lab/config/cli-channel-picker.ts
Normal file
300
test/e2e/qa-lab/config/cli-channel-picker.ts
Normal file
@@ -0,0 +1,300 @@
|
||||
// CLI channel picker producer drives the real onboarding prompt in an isolated home.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { stripAnsiSequences } from "../../../../packages/terminal-core/src/ansi.js";
|
||||
import { createQaScriptEvidenceWriter } from "../runtime/script-evidence.js";
|
||||
|
||||
const SCENARIO_ID = "cli-channel-picker";
|
||||
const SOURCE_PATH = "test/e2e/qa-lab/config/cli-channel-picker.ts";
|
||||
const TEST_BOT_TOKEN = "123456:QA_CHANNEL_PICKER_TEST_TOKEN_ABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
|
||||
type ProducerOptions = {
|
||||
artifactBase: string;
|
||||
repoRoot: string;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
function formatErrorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function sanitizePickerTranscript(transcript: string) {
|
||||
return stripAnsiSequences(transcript).replaceAll(
|
||||
/123456(?:(?::|…)[A-Za-z0-9_…-]*)?/gu,
|
||||
"<test-token>",
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInt(value: string, label: string) {
|
||||
if (!/^[1-9]\d*$/.test(value)) {
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed)) {
|
||||
throw new Error(`${label} must be a safe integer`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseOptions(args: string[]): ProducerOptions {
|
||||
let artifactBase: string | undefined;
|
||||
let timeoutMs = DEFAULT_TIMEOUT_MS;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--artifact-base") {
|
||||
artifactBase = args[++index];
|
||||
} else if (arg === "--timeout-ms") {
|
||||
timeoutMs = parsePositiveInt(args[++index] ?? "", "--timeout-ms");
|
||||
} else {
|
||||
throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
if (!artifactBase) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
return { artifactBase: path.resolve(artifactBase), repoRoot: process.cwd(), timeoutMs };
|
||||
}
|
||||
|
||||
function buildCliStartup(repoRoot: string) {
|
||||
const result = spawnSync(process.execPath, ["scripts/build-all.mjs", "cliStartup"], {
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
stdio: "inherit",
|
||||
});
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`cliStartup build failed with exit code ${String(result.status)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function runRealPicker(options: ProducerOptions, openclawHome: string) {
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + options.timeoutMs;
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"scripts/e2e/lib/run-with-pty.mjs",
|
||||
path.join(openclawHome, "picker.raw.log"),
|
||||
process.execPath,
|
||||
"openclaw.mjs",
|
||||
"configure",
|
||||
"--section",
|
||||
"channels",
|
||||
],
|
||||
{
|
||||
cwd: options.repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
CI: undefined,
|
||||
COLUMNS: "120",
|
||||
HOME: openclawHome,
|
||||
LANG: "en_US.UTF-8",
|
||||
LC_ALL: "en_US.UTF-8",
|
||||
LC_MESSAGES: "en_US.UTF-8",
|
||||
LINES: "40",
|
||||
OPENCLAW_CONFIG_PATH: undefined,
|
||||
OPENCLAW_HOME: openclawHome,
|
||||
OPENCLAW_LOCALE: "en",
|
||||
OPENCLAW_STATE_DIR: undefined,
|
||||
TELEGRAM_BOT_TOKEN: undefined,
|
||||
TERM: "xterm-256color",
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
let output = "";
|
||||
let exit: { code: number | null; signal: NodeJS.Signals | null } | undefined;
|
||||
let spawnError: Error | undefined;
|
||||
child.stdout.on("data", (chunk: Buffer) => (output += chunk.toString("utf8")));
|
||||
child.stderr.on("data", (chunk: Buffer) => (output += chunk.toString("utf8")));
|
||||
child.on("error", (error) => {
|
||||
spawnError = error;
|
||||
});
|
||||
child.on("exit", (code, signal) => {
|
||||
exit = { code, signal };
|
||||
});
|
||||
const remainingMs = () => Math.max(0, deadline - Date.now());
|
||||
const waitFor = async (matcher: RegExp, fromIndex = 0) => {
|
||||
while (!matcher.test(stripAnsiSequences(output.slice(fromIndex)))) {
|
||||
if (spawnError) {
|
||||
throw spawnError;
|
||||
}
|
||||
if (exit) {
|
||||
throw new Error(
|
||||
`picker exited before output ${matcher}: code=${String(exit.code)} signal=${String(exit.signal)}`,
|
||||
);
|
||||
}
|
||||
if (remainingMs() === 0) {
|
||||
throw new Error(`picker timed out waiting for output: ${matcher}`);
|
||||
}
|
||||
await delay(Math.min(25, remainingMs()));
|
||||
}
|
||||
};
|
||||
const send = (input: string) => child.stdin.write(input);
|
||||
const sendAndWait = async (input: string, matcher: RegExp) => {
|
||||
const checkpoint = output.length;
|
||||
send(input);
|
||||
await waitFor(matcher, checkpoint);
|
||||
};
|
||||
|
||||
try {
|
||||
await waitFor(/Channel setup[\s\S]*Add or update channels/u);
|
||||
await sendAndWait("\r", /Select a channel/u);
|
||||
|
||||
for (let attempt = 0; attempt < 64; attempt += 1) {
|
||||
const checkpoint = output.length;
|
||||
send("\u001b[B");
|
||||
await waitFor(/●\s+[^\r\n]+/u, checkpoint);
|
||||
if (/●\s+Telegram \(Bot API\)/u.test(stripAnsiSequences(output.slice(checkpoint)))) {
|
||||
break;
|
||||
}
|
||||
if (attempt === 63) {
|
||||
throw new Error("Telegram was not reachable from the real channel picker");
|
||||
}
|
||||
}
|
||||
|
||||
await sendAndWait("\r", /How do you want to provide this Telegram bot token\?/u);
|
||||
await sendAndWait("\r", /◆\s+Enter Telegram bot token[\s\S]*│\s+_/u);
|
||||
await sendAndWait(`${TEST_BOT_TOKEN}\r`, /Telegram DM access warning[\s\S]*Select a channel/u);
|
||||
await sendAndWait("\u001b[A", /●\s+Finished \(Done\)/u);
|
||||
await sendAndWait("\r", /Configure DM access policies now\?/u);
|
||||
await sendAndWait("\r", /Configuration updated\./u);
|
||||
|
||||
while (!exit) {
|
||||
if (remainingMs() === 0) {
|
||||
throw new Error(`picker timed out after ${options.timeoutMs}ms`);
|
||||
}
|
||||
await delay(Math.min(25, remainingMs()));
|
||||
}
|
||||
if (exit.code !== 0) {
|
||||
throw new Error(
|
||||
`picker exited unsuccessfully: code=${String(exit.code)} signal=${String(exit.signal)}`,
|
||||
);
|
||||
}
|
||||
return { durationMs: Math.max(1, Date.now() - startedAt), transcript: output };
|
||||
} catch (error) {
|
||||
if (!exit) {
|
||||
child.kill("SIGTERM");
|
||||
const cleanupDeadline = Date.now() + 5_000;
|
||||
while (!exit && Date.now() < cleanupDeadline) {
|
||||
await delay(25);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function assertPickerConfig(config: unknown) {
|
||||
const value = config as {
|
||||
channels?: {
|
||||
telegram?: { botToken?: string; enabled?: boolean; groups?: Record<string, unknown> };
|
||||
};
|
||||
plugins?: { entries?: { telegram?: { enabled?: boolean } } };
|
||||
wizard?: { lastRunCommand?: string; lastRunMode?: string };
|
||||
};
|
||||
const telegram = value.channels?.telegram;
|
||||
const defaultGroup = telegram?.groups?.["*"] as { requireMention?: boolean } | undefined;
|
||||
if (value.plugins?.entries?.telegram?.enabled !== true) {
|
||||
throw new Error("picker did not enable the Telegram plugin");
|
||||
}
|
||||
if (telegram?.enabled !== true) {
|
||||
throw new Error("picker did not enable the Telegram channel");
|
||||
}
|
||||
if (telegram.botToken !== TEST_BOT_TOKEN) {
|
||||
throw new Error("picker did not write the entered Telegram bot token");
|
||||
}
|
||||
if (defaultGroup?.requireMention !== true) {
|
||||
throw new Error("picker did not write the Telegram default mention gate");
|
||||
}
|
||||
if (value.wizard?.lastRunCommand !== "configure" || value.wizard.lastRunMode !== "local") {
|
||||
throw new Error("picker did not persist configure wizard metadata");
|
||||
}
|
||||
return {
|
||||
channelEnabled: true,
|
||||
configPath: ".openclaw/openclaw.json",
|
||||
defaultGroupRequiresMention: true,
|
||||
pluginEnabled: true,
|
||||
selectedChannel: "telegram",
|
||||
wizardCommand: "configure",
|
||||
wizardMode: "local",
|
||||
};
|
||||
}
|
||||
|
||||
function createEvidenceWriter(options: ProducerOptions) {
|
||||
return createQaScriptEvidenceWriter({
|
||||
artifactBase: options.artifactBase,
|
||||
logFileName: "cli-channel-picker.log",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: options.repoRoot,
|
||||
target: {
|
||||
id: SCENARIO_ID,
|
||||
title: "CLI channel picker",
|
||||
sourcePath: SOURCE_PATH,
|
||||
primaryCoverageIds: ["cli.channel-picker"],
|
||||
docsRefs: ["docs/channels/telegram.md", "docs/help/testing.md"],
|
||||
codeRefs: [SOURCE_PATH, "scripts/e2e/lib/run-with-pty.mjs", "src/flows/channel-setup.ts"],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function runCliChannelPickerProducer(options: ProducerOptions) {
|
||||
const startedAt = Date.now();
|
||||
const writer = createEvidenceWriter(options);
|
||||
const workDir = path.join(options.artifactBase, ".work");
|
||||
const openclawHome = path.join(workDir, "openclaw-home");
|
||||
|
||||
try {
|
||||
await fs.rm(workDir, { force: true, recursive: true });
|
||||
await fs.mkdir(openclawHome, { recursive: true });
|
||||
buildCliStartup(options.repoRoot);
|
||||
const result = await runRealPicker(options, openclawHome);
|
||||
writer.appendLog(sanitizePickerTranscript(result.transcript));
|
||||
const configPath = path.join(openclawHome, ".openclaw", "openclaw.json");
|
||||
const assertion = assertPickerConfig(JSON.parse(await fs.readFile(configPath, "utf8")));
|
||||
await fs.writeFile(
|
||||
path.join(options.artifactBase, "config-assertion.json"),
|
||||
`${JSON.stringify(assertion, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
return await writer.write({
|
||||
artifacts: [{ kind: "config-assertion", filePath: "config-assertion.json" }],
|
||||
details: "real channel picker completed and persisted isolated Telegram configuration",
|
||||
durationMs: result.durationMs,
|
||||
status: "pass",
|
||||
});
|
||||
} catch (error) {
|
||||
const details = formatErrorMessage(error);
|
||||
writer.appendLog(`\nfail: ${details}\n`);
|
||||
return await writer.write({
|
||||
details,
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "fail",
|
||||
});
|
||||
} finally {
|
||||
await fs.rm(workDir, { force: true, recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
export const cliChannelPickerTestApi = {
|
||||
assertPickerConfig,
|
||||
sanitizePickerTranscript,
|
||||
testBotToken: TEST_BOT_TOKEN,
|
||||
};
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
runCliChannelPickerProducer(parseOptions(process.argv.slice(2)))
|
||||
.then((evidence) => {
|
||||
console.log(`CLI channel picker status: ${evidence.entries[0]?.result.status}`);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
console.error(formatErrorMessage(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
282
test/e2e/qa-lab/media/hosted-media-provider-live.test.ts
Normal file
282
test/e2e/qa-lab/media/hosted-media-provider-live.test.ts
Normal file
@@ -0,0 +1,282 @@
|
||||
// Hosted media provider live producer tests cover QA evidence wiring.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
MEDIA_SUITES,
|
||||
buildRunPlan,
|
||||
buildHostedMediaCommand,
|
||||
buildHostedMediaEvidence,
|
||||
classifyHostedMediaFailureStatus,
|
||||
findSkippedExplicitProviderSelections,
|
||||
parseArgs,
|
||||
parseHostedMediaOptions,
|
||||
runCli,
|
||||
type SuiteRunPlan,
|
||||
} from "./hosted-media-provider-live.js";
|
||||
|
||||
const SOURCE_PATH = "test/e2e/qa-lab/media/hosted-media-provider-live.ts";
|
||||
const loadShellEnvFallbackMock = vi.fn();
|
||||
const collectProviderApiKeysMock = vi.fn((provider: string) =>
|
||||
process.env[`TEST_AUTH_${provider.toUpperCase()}`] ? ["test-key"] : [],
|
||||
);
|
||||
|
||||
function requirePlanEntry(plan: SuiteRunPlan[], suiteId: string) {
|
||||
const entry = plan.find((candidate) => candidate.suite.id === suiteId);
|
||||
if (!entry) {
|
||||
throw new Error(`expected ${suiteId} run plan entry`);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
collectProviderApiKeysMock.mockClear();
|
||||
loadShellEnvFallbackMock.mockReset();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("hosted media provider live QA producer", () => {
|
||||
it("builds the image live media command with provider filters from env", () => {
|
||||
const options = parseHostedMediaOptions([
|
||||
"--suite",
|
||||
"image",
|
||||
"--artifact-base",
|
||||
".artifacts/qa-e2e/image",
|
||||
"--providers-env",
|
||||
"OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS",
|
||||
]);
|
||||
const command = buildHostedMediaCommand({
|
||||
env: { OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS: "openai,google" },
|
||||
options,
|
||||
});
|
||||
|
||||
expect(command.args).toContain(SOURCE_PATH);
|
||||
expect(command.args).toContain("image");
|
||||
expect(command.args).toContain("--image-providers");
|
||||
expect(command.args).toContain("openai,google");
|
||||
expect(command.env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES).toBeUndefined();
|
||||
});
|
||||
|
||||
it("forces full video modes so reference image and video inputs are covered", () => {
|
||||
const options = parseHostedMediaOptions([
|
||||
"--suite",
|
||||
"video",
|
||||
"--artifact-base",
|
||||
".artifacts/qa-e2e/video",
|
||||
]);
|
||||
const command = buildHostedMediaCommand({ env: {}, options });
|
||||
|
||||
expect(command.args).toContain("video");
|
||||
expect(command.env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES).toBe("1");
|
||||
});
|
||||
|
||||
it("classifies missing live media auth as blocked evidence", () => {
|
||||
expect(
|
||||
classifyHostedMediaFailureStatus(
|
||||
"[live:media] no runnable providers matched available auth; pass --allow-empty",
|
||||
),
|
||||
).toBe("blocked");
|
||||
expect(classifyHostedMediaFailureStatus("provider response was malformed")).toBe("fail");
|
||||
});
|
||||
|
||||
it("maps video provider live coverage roles without making tool invocation primary", () => {
|
||||
const artifactBase = path.join(os.tmpdir(), "openclaw-hosted-media-live-test");
|
||||
const options = parseHostedMediaOptions(["--suite", "video", "--artifact-base", artifactBase]);
|
||||
const evidence = buildHostedMediaEvidence({
|
||||
options,
|
||||
result: {
|
||||
durationMs: 10,
|
||||
status: "pass",
|
||||
},
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]?.coverage).toEqual([
|
||||
{ id: "hosted-providers.video-generation-providers", role: "primary" },
|
||||
{ id: "media.reference-image-video-and-audio-inputs", role: "primary" },
|
||||
{ id: "media.video-generation-tool-invocation", role: "secondary" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("hosted media provider live CLI", () => {
|
||||
it("prints help through the real node --import tsx entrypoint", () => {
|
||||
const result = spawnSync(process.execPath, ["--import", "tsx", SOURCE_PATH, "--help"], {
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
});
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain("Media live harness");
|
||||
expect(result.stdout).toContain("pnpm test:live:media");
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("rejects unknown global providers for the selected suites", () => {
|
||||
expect(() =>
|
||||
parseArgs(["image", "--providers", "definitely-not-a-provider", "--all-providers"]),
|
||||
).toThrow("Unknown provider(s) for selected media suite(s): definitely-not-a-provider");
|
||||
});
|
||||
|
||||
it("rejects unknown suite-specific providers", () => {
|
||||
expect(() => parseArgs(["image", "--image-providers", "runway", "--all-providers"])).toThrow(
|
||||
"Unknown image provider(s): runway",
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts providers supported by the wrapped live suites", () => {
|
||||
expect(
|
||||
parseArgs(["image", "--image-providers", "openrouter", "--all-providers"]).suiteProviders
|
||||
.image,
|
||||
).toEqual(new Set(["openrouter"]));
|
||||
expect(
|
||||
parseArgs(["music", "--music-providers", "fal,openrouter", "--all-providers"]).suiteProviders
|
||||
.music,
|
||||
).toEqual(new Set(["fal", "openrouter"]));
|
||||
expect(
|
||||
parseArgs(["video", "--video-providers", "openrouter", "--all-providers"]).suiteProviders
|
||||
.video,
|
||||
).toEqual(new Set(["openrouter"]));
|
||||
});
|
||||
|
||||
it("rejects suite-specific provider filters for unselected suites", () => {
|
||||
expect(() => parseArgs(["image", "--music-providers", "fal", "--all-providers"])).toThrow(
|
||||
"Provider filter(s) target unselected media suite(s): music",
|
||||
);
|
||||
});
|
||||
|
||||
it("passes single-dash Vitest args after the option separator", () => {
|
||||
expect(
|
||||
parseArgs(["image", "--all-providers", "--project", "tooling", "--", "-t", "media-smoke"]),
|
||||
).toMatchObject({
|
||||
suites: ["image"],
|
||||
requireAuth: false,
|
||||
passthroughArgs: ["--project", "tooling", "-t", "media-smoke"],
|
||||
});
|
||||
});
|
||||
|
||||
it("parses the explicit empty-run escape hatch", () => {
|
||||
expect(parseArgs(["--allow-empty"])).toMatchObject({
|
||||
allowEmpty: true,
|
||||
requireAuth: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("fails explicit suite selections that auth filtering would skip", () => {
|
||||
const options = parseArgs([
|
||||
"image",
|
||||
"music",
|
||||
"--image-providers",
|
||||
"openai",
|
||||
"--music-providers",
|
||||
"minimax",
|
||||
]);
|
||||
const skipped = findSkippedExplicitProviderSelections(options, [
|
||||
{ suite: MEDIA_SUITES.image, providers: ["openai"] },
|
||||
{
|
||||
suite: MEDIA_SUITES.music,
|
||||
providers: [],
|
||||
skippedReason: "no providers with usable auth",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(skipped.map((entry) => entry.suite.id)).toEqual(["music"]);
|
||||
});
|
||||
|
||||
it("does not fail global provider filters for suites without provider overlap", () => {
|
||||
const options = parseArgs(["image", "music", "video", "--providers", "openai"]);
|
||||
const skipped = findSkippedExplicitProviderSelections(options, [
|
||||
{ suite: MEDIA_SUITES.image, providers: ["openai"] },
|
||||
{
|
||||
suite: MEDIA_SUITES.music,
|
||||
providers: [],
|
||||
skippedReason: "no providers selected",
|
||||
},
|
||||
{ suite: MEDIA_SUITES.video, providers: ["openai"] },
|
||||
]);
|
||||
|
||||
expect(skipped).toEqual([]);
|
||||
});
|
||||
|
||||
it("fails default live media runs when auth filtering leaves no providers", async () => {
|
||||
await expect(
|
||||
runCli(["image"], {
|
||||
buildRunPlanImpl: () => [
|
||||
{
|
||||
providers: [],
|
||||
skippedReason: "no providers with usable auth",
|
||||
suite: MEDIA_SUITES.image,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).resolves.toBe(1);
|
||||
});
|
||||
|
||||
it("allows empty live media runs only with an explicit escape hatch", async () => {
|
||||
await expect(
|
||||
runCli(["image", "--allow-empty"], {
|
||||
buildRunPlanImpl: () => [
|
||||
{
|
||||
providers: [],
|
||||
skippedReason: "no providers with usable auth",
|
||||
suite: MEDIA_SUITES.image,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it("defaults to all suites with auth filtering", async () => {
|
||||
vi.stubEnv("TEST_AUTH_OPENAI", "1");
|
||||
vi.stubEnv("TEST_AUTH_GOOGLE", "1");
|
||||
vi.stubEnv("TEST_AUTH_MINIMAX", "1");
|
||||
vi.stubEnv("TEST_AUTH_FAL", "1");
|
||||
vi.stubEnv("TEST_AUTH_VYDRA", "1");
|
||||
|
||||
const plan = await buildRunPlan(parseArgs([]), {
|
||||
collectProviderApiKeysImpl: collectProviderApiKeysMock,
|
||||
getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`],
|
||||
loadShellEnvFallbackImpl: loadShellEnvFallbackMock,
|
||||
});
|
||||
|
||||
expect(plan.map((entry) => entry.suite.id)).toEqual(["image", "music", "video"]);
|
||||
expect(requirePlanEntry(plan, "image").providers).toEqual([
|
||||
"fal",
|
||||
"google",
|
||||
"minimax",
|
||||
"openai",
|
||||
"vydra",
|
||||
]);
|
||||
expect(requirePlanEntry(plan, "music").providers).toEqual(["fal", "google", "minimax"]);
|
||||
expect(requirePlanEntry(plan, "video").providers).toEqual([
|
||||
"google",
|
||||
"minimax",
|
||||
"openai",
|
||||
"vydra",
|
||||
]);
|
||||
});
|
||||
|
||||
it("supports suite-specific provider filters without auth narrowing", async () => {
|
||||
const plan = await buildRunPlan(
|
||||
parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]),
|
||||
{
|
||||
collectProviderApiKeysImpl: collectProviderApiKeysMock,
|
||||
getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`],
|
||||
loadShellEnvFallbackImpl: loadShellEnvFallbackMock,
|
||||
},
|
||||
);
|
||||
|
||||
expect(plan).toHaveLength(1);
|
||||
const [entry] = plan;
|
||||
expect(entry?.suite.id).toBe("video");
|
||||
expect(entry?.providers).toEqual(["fal", "openai", "runway"]);
|
||||
});
|
||||
|
||||
it("forwards quiet flags separately from passthrough args", () => {
|
||||
const options = parseArgs(["image", "--quiet", "--reporter", "dot"]);
|
||||
|
||||
expect(options.suites).toEqual(["image"]);
|
||||
expect(options.quietArgs).toEqual(["--quiet"]);
|
||||
expect(options.passthroughArgs).toEqual(["--reporter", "dot"]);
|
||||
});
|
||||
});
|
||||
819
test/e2e/qa-lab/media/hosted-media-provider-live.ts
Normal file
819
test/e2e/qa-lab/media/hosted-media-provider-live.ts
Normal file
@@ -0,0 +1,819 @@
|
||||
// Hosted media provider live runner and QA Lab evidence producer.
|
||||
import { spawn } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import {
|
||||
QA_EVIDENCE_FILENAME,
|
||||
type QaEvidenceSummaryJson,
|
||||
} from "../../../../extensions/qa-lab/api.js";
|
||||
import { spawnPnpmRunner as _spawnPnpmRunner } from "../../../../scripts/pnpm-runner.mjs";
|
||||
import {
|
||||
createQaScriptBlockedStatusTracker,
|
||||
createQaScriptEvidenceWriter,
|
||||
type QaScriptEvidenceStatus,
|
||||
} from "../runtime/script-evidence.js";
|
||||
|
||||
const SOURCE_PATH = "test/e2e/qa-lab/media/hosted-media-provider-live.ts";
|
||||
const DEFAULT_PROVIDERS_ENV = "OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS";
|
||||
|
||||
export type MediaSuiteId = "image" | "music" | "video";
|
||||
type EvidenceSuiteId = "image" | "video";
|
||||
|
||||
export type MediaSuiteConfig = {
|
||||
id: MediaSuiteId;
|
||||
testFile: string;
|
||||
providerEnvVar: string;
|
||||
providers: string[];
|
||||
defaultProviders?: string[];
|
||||
};
|
||||
|
||||
export const MEDIA_SUITES: Record<MediaSuiteId, MediaSuiteConfig> = {
|
||||
image: {
|
||||
id: "image",
|
||||
testFile: "test/image-generation.runtime.live.test.ts",
|
||||
providerEnvVar: "OPENCLAW_LIVE_IMAGE_GENERATION_PROVIDERS",
|
||||
providers: ["deepinfra", "fal", "google", "minimax", "openai", "openrouter", "vydra", "xai"],
|
||||
},
|
||||
music: {
|
||||
id: "music",
|
||||
testFile: "extensions/music-generation-providers.live.test.ts",
|
||||
providerEnvVar: "OPENCLAW_LIVE_MUSIC_GENERATION_PROVIDERS",
|
||||
providers: ["fal", "google", "minimax", "openrouter"],
|
||||
},
|
||||
video: {
|
||||
id: "video",
|
||||
testFile: "extensions/video-generation-providers.live.test.ts",
|
||||
providerEnvVar: "OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS",
|
||||
providers: [
|
||||
"alibaba",
|
||||
"byteplus",
|
||||
"deepinfra",
|
||||
"fal",
|
||||
"google",
|
||||
"minimax",
|
||||
"openai",
|
||||
"openrouter",
|
||||
"qwen",
|
||||
"runway",
|
||||
"together",
|
||||
"vydra",
|
||||
"xai",
|
||||
],
|
||||
defaultProviders: [
|
||||
"alibaba",
|
||||
"byteplus",
|
||||
"deepinfra",
|
||||
"google",
|
||||
"minimax",
|
||||
"openai",
|
||||
"openrouter",
|
||||
"qwen",
|
||||
"runway",
|
||||
"together",
|
||||
"vydra",
|
||||
"xai",
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const DEFAULT_SUITES: MediaSuiteId[] = ["image", "music", "video"];
|
||||
|
||||
export type CliOptions = {
|
||||
allowEmpty: boolean;
|
||||
globalProviders: Set<string> | null;
|
||||
help: boolean;
|
||||
passthroughArgs: string[];
|
||||
quietArgs: string[];
|
||||
requireAuth: boolean;
|
||||
suiteProviders: Partial<Record<MediaSuiteId, Set<string>>>;
|
||||
suites: MediaSuiteId[];
|
||||
};
|
||||
|
||||
export type SuiteRunPlan = {
|
||||
suite: MediaSuiteConfig;
|
||||
providers: string[];
|
||||
skippedReason?: string;
|
||||
};
|
||||
|
||||
export type BuildRunPlanDeps = {
|
||||
collectProviderApiKeysImpl?: (provider: string) => Promise<unknown[]> | unknown[];
|
||||
getProviderEnvVarsImpl?: (provider: string) => Promise<string[]> | string[];
|
||||
loadShellEnvFallbackImpl?: (params: {
|
||||
enabled: true;
|
||||
env: NodeJS.ProcessEnv;
|
||||
expectedKeys: string[];
|
||||
logger: { warn: (message: string) => void };
|
||||
}) => Promise<void> | void;
|
||||
};
|
||||
|
||||
export type RunCliDeps = {
|
||||
buildRunPlanImpl?: (options: CliOptions) => Promise<SuiteRunPlan[]> | SuiteRunPlan[];
|
||||
runSuiteImpl?: typeof runSuite;
|
||||
};
|
||||
|
||||
type HostedMediaOptions = {
|
||||
artifactBase: string;
|
||||
providersEnv: string;
|
||||
repoRoot: string;
|
||||
suiteId: EvidenceSuiteId;
|
||||
};
|
||||
|
||||
type HostedMediaSuiteDefinition = {
|
||||
codeRefs: string[];
|
||||
docsRefs: string[];
|
||||
primaryCoverageIds: string[];
|
||||
secondaryCoverageIds?: string[];
|
||||
scenarioId: string;
|
||||
title: string;
|
||||
videoFullModes?: boolean;
|
||||
};
|
||||
|
||||
type HostedMediaProofResult = {
|
||||
artifacts?: Array<{ filePath: string; kind: string }>;
|
||||
details?: string;
|
||||
durationMs: number;
|
||||
status: QaScriptEvidenceStatus;
|
||||
};
|
||||
|
||||
const EVIDENCE_SUITES: Record<EvidenceSuiteId, HostedMediaSuiteDefinition> = {
|
||||
image: {
|
||||
scenarioId: "hosted-image-generation-providers-live",
|
||||
title: "Hosted image generation providers live",
|
||||
primaryCoverageIds: ["hosted-providers.image-generation-providers"],
|
||||
docsRefs: [
|
||||
"docs/help/testing.md",
|
||||
"docs/tools/image-generation.md",
|
||||
"docs/tools/media-overview.md",
|
||||
],
|
||||
codeRefs: [
|
||||
SOURCE_PATH,
|
||||
"test/image-generation.runtime.live.test.ts",
|
||||
"src/image-generation/live-test-helpers.ts",
|
||||
],
|
||||
},
|
||||
video: {
|
||||
scenarioId: "hosted-video-generation-providers-live",
|
||||
title: "Hosted video generation providers live",
|
||||
primaryCoverageIds: [
|
||||
"hosted-providers.video-generation-providers",
|
||||
"media.reference-image-video-and-audio-inputs",
|
||||
],
|
||||
secondaryCoverageIds: ["media.video-generation-tool-invocation"],
|
||||
docsRefs: [
|
||||
"docs/help/testing.md",
|
||||
"docs/tools/video-generation.md",
|
||||
"docs/tools/media-overview.md",
|
||||
],
|
||||
codeRefs: [
|
||||
SOURCE_PATH,
|
||||
"extensions/video-generation-providers.live.test.ts",
|
||||
"src/video-generation/runtime.ts",
|
||||
"src/agents/tools/video-generate-tool.ts",
|
||||
],
|
||||
videoFullModes: true,
|
||||
},
|
||||
};
|
||||
|
||||
function formatProviderList(providers: Iterable<string>): string {
|
||||
return [...providers].toSorted().join(", ");
|
||||
}
|
||||
|
||||
function formatErrorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function spawnLivePnpm(params: { pnpmArgs: string[]; env: NodeJS.ProcessEnv }) {
|
||||
return _spawnPnpmRunner({
|
||||
pnpmArgs: params.pnpmArgs,
|
||||
stdio: "inherit",
|
||||
env: params.env,
|
||||
});
|
||||
}
|
||||
|
||||
async function collectProviderApiKeysForLiveMedia(provider: string): Promise<unknown[]> {
|
||||
const { collectProviderApiKeys } = await import("../../../../src/agents/live-auth-keys.js");
|
||||
return collectProviderApiKeys(provider);
|
||||
}
|
||||
|
||||
async function getProviderEnvVarsForLiveMedia(provider: string): Promise<string[]> {
|
||||
const { getProviderEnvVars } = await import("../../../../src/secrets/provider-env-vars.js");
|
||||
return getProviderEnvVars(provider);
|
||||
}
|
||||
|
||||
async function loadShellEnvFallbackForLiveMedia(params: {
|
||||
enabled: true;
|
||||
env: NodeJS.ProcessEnv;
|
||||
expectedKeys: string[];
|
||||
logger: { warn: (message: string) => void };
|
||||
}): Promise<void> {
|
||||
const { loadShellEnvFallback } = await import("../../../../src/infra/shell-env.js");
|
||||
loadShellEnvFallback(params);
|
||||
}
|
||||
|
||||
function parseCsv(raw: string | undefined): Set<string> | null {
|
||||
const trimmed = raw?.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
const values = trimmed
|
||||
.split(",")
|
||||
.map((entry) => entry.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
return values.length ? new Set(values) : null;
|
||||
}
|
||||
|
||||
function parseSuiteToken(raw: string): MediaSuiteId | null {
|
||||
const normalized = raw.trim().toLowerCase();
|
||||
if (normalized === "image" || normalized === "music" || normalized === "video") {
|
||||
return normalized;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseEvidenceSuiteToken(raw: string): EvidenceSuiteId {
|
||||
const normalized = raw.trim().toLowerCase();
|
||||
if (normalized === "image" || normalized === "video") {
|
||||
return normalized;
|
||||
}
|
||||
throw new Error(`unsupported hosted media evidence suite: ${raw}`);
|
||||
}
|
||||
|
||||
function readOptionValue(argv: readonly string[], index: number, arg: string) {
|
||||
const value = argv[index + 1] ?? "";
|
||||
if (!value || value.startsWith("-")) {
|
||||
throw new Error(`${arg} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): CliOptions {
|
||||
const separatorIndex = argv.indexOf("--");
|
||||
const optionArgs = separatorIndex >= 0 ? argv.slice(0, separatorIndex) : argv;
|
||||
const separatorPassthroughArgs = separatorIndex >= 0 ? argv.slice(separatorIndex + 1) : [];
|
||||
const suites = new Set<MediaSuiteId>();
|
||||
const suiteProviders: Partial<Record<MediaSuiteId, Set<string>>> = {};
|
||||
const passthroughArgs: string[] = [];
|
||||
const quietArgs: string[] = [];
|
||||
let allowEmpty = false;
|
||||
let globalProviders: Set<string> | null = null;
|
||||
let help = false;
|
||||
let requireAuth = true;
|
||||
|
||||
for (let index = 0; index < optionArgs.length; index += 1) {
|
||||
const arg = optionArgs[index] ?? "";
|
||||
if (!arg) {
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
help = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
arg === "--quiet" ||
|
||||
arg === "--quiet-live" ||
|
||||
arg === "--no-quiet" ||
|
||||
arg === "--no-quiet-live"
|
||||
) {
|
||||
quietArgs.push(arg);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--providers") {
|
||||
globalProviders = parseCsv(readOptionValue(optionArgs, index, arg));
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--image-providers" || arg === "--music-providers" || arg === "--video-providers") {
|
||||
const suite = parseSuiteToken(arg.slice(2, arg.indexOf("-providers")));
|
||||
if (!suite) {
|
||||
throw new Error(`Unknown suite flag: ${arg}`);
|
||||
}
|
||||
suiteProviders[suite] =
|
||||
parseCsv(readOptionValue(optionArgs, index, arg)) ?? new Set<string>();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--with-auth" || arg === "--require-auth") {
|
||||
requireAuth = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--allow-empty") {
|
||||
allowEmpty = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--all-providers" || arg === "--no-auth-filter") {
|
||||
requireAuth = false;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--")) {
|
||||
passthroughArgs.push(arg);
|
||||
const next = optionArgs[index + 1];
|
||||
if (next && !next.startsWith("--")) {
|
||||
passthroughArgs.push(next);
|
||||
index += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const suite = parseSuiteToken(arg);
|
||||
if (suite) {
|
||||
suites.add(suite);
|
||||
continue;
|
||||
}
|
||||
if (arg === "all") {
|
||||
suites.add("image");
|
||||
suites.add("music");
|
||||
suites.add("video");
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
const options = {
|
||||
allowEmpty,
|
||||
globalProviders,
|
||||
help,
|
||||
passthroughArgs: [...passthroughArgs, ...separatorPassthroughArgs],
|
||||
quietArgs,
|
||||
requireAuth,
|
||||
suiteProviders,
|
||||
suites: (suites.size ? [...suites] : DEFAULT_SUITES).toSorted(),
|
||||
};
|
||||
validateProviderFilters(options);
|
||||
return options;
|
||||
}
|
||||
|
||||
function validateProviderFilters(options: CliOptions): void {
|
||||
const selectedSuites = new Set(options.suites);
|
||||
const unselectedSuiteFilters = Object.keys(options.suiteProviders).filter(
|
||||
(suiteId) => !selectedSuites.has(suiteId as MediaSuiteId),
|
||||
);
|
||||
if (unselectedSuiteFilters.length > 0) {
|
||||
throw new Error(
|
||||
`Provider filter(s) target unselected media suite(s): ${unselectedSuiteFilters.toSorted().join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (options.globalProviders) {
|
||||
const selectedProviders = new Set(
|
||||
options.suites.flatMap((suiteId) => MEDIA_SUITES[suiteId].providers),
|
||||
);
|
||||
const unknown = [...options.globalProviders].filter(
|
||||
(provider) => !selectedProviders.has(provider),
|
||||
);
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(
|
||||
`Unknown provider(s) for selected media suite(s): ${formatProviderList(unknown)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [suiteId, providers] of Object.entries(options.suiteProviders) as [
|
||||
MediaSuiteId,
|
||||
Set<string>,
|
||||
][]) {
|
||||
const suite = MEDIA_SUITES[suiteId];
|
||||
const supported = new Set(suite.providers);
|
||||
const unknown = [...providers].filter((provider) => !supported.has(provider));
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`Unknown ${suiteId} provider(s): ${formatProviderList(unknown)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasExplicitProviderSelection(options: CliOptions): boolean {
|
||||
return options.globalProviders !== null || Object.keys(options.suiteProviders).length > 0;
|
||||
}
|
||||
|
||||
function hasExplicitProviderSelectionForSuite(options: CliOptions, suiteId: MediaSuiteId): boolean {
|
||||
if (Object.hasOwn(options.suiteProviders, suiteId)) {
|
||||
return true;
|
||||
}
|
||||
if (!options.globalProviders) {
|
||||
return false;
|
||||
}
|
||||
return MEDIA_SUITES[suiteId].providers.some((provider) => options.globalProviders?.has(provider));
|
||||
}
|
||||
|
||||
export function findSkippedExplicitProviderSelections(
|
||||
options: CliOptions,
|
||||
plan: SuiteRunPlan[],
|
||||
): SuiteRunPlan[] {
|
||||
return plan.filter(
|
||||
(entry) =>
|
||||
entry.providers.length === 0 && hasExplicitProviderSelectionForSuite(options, entry.suite.id),
|
||||
);
|
||||
}
|
||||
|
||||
async function selectProviders(params: {
|
||||
collectProviderApiKeysImpl?: BuildRunPlanDeps["collectProviderApiKeysImpl"];
|
||||
globalProviders: Set<string> | null;
|
||||
requireAuth: boolean;
|
||||
suite: MediaSuiteConfig;
|
||||
suiteProviders: Set<string> | undefined;
|
||||
}): Promise<string[]> {
|
||||
const explicit = params.suiteProviders ?? params.globalProviders;
|
||||
const candidates = explicit
|
||||
? params.suite.providers
|
||||
: (params.suite.defaultProviders ?? params.suite.providers);
|
||||
let providers = candidates.filter((provider) => (explicit ? explicit.has(provider) : true));
|
||||
if (!params.requireAuth) {
|
||||
return providers;
|
||||
}
|
||||
const providerAuth = await Promise.all(
|
||||
providers.map(async (provider) => ({
|
||||
provider,
|
||||
hasAuth:
|
||||
(await (params.collectProviderApiKeysImpl ?? collectProviderApiKeysForLiveMedia)(provider))
|
||||
.length > 0,
|
||||
})),
|
||||
);
|
||||
return providerAuth.filter((entry) => entry.hasAuth).map((entry) => entry.provider);
|
||||
}
|
||||
|
||||
export async function buildRunPlan(
|
||||
options: CliOptions,
|
||||
deps: BuildRunPlanDeps = {},
|
||||
): Promise<SuiteRunPlan[]> {
|
||||
const getProviderEnvVarsImpl = deps.getProviderEnvVarsImpl ?? getProviderEnvVarsForLiveMedia;
|
||||
const expectedKeys = [
|
||||
...new Set(
|
||||
(
|
||||
await Promise.all(
|
||||
options.suites.flatMap((suiteId) =>
|
||||
MEDIA_SUITES[suiteId].providers.map(
|
||||
async (provider) => await getProviderEnvVarsImpl(provider),
|
||||
),
|
||||
),
|
||||
)
|
||||
).flat(),
|
||||
),
|
||||
];
|
||||
if (expectedKeys.length) {
|
||||
await (deps.loadShellEnvFallbackImpl ?? loadShellEnvFallbackForLiveMedia)({
|
||||
enabled: true,
|
||||
env: process.env,
|
||||
expectedKeys,
|
||||
logger: { warn: (message: string) => console.warn(message) },
|
||||
});
|
||||
}
|
||||
|
||||
return await Promise.all(
|
||||
options.suites.map(async (suiteId) => {
|
||||
const suite = MEDIA_SUITES[suiteId];
|
||||
const providers = await selectProviders({
|
||||
collectProviderApiKeysImpl: deps.collectProviderApiKeysImpl,
|
||||
globalProviders: options.globalProviders,
|
||||
requireAuth: options.requireAuth,
|
||||
suite,
|
||||
suiteProviders: options.suiteProviders[suiteId],
|
||||
});
|
||||
return {
|
||||
suite,
|
||||
providers,
|
||||
...(providers.length === 0
|
||||
? {
|
||||
skippedReason: options.requireAuth
|
||||
? "no providers with usable auth"
|
||||
: "no providers selected",
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`Media live harness
|
||||
|
||||
Usage:
|
||||
pnpm test:live:media
|
||||
pnpm test:live:media image
|
||||
pnpm test:live:media image video --providers openai,google,minimax
|
||||
pnpm test:live:media video --video-providers openai,runway --all-providers
|
||||
|
||||
QA evidence mode:
|
||||
node --import tsx ${SOURCE_PATH} --qa-evidence --suite image --artifact-base <dir>
|
||||
node --import tsx ${SOURCE_PATH} --qa-evidence --suite video --artifact-base <dir>
|
||||
|
||||
Defaults:
|
||||
- runs image + music + video
|
||||
- auto-loads missing provider env vars from ~/.profile
|
||||
- narrows each suite to providers that currently have usable auth
|
||||
- skips the slow fal video smoke by default; pass --video-providers fal to run it
|
||||
- forwards extra args to scripts/test-live.mjs
|
||||
|
||||
Flags:
|
||||
--providers <csv> global provider filter
|
||||
--image-providers <csv> image-suite provider filter
|
||||
--music-providers <csv> music-suite provider filter
|
||||
--video-providers <csv> video-suite provider filter
|
||||
--all-providers do not auto-filter by available auth
|
||||
--allow-empty exit 0 when auth filtering leaves no runnable providers
|
||||
--quiet | --no-quiet passed through to test:live
|
||||
`);
|
||||
}
|
||||
|
||||
export async function runSuite(params: {
|
||||
passthroughArgs: string[];
|
||||
plan: SuiteRunPlan;
|
||||
quietArgs: string[];
|
||||
}): Promise<number> {
|
||||
const { plan } = params;
|
||||
if (!plan.providers.length) {
|
||||
console.log(
|
||||
`[live:media] skip ${plan.suite.id}: ${plan.skippedReason ?? "no providers selected"}`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const env = {
|
||||
...process.env,
|
||||
[plan.suite.providerEnvVar]: plan.providers.join(","),
|
||||
};
|
||||
const args = [
|
||||
"test:live",
|
||||
...params.quietArgs,
|
||||
"--",
|
||||
plan.suite.testFile,
|
||||
...params.passthroughArgs,
|
||||
];
|
||||
console.log(
|
||||
`[live:media] run ${plan.suite.id}: ${plan.suite.testFile} providers=${plan.providers.join(",")}`,
|
||||
);
|
||||
|
||||
const child = spawnLivePnpm({ pnpmArgs: args, env });
|
||||
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("exit", (code: number | null, signal: NodeJS.Signals | null) => {
|
||||
if (signal) {
|
||||
reject(new Error(`${plan.suite.id} exited via signal ${signal}`));
|
||||
return;
|
||||
}
|
||||
resolve(code ?? 1);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function runCli(argv: string[], deps: RunCliDeps = {}): Promise<number> {
|
||||
const options = parseArgs(argv);
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return 0;
|
||||
}
|
||||
const plan = await (deps.buildRunPlanImpl ?? buildRunPlan)(options);
|
||||
const runnable = plan.filter((entry) => entry.providers.length > 0);
|
||||
const skipped = plan.filter((entry) => entry.providers.length === 0);
|
||||
|
||||
for (const entry of skipped) {
|
||||
console.log(
|
||||
`[live:media] skip ${entry.suite.id}: ${entry.skippedReason ?? "no providers selected"}`,
|
||||
);
|
||||
}
|
||||
const skippedExplicit = findSkippedExplicitProviderSelections(options, plan);
|
||||
if (skippedExplicit.length > 0) {
|
||||
console.error(
|
||||
`[live:media] no runnable providers matched explicit provider selection for: ${skippedExplicit.map((entry) => entry.suite.id).join(", ")}`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
if (runnable.length === 0) {
|
||||
console.log("[live:media] nothing to run");
|
||||
if (options.allowEmpty) {
|
||||
return 0;
|
||||
}
|
||||
console.error(
|
||||
hasExplicitProviderSelection(options)
|
||||
? "[live:media] no runnable providers matched the explicit provider selection"
|
||||
: "[live:media] no runnable providers matched available auth; pass --allow-empty to accept an empty live-media run",
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
|
||||
for (const entry of runnable) {
|
||||
const exitCode = await (deps.runSuiteImpl ?? runSuite)({
|
||||
passthroughArgs: options.passthroughArgs,
|
||||
plan: entry,
|
||||
quietArgs: options.quietArgs,
|
||||
});
|
||||
if (exitCode !== 0) {
|
||||
return exitCode;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function parseHostedMediaOptions(argv: readonly string[]): HostedMediaOptions {
|
||||
let artifactBase = "";
|
||||
let providersEnv = DEFAULT_PROVIDERS_ENV;
|
||||
let repoRoot = process.cwd();
|
||||
let suiteId: EvidenceSuiteId | undefined;
|
||||
const seen = new Set<string>();
|
||||
const recordOnce = (flag: string) => {
|
||||
if (seen.has(flag)) {
|
||||
throw new Error(`${flag} was provided more than once`);
|
||||
}
|
||||
seen.add(flag);
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
if (arg === "--qa-evidence") {
|
||||
continue;
|
||||
}
|
||||
if (arg === "--artifact-base") {
|
||||
recordOnce(arg);
|
||||
artifactBase = readOptionValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--repo-root") {
|
||||
recordOnce(arg);
|
||||
repoRoot = readOptionValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--providers-env") {
|
||||
recordOnce(arg);
|
||||
providersEnv = readOptionValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--suite") {
|
||||
recordOnce(arg);
|
||||
suiteId = parseEvidenceSuiteToken(readOptionValue(argv, index, arg));
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unsupported hosted media evidence arg: ${arg}`);
|
||||
}
|
||||
|
||||
if (!artifactBase.trim()) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
if (!suiteId) {
|
||||
throw new Error("--suite is required");
|
||||
}
|
||||
if (!providersEnv.trim()) {
|
||||
throw new Error("--providers-env requires a non-empty env var name");
|
||||
}
|
||||
|
||||
return {
|
||||
artifactBase: path.resolve(repoRoot, artifactBase),
|
||||
providersEnv,
|
||||
repoRoot: path.resolve(repoRoot),
|
||||
suiteId,
|
||||
};
|
||||
}
|
||||
|
||||
function suiteProviderFilter(options: HostedMediaOptions, env: NodeJS.ProcessEnv) {
|
||||
const suiteEnv = `OPENCLAW_QA_HOSTED_${options.suiteId.toUpperCase()}_PROVIDERS`;
|
||||
return env[suiteEnv]?.trim() || env[options.providersEnv]?.trim() || "";
|
||||
}
|
||||
|
||||
export function buildHostedMediaCommand(params: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
options: HostedMediaOptions;
|
||||
}) {
|
||||
const definition = EVIDENCE_SUITES[params.options.suiteId];
|
||||
const env = { ...(params.env ?? process.env) };
|
||||
const args = ["--import", "tsx", SOURCE_PATH, params.options.suiteId];
|
||||
const providerFilter = suiteProviderFilter(params.options, env);
|
||||
if (providerFilter) {
|
||||
args.push(`--${params.options.suiteId}-providers`, providerFilter);
|
||||
}
|
||||
if (definition.videoFullModes) {
|
||||
env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES = "1";
|
||||
}
|
||||
return {
|
||||
args,
|
||||
command: process.execPath,
|
||||
env,
|
||||
};
|
||||
}
|
||||
|
||||
const HOSTED_MEDIA_BLOCKED_PATTERNS = [
|
||||
/no runnable providers matched available auth/i,
|
||||
/no runnable providers matched the explicit provider selection/i,
|
||||
/no runnable providers matched explicit provider selection/i,
|
||||
/no providers with usable auth/i,
|
||||
];
|
||||
|
||||
export function classifyHostedMediaFailureStatus(message: string): QaScriptEvidenceStatus {
|
||||
const tracker = createQaScriptBlockedStatusTracker(HOSTED_MEDIA_BLOCKED_PATTERNS);
|
||||
tracker.append(message);
|
||||
return tracker.status();
|
||||
}
|
||||
|
||||
function formatCommand(command: string, args: readonly string[]) {
|
||||
return [command, ...args].map((arg) => JSON.stringify(arg)).join(" ");
|
||||
}
|
||||
|
||||
function createHostedMediaEvidenceWriter(options: HostedMediaOptions) {
|
||||
const definition = EVIDENCE_SUITES[options.suiteId];
|
||||
return createQaScriptEvidenceWriter({
|
||||
artifactBase: options.artifactBase,
|
||||
logFileName: "hosted-media-live.log",
|
||||
primaryModel: "live-media/hosted-media-provider",
|
||||
providerMode: "live-frontier",
|
||||
repoRoot: options.repoRoot,
|
||||
target: {
|
||||
id: definition.scenarioId,
|
||||
title: definition.title,
|
||||
sourcePath: SOURCE_PATH,
|
||||
primaryCoverageIds: definition.primaryCoverageIds,
|
||||
secondaryCoverageIds: definition.secondaryCoverageIds,
|
||||
docsRefs: definition.docsRefs,
|
||||
codeRefs: definition.codeRefs,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runHostedMediaProof(
|
||||
options: HostedMediaOptions,
|
||||
writer: ReturnType<typeof createHostedMediaEvidenceWriter>,
|
||||
): Promise<HostedMediaProofResult> {
|
||||
const startedAt = Date.now();
|
||||
const command = buildHostedMediaCommand({ options });
|
||||
|
||||
writer.appendLog(`$ ${formatCommand(command.command, command.args)}\n`);
|
||||
writer.appendLog(
|
||||
`suite: ${options.suiteId}\nprovidersEnv: ${options.providersEnv}\nvideoFullModes: ${String(EVIDENCE_SUITES[options.suiteId].videoFullModes === true)}\n`,
|
||||
);
|
||||
|
||||
return await new Promise<HostedMediaProofResult>((resolve, reject) => {
|
||||
const child = spawn(command.command, command.args, {
|
||||
cwd: options.repoRoot,
|
||||
env: command.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const statusTracker = createQaScriptBlockedStatusTracker(HOSTED_MEDIA_BLOCKED_PATTERNS);
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
writer.appendLog(chunk);
|
||||
statusTracker.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
writer.appendLog(chunk);
|
||||
statusTracker.append(chunk);
|
||||
});
|
||||
child.on("error", reject);
|
||||
child.on("close", (status, signal) => {
|
||||
const durationMs = Math.max(1, Date.now() - startedAt);
|
||||
if (status === 0 && !signal) {
|
||||
resolve({
|
||||
details: `${options.suiteId} hosted media live suite passed`,
|
||||
durationMs,
|
||||
status: "pass",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const details = signal
|
||||
? `${options.suiteId} hosted media live suite terminated by ${signal}`
|
||||
: `${options.suiteId} hosted media live suite exited with ${status ?? 1}`;
|
||||
resolve({
|
||||
details,
|
||||
durationMs,
|
||||
status: statusTracker.status(),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function buildHostedMediaEvidence(params: {
|
||||
options: HostedMediaOptions;
|
||||
result: HostedMediaProofResult;
|
||||
}): QaEvidenceSummaryJson {
|
||||
return createHostedMediaEvidenceWriter(params.options).build(params.result);
|
||||
}
|
||||
|
||||
export async function runHostedMediaProviderLiveProducer(
|
||||
options: HostedMediaOptions,
|
||||
): Promise<QaEvidenceSummaryJson> {
|
||||
const writer = createHostedMediaEvidenceWriter(options);
|
||||
const result = await runHostedMediaProof(options, writer);
|
||||
return await writer.write(result);
|
||||
}
|
||||
|
||||
async function main(argv: string[]) {
|
||||
if (argv.includes("--qa-evidence")) {
|
||||
const evidence = await runHostedMediaProviderLiveProducer(parseHostedMediaOptions(argv));
|
||||
console.log(`Hosted media provider live evidence: ${QA_EVIDENCE_FILENAME}`);
|
||||
console.log(`Hosted media provider live status: ${evidence.entries[0]?.result.status}`);
|
||||
return evidence.entries[0]?.result.status === "fail" ? 1 : 0;
|
||||
}
|
||||
return await runCli(argv);
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
main(process.argv.slice(2))
|
||||
.then((code) => process.exit(code))
|
||||
.catch((error: unknown) => {
|
||||
console.error(formatErrorMessage(error));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
193
test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts
Normal file
193
test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { maybeApplyTtsToPayload } from "../../../../packages/speech-core/src/tts.ts";
|
||||
import { setRuntimeConfigSnapshot } from "../../../../src/config/config.ts";
|
||||
import { buildWebchatAudioContentBlocksFromReplyPayloads } from "../../../../src/gateway/server-methods/chat-webchat-media.ts";
|
||||
import {
|
||||
installGatewayTestHooks,
|
||||
setTestPluginRegistry,
|
||||
testState,
|
||||
withGatewayServer,
|
||||
} from "../../../../src/gateway/test-helpers.ts";
|
||||
import { createPluginRecord } from "../../../../src/plugins/loader-records.ts";
|
||||
import { createPluginRegistry } from "../../../../src/plugins/registry.ts";
|
||||
import { getActivePluginRegistry } from "../../../../src/plugins/runtime.ts";
|
||||
import { resetPluginRuntimeStateForTest } from "../../../../src/plugins/runtime.ts";
|
||||
import { getSpeechProvider } from "../../../../src/tts/provider-registry.ts";
|
||||
|
||||
installGatewayTestHooks({ scope: "suite" });
|
||||
|
||||
const CONTROL_UI_E2E_TOKEN = "test-gateway-token-1234567890";
|
||||
|
||||
const noopLogger = {
|
||||
info() {},
|
||||
warn() {},
|
||||
error() {},
|
||||
debug() {},
|
||||
};
|
||||
|
||||
function installMockTtsProvider() {
|
||||
const registry = createPluginRegistry({
|
||||
logger: noopLogger,
|
||||
runtime: {},
|
||||
activateGlobalSideEffects: false,
|
||||
});
|
||||
const record = createPluginRecord({
|
||||
id: "qa-webchat-auto-tts",
|
||||
name: "QA WebChat Auto TTS",
|
||||
source: "test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts",
|
||||
origin: "global",
|
||||
enabled: true,
|
||||
configSchema: false,
|
||||
});
|
||||
const synthesizeCalls: string[] = [];
|
||||
|
||||
registry.registerSpeechProvider(record, {
|
||||
id: "mock",
|
||||
label: "Mock",
|
||||
autoSelectOrder: 1,
|
||||
isConfigured: () => true,
|
||||
synthesize: async (request) => {
|
||||
synthesizeCalls.push(request.text);
|
||||
return {
|
||||
audioBuffer: Buffer.from("voice"),
|
||||
fileExtension: ".ogg",
|
||||
outputFormat: "ogg",
|
||||
voiceCompatible: request.target === "voice-note",
|
||||
};
|
||||
},
|
||||
});
|
||||
setTestPluginRegistry(registry.registry);
|
||||
return synthesizeCalls;
|
||||
}
|
||||
|
||||
describe("QA WebChat auto TTS", () => {
|
||||
afterEach(() => {
|
||||
resetPluginRuntimeStateForTest();
|
||||
});
|
||||
|
||||
it("synthesizes only the final WebChat tail and exposes trusted local audio", async () => {
|
||||
resetPluginRuntimeStateForTest();
|
||||
const synthesizeCalls = installMockTtsProvider();
|
||||
const prefsPath = path.join(os.tmpdir(), `openclaw-webchat-tts-${process.pid}.json`);
|
||||
let mediaPath: string | undefined;
|
||||
|
||||
try {
|
||||
const text = "WebChat streams block text; dispatch synthesizes one TTS tail with kind final.";
|
||||
const cfg = {
|
||||
messages: {
|
||||
tts: {
|
||||
enabled: true,
|
||||
provider: "mock",
|
||||
prefsPath,
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
setRuntimeConfigSnapshot(cfg, cfg);
|
||||
|
||||
expect(getActivePluginRegistry()?.speechProviders.map((entry) => entry.provider.id)).toEqual([
|
||||
"mock",
|
||||
]);
|
||||
expect(getSpeechProvider("mock", cfg)?.id).toBe("mock");
|
||||
|
||||
const blockResult = await maybeApplyTtsToPayload({
|
||||
payload: { text },
|
||||
cfg,
|
||||
channel: "webchat",
|
||||
kind: "block",
|
||||
});
|
||||
expect(blockResult.mediaUrl).toBeUndefined();
|
||||
expect(blockResult.text).toBe(text);
|
||||
expect(synthesizeCalls).toEqual([]);
|
||||
|
||||
const tailResult = await maybeApplyTtsToPayload({
|
||||
payload: { text },
|
||||
cfg,
|
||||
channel: "webchat",
|
||||
kind: "final",
|
||||
});
|
||||
mediaPath = tailResult.mediaUrl;
|
||||
|
||||
expect(synthesizeCalls).toEqual([text]);
|
||||
expect(mediaPath).toMatch(/voice-\d+\.ogg$/);
|
||||
if (!mediaPath || !fs.existsSync(mediaPath)) {
|
||||
throw new Error("expected final WebChat TTS to write local audio");
|
||||
}
|
||||
expect(tailResult).toMatchObject({
|
||||
spokenText: text,
|
||||
trustedLocalMedia: true,
|
||||
});
|
||||
|
||||
const trustedBlocks = await buildWebchatAudioContentBlocksFromReplyPayloads(
|
||||
[
|
||||
{
|
||||
mediaUrl: mediaPath,
|
||||
audioAsVoice: tailResult.audioAsVoice,
|
||||
spokenText: text,
|
||||
trustedLocalMedia: true,
|
||||
},
|
||||
],
|
||||
{ localRoots: [path.dirname(mediaPath)] },
|
||||
);
|
||||
const untrustedBlocks = await buildWebchatAudioContentBlocksFromReplyPayloads(
|
||||
[{ mediaUrl: mediaPath }],
|
||||
{ localRoots: [path.dirname(mediaPath)] },
|
||||
);
|
||||
|
||||
expect(trustedBlocks).toHaveLength(1);
|
||||
expect(trustedBlocks[0]).toMatchObject({
|
||||
type: "attachment",
|
||||
attachment: {
|
||||
kind: "audio",
|
||||
label: path.basename(mediaPath),
|
||||
mimeType: "audio/ogg",
|
||||
url: fs.realpathSync(mediaPath),
|
||||
},
|
||||
});
|
||||
expect(untrustedBlocks).toHaveLength(0);
|
||||
|
||||
const source = trustedBlocks[0]?.type === "attachment" ? trustedBlocks[0].attachment.url : "";
|
||||
testState.gatewayAuth = { mode: "token", token: CONTROL_UI_E2E_TOKEN };
|
||||
await withGatewayServer(
|
||||
async ({ port }) => {
|
||||
const route = `http://127.0.0.1:${port}/__openclaw__/assistant-media`;
|
||||
const sourceParam = encodeURIComponent(source);
|
||||
const metadata = await fetch(`${route}?meta=1&source=${sourceParam}`, {
|
||||
headers: { Authorization: `Bearer ${CONTROL_UI_E2E_TOKEN}` },
|
||||
});
|
||||
expect(metadata.status).toBe(200);
|
||||
const ticket = (await metadata.json()) as {
|
||||
available?: boolean;
|
||||
mediaTicket?: string;
|
||||
};
|
||||
expect(ticket.available).toBe(true);
|
||||
expect(ticket.mediaTicket).toMatch(/^v1\./);
|
||||
|
||||
const withoutTicket = await fetch(`${route}?source=${sourceParam}`);
|
||||
expect(withoutTicket.status).toBe(401);
|
||||
|
||||
const ticketed = await fetch(
|
||||
`${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(ticket.mediaTicket ?? "")}`,
|
||||
);
|
||||
expect(ticketed.status).toBe(200);
|
||||
expect(ticketed.headers.get("content-type")).toContain("audio/ogg");
|
||||
expect(Buffer.from(await ticketed.arrayBuffer())).toEqual(Buffer.from("voice"));
|
||||
},
|
||||
{
|
||||
serverOptions: {
|
||||
auth: { mode: "token", token: CONTROL_UI_E2E_TOKEN },
|
||||
controlUiEnabled: true,
|
||||
},
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
if (mediaPath) {
|
||||
fs.rmSync(path.dirname(mediaPath), { recursive: true, force: true });
|
||||
}
|
||||
fs.rmSync(prefsPath, { force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
// ClawHub release candidate producer tests cover blocked script evidence output.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { validateQaEvidenceSummaryJson } from "../../../../extensions/qa-lab/api.js";
|
||||
|
||||
const SOURCE_PATH = "test/e2e/qa-lab/plugins/clawhub-release-candidate-install.ts";
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe("ClawHub release candidate install producer", () => {
|
||||
it("writes blocked evidence when no candidate tarball is available", async () => {
|
||||
const artifactBase = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), "openclaw-clawhub-release-evidence-"),
|
||||
);
|
||||
tempRoots.push(artifactBase);
|
||||
const missingTarballEnv = "OPENCLAW_TEST_MISSING_RELEASE_CANDIDATE_TARBALL";
|
||||
const env = { ...process.env };
|
||||
delete env[missingTarballEnv];
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
SOURCE_PATH,
|
||||
"--artifact-base",
|
||||
artifactBase,
|
||||
"--tarball-env",
|
||||
missingTarballEnv,
|
||||
],
|
||||
{ cwd: process.cwd(), encoding: "utf8", env },
|
||||
);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
const evidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(path.join(artifactBase, "qa-evidence.json"), "utf8")),
|
||||
);
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
execution: {
|
||||
artifacts: [{ kind: "log", path: "parallels-npm-update.log", source: "script" }],
|
||||
},
|
||||
result: {
|
||||
status: "blocked",
|
||||
failure: {
|
||||
reason: expect.stringContaining(`${missingTarballEnv} is not set`),
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.stdout).toContain("ClawHub release-candidate install status: blocked");
|
||||
});
|
||||
});
|
||||
456
test/e2e/qa-lab/plugins/clawhub-release-candidate-install.ts
Normal file
456
test/e2e/qa-lab/plugins/clawhub-release-candidate-install.ts
Normal file
@@ -0,0 +1,456 @@
|
||||
// Produces QA Lab evidence for release-candidate npm package install proof.
|
||||
import { execFile, spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import {
|
||||
QA_EVIDENCE_FILENAME,
|
||||
type QaEvidenceSummaryJson,
|
||||
} from "../../../../extensions/qa-lab/api.js";
|
||||
import { createBoundedChildOutput } from "../../../helpers/bounded-child-output.js";
|
||||
import {
|
||||
createQaScriptBlockedStatusTracker,
|
||||
createQaScriptEvidenceWriter,
|
||||
type QaScriptEvidenceStatus,
|
||||
} from "../runtime/script-evidence.js";
|
||||
|
||||
const SCENARIO_ID = "clawhub-release-candidate-checklist";
|
||||
const SCENARIO_TITLE = "ClawHub release candidate npm package install proof";
|
||||
const SOURCE_PATH = "test/e2e/qa-lab/plugins/clawhub-release-candidate-install.ts";
|
||||
const COVERAGE_ID = "clawhub.npm-pack-local-release-candidate-installs";
|
||||
const DEFAULT_TARBALL_ENV = "OPENCLAW_QA_RELEASE_CANDIDATE_TARBALL";
|
||||
const CHECKOUT_BUILD_RESULT_PREFIX = "__OPENCLAW_QA_RELEASE_CANDIDATE_TARBALL__";
|
||||
const execFileAsync = promisify(execFile);
|
||||
const CLAWHUB_BLOCKED_PREREQUISITE_PATTERNS = [
|
||||
/\bprlctl\b/i,
|
||||
/failed to detect parallels host ip/i,
|
||||
/vm .*not found/i,
|
||||
/could not resolve .*vm/i,
|
||||
/no .*vm/i,
|
||||
/parallels desktop .*not/i,
|
||||
/api key/i,
|
||||
/provider auth/i,
|
||||
];
|
||||
|
||||
type ProducerOptions = {
|
||||
artifactBase: string;
|
||||
buildFromCheckout: boolean;
|
||||
platform?: string;
|
||||
repoRoot: string;
|
||||
tarballEnv: string;
|
||||
};
|
||||
|
||||
type ParallelsSummary = {
|
||||
freshTarget?: Record<string, string>;
|
||||
freshTargetSpec?: string;
|
||||
update?: Record<string, { status?: string; version?: string }>;
|
||||
updateTargetPackageVersion?: string;
|
||||
updateTargetTarball?: string;
|
||||
};
|
||||
|
||||
type ProofResult = {
|
||||
artifacts?: Array<{ filePath: string; kind: string }>;
|
||||
details?: string;
|
||||
durationMs: number;
|
||||
status: QaScriptEvidenceStatus;
|
||||
};
|
||||
|
||||
class ParallelsProofError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly evidenceStatus: QaScriptEvidenceStatus,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ParallelsProofError";
|
||||
}
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return `Usage: node --import tsx ${SOURCE_PATH} --artifact-base <dir> [options]
|
||||
|
||||
Produces QA Lab evidence for ClawHub release-candidate package install proof.
|
||||
|
||||
Options:
|
||||
--artifact-base <dir> Evidence artifact directory
|
||||
--repo-root <dir> Repository root
|
||||
--tarball-env <name> Env var containing candidate .tgz path
|
||||
Default: ${DEFAULT_TARBALL_ENV}
|
||||
--build-from-checkout Build a candidate .tgz from this checkout when no
|
||||
tarball env is set
|
||||
--platform <list> Optional Parallels platform list passed through
|
||||
-h, --help Show this help
|
||||
`;
|
||||
}
|
||||
|
||||
function readOptionValue(argv: readonly string[], index: number, arg: string) {
|
||||
const value = argv[index + 1] ?? "";
|
||||
if (!value || value.startsWith("-")) {
|
||||
throw new Error(`${arg} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseOptions(
|
||||
argv: readonly string[],
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): ProducerOptions {
|
||||
let artifactBase = "";
|
||||
let buildFromCheckout = env.OPENCLAW_QA_RELEASE_CANDIDATE_BUILD === "1";
|
||||
let platform: string | undefined;
|
||||
let repoRoot = process.cwd();
|
||||
let tarballEnv = DEFAULT_TARBALL_ENV;
|
||||
const seen = new Set<string>();
|
||||
const recordOnce = (flag: string) => {
|
||||
if (seen.has(flag)) {
|
||||
throw new Error(`${flag} was provided more than once`);
|
||||
}
|
||||
seen.add(flag);
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
console.log(usage());
|
||||
process.exit(0);
|
||||
}
|
||||
if (arg === "--artifact-base") {
|
||||
recordOnce(arg);
|
||||
artifactBase = readOptionValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--repo-root") {
|
||||
recordOnce(arg);
|
||||
repoRoot = readOptionValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--tarball-env") {
|
||||
recordOnce(arg);
|
||||
tarballEnv = readOptionValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--build-from-checkout") {
|
||||
recordOnce(arg);
|
||||
buildFromCheckout = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--platform") {
|
||||
recordOnce(arg);
|
||||
platform = readOptionValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unsupported release-candidate install producer arg: ${arg}`);
|
||||
}
|
||||
|
||||
if (!artifactBase.trim()) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
if (!tarballEnv.trim()) {
|
||||
throw new Error("--tarball-env requires a non-empty env var name");
|
||||
}
|
||||
return {
|
||||
artifactBase: path.resolve(repoRoot, artifactBase),
|
||||
buildFromCheckout,
|
||||
platform,
|
||||
repoRoot: path.resolve(repoRoot),
|
||||
tarballEnv,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeJson(filePath: string, value: unknown) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function formatErrorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
async function resolveCandidateTarball(options: ProducerOptions) {
|
||||
const explicitTarball = process.env[options.tarballEnv]?.trim();
|
||||
if (explicitTarball) {
|
||||
return path.resolve(options.repoRoot, explicitTarball);
|
||||
}
|
||||
if (!options.buildFromCheckout) {
|
||||
return undefined;
|
||||
}
|
||||
return await buildCandidateTarballFromCheckout(options);
|
||||
}
|
||||
|
||||
async function buildCandidateTarballFromCheckout(options: ProducerOptions) {
|
||||
const destination = path.join(options.artifactBase, "package");
|
||||
await fs.mkdir(destination, { recursive: true });
|
||||
const evalScript = `
|
||||
import { packOpenClaw } from "./scripts/e2e/parallels/package-artifact.ts";
|
||||
const artifact = await packOpenClaw({ destination: ${JSON.stringify(destination)} });
|
||||
process.stdout.write(${JSON.stringify(CHECKOUT_BUILD_RESULT_PREFIX)} + JSON.stringify({ path: artifact.path }) + "\\n");
|
||||
`;
|
||||
const result = await execFileAsync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "--input-type=module", "--eval", evalScript],
|
||||
{
|
||||
cwd: options.repoRoot,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
const resultLine = result.stdout
|
||||
.split("\n")
|
||||
.find((line) => line.startsWith(CHECKOUT_BUILD_RESULT_PREFIX));
|
||||
if (!resultLine) {
|
||||
throw new Error("checkout package build did not report a tarball marker");
|
||||
}
|
||||
const parsed = JSON.parse(resultLine.slice(CHECKOUT_BUILD_RESULT_PREFIX.length)) as {
|
||||
path?: unknown;
|
||||
};
|
||||
if (typeof parsed.path !== "string" || !parsed.path.trim()) {
|
||||
throw new Error("checkout package build did not report a tarball path");
|
||||
}
|
||||
return parsed.path;
|
||||
}
|
||||
|
||||
async function extractPackageJsonFromTgz<T>(tgzPath: string, entry: string): Promise<T> {
|
||||
const result = await execFileAsync("tar", ["-xOf", tgzPath, entry], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
return JSON.parse(result.stdout) as T;
|
||||
}
|
||||
|
||||
async function validateCandidateTarball(tarballPath: string) {
|
||||
const [version, buildCommit] = await Promise.all([
|
||||
extractPackageJsonFromTgz<{ version?: string }>(tarballPath, "package/package.json").then(
|
||||
(pkg) => pkg.version ?? "",
|
||||
),
|
||||
extractPackageJsonFromTgz<{ commit?: string }>(
|
||||
tarballPath,
|
||||
"package/dist/build-info.json",
|
||||
).then((info) => info.commit ?? ""),
|
||||
]);
|
||||
if (!version || !buildCommit) {
|
||||
throw new Error(`target tarball is missing package or build metadata: ${tarballPath}`);
|
||||
}
|
||||
return { buildCommit, version };
|
||||
}
|
||||
|
||||
async function runParallelsProof(params: {
|
||||
options: ProducerOptions;
|
||||
tarballPath: string;
|
||||
writer: ReturnType<typeof createClawHubEvidenceWriter>;
|
||||
}) {
|
||||
const args = [
|
||||
"scripts/e2e/parallels-npm-update-smoke.sh",
|
||||
"--target-tarball",
|
||||
params.tarballPath,
|
||||
"--json",
|
||||
];
|
||||
if (params.options.platform) {
|
||||
args.push("--platform", params.options.platform);
|
||||
}
|
||||
params.writer.appendLog(`$ bash ${args.map((arg) => JSON.stringify(arg)).join(" ")}\n`);
|
||||
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
const child = spawn("bash", args, {
|
||||
cwd: params.options.repoRoot,
|
||||
env: process.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = createBoundedChildOutput(1024 * 1024);
|
||||
const statusTracker = createQaScriptBlockedStatusTracker(CLAWHUB_BLOCKED_PREREQUISITE_PATTERNS);
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
params.writer.appendLog(chunk);
|
||||
stdout.append(chunk);
|
||||
statusTracker.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
params.writer.appendLog(chunk);
|
||||
statusTracker.append(chunk);
|
||||
});
|
||||
child.on("error", reject);
|
||||
child.on("close", (status, signal) => {
|
||||
const stdoutText = stdout.text();
|
||||
if (status === 0 && !signal) {
|
||||
resolve(stdoutText);
|
||||
return;
|
||||
}
|
||||
const reason = signal
|
||||
? `Parallels npm-update proof terminated by ${signal}`
|
||||
: `Parallels npm-update proof exited with ${status ?? 1}`;
|
||||
reject(new ParallelsProofError(reason, statusTracker.status()));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function parseParallelsSummary(stdout: string): ParallelsSummary {
|
||||
try {
|
||||
return JSON.parse(stdout) as ParallelsSummary;
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Parallels npm-update proof did not print JSON summary: ${formatErrorMessage(error)}`,
|
||||
{
|
||||
cause: error,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function requireHostedCandidateTarball(params: { summary: ParallelsSummary; tarballPath: string }) {
|
||||
const rawUrl = params.summary.updateTargetTarball?.trim();
|
||||
if (!rawUrl) {
|
||||
throw new Error("summary missing updateTargetTarball");
|
||||
}
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(rawUrl);
|
||||
} catch (error) {
|
||||
throw new Error(`updateTargetTarball is not a URL: ${rawUrl}`, { cause: error });
|
||||
}
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
throw new Error(`updateTargetTarball is not hosted over HTTP(S): ${rawUrl}`);
|
||||
}
|
||||
const hostedName = decodeURIComponent(path.posix.basename(url.pathname));
|
||||
const expectedName = path.basename(params.tarballPath);
|
||||
if (hostedName !== expectedName) {
|
||||
throw new Error(
|
||||
`updateTargetTarball does not point at the candidate tarball: expected ${expectedName}, got ${hostedName}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertParallelsSummary(params: {
|
||||
summary: ParallelsSummary;
|
||||
tarballPath: string;
|
||||
version: string;
|
||||
}) {
|
||||
requireHostedCandidateTarball(params);
|
||||
if (!params.summary.updateTargetPackageVersion) {
|
||||
throw new Error("summary missing updateTargetPackageVersion");
|
||||
}
|
||||
if (params.summary.updateTargetPackageVersion !== params.version) {
|
||||
throw new Error(
|
||||
`summary target version ${params.summary.updateTargetPackageVersion} does not match candidate ${params.version}`,
|
||||
);
|
||||
}
|
||||
|
||||
const freshTargetPasses = Object.entries(params.summary.freshTarget ?? {}).filter(
|
||||
([, status]) => status === "pass",
|
||||
);
|
||||
if (freshTargetPasses.length === 0) {
|
||||
throw new Error("summary has no passing freshTarget platform");
|
||||
}
|
||||
|
||||
const updatePasses = Object.entries(params.summary.update ?? {}).filter(
|
||||
([, result]) => result?.status === "pass",
|
||||
);
|
||||
if (updatePasses.length === 0) {
|
||||
throw new Error("summary has no passing update platform");
|
||||
}
|
||||
}
|
||||
|
||||
function isBlockedPrerequisiteFailure(message: string) {
|
||||
return CLAWHUB_BLOCKED_PREREQUISITE_PATTERNS.some((pattern) => pattern.test(message));
|
||||
}
|
||||
|
||||
function createClawHubEvidenceWriter(options: ProducerOptions) {
|
||||
return createQaScriptEvidenceWriter({
|
||||
artifactBase: options.artifactBase,
|
||||
logFileName: "parallels-npm-update.log",
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: options.repoRoot,
|
||||
target: {
|
||||
id: SCENARIO_ID,
|
||||
title: SCENARIO_TITLE,
|
||||
sourcePath: SOURCE_PATH,
|
||||
primaryCoverageIds: [COVERAGE_ID],
|
||||
docsRefs: ["docs/help/testing.md", "docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: [
|
||||
SOURCE_PATH,
|
||||
"scripts/e2e/parallels-npm-update-smoke.sh",
|
||||
"scripts/e2e/parallels/npm-update-smoke.ts",
|
||||
"test/scripts/release-candidate-checklist.test.ts",
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function produceProof(
|
||||
options: ProducerOptions,
|
||||
writer: ReturnType<typeof createClawHubEvidenceWriter>,
|
||||
): Promise<ProofResult> {
|
||||
const startedAt = Date.now();
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
const summaryPath = path.join(options.artifactBase, "parallels-summary.json");
|
||||
|
||||
try {
|
||||
const tarballPath = await resolveCandidateTarball(options);
|
||||
if (!tarballPath) {
|
||||
return {
|
||||
details: `${options.tarballEnv} is not set; provide a candidate .tgz or pass --build-from-checkout.`,
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "blocked",
|
||||
};
|
||||
}
|
||||
await fs.access(tarballPath);
|
||||
const metadata = await validateCandidateTarball(tarballPath);
|
||||
writer.appendLog(
|
||||
`candidate: ${tarballPath}\nversion: ${metadata.version}\nbuild commit: ${metadata.buildCommit}\n`,
|
||||
);
|
||||
const commandOutput = await runParallelsProof({ options, tarballPath, writer });
|
||||
const summary = parseParallelsSummary(commandOutput);
|
||||
assertParallelsSummary({
|
||||
summary,
|
||||
tarballPath,
|
||||
version: metadata.version,
|
||||
});
|
||||
await writeJson(summaryPath, summary);
|
||||
return {
|
||||
artifacts: [{ kind: "summary", filePath: "parallels-summary.json" }],
|
||||
details: `candidate ${metadata.version} installed fresh and updated through Parallels npm semantics`,
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "pass",
|
||||
};
|
||||
} catch (error) {
|
||||
const details = formatErrorMessage(error);
|
||||
const status: QaScriptEvidenceStatus =
|
||||
error instanceof ParallelsProofError
|
||||
? error.evidenceStatus
|
||||
: isBlockedPrerequisiteFailure(details)
|
||||
? "blocked"
|
||||
: "fail";
|
||||
writer.appendLog(`\n${status}: ${details}\n`);
|
||||
return {
|
||||
details,
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function runClawHubReleaseCandidateInstallProducer(
|
||||
options: ProducerOptions,
|
||||
): Promise<QaEvidenceSummaryJson> {
|
||||
const writer = createClawHubEvidenceWriter(options);
|
||||
const result = await produceProof(options, writer);
|
||||
return await writer.write(result);
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
runClawHubReleaseCandidateInstallProducer(parseOptions(process.argv.slice(2)))
|
||||
.then((evidence) => {
|
||||
console.log(`ClawHub release-candidate install evidence: ${QA_EVIDENCE_FILENAME}`);
|
||||
console.log(
|
||||
`ClawHub release-candidate install status: ${evidence.entries[0]?.result.status}`,
|
||||
);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
console.error(formatErrorMessage(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
650
test/e2e/qa-lab/plugins/plugin-lifecycle-probe-runtime.ts
Normal file
650
test/e2e/qa-lab/plugins/plugin-lifecycle-probe-runtime.ts
Normal file
@@ -0,0 +1,650 @@
|
||||
// Plugin Lifecycle Probe tests cover QA Lab plugin lifecycle evidence.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { readPluginInstallRecords } from "../../../../scripts/e2e/lib/plugin-index-sqlite.mjs";
|
||||
import { resolveWindowsTaskkillPath } from "../../../../scripts/lib/windows-taskkill.mjs";
|
||||
|
||||
// The Docker entrypoint runs without Vitest installed, so keep cleanup local to this runtime probe.
|
||||
const tempDirs = (() => {
|
||||
const dirs = new Set<string>();
|
||||
return {
|
||||
make(prefix: string): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
dirs.add(dir);
|
||||
return dir;
|
||||
},
|
||||
cleanup(): void {
|
||||
for (const dir of dirs) {
|
||||
fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 });
|
||||
}
|
||||
dirs.clear();
|
||||
},
|
||||
};
|
||||
})();
|
||||
|
||||
type ProbeEnv = Pick<NodeJS.ProcessEnv, "HOME" | "OPENCLAW_CONFIG_PATH" | "OPENCLAW_STATE_DIR">;
|
||||
|
||||
type MatrixEnv = NodeJS.ProcessEnv & ProbeEnv;
|
||||
|
||||
interface CommandOptions {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
outputFile?: string;
|
||||
spawnImpl?: typeof spawn;
|
||||
taskkillImpl?: typeof spawnSync;
|
||||
timeoutKillGraceMs?: number;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
interface RegistryServer {
|
||||
env: NodeJS.ProcessEnv;
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
function stateDir(env: ProbeEnv = process.env) {
|
||||
return env.OPENCLAW_STATE_DIR || path.join(env.HOME ?? os.homedir(), ".openclaw");
|
||||
}
|
||||
|
||||
function configPath(env: ProbeEnv = process.env) {
|
||||
return env.OPENCLAW_CONFIG_PATH || path.join(stateDir(env), "openclaw.json");
|
||||
}
|
||||
|
||||
function readJson(file: string) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8")) as Record<string, unknown>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function readRequiredJson(file: string) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8")) as Record<string, unknown>;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`failed to read JSON from ${file}: ${message}`, { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
function records(env: ProbeEnv = process.env) {
|
||||
return readPluginInstallRecords({
|
||||
configPath: configPath(env),
|
||||
stateDir: stateDir(env),
|
||||
}) as Record<string, Record<string, unknown>>;
|
||||
}
|
||||
|
||||
function recordFor(pluginId: string, env: ProbeEnv = process.env) {
|
||||
return records(env)[pluginId];
|
||||
}
|
||||
|
||||
function config(env: ProbeEnv = process.env) {
|
||||
return readJson(configPath(env));
|
||||
}
|
||||
|
||||
function requiredConfig(env: ProbeEnv = process.env) {
|
||||
return readRequiredJson(configPath(env));
|
||||
}
|
||||
|
||||
function assertProbe(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function assertVersion(pluginId: string, version: string, env: ProbeEnv = process.env) {
|
||||
const record = recordFor(pluginId, env);
|
||||
assertProbe(record, `install record missing for ${pluginId}`);
|
||||
assertProbe(record.source === "npm", `expected npm source for ${pluginId}, got ${record.source}`);
|
||||
assertProbe(
|
||||
record.resolvedVersion === version || record.version === version,
|
||||
`expected ${pluginId} record version ${version}, got ${JSON.stringify(record)}`,
|
||||
);
|
||||
assertProbe(record.installPath, `install path missing for ${pluginId}`);
|
||||
const packageJson = readJson(path.join(String(record.installPath), "package.json"));
|
||||
assertProbe(
|
||||
packageJson.version === version,
|
||||
`expected installed package version ${version}, got ${packageJson.version}`,
|
||||
);
|
||||
}
|
||||
|
||||
function assertNpmProjectRoot(pluginId: string, packageName: string, env: ProbeEnv = process.env) {
|
||||
const record = recordFor(pluginId, env);
|
||||
assertProbe(record?.installPath, `install path missing for ${pluginId}`);
|
||||
const installPath = String(record.installPath);
|
||||
const relative = path.relative(path.join(stateDir(env), "npm", "projects"), installPath);
|
||||
assertProbe(
|
||||
!relative.startsWith("..") && !path.isAbsolute(relative),
|
||||
`install path outside npm projects: ${installPath}`,
|
||||
);
|
||||
const segments = relative.split(path.sep);
|
||||
const packageSegments = packageName.split("/");
|
||||
assertProbe(
|
||||
segments.length === 2 + packageSegments.length,
|
||||
`unexpected npm project install path: ${installPath}`,
|
||||
);
|
||||
assertProbe(Boolean(segments[0]), `missing npm project directory: ${installPath}`);
|
||||
assertProbe(
|
||||
segments[1] === "node_modules",
|
||||
`missing project node_modules segment: ${installPath}`,
|
||||
);
|
||||
for (let index = 0; index < packageSegments.length; index++) {
|
||||
assertProbe(
|
||||
segments[index + 2] === packageSegments[index],
|
||||
`package path mismatch: ${installPath}`,
|
||||
);
|
||||
}
|
||||
assertProbe(
|
||||
!fs.existsSync(path.join(stateDir(env), "npm", "node_modules", ...packageSegments)),
|
||||
`legacy flat npm install path exists for ${packageName}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function assertInspectLoaded(pluginId: string, inspectPath: string | undefined) {
|
||||
assertProbe(inspectPath, "inspect JSON path is required");
|
||||
const inspect = readRequiredJson(inspectPath);
|
||||
const plugin = inspect.plugin as
|
||||
| { enabled?: boolean; id?: string; status?: string }
|
||||
| null
|
||||
| undefined;
|
||||
assertProbe(
|
||||
plugin?.id === pluginId,
|
||||
`expected inspected plugin id ${pluginId}, got ${plugin?.id}`,
|
||||
);
|
||||
assertProbe(plugin.enabled === true, `expected ${pluginId} inspect enabled=true`);
|
||||
assertProbe(
|
||||
plugin.status === "loaded",
|
||||
`expected ${pluginId} inspect status loaded, got ${plugin.status}`,
|
||||
);
|
||||
}
|
||||
|
||||
function assertEnabled(pluginId: string, expected: boolean, env: ProbeEnv = process.env) {
|
||||
const cfg = config(env) as {
|
||||
plugins?: { entries?: Record<string, { enabled?: boolean }> };
|
||||
};
|
||||
const entry = cfg.plugins?.entries?.[pluginId];
|
||||
assertProbe(entry?.enabled === expected, `expected ${pluginId} enabled=${expected}`);
|
||||
}
|
||||
|
||||
function installPath(pluginId: string, env: ProbeEnv = process.env) {
|
||||
const record = recordFor(pluginId, env);
|
||||
assertProbe(record?.installPath, `install path missing for ${pluginId}`);
|
||||
return String(record.installPath);
|
||||
}
|
||||
|
||||
export function assertUninstalled(pluginId: string, env: ProbeEnv = process.env) {
|
||||
const cfg = requiredConfig(env) as {
|
||||
plugins?: {
|
||||
allow?: string[];
|
||||
deny?: string[];
|
||||
entries?: Record<string, unknown>;
|
||||
load?: { paths?: unknown[] };
|
||||
};
|
||||
};
|
||||
const record = recordFor(pluginId, env);
|
||||
assertProbe(!record, `install record still present for ${pluginId}`);
|
||||
assertProbe(
|
||||
!cfg.plugins?.entries?.[pluginId],
|
||||
`plugin config entry still present for ${pluginId}`,
|
||||
);
|
||||
assertProbe(
|
||||
!(cfg.plugins?.allow ?? []).includes(pluginId),
|
||||
`allowlist still contains ${pluginId}`,
|
||||
);
|
||||
assertProbe(!(cfg.plugins?.deny ?? []).includes(pluginId), `denylist still contains ${pluginId}`);
|
||||
const loadPaths = cfg.plugins?.load?.paths ?? [];
|
||||
assertProbe(
|
||||
!loadPaths.some((entry) => String(entry).includes(pluginId)),
|
||||
`load path still references ${pluginId}: ${loadPaths.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function parseDurationMs(value: string | undefined, fallback: string) {
|
||||
const text = (value || fallback).trim();
|
||||
if (text === "0") {
|
||||
return undefined;
|
||||
}
|
||||
const match = /^([0-9]+(?:\.[0-9]+)?)(ms|s|m|h)?$/u.exec(text);
|
||||
if (!match) {
|
||||
throw new Error(`unsupported duration value: ${text}`);
|
||||
}
|
||||
const amount = Number(match[1]);
|
||||
const unit = match[2] ?? "s";
|
||||
const multiplier = unit === "ms" ? 1 : unit === "s" ? 1_000 : unit === "m" ? 60_000 : 3_600_000;
|
||||
return Math.max(1, Math.ceil(amount * multiplier));
|
||||
}
|
||||
|
||||
function createMatrixStateEnv(resourceDir: string): MatrixEnv {
|
||||
const home = fs.mkdtempSync(path.join(resourceDir, "home."));
|
||||
const stateDir = path.join(home, ".openclaw");
|
||||
const workspaceDir = path.join(home, "workspace");
|
||||
const configFile = path.join(stateDir, "openclaw.json");
|
||||
fs.mkdirSync(stateDir, { recursive: true });
|
||||
fs.mkdirSync(workspaceDir, { recursive: true });
|
||||
return {
|
||||
...process.env,
|
||||
HOME: home,
|
||||
USERPROFILE: home,
|
||||
OPENCLAW_HOME: home,
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
OPENCLAW_CONFIG_PATH: configFile,
|
||||
OPENCLAW_TEST_WORKSPACE_DIR: workspaceDir,
|
||||
OPENCLAW_AUTH_PROFILE_SECRET_KEY: randomBytes(32).toString("hex"),
|
||||
};
|
||||
}
|
||||
|
||||
function packageEntrypoint(prefix: string) {
|
||||
const packageRoot = path.join(prefix, "lib", "node_modules", "openclaw");
|
||||
for (const entry of ["dist/index.mjs", "dist/index.js"]) {
|
||||
const candidate = path.join(packageRoot, entry);
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
throw new Error(`OpenClaw package entrypoint not found under ${packageRoot}/dist/`);
|
||||
}
|
||||
|
||||
async function runCommand(command: string, args: readonly string[], options: CommandOptions = {}) {
|
||||
const outputFd =
|
||||
options.outputFile === undefined ? undefined : fs.openSync(options.outputFile, "a");
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const spawnImpl = options.spawnImpl ?? spawn;
|
||||
const useProcessGroup = process.platform !== "win32";
|
||||
const child = spawnImpl(command, args, {
|
||||
cwd: process.cwd(),
|
||||
detached: useProcessGroup,
|
||||
env: options.env ?? process.env,
|
||||
stdio: outputFd === undefined ? "inherit" : (["ignore", outputFd, outputFd] as const),
|
||||
});
|
||||
let settled = false;
|
||||
let forceKillTimer: NodeJS.Timeout | undefined;
|
||||
let forceSettleTimer: NodeJS.Timeout | undefined;
|
||||
let timeoutTimer: NodeJS.Timeout | undefined;
|
||||
let timeoutError: Error | undefined;
|
||||
const clearTimers = () => {
|
||||
if (timeoutTimer) {
|
||||
clearTimeout(timeoutTimer);
|
||||
}
|
||||
if (forceKillTimer) {
|
||||
clearTimeout(forceKillTimer);
|
||||
}
|
||||
if (forceSettleTimer) {
|
||||
clearTimeout(forceSettleTimer);
|
||||
}
|
||||
};
|
||||
const signalChild = (signal: NodeJS.Signals) => {
|
||||
if (useProcessGroup && child.pid) {
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
return;
|
||||
} catch {
|
||||
// The process group may already be gone; fall back to the direct child.
|
||||
}
|
||||
}
|
||||
if (!useProcessGroup && child.pid) {
|
||||
const runTaskkill = options.taskkillImpl ?? spawnSync;
|
||||
const taskkillPath = resolveWindowsTaskkillPath();
|
||||
const args = ["/PID", String(child.pid), "/T"];
|
||||
if (signal === "SIGKILL") {
|
||||
args.push("/F");
|
||||
}
|
||||
const result = runTaskkill(taskkillPath, args, { stdio: "ignore", windowsHide: true });
|
||||
if (!result.error && result.status === 0) {
|
||||
return;
|
||||
}
|
||||
if (signal !== "SIGKILL") {
|
||||
const forceResult = runTaskkill(taskkillPath, [...args, "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
if (!forceResult.error && forceResult.status === 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
child.kill(signal);
|
||||
};
|
||||
const isProcessGroupRunning = () => {
|
||||
if (!useProcessGroup || !child.pid) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
process.kill(-child.pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return (error as NodeJS.ErrnoException).code === "EPERM";
|
||||
}
|
||||
};
|
||||
const finish = (error?: Error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimers();
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
timeoutTimer =
|
||||
options.timeoutMs === undefined
|
||||
? undefined
|
||||
: setTimeout(() => {
|
||||
timeoutError = new Error(
|
||||
`${command} ${args.join(" ")} timed out after ${options.timeoutMs}ms`,
|
||||
);
|
||||
signalChild("SIGTERM");
|
||||
forceKillTimer = setTimeout(() => {
|
||||
forceKillTimer = undefined;
|
||||
signalChild("SIGKILL");
|
||||
forceSettleTimer = setTimeout(
|
||||
() => finish(timeoutError),
|
||||
options.timeoutKillGraceMs ?? 2_000,
|
||||
);
|
||||
forceSettleTimer.unref();
|
||||
}, options.timeoutKillGraceMs ?? 2_000);
|
||||
forceKillTimer.unref();
|
||||
}, options.timeoutMs);
|
||||
timeoutTimer?.unref();
|
||||
child.once("error", (error) => {
|
||||
finish(error);
|
||||
});
|
||||
child.once("exit", (code, signal) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
if (timeoutError) {
|
||||
if (isProcessGroupRunning()) {
|
||||
return;
|
||||
}
|
||||
finish(timeoutError);
|
||||
return;
|
||||
}
|
||||
if (code === 0 && !signal) {
|
||||
finish();
|
||||
return;
|
||||
}
|
||||
finish(new Error(`${command} ${args.join(" ")} failed with ${signal ?? `exit ${code}`}`));
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
if (options.outputFile && fs.existsSync(options.outputFile)) {
|
||||
const log = fs.readFileSync(options.outputFile, "utf8");
|
||||
if (log.trim()) {
|
||||
process.stderr.write(`--- ${options.outputFile} ---\n${log}`);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (outputFd !== undefined) {
|
||||
fs.closeSync(outputFd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function installOpenClawPackage(prefix: string, env: MatrixEnv) {
|
||||
const packageTgz = env.OPENCLAW_CURRENT_PACKAGE_TGZ;
|
||||
assertProbe(packageTgz, "OPENCLAW_CURRENT_PACKAGE_TGZ is required");
|
||||
const installLog = "/tmp/openclaw-plugin-lifecycle-install.log";
|
||||
process.stdout.write("Installing mounted OpenClaw package...\n");
|
||||
await runCommand(
|
||||
"npm",
|
||||
["install", "-g", "--prefix", prefix, packageTgz, "--no-fund", "--no-audit"],
|
||||
{
|
||||
env,
|
||||
outputFile: installLog,
|
||||
timeoutMs: parseDurationMs(env.OPENCLAW_E2E_NPM_INSTALL_TIMEOUT, "600s"),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function packFixturePlugin(
|
||||
packDir: string,
|
||||
outputTgz: string,
|
||||
pluginId: string,
|
||||
version: string,
|
||||
method: string,
|
||||
name: string,
|
||||
) {
|
||||
const packageDir = path.join(packDir, "package");
|
||||
fs.mkdirSync(packageDir, { recursive: true });
|
||||
await runCommand("node", [
|
||||
"scripts/e2e/lib/fixture.mjs",
|
||||
"plugin",
|
||||
packageDir,
|
||||
pluginId,
|
||||
version,
|
||||
method,
|
||||
name,
|
||||
]);
|
||||
await runCommand("tar", ["-czf", outputTgz, "-C", packDir, "package"]);
|
||||
}
|
||||
|
||||
async function startNpmFixtureRegistry(
|
||||
registryRoot: string,
|
||||
packages: readonly [packageName: string, version: string, tarball: string][],
|
||||
env: MatrixEnv,
|
||||
): Promise<RegistryServer> {
|
||||
const serverLog = path.join(registryRoot, "npm-registry.log");
|
||||
const serverPortFile = path.join(registryRoot, "npm-registry-port");
|
||||
const logFd = fs.openSync(serverLog, "a");
|
||||
const child = spawn(
|
||||
"node",
|
||||
[
|
||||
"scripts/e2e/lib/plugins/npm-registry-server.mjs",
|
||||
serverPortFile,
|
||||
...packages.flatMap(([packageName, version, tarball]) => [packageName, version, tarball]),
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env,
|
||||
stdio: ["ignore", logFd, logFd],
|
||||
},
|
||||
);
|
||||
fs.closeSync(logFd);
|
||||
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
if (fs.existsSync(serverPortFile) && fs.statSync(serverPortFile).size > 0) {
|
||||
const port = fs.readFileSync(serverPortFile, "utf8").trim();
|
||||
return {
|
||||
env: {
|
||||
...env,
|
||||
NPM_CONFIG_REGISTRY: `http://127.0.0.1:${port}`,
|
||||
},
|
||||
stop() {
|
||||
child.kill();
|
||||
},
|
||||
};
|
||||
}
|
||||
if (child.exitCode !== null) {
|
||||
const log = fs.existsSync(serverLog) ? fs.readFileSync(serverLog, "utf8") : "";
|
||||
throw new Error(`npm fixture registry exited early${log ? `\n${log}` : ""}`);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
child.kill();
|
||||
const log = fs.existsSync(serverLog) ? fs.readFileSync(serverLog, "utf8") : "";
|
||||
throw new Error(`timed out waiting for npm fixture registry${log ? `\n${log}` : ""}`);
|
||||
}
|
||||
|
||||
async function runMeasured(
|
||||
summaryTsv: string,
|
||||
phase: string,
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
env: MatrixEnv,
|
||||
) {
|
||||
process.stdout.write(`Running plugin lifecycle phase: ${phase}\n`);
|
||||
await runCommand(
|
||||
"node",
|
||||
[
|
||||
"scripts/e2e/lib/plugin-lifecycle-matrix/measure.mjs",
|
||||
summaryTsv,
|
||||
phase,
|
||||
"--",
|
||||
command,
|
||||
...args,
|
||||
],
|
||||
{ env },
|
||||
);
|
||||
}
|
||||
|
||||
export async function runPluginLifecycleMatrix() {
|
||||
const pluginId = "lifecycle-claw";
|
||||
const packageName = "@openclaw/lifecycle-claw";
|
||||
const resourceDir = tempDirs.make("openclaw-plugin-lifecycle-matrix-");
|
||||
const npmPrefix = "/tmp/npm-prefix";
|
||||
const env = createMatrixStateEnv(resourceDir);
|
||||
const tarballV1 = path.join(resourceDir, "lifecycle-claw-1.0.0.tgz");
|
||||
const tarballV2 = path.join(resourceDir, "lifecycle-claw-2.0.0.tgz");
|
||||
const inspectV1 = path.join(resourceDir, "plugin-lifecycle-inspect-v1.json");
|
||||
const summaryTsv = path.join(resourceDir, "resource-summary.tsv");
|
||||
let registry: RegistryServer | undefined;
|
||||
|
||||
fs.writeFileSync(
|
||||
summaryTsv,
|
||||
"phase\tmax_rss_kb\tcpu_seconds\twall_ms\tcpu_core_ratio\tsignal\n",
|
||||
"utf8",
|
||||
);
|
||||
fs.rmSync(npmPrefix, { recursive: true, force: true });
|
||||
|
||||
try {
|
||||
await installOpenClawPackage(npmPrefix, env);
|
||||
const entry = packageEntrypoint(npmPrefix);
|
||||
const matrixEnv: MatrixEnv = {
|
||||
...env,
|
||||
PATH: `${path.join(npmPrefix, "bin")}:${env.PATH ?? ""}`,
|
||||
npm_config_audit: "false",
|
||||
npm_config_fund: "false",
|
||||
npm_config_loglevel: "error",
|
||||
};
|
||||
const packRoot = fs.mkdtempSync(path.join(resourceDir, "pack."));
|
||||
const registryRoot = fs.mkdtempSync(path.join(resourceDir, "registry."));
|
||||
await packFixturePlugin(
|
||||
path.join(packRoot, "v1"),
|
||||
tarballV1,
|
||||
pluginId,
|
||||
"1.0.0",
|
||||
"lifecycle.v1",
|
||||
"Lifecycle Claw",
|
||||
);
|
||||
await packFixturePlugin(
|
||||
path.join(packRoot, "v2"),
|
||||
tarballV2,
|
||||
pluginId,
|
||||
"2.0.0",
|
||||
"lifecycle.v2",
|
||||
"Lifecycle Claw",
|
||||
);
|
||||
registry = await startNpmFixtureRegistry(
|
||||
registryRoot,
|
||||
[
|
||||
[packageName, "1.0.0", tarballV1],
|
||||
[packageName, "2.0.0", tarballV2],
|
||||
],
|
||||
matrixEnv,
|
||||
);
|
||||
const runEnv = registry.env as MatrixEnv;
|
||||
|
||||
await runMeasured(
|
||||
summaryTsv,
|
||||
"install-v1",
|
||||
"node",
|
||||
[entry, "plugins", "install", `npm:${packageName}@1.0.0`],
|
||||
runEnv,
|
||||
);
|
||||
assertVersion(pluginId, "1.0.0", runEnv);
|
||||
assertNpmProjectRoot(pluginId, packageName, runEnv);
|
||||
|
||||
await runMeasured(
|
||||
summaryTsv,
|
||||
"inspect-v1",
|
||||
"bash",
|
||||
[
|
||||
"-c",
|
||||
'node "$1" plugins inspect "$2" --runtime --json >"$3"',
|
||||
"bash",
|
||||
entry,
|
||||
pluginId,
|
||||
inspectV1,
|
||||
],
|
||||
runEnv,
|
||||
);
|
||||
assertInspectLoaded(pluginId, inspectV1);
|
||||
|
||||
await runMeasured(
|
||||
summaryTsv,
|
||||
"disable",
|
||||
"node",
|
||||
[entry, "plugins", "disable", pluginId],
|
||||
runEnv,
|
||||
);
|
||||
assertEnabled(pluginId, false, runEnv);
|
||||
|
||||
await runMeasured(summaryTsv, "enable", "node", [entry, "plugins", "enable", pluginId], runEnv);
|
||||
assertEnabled(pluginId, true, runEnv);
|
||||
|
||||
await runMeasured(
|
||||
summaryTsv,
|
||||
"upgrade-v2",
|
||||
"node",
|
||||
[entry, "plugins", "update", `${packageName}@2.0.0`],
|
||||
runEnv,
|
||||
);
|
||||
assertVersion(pluginId, "2.0.0", runEnv);
|
||||
assertNpmProjectRoot(pluginId, packageName, runEnv);
|
||||
|
||||
await runMeasured(
|
||||
summaryTsv,
|
||||
"downgrade-v1",
|
||||
"node",
|
||||
[entry, "plugins", "update", `${packageName}@1.0.0`],
|
||||
runEnv,
|
||||
);
|
||||
assertVersion(pluginId, "1.0.0", runEnv);
|
||||
assertNpmProjectRoot(pluginId, packageName, runEnv);
|
||||
|
||||
const installedPath = installPath(pluginId, runEnv);
|
||||
fs.rmSync(installedPath, { recursive: true, force: true });
|
||||
assertProbe(
|
||||
!fs.existsSync(installedPath),
|
||||
`failed to remove plugin code before missing-code uninstall: ${installedPath}`,
|
||||
);
|
||||
|
||||
await runMeasured(
|
||||
summaryTsv,
|
||||
"missing-code-uninstall",
|
||||
"node",
|
||||
[entry, "plugins", "uninstall", pluginId, "--force"],
|
||||
runEnv,
|
||||
);
|
||||
assertUninstalled(pluginId, runEnv);
|
||||
|
||||
process.stdout.write(
|
||||
`Plugin lifecycle resource summary:\n${fs.readFileSync(summaryTsv, "utf8")}`,
|
||||
);
|
||||
process.stdout.write("Plugin lifecycle matrix passed.\n");
|
||||
} finally {
|
||||
registry?.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export const testing = { runCommand };
|
||||
|
||||
const isLifecycleMatrixCli = process.argv[2] === "--lifecycle-matrix";
|
||||
|
||||
if (isLifecycleMatrixCli) {
|
||||
void (async () => {
|
||||
try {
|
||||
await runPluginLifecycleMatrix();
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
tempDirs.cleanup();
|
||||
}
|
||||
})();
|
||||
}
|
||||
240
test/e2e/qa-lab/plugins/plugin-lifecycle-probe.e2e.test.ts
Normal file
240
test/e2e/qa-lab/plugins/plugin-lifecycle-probe.e2e.test.ts
Normal file
@@ -0,0 +1,240 @@
|
||||
// Plugin Lifecycle Probe tests cover QA Lab plugin lifecycle evidence.
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveWindowsTaskkillPath } from "../../../../scripts/lib/windows-taskkill.mjs";
|
||||
import { createTempDirTracker } from "../../../helpers/temp-dir.js";
|
||||
import {
|
||||
assertInspectLoaded,
|
||||
assertUninstalled,
|
||||
parseDurationMs,
|
||||
testing as probeTesting,
|
||||
} from "./plugin-lifecycle-probe-runtime.js";
|
||||
|
||||
const tempDirs = createTempDirTracker();
|
||||
|
||||
function expectedTaskkillPath(): string {
|
||||
return resolveWindowsTaskkillPath();
|
||||
}
|
||||
|
||||
function makeTempDir(): string {
|
||||
return tempDirs.make("openclaw-plugin-lifecycle-probe-");
|
||||
}
|
||||
|
||||
function isProcessRunning(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForFile(pathToCheck: string, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (existsSync(pathToCheck)) {
|
||||
return;
|
||||
}
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${pathToCheck}`);
|
||||
}
|
||||
|
||||
class FakeCommandChild extends EventEmitter {
|
||||
readonly signals: string[] = [];
|
||||
|
||||
kill(signal?: NodeJS.Signals | number): boolean {
|
||||
this.signals.push(String(signal));
|
||||
if (signal === "SIGTERM") {
|
||||
queueMicrotask(() => this.emit("exit", 0, null));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(tempDirs.cleanup);
|
||||
|
||||
describe("plugin lifecycle matrix probe", () => {
|
||||
it("accepts inspect JSON for an enabled loaded plugin", async () => {
|
||||
const dir = makeTempDir();
|
||||
const inspectPath = path.join(dir, "inspect.json");
|
||||
writeFileSync(
|
||||
inspectPath,
|
||||
`${JSON.stringify({ plugin: { enabled: true, id: "lifecycle-claw", status: "loaded" } })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
expect(() => assertInspectLoaded("lifecycle-claw", inspectPath)).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects inspect JSON that does not prove the runtime loaded", async () => {
|
||||
const dir = makeTempDir();
|
||||
const inspectPath = path.join(dir, "inspect.json");
|
||||
writeFileSync(
|
||||
inspectPath,
|
||||
`${JSON.stringify({ plugin: { enabled: true, id: "lifecycle-claw", status: "pending" } })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
expect(() => assertInspectLoaded("lifecycle-claw", inspectPath)).toThrow(
|
||||
"expected lifecycle-claw inspect status loaded, got pending",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects missing inspect JSON instead of treating it as an empty object", async () => {
|
||||
const dir = makeTempDir();
|
||||
const inspectPath = path.join(dir, "missing.json");
|
||||
|
||||
expect(() => assertInspectLoaded("lifecycle-claw", inspectPath)).toThrow(
|
||||
`failed to read JSON from ${inspectPath}`,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects unreadable config during uninstall proof", async () => {
|
||||
const dir = makeTempDir();
|
||||
const configFile = path.join(dir, ".openclaw", "openclaw.json");
|
||||
mkdirSync(path.dirname(configFile), { recursive: true });
|
||||
writeFileSync(configFile, "{ malformed\n", "utf8");
|
||||
|
||||
expect(() =>
|
||||
assertUninstalled("lifecycle-claw", {
|
||||
HOME: dir,
|
||||
OPENCLAW_CONFIG_PATH: configFile,
|
||||
}),
|
||||
).toThrow(`failed to read JSON from ${configFile}`);
|
||||
});
|
||||
|
||||
it("preserves disabled npm install timeout semantics", () => {
|
||||
expect(parseDurationMs("0", "600s")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects timed commands that exit cleanly during kill grace", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const child = new FakeCommandChild();
|
||||
const runPromise = probeTesting.runCommand("fake-command", ["install"], {
|
||||
spawnImpl: (() => child) as unknown as typeof import("node:child_process").spawn,
|
||||
timeoutKillGraceMs: 100,
|
||||
timeoutMs: 10,
|
||||
});
|
||||
const runError = runPromise.catch((error: unknown) => error);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
const error = await runError;
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as Error).message).toBe("fake-command install timed out after 10ms");
|
||||
expect(child.signals).toEqual(["SIGTERM"]);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(child.signals).toEqual(["SIGTERM"]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("force-kills timed Windows commands with taskkill when graceful taskkill fails", async () => {
|
||||
vi.useFakeTimers();
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
Object.defineProperty(process, "platform", { value: "win32", configurable: true });
|
||||
try {
|
||||
const child = Object.assign(new FakeCommandChild(), { pid: 12345 });
|
||||
const taskkillImpl = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce({ status: 1 })
|
||||
.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => child.emit("exit", null, "SIGTERM"));
|
||||
return { status: 0 };
|
||||
});
|
||||
const runPromise = probeTesting.runCommand("fake-command", ["install"], {
|
||||
spawnImpl: (() => child) as unknown as typeof import("node:child_process").spawn,
|
||||
taskkillImpl,
|
||||
timeoutKillGraceMs: 100,
|
||||
timeoutMs: 10,
|
||||
});
|
||||
const runError = runPromise.catch((error: unknown) => error);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
expect(taskkillImpl).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expectedTaskkillPath(),
|
||||
["/PID", "12345", "/T"],
|
||||
{
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
expect(taskkillImpl).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expectedTaskkillPath(),
|
||||
["/PID", "12345", "/T", "/F"],
|
||||
{
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
expect(child.signals).toEqual([]);
|
||||
|
||||
const error = await runError;
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as Error).message).toBe("fake-command install timed out after 10ms");
|
||||
} finally {
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, "platform", platformDescriptor);
|
||||
}
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps fallback SIGKILL armed for ignored-stdio descendants", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const dir = makeTempDir();
|
||||
const descendantPidPath = path.join(dir, "descendant.pid");
|
||||
let descendantPid: number | undefined;
|
||||
try {
|
||||
const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
|
||||
const parentScript = [
|
||||
"import { spawn } from 'node:child_process';",
|
||||
"import { writeFileSync } from 'node:fs';",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"child.unref();",
|
||||
"writeFileSync(process.env.OPENCLAW_TEST_DESCENDANT_PID, String(child.pid));",
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n");
|
||||
|
||||
const run = probeTesting.runCommand(
|
||||
process.execPath,
|
||||
["--input-type=module", "-e", parentScript],
|
||||
{
|
||||
env: { ...process.env, OPENCLAW_TEST_DESCENDANT_PID: descendantPidPath },
|
||||
timeoutKillGraceMs: 250,
|
||||
timeoutMs: 500,
|
||||
},
|
||||
);
|
||||
await waitForFile(descendantPidPath, 2_000);
|
||||
await sleep(300);
|
||||
|
||||
await expect(run).rejects.toThrow(/timed out after 500ms/u);
|
||||
|
||||
descendantPid = Number(readFileSync(descendantPidPath, "utf8"));
|
||||
expect(isProcessRunning(descendantPid)).toBe(false);
|
||||
} finally {
|
||||
if (descendantPid && isProcessRunning(descendantPid)) {
|
||||
process.kill(descendantPid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
211
test/e2e/qa-lab/runtime/agent-bundle-mcp-tools-docker-client.ts
Normal file
211
test/e2e/qa-lab/runtime/agent-bundle-mcp-tools-docker-client.ts
Normal file
@@ -0,0 +1,211 @@
|
||||
// OpenClaw bundle MCP tools Docker harness.
|
||||
// Imports packaged dist modules so tool materialization is verified against the
|
||||
// npm tarball installed in the functional image.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { materializeBundleMcpToolsForRun } from "../../../../dist/agents/agent-bundle-mcp-materialize.js";
|
||||
import {
|
||||
disposeAllSessionMcpRuntimes,
|
||||
getOrCreateSessionMcpRuntime,
|
||||
} from "../../../../dist/agents/agent-bundle-mcp-runtime.js";
|
||||
import {
|
||||
applyFinalEffectiveToolPolicy,
|
||||
resolveConversationCapabilityProfile,
|
||||
} from "../../../../dist/agents/embedded-agent-runner/effective-tool-policy.js";
|
||||
import { splitSdkTools } from "../../../../dist/agents/embedded-agent-runner/tool-split.js";
|
||||
import type { OpenClawConfig } from "../../../../dist/config/types.openclaw.js";
|
||||
import { getPluginToolMeta } from "../../../../dist/plugins/tools.js";
|
||||
import { createE2eStateDir } from "../../../../scripts/e2e/lib/temp-state-dir.ts";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeProbeServer(serverPath: string) {
|
||||
const sdkMcpServerPath = require.resolve("@modelcontextprotocol/sdk/server/mcp.js");
|
||||
const sdkStdioServerPath = require.resolve("@modelcontextprotocol/sdk/server/stdio.js");
|
||||
await fs.writeFile(
|
||||
serverPath,
|
||||
`#!/usr/bin/env node
|
||||
import { McpServer } from ${JSON.stringify(sdkMcpServerPath)};
|
||||
import { StdioServerTransport } from ${JSON.stringify(sdkStdioServerPath)};
|
||||
|
||||
const server = new McpServer({ name: "agent-bundle-mcp-tools-probe", version: "1.0.0" });
|
||||
server.tool("docker_probe", "Docker OpenClaw MCP tool availability probe", async () => ({
|
||||
content: [{ type: "text", text: "agent-bundle-mcp-tools-ok" }],
|
||||
}));
|
||||
|
||||
await server.connect(new StdioServerTransport());
|
||||
`,
|
||||
{ encoding: "utf-8", mode: 0o755 },
|
||||
);
|
||||
}
|
||||
|
||||
function applyPolicy(params: {
|
||||
tools: Awaited<ReturnType<typeof materializeBundleMcpToolsForRun>>["tools"];
|
||||
config: OpenClawConfig;
|
||||
}) {
|
||||
const warnings: string[] = [];
|
||||
return {
|
||||
tools: applyFinalEffectiveToolPolicy({
|
||||
bundledTools: params.tools,
|
||||
config: params.config,
|
||||
conversationCapabilityProfile: resolveConversationCapabilityProfile({
|
||||
config: params.config,
|
||||
sessionKey: "agent:main:docker-agent-bundle-mcp",
|
||||
agentId: "main",
|
||||
}),
|
||||
warn: (message) => {
|
||||
warnings.push(message);
|
||||
},
|
||||
}),
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tempState = await createE2eStateDir("openclaw-agent-bundle-mcp-");
|
||||
tempState.registerExitCleanup();
|
||||
const stateDir = tempState.stateDir;
|
||||
const probeDir = path.join(stateDir, "agent-bundle-mcp-tools");
|
||||
const serverPath = path.join(probeDir, "probe-server.mjs");
|
||||
await fs.mkdir(probeDir, { recursive: true });
|
||||
await writeProbeServer(serverPath);
|
||||
|
||||
const cfg: OpenClawConfig = {
|
||||
tools: {
|
||||
profile: "coding",
|
||||
},
|
||||
mcp: {
|
||||
servers: {
|
||||
dockerProbe: {
|
||||
command: "node",
|
||||
args: [serverPath],
|
||||
cwd: probeDir,
|
||||
connectionTimeoutMs: 5000,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const runtime = await getOrCreateSessionMcpRuntime({
|
||||
sessionId: `docker-agent-bundle-mcp-${randomUUID()}`,
|
||||
sessionKey: "agent:main:docker-agent-bundle-mcp",
|
||||
workspaceDir: probeDir,
|
||||
cfg,
|
||||
});
|
||||
const materialized = await materializeBundleMcpToolsForRun({ runtime });
|
||||
const probeTool = materialized.tools.find((tool) => tool.name === "dockerProbe__docker_probe");
|
||||
assert(probeTool, "expected dockerProbe__docker_probe to materialize");
|
||||
assert(
|
||||
getPluginToolMeta(probeTool)?.pluginId === "bundle-mcp",
|
||||
"expected materialized MCP tool to be tagged as bundle-mcp",
|
||||
);
|
||||
|
||||
const result = await probeTool.execute("docker-mcp-probe", {}, undefined, undefined);
|
||||
assert(
|
||||
result.content.some(
|
||||
(item) => item.type === "text" && item.text === "agent-bundle-mcp-tools-ok",
|
||||
),
|
||||
"expected materialized MCP tool execution result",
|
||||
);
|
||||
|
||||
const coding = applyPolicy({ tools: materialized.tools, config: cfg });
|
||||
assert(
|
||||
coding.tools.some((tool) => tool.name === probeTool.name),
|
||||
"expected coding profile to keep bundle MCP tools",
|
||||
);
|
||||
|
||||
const messaging = applyPolicy({
|
||||
tools: materialized.tools,
|
||||
config: { ...cfg, tools: { profile: "messaging" } },
|
||||
});
|
||||
assert(
|
||||
messaging.tools.some((tool) => tool.name === probeTool.name),
|
||||
"expected messaging profile to keep bundle MCP tools",
|
||||
);
|
||||
|
||||
const minimal = applyPolicy({
|
||||
tools: materialized.tools,
|
||||
config: { ...cfg, tools: { profile: "minimal" } },
|
||||
});
|
||||
assert(minimal.tools.length === 0, "expected minimal profile to filter bundle MCP tools");
|
||||
|
||||
const denied = applyPolicy({
|
||||
tools: materialized.tools,
|
||||
config: { ...cfg, tools: { profile: "coding", deny: ["bundle-mcp"] } },
|
||||
});
|
||||
assert(denied.tools.length === 0, "expected tools.deny bundle-mcp to filter MCP tools");
|
||||
|
||||
// The disputed boundary on #76063 is what reaches the SDK as `customTools`,
|
||||
// since that is the exact value serialized to the outbound provider request.
|
||||
// Prove the live stdio probe survives the materialize -> filter -> split chain
|
||||
// through `splitSdkTools` for the same four profiles already asserted above.
|
||||
const codingCustom = splitSdkTools({ tools: coding.tools, sandboxEnabled: false }).customTools;
|
||||
const messagingCustom = splitSdkTools({
|
||||
tools: messaging.tools,
|
||||
sandboxEnabled: false,
|
||||
}).customTools;
|
||||
const minimalCustom = splitSdkTools({
|
||||
tools: minimal.tools,
|
||||
sandboxEnabled: false,
|
||||
}).customTools;
|
||||
const deniedCustom = splitSdkTools({ tools: denied.tools, sandboxEnabled: false }).customTools;
|
||||
assert(
|
||||
codingCustom.some((tool) => tool.name === probeTool.name),
|
||||
"expected coding profile customTools to include bundle MCP tools",
|
||||
);
|
||||
assert(
|
||||
messagingCustom.some((tool) => tool.name === probeTool.name),
|
||||
"expected messaging profile customTools to include bundle MCP tools",
|
||||
);
|
||||
assert(
|
||||
minimalCustom.length === 0,
|
||||
"expected minimal profile customTools to exclude bundle MCP tools",
|
||||
);
|
||||
assert(
|
||||
deniedCustom.length === 0,
|
||||
"expected tools.deny bundle-mcp customTools to exclude bundle MCP tools",
|
||||
);
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: true,
|
||||
tool: probeTool.name,
|
||||
profileCounts: {
|
||||
coding: coding.tools.length,
|
||||
messaging: messaging.tools.length,
|
||||
minimal: minimal.tools.length,
|
||||
denied: denied.tools.length,
|
||||
},
|
||||
customToolsCounts: {
|
||||
coding: codingCustom.length,
|
||||
messaging: messagingCustom.length,
|
||||
minimal: minimalCustom.length,
|
||||
denied: deniedCustom.length,
|
||||
},
|
||||
customToolNames: {
|
||||
coding: codingCustom.map((tool) => tool.name),
|
||||
messaging: messagingCustom.map((tool) => tool.name),
|
||||
minimal: minimalCustom.map((tool) => tool.name),
|
||||
denied: deniedCustom.map((tool) => tool.name),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
);
|
||||
} finally {
|
||||
await disposeAllSessionMcpRuntimes();
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
187
test/e2e/qa-lab/runtime/crestodian-first-run-docker-client.ts
Normal file
187
test/e2e/qa-lab/runtime/crestodian-first-run-docker-client.ts
Normal file
@@ -0,0 +1,187 @@
|
||||
// Crestodian first-run Docker harness.
|
||||
// Imports packaged dist modules so the Docker lane verifies the npm tarball,
|
||||
// while this small test driver stays mounted from the checkout.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
runCli,
|
||||
shouldStartCrestodianForModernOnboard,
|
||||
shouldStartOnboardingForFreshInstall,
|
||||
} from "../../../../dist/cli/run-main.js";
|
||||
import { clearConfigCache } from "../../../../dist/config/config.js";
|
||||
import type { OpenClawConfig } from "../../../../dist/config/types.openclaw.js";
|
||||
import { runCrestodian } from "../../../../dist/crestodian/crestodian.js";
|
||||
import type { RuntimeEnv } from "../../../../dist/runtime.js";
|
||||
import { createE2eStateDir } from "../../../../scripts/e2e/lib/temp-state-dir.ts";
|
||||
|
||||
type CrestodianFirstRunCommand = {
|
||||
id: string;
|
||||
message: string;
|
||||
expectOutput: string;
|
||||
approve: boolean;
|
||||
};
|
||||
|
||||
type CrestodianFirstRunSpec = {
|
||||
dockerDefaultWorkspace: string;
|
||||
dockerAgentWorkspace: string;
|
||||
agentId: string;
|
||||
model: string;
|
||||
discordEnv: string;
|
||||
discordToken: string;
|
||||
commands: CrestodianFirstRunCommand[];
|
||||
auditOperations: string[];
|
||||
};
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function setEnvValue(key: string, value: string): void {
|
||||
Reflect.set(process.env, key, value);
|
||||
}
|
||||
|
||||
function createRuntime(): { runtime: RuntimeEnv; lines: string[] } {
|
||||
const lines: string[] = [];
|
||||
return {
|
||||
lines,
|
||||
runtime: {
|
||||
log: (...args) => lines.push(args.join(" ")),
|
||||
error: (...args) => lines.push(args.join(" ")),
|
||||
exit: (code) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readFirstRunSpec(): Promise<CrestodianFirstRunSpec> {
|
||||
return JSON.parse(
|
||||
await fs.readFile(
|
||||
path.join(process.cwd(), "scripts", "e2e", "crestodian-first-run-spec.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as CrestodianFirstRunSpec;
|
||||
}
|
||||
|
||||
function renderCommandTemplate(template: string, vars: Record<string, string>): string {
|
||||
return template.replace(/\{([A-Za-z0-9_]+)\}/g, (match, key: string) => vars[key] ?? match);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const spec = await readFirstRunSpec();
|
||||
const tempState = await createE2eStateDir("openclaw-crestodian-first-run-");
|
||||
tempState.registerExitCleanup();
|
||||
const stateDir = tempState.stateDir;
|
||||
const configPath = process.env.OPENCLAW_CONFIG_PATH ?? path.join(stateDir, "openclaw.json");
|
||||
setEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
setEnvValue("OPENCLAW_CONFIG_PATH", configPath);
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
await fs.mkdir(stateDir, { recursive: true });
|
||||
clearConfigCache();
|
||||
|
||||
assert(
|
||||
await shouldStartOnboardingForFreshInstall(["node", "openclaw"]),
|
||||
"fresh bare OpenClaw invocation did not route to onboarding",
|
||||
);
|
||||
assert(
|
||||
shouldStartCrestodianForModernOnboard(["node", "openclaw", "onboard", "--modern"]),
|
||||
"modern onboard invocation did not route to Crestodian",
|
||||
);
|
||||
process.exitCode = undefined;
|
||||
await runCli(["node", "openclaw", "onboard", "--modern", "--non-interactive", "--json"]);
|
||||
assert(
|
||||
process.exitCode === undefined || process.exitCode === 0,
|
||||
"modern onboard overview exited nonzero",
|
||||
);
|
||||
|
||||
const overviewRuntime = createRuntime();
|
||||
await runCrestodian({ message: "overview", interactive: false }, overviewRuntime.runtime);
|
||||
const overviewOutput = overviewRuntime.lines.join("\n");
|
||||
assert(
|
||||
overviewOutput.includes("Config: missing"),
|
||||
"fresh overview did not report missing config",
|
||||
);
|
||||
assert(
|
||||
overviewOutput.includes('Next: run "setup" to create a starter config'),
|
||||
"fresh overview did not include setup recommendation",
|
||||
);
|
||||
|
||||
setEnvValue(spec.discordEnv, spec.discordToken);
|
||||
|
||||
const commandVars = {
|
||||
defaultWorkspace: spec.dockerDefaultWorkspace,
|
||||
agentWorkspace: spec.dockerAgentWorkspace,
|
||||
agentId: spec.agentId,
|
||||
model: spec.model,
|
||||
discordEnv: spec.discordEnv,
|
||||
};
|
||||
for (const command of spec.commands) {
|
||||
clearConfigCache();
|
||||
const commandRuntime = createRuntime();
|
||||
await runCrestodian(
|
||||
{
|
||||
message: renderCommandTemplate(command.message, commandVars),
|
||||
yes: command.approve,
|
||||
interactive: false,
|
||||
},
|
||||
commandRuntime.runtime,
|
||||
);
|
||||
const output = commandRuntime.lines.join("\n");
|
||||
assert(
|
||||
output.includes(command.expectOutput),
|
||||
`Crestodian first-run command ${command.id} did not apply: ${output}`,
|
||||
);
|
||||
}
|
||||
|
||||
const config = JSON.parse(await fs.readFile(configPath, "utf8")) as OpenClawConfig;
|
||||
assert(
|
||||
config.agents?.defaults?.workspace === spec.dockerDefaultWorkspace,
|
||||
"first-run setup did not write default workspace",
|
||||
);
|
||||
assert(
|
||||
config.agents?.defaults?.model &&
|
||||
typeof config.agents.defaults.model === "object" &&
|
||||
"primary" in config.agents.defaults.model &&
|
||||
config.agents.defaults.model.primary === spec.model,
|
||||
"first-run setup did not write default model",
|
||||
);
|
||||
const reef = config.agents?.list?.find((agent) => agent.id === spec.agentId);
|
||||
assert(reef, "Crestodian did not create reef agent");
|
||||
assert(reef.workspace === spec.dockerAgentWorkspace, "Crestodian did not write reef workspace");
|
||||
assert(reef.model === spec.model, "Crestodian did not write reef model");
|
||||
assert(config.plugins?.allow?.includes("discord"), "Crestodian did not allow Discord plugin");
|
||||
assert(
|
||||
config.plugins?.entries?.discord?.enabled === true,
|
||||
"Crestodian did not enable Discord plugin entry",
|
||||
);
|
||||
assert(config.channels?.discord?.enabled === true, "Crestodian did not enable Discord");
|
||||
const discordToken = config.channels?.discord?.token;
|
||||
assert(
|
||||
discordToken &&
|
||||
typeof discordToken === "object" &&
|
||||
"source" in discordToken &&
|
||||
discordToken.source === "env" &&
|
||||
"id" in discordToken &&
|
||||
discordToken.id === spec.discordEnv,
|
||||
"Crestodian did not write Discord token SecretRef",
|
||||
);
|
||||
assert(
|
||||
!JSON.stringify(config.channels.discord).includes(spec.discordToken),
|
||||
"Crestodian persisted the raw Discord token",
|
||||
);
|
||||
|
||||
const auditPath = path.join(stateDir, "audit", "crestodian.jsonl");
|
||||
const audit = (await fs.readFile(auditPath, "utf8")).trim();
|
||||
for (const operation of spec.auditOperations) {
|
||||
assert(audit.includes(`"operation":"${operation}"`), `${operation} audit entry missing`);
|
||||
}
|
||||
|
||||
console.log("Crestodian first-run Docker E2E passed");
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
84
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.test.ts
Normal file
84
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.test.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
// Docker E2E lane fixture tests keep QA scenario dispatch policy reusable.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
formatQaDockerE2eLaneUsage,
|
||||
listQaDockerE2eLaneNames,
|
||||
parseQaDockerE2eLaneArgs,
|
||||
resolveQaDockerE2eLane,
|
||||
runQaDockerE2eLane,
|
||||
} from "./docker-e2e-lane.fixture.ts";
|
||||
|
||||
describe("QA Docker E2E lane fixture", () => {
|
||||
it("lists known Docker lanes for scenario wrappers", () => {
|
||||
expect(listQaDockerE2eLaneNames()).toEqual(
|
||||
expect.arrayContaining([
|
||||
"agent-bundle-mcp-tools",
|
||||
"crestodian-first-run",
|
||||
"gateway-network",
|
||||
"release-plugin-marketplace",
|
||||
"update-migration",
|
||||
"update-restart-auth",
|
||||
]),
|
||||
);
|
||||
expect(listQaDockerE2eLaneNames()).toEqual([...listQaDockerE2eLaneNames()].sort());
|
||||
});
|
||||
|
||||
it("parses help, list, and lane arguments", () => {
|
||||
expect(parseQaDockerE2eLaneArgs(["--help"])).toEqual({ kind: "help" });
|
||||
expect(parseQaDockerE2eLaneArgs(["--list"])).toEqual({ kind: "list" });
|
||||
expect(parseQaDockerE2eLaneArgs(["--lane", "gateway-network"])).toEqual({
|
||||
kind: "run",
|
||||
laneName: "gateway-network",
|
||||
});
|
||||
|
||||
expect(() => parseQaDockerE2eLaneArgs([])).toThrow("--lane is required");
|
||||
expect(() => parseQaDockerE2eLaneArgs(["--lane"])).toThrow("--lane requires a value");
|
||||
});
|
||||
|
||||
it("renders usage from the shared lane registry", () => {
|
||||
const usage = formatQaDockerE2eLaneUsage("node qa-docker.js");
|
||||
|
||||
expect(usage).toContain("Usage: node qa-docker.js --lane <name>");
|
||||
expect(usage).toContain(" - gateway-network");
|
||||
expect(usage).toContain(" - update-restart-auth");
|
||||
});
|
||||
|
||||
it("resolves lane-specific environment overlays at run time", () => {
|
||||
const updateMigration = resolveQaDockerE2eLane("update-migration", {
|
||||
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: "openclaw@custom",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: "custom-scenario",
|
||||
});
|
||||
|
||||
expect(updateMigration.script).toBe("scripts/e2e/upgrade-survivor-docker.sh");
|
||||
expect(updateMigration.env.OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE).toBe("1");
|
||||
expect(updateMigration.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC).toBe("openclaw@custom");
|
||||
expect(updateMigration.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO).toBe("custom-scenario");
|
||||
|
||||
const updateRestartAuth = resolveQaDockerE2eLane("update-restart-auth", {});
|
||||
expect(updateRestartAuth.env.OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE).toBe("auto-auth");
|
||||
expect(updateRestartAuth.env.OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT).toBe("1500s");
|
||||
});
|
||||
|
||||
it("dispatches through bash without running Docker in fixture tests", () => {
|
||||
const spawn = vi.fn(() => ({ signal: null, status: 0 }));
|
||||
|
||||
expect(runQaDockerE2eLane("gateway-network", { env: { EXTRA: "1" }, spawn })).toEqual({
|
||||
signal: null,
|
||||
status: 0,
|
||||
});
|
||||
|
||||
expect(spawn).toHaveBeenCalledWith("bash", ["scripts/e2e/gateway-network-docker.sh"], {
|
||||
env: { EXTRA: "1" },
|
||||
stdio: "inherit",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects unknown lanes before spawning", () => {
|
||||
const spawn = vi.fn(() => ({ signal: null, status: 0 }));
|
||||
|
||||
expect(() => runQaDockerE2eLane("missing-lane", { spawn })).toThrow(
|
||||
"unknown Docker E2E lane: missing-lane",
|
||||
);
|
||||
expect(spawn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
169
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.ts
Normal file
169
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
// Shared QA Lab fixture for dispatching existing Docker E2E lanes.
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
export type QaDockerE2eLaneDefinition = {
|
||||
env?: (env: NodeJS.ProcessEnv) => Record<string, string>;
|
||||
script: string;
|
||||
};
|
||||
|
||||
type QaDockerE2eLaneRunResult = {
|
||||
error?: Error;
|
||||
signal: NodeJS.Signals | null;
|
||||
status: number | null;
|
||||
};
|
||||
|
||||
type SpawnQaDockerE2eLane = (
|
||||
command: string,
|
||||
args: string[],
|
||||
options: { env: NodeJS.ProcessEnv; stdio: "inherit" },
|
||||
) => QaDockerE2eLaneRunResult;
|
||||
|
||||
export const QA_DOCKER_E2E_LANES = {
|
||||
"agent-bundle-mcp-tools": {
|
||||
script: "scripts/e2e/agent-bundle-mcp-tools-docker.sh",
|
||||
},
|
||||
"agents-delete-shared-workspace": {
|
||||
script: "scripts/e2e/agents-delete-shared-workspace-docker.sh",
|
||||
},
|
||||
"bundled-plugin-install-uninstall": {
|
||||
script: "scripts/e2e/bundled-plugin-install-uninstall-docker.sh",
|
||||
},
|
||||
"crestodian-first-run": {
|
||||
script: "scripts/e2e/crestodian-first-run-docker.sh",
|
||||
},
|
||||
"docker-build-image": {
|
||||
script: "scripts/e2e/build-image.sh",
|
||||
},
|
||||
"gateway-network": {
|
||||
script: "scripts/e2e/gateway-network-docker.sh",
|
||||
},
|
||||
"npm-onboard-channel-agent": {
|
||||
script: "scripts/e2e/npm-onboard-channel-agent-docker.sh",
|
||||
},
|
||||
"openai-chat-tools": {
|
||||
script: "scripts/e2e/openai-chat-tools-docker.sh",
|
||||
},
|
||||
"openai-web-search-minimal": {
|
||||
script: "scripts/e2e/openai-web-search-minimal-docker.sh",
|
||||
},
|
||||
openwebui: {
|
||||
script: "scripts/e2e/openwebui-docker.sh",
|
||||
},
|
||||
"plugin-lifecycle-matrix": {
|
||||
script: "scripts/e2e/plugin-lifecycle-matrix-docker.sh",
|
||||
},
|
||||
"release-plugin-marketplace": {
|
||||
script: "scripts/e2e/release-plugin-marketplace-docker.sh",
|
||||
},
|
||||
"release-upgrade-user-journey": {
|
||||
script: "scripts/e2e/release-upgrade-user-journey-docker.sh",
|
||||
},
|
||||
"release-user-journey": {
|
||||
script: "scripts/e2e/release-user-journey-docker.sh",
|
||||
},
|
||||
"update-channel-switch": {
|
||||
script: "scripts/e2e/update-channel-switch-docker.sh",
|
||||
},
|
||||
"update-migration": {
|
||||
env: (env) => ({
|
||||
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC:
|
||||
env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC ?? "openclaw@2026.4.23",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "1",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO:
|
||||
env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO ?? "plugin-deps-cleanup",
|
||||
}),
|
||||
script: "scripts/e2e/upgrade-survivor-docker.sh",
|
||||
},
|
||||
"update-restart-auth": {
|
||||
env: (env) => ({
|
||||
OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT:
|
||||
env.OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT ?? "1500s",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE: "auto-auth",
|
||||
}),
|
||||
script: "scripts/e2e/upgrade-survivor-docker.sh",
|
||||
},
|
||||
"upgrade-survivor": {
|
||||
script: "scripts/e2e/upgrade-survivor-docker.sh",
|
||||
},
|
||||
} satisfies Record<string, QaDockerE2eLaneDefinition>;
|
||||
|
||||
export type QaDockerE2eLaneName = keyof typeof QA_DOCKER_E2E_LANES;
|
||||
|
||||
export type QaDockerE2eLaneArgs =
|
||||
| { kind: "help" }
|
||||
| { kind: "list" }
|
||||
| { kind: "run"; laneName: string };
|
||||
|
||||
export type ResolvedQaDockerE2eLane = {
|
||||
env: NodeJS.ProcessEnv;
|
||||
name: QaDockerE2eLaneName;
|
||||
script: string;
|
||||
};
|
||||
|
||||
export function listQaDockerE2eLaneNames(): QaDockerE2eLaneName[] {
|
||||
return Object.keys(QA_DOCKER_E2E_LANES).toSorted() as QaDockerE2eLaneName[];
|
||||
}
|
||||
|
||||
export function formatQaDockerE2eLaneUsage(
|
||||
entrypoint = "node --import tsx test/e2e/qa-lab/runtime/docker-e2e-lane.ts",
|
||||
): string {
|
||||
return [
|
||||
`Usage: ${entrypoint} --lane <name>`,
|
||||
"",
|
||||
"Known lanes:",
|
||||
...listQaDockerE2eLaneNames().map((lane) => ` - ${lane}`),
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function parseQaDockerE2eLaneArgs(argv: string[]): QaDockerE2eLaneArgs {
|
||||
if (argv.includes("--help") || argv.includes("-h")) {
|
||||
return { kind: "help" };
|
||||
}
|
||||
if (argv.includes("--list")) {
|
||||
return { kind: "list" };
|
||||
}
|
||||
const index = argv.indexOf("--lane");
|
||||
if (index === -1) {
|
||||
throw new Error("--lane is required");
|
||||
}
|
||||
const laneName = argv[index + 1];
|
||||
if (!laneName || laneName.startsWith("-")) {
|
||||
throw new Error("--lane requires a value");
|
||||
}
|
||||
return { kind: "run", laneName };
|
||||
}
|
||||
|
||||
export function resolveQaDockerE2eLane(
|
||||
laneName: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): ResolvedQaDockerE2eLane {
|
||||
if (!isQaDockerE2eLaneName(laneName)) {
|
||||
throw new Error(`unknown Docker E2E lane: ${laneName}\n\n${formatQaDockerE2eLaneUsage()}`);
|
||||
}
|
||||
const lane = QA_DOCKER_E2E_LANES[laneName];
|
||||
return {
|
||||
env: { ...env, ...lane.env?.(env) },
|
||||
name: laneName,
|
||||
script: lane.script,
|
||||
};
|
||||
}
|
||||
|
||||
export function runQaDockerE2eLane(
|
||||
laneName: string,
|
||||
deps: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
spawn?: SpawnQaDockerE2eLane;
|
||||
} = {},
|
||||
): QaDockerE2eLaneRunResult {
|
||||
const lane = resolveQaDockerE2eLane(laneName, deps.env);
|
||||
const spawn = deps.spawn ?? spawnSync;
|
||||
return spawn("bash", [lane.script], {
|
||||
env: lane.env,
|
||||
stdio: "inherit",
|
||||
});
|
||||
}
|
||||
|
||||
function isQaDockerE2eLaneName(laneName: string): laneName is QaDockerE2eLaneName {
|
||||
return Object.hasOwn(QA_DOCKER_E2E_LANES, laneName);
|
||||
}
|
||||
28
test/e2e/qa-lab/runtime/docker-e2e-lane.ts
Normal file
28
test/e2e/qa-lab/runtime/docker-e2e-lane.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
// Runs an existing Docker E2E lane through the QA Lab script scenario contract.
|
||||
import {
|
||||
formatQaDockerE2eLaneUsage,
|
||||
listQaDockerE2eLaneNames,
|
||||
parseQaDockerE2eLaneArgs,
|
||||
runQaDockerE2eLane,
|
||||
} from "./docker-e2e-lane.fixture.ts";
|
||||
|
||||
const args = parseQaDockerE2eLaneArgs(process.argv.slice(2));
|
||||
if (args.kind === "help") {
|
||||
console.log(formatQaDockerE2eLaneUsage());
|
||||
process.exit(0);
|
||||
}
|
||||
if (args.kind === "list") {
|
||||
console.log(listQaDockerE2eLaneNames().join("\n"));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const result = runQaDockerE2eLane(args.laneName);
|
||||
|
||||
if (result.error) {
|
||||
console.error(result.error);
|
||||
process.exit(1);
|
||||
}
|
||||
if (result.signal) {
|
||||
process.kill(process.pid, result.signal);
|
||||
}
|
||||
process.exit(result.status ?? 1);
|
||||
99
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts
Normal file
99
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../../helpers/temp-dir.js";
|
||||
import { testing } from "./gateway-mcp-real-transports.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
function createRepoRoot() {
|
||||
return tempDirs.make("openclaw-qalab-cli-entry-");
|
||||
}
|
||||
|
||||
async function writeEntry(root: string, relativePath: string) {
|
||||
const entryPath = path.join(root, relativePath);
|
||||
await mkdir(path.dirname(entryPath), { recursive: true });
|
||||
await writeFile(entryPath, "", "utf8");
|
||||
return entryPath;
|
||||
}
|
||||
|
||||
describe("gateway MCP real transport producer", () => {
|
||||
it("uses the source CLI entry when build output is absent", async () => {
|
||||
const root = createRepoRoot();
|
||||
const entryPath = await writeEntry(root, "src/entry.ts");
|
||||
|
||||
const cli = testing.resolveOpenClawCliInvocation(root);
|
||||
|
||||
expect(cli.command).toBe(process.execPath);
|
||||
expect(cli.argsPrefix).toStrictEqual(["--import", "tsx", entryPath]);
|
||||
expect(cli.cwd).toBe(root);
|
||||
expect(cli.gatewayCommand).toMatchObject({
|
||||
executablePath: process.execPath,
|
||||
argsPrefix: ["--import", "tsx", entryPath],
|
||||
cwd: root,
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers built package output when it exists", async () => {
|
||||
const root = createRepoRoot();
|
||||
const distEntry = await writeEntry(root, "dist/index.mjs");
|
||||
await writeEntry(root, "src/entry.ts");
|
||||
|
||||
const cli = testing.resolveOpenClawCliInvocation(root);
|
||||
|
||||
expect(cli.argsPrefix).toStrictEqual([distEntry]);
|
||||
expect(cli.gatewayCommand).toMatchObject({
|
||||
argsPrefix: [distEntry],
|
||||
usePackagedPlugins: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the source channel MCP module when build output is absent", async () => {
|
||||
const root = createRepoRoot();
|
||||
const channelServerPath = await writeEntry(root, "src/mcp/channel-server.ts");
|
||||
|
||||
const mcp = testing.resolveChannelMcpInvocation({
|
||||
gatewayToken: "secret-token",
|
||||
gatewayUrl: "ws://127.0.0.1:12345",
|
||||
repoRoot: root,
|
||||
tokenFile: "/tmp/token-file",
|
||||
});
|
||||
|
||||
expect(mcp.command).toBe(process.execPath);
|
||||
expect(mcp.args.slice(0, 3)).toStrictEqual(["--import", "tsx", "--eval"]);
|
||||
expect(mcp.args[3]).toContain(channelServerPath);
|
||||
expect(mcp.args[3]).toContain("serveOpenClawChannelMcp");
|
||||
expect(mcp.cwd).toBe(root);
|
||||
expect(mcp.envPatch).toStrictEqual({
|
||||
OPENCLAW_QA_GATEWAY_TOKEN: "secret-token",
|
||||
OPENCLAW_QA_GATEWAY_URL: "ws://127.0.0.1:12345",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the packaged CLI for channel MCP when build output exists", async () => {
|
||||
const root = createRepoRoot();
|
||||
const distEntry = await writeEntry(root, "dist/index.js");
|
||||
await writeEntry(root, "src/mcp/channel-server.ts");
|
||||
|
||||
const mcp = testing.resolveChannelMcpInvocation({
|
||||
gatewayToken: "secret-token",
|
||||
gatewayUrl: "ws://127.0.0.1:12345",
|
||||
repoRoot: root,
|
||||
tokenFile: "/tmp/token-file",
|
||||
});
|
||||
|
||||
expect(mcp.args).toStrictEqual([
|
||||
distEntry,
|
||||
"mcp",
|
||||
"serve",
|
||||
"--url",
|
||||
"ws://127.0.0.1:12345",
|
||||
"--token-file",
|
||||
"/tmp/token-file",
|
||||
"--claude-channel-mode",
|
||||
"off",
|
||||
"--verbose",
|
||||
]);
|
||||
expect(mcp.envPatch).toStrictEqual({});
|
||||
});
|
||||
});
|
||||
817
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
Normal file
817
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
Normal file
@@ -0,0 +1,817 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
// QA Lab producer proves Gateway and MCP scenarios across real process and protocol boundaries.
|
||||
import { createServer, type Server } from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
||||
import {
|
||||
QA_EVIDENCE_FILENAME,
|
||||
startQaGatewayChild,
|
||||
type QaGatewayChildCommand,
|
||||
type QaEvidenceSummaryJson,
|
||||
type QaGatewayChildListeningContext,
|
||||
} from "../../../../extensions/qa-lab/api.js";
|
||||
import {
|
||||
PROTOCOL_VERSION,
|
||||
MIN_CLIENT_PROTOCOL_VERSION,
|
||||
} from "../../../../packages/gateway-protocol/src/version.js";
|
||||
import { runGatewaySmoke } from "../../../../scripts/dev/gateway-smoke.js";
|
||||
import type { OpenClawConfig } from "../../../../src/config/types.openclaw.js";
|
||||
import { formatErrorMessage } from "../../../../src/infra/errors.js";
|
||||
import { createMcpClientTempState } from "./mcp-client-temp-state.fixture.ts";
|
||||
import { createQaScriptEvidenceWriter, type QaScriptEvidenceStatus } from "./script-evidence.ts";
|
||||
|
||||
const FIXTURE_PLUGIN_ID = "qa-real-transports-fixture";
|
||||
const FIXTURE_TOOL_NAME = "memory_search";
|
||||
const FIXTURE_FACT = "MCP fact: the codename is ORBIT-9.";
|
||||
const STARTUP_GATE_TIMEOUT_MS = 30_000;
|
||||
const MCP_CONNECT_TIMEOUT_MS = 30_000;
|
||||
const SOURCE_PATH = "test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts";
|
||||
|
||||
type ScenarioId = "gateway-smoke" | "mcp-gateway-connect-startup-retry" | "mcp-plugin-tools-call";
|
||||
|
||||
type ProducerOptions = {
|
||||
artifactBase: string;
|
||||
repoRoot: string;
|
||||
scenarioId: ScenarioId;
|
||||
};
|
||||
|
||||
type ProofResult = {
|
||||
details?: string;
|
||||
durationMs: number;
|
||||
status: QaScriptEvidenceStatus;
|
||||
};
|
||||
|
||||
type GatewayFrameCapture = {
|
||||
connectFrames: Array<{ minProtocol: number; maxProtocol: number }>;
|
||||
helloProtocols: number[];
|
||||
startupUnavailableResponses: number;
|
||||
};
|
||||
|
||||
type GatewayProxy = {
|
||||
capture: GatewayFrameCapture;
|
||||
stop: () => Promise<void>;
|
||||
url: string;
|
||||
};
|
||||
|
||||
type OpenClawCliInvocation = {
|
||||
argsPrefix: string[];
|
||||
command: string;
|
||||
cwd: string;
|
||||
gatewayCommand: QaGatewayChildCommand;
|
||||
};
|
||||
|
||||
type ChannelMcpInvocation = {
|
||||
args: string[];
|
||||
command: string;
|
||||
cwd: string;
|
||||
envPatch: NodeJS.ProcessEnv;
|
||||
};
|
||||
|
||||
type McpClientHandle = {
|
||||
client: Client;
|
||||
cleanup: () => void;
|
||||
stderr: () => string;
|
||||
transport: StdioClientTransport;
|
||||
};
|
||||
|
||||
const SCENARIOS = {
|
||||
"gateway-smoke": {
|
||||
title: "Gateway smoke evidence",
|
||||
sourcePath: "qa/scenarios/runtime/gateway-smoke.yaml",
|
||||
primaryCoverageIds: [
|
||||
"gateway.websocket-transport",
|
||||
"gateway.health-apis",
|
||||
"gateway.hello-ok-snapshot",
|
||||
],
|
||||
docsRefs: ["docs/gateway/index.md", "docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: [
|
||||
SOURCE_PATH,
|
||||
"extensions/qa-lab/src/gateway-child.ts",
|
||||
"scripts/dev/gateway-smoke.ts",
|
||||
],
|
||||
},
|
||||
"mcp-gateway-connect-startup-retry": {
|
||||
title: "MCP Gateway connect startup retry",
|
||||
sourcePath: "qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml",
|
||||
primaryCoverageIds: [
|
||||
"gateway.connect-request",
|
||||
"gateway.protocol-version-negotiation",
|
||||
"gateway.startup-retry",
|
||||
],
|
||||
docsRefs: ["docs/gateway/protocol.md", "docs/cli/mcp.md"],
|
||||
codeRefs: [SOURCE_PATH, "extensions/qa-lab/src/gateway-child.ts", "src/mcp/channel-bridge.ts"],
|
||||
},
|
||||
"mcp-plugin-tools-call": {
|
||||
title: "MCP plugin-tools call",
|
||||
sourcePath: "qa/scenarios/plugins/mcp-plugin-tools-call.yaml",
|
||||
primaryCoverageIds: ["plugins.mcp-tools", "tools.invocation"],
|
||||
docsRefs: ["docs/cli/mcp.md", "docs/gateway/protocol.md"],
|
||||
codeRefs: [SOURCE_PATH, "src/mcp/plugin-tools-serve.ts", "src/mcp/plugin-tools-handlers.ts"],
|
||||
},
|
||||
} as const;
|
||||
|
||||
function parseOptions(argv: readonly string[]): ProducerOptions {
|
||||
const readValue = (name: string) => {
|
||||
const index = argv.indexOf(name);
|
||||
return index >= 0 ? argv[index + 1] : undefined;
|
||||
};
|
||||
const scenarioId = readValue("--scenario");
|
||||
if (!scenarioId || !(scenarioId in SCENARIOS)) {
|
||||
throw new Error(`--scenario must be one of: ${Object.keys(SCENARIOS).join(", ")}`);
|
||||
}
|
||||
const artifactBase = readValue("--artifact-base");
|
||||
if (!artifactBase) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
return {
|
||||
artifactBase: path.resolve(artifactBase),
|
||||
repoRoot: path.resolve(readValue("--repo-root") ?? process.cwd()),
|
||||
scenarioId: scenarioId as ScenarioId,
|
||||
};
|
||||
}
|
||||
|
||||
async function createFixturePlugin() {
|
||||
// openclaw-temp-dir: allow standalone producer cleans this root in each scenario finally block
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gateway-mcp-fixture-"));
|
||||
const pluginDir = path.join(root, FIXTURE_PLUGIN_ID);
|
||||
const startupGatePath = path.join(root, "startup-connect-observed");
|
||||
await fs.mkdir(pluginDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(pluginDir, "openclaw.plugin.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
id: FIXTURE_PLUGIN_ID,
|
||||
activation: { onStartup: true },
|
||||
configSchema: { type: "object", additionalProperties: false, properties: {} },
|
||||
contracts: { tools: [FIXTURE_TOOL_NAME] },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(pluginDir, "index.js"),
|
||||
`const fs = require("node:fs");
|
||||
|
||||
module.exports = {
|
||||
id: ${JSON.stringify(FIXTURE_PLUGIN_ID)},
|
||||
register(api) {
|
||||
api.registerTool({
|
||||
name: ${JSON.stringify(FIXTURE_TOOL_NAME)},
|
||||
description: "Search fixture memory",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { query: { type: "string" }, maxResults: { type: "number" } },
|
||||
required: ["query"],
|
||||
},
|
||||
async execute(_toolCallId, params) {
|
||||
return { content: [{ type: "text", text: ${JSON.stringify(FIXTURE_FACT)} + " query=" + String(params.query) }] };
|
||||
},
|
||||
});
|
||||
api.registerService({
|
||||
id: "qa-startup-delay",
|
||||
async start() {
|
||||
const deadline = Date.now() + ${STARTUP_GATE_TIMEOUT_MS};
|
||||
while (!fs.existsSync(${JSON.stringify(startupGatePath)})) {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error("timed out waiting for the QA MCP startup connect frame");
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
},
|
||||
stop() {},
|
||||
});
|
||||
},
|
||||
};\n`,
|
||||
"utf8",
|
||||
);
|
||||
return {
|
||||
pluginDir,
|
||||
startupGatePath,
|
||||
cleanup: () => fs.rm(root, { force: true, recursive: true }),
|
||||
};
|
||||
}
|
||||
|
||||
function withFixturePlugin(config: OpenClawConfig, pluginDir: string): OpenClawConfig {
|
||||
const existingPaths = config.plugins?.load?.paths ?? [];
|
||||
const existingAllow = config.plugins?.allow ?? [];
|
||||
return {
|
||||
...config,
|
||||
plugins: {
|
||||
...config.plugins,
|
||||
enabled: true,
|
||||
allow: [...new Set([...existingAllow, FIXTURE_PLUGIN_ID])],
|
||||
load: {
|
||||
...config.plugins?.load,
|
||||
paths: [...new Set([...existingPaths, pluginDir])],
|
||||
},
|
||||
entries: {
|
||||
...config.plugins?.entries,
|
||||
[FIXTURE_PLUGIN_ID]: { enabled: true },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function emptyTransport() {
|
||||
return {
|
||||
requiredPluginIds: [] as string[],
|
||||
createGatewayConfig: () => ({}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveOpenClawCliInvocation(repoRoot: string): OpenClawCliInvocation {
|
||||
for (const relativePath of ["dist/index.mjs", "dist/index.js"]) {
|
||||
const entryPath = path.join(repoRoot, relativePath);
|
||||
if (existsSync(entryPath)) {
|
||||
const argsPrefix = [entryPath];
|
||||
return {
|
||||
argsPrefix,
|
||||
command: process.execPath,
|
||||
cwd: repoRoot,
|
||||
gatewayCommand: {
|
||||
executablePath: process.execPath,
|
||||
argsPrefix,
|
||||
cwd: repoRoot,
|
||||
usePackagedPlugins: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const sourceEntryPath = path.join(repoRoot, "src/entry.ts");
|
||||
if (existsSync(sourceEntryPath)) {
|
||||
const argsPrefix = ["--import", "tsx", sourceEntryPath];
|
||||
return {
|
||||
argsPrefix,
|
||||
command: process.execPath,
|
||||
cwd: repoRoot,
|
||||
gatewayCommand: {
|
||||
executablePath: process.execPath,
|
||||
argsPrefix,
|
||||
cwd: repoRoot,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error("OpenClaw CLI entry not found: expected dist/index.(m)js or src/entry.ts");
|
||||
}
|
||||
|
||||
function resolveChannelMcpInvocation(params: {
|
||||
gatewayToken: string;
|
||||
gatewayUrl: string;
|
||||
repoRoot: string;
|
||||
tokenFile: string;
|
||||
}): ChannelMcpInvocation {
|
||||
for (const relativePath of ["dist/index.mjs", "dist/index.js"]) {
|
||||
const entryPath = path.join(params.repoRoot, relativePath);
|
||||
if (existsSync(entryPath)) {
|
||||
return {
|
||||
args: [
|
||||
entryPath,
|
||||
"mcp",
|
||||
"serve",
|
||||
"--url",
|
||||
params.gatewayUrl,
|
||||
"--token-file",
|
||||
params.tokenFile,
|
||||
"--claude-channel-mode",
|
||||
"off",
|
||||
"--verbose",
|
||||
],
|
||||
command: process.execPath,
|
||||
cwd: params.repoRoot,
|
||||
envPatch: {},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const channelServerPath = path.join(params.repoRoot, "src/mcp/channel-server.ts");
|
||||
if (existsSync(channelServerPath)) {
|
||||
const channelServerUrl = pathToFileURL(channelServerPath).href;
|
||||
return {
|
||||
args: [
|
||||
"--import",
|
||||
"tsx",
|
||||
"--eval",
|
||||
[
|
||||
`import(${JSON.stringify(channelServerUrl)})`,
|
||||
`.then((module) => module.serveOpenClawChannelMcp({`,
|
||||
`gatewayUrl: process.env.OPENCLAW_QA_GATEWAY_URL,`,
|
||||
`gatewayToken: process.env.OPENCLAW_QA_GATEWAY_TOKEN,`,
|
||||
`claudeChannelMode: "off",`,
|
||||
`verbose: true`,
|
||||
`}))`,
|
||||
].join(""),
|
||||
],
|
||||
command: process.execPath,
|
||||
cwd: params.repoRoot,
|
||||
envPatch: {
|
||||
OPENCLAW_QA_GATEWAY_TOKEN: params.gatewayToken,
|
||||
OPENCLAW_QA_GATEWAY_URL: params.gatewayUrl,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"OpenClaw channel MCP entry not found: expected dist/index.(m)js or src/mcp/channel-server.ts",
|
||||
);
|
||||
}
|
||||
|
||||
function parseJsonFrame(data: RawData): Record<string, unknown> | null {
|
||||
try {
|
||||
const text = Array.isArray(data)
|
||||
? Buffer.concat(data).toString("utf8")
|
||||
: Buffer.from(data).toString("utf8");
|
||||
const value = JSON.parse(text);
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function startGatewayProxy(
|
||||
upstreamUrl: string,
|
||||
onConnectFrame?: () => void,
|
||||
): Promise<GatewayProxy> {
|
||||
const capture: GatewayFrameCapture = {
|
||||
connectFrames: [],
|
||||
helloProtocols: [],
|
||||
startupUnavailableResponses: 0,
|
||||
};
|
||||
const connectRequestIds = new Set<string>();
|
||||
const server: Server = createServer();
|
||||
const wss = new WebSocketServer({ server });
|
||||
wss.on("connection", (downstream) => {
|
||||
const upstream = new WebSocket(upstreamUrl);
|
||||
const pending: RawData[] = [];
|
||||
downstream.on("message", (data) => {
|
||||
const frame = parseJsonFrame(data);
|
||||
if (frame?.method === "connect" && typeof frame.id === "string") {
|
||||
const params = frame.params as Record<string, unknown> | undefined;
|
||||
if (typeof params?.minProtocol === "number" && typeof params.maxProtocol === "number") {
|
||||
capture.connectFrames.push({
|
||||
minProtocol: params.minProtocol,
|
||||
maxProtocol: params.maxProtocol,
|
||||
});
|
||||
connectRequestIds.add(frame.id);
|
||||
onConnectFrame?.();
|
||||
}
|
||||
}
|
||||
if (upstream.readyState === WebSocket.OPEN) {
|
||||
upstream.send(data);
|
||||
} else {
|
||||
pending.push(data);
|
||||
}
|
||||
});
|
||||
upstream.on("open", () => {
|
||||
for (const data of pending.splice(0)) {
|
||||
upstream.send(data);
|
||||
}
|
||||
});
|
||||
upstream.on("message", (data) => {
|
||||
const frame = parseJsonFrame(data);
|
||||
if (typeof frame?.id === "string" && connectRequestIds.has(frame.id)) {
|
||||
const error = frame.error as Record<string, unknown> | undefined;
|
||||
const details = error?.details as Record<string, unknown> | undefined;
|
||||
if (error?.retryable === true && details?.reason === "startup-sidecars") {
|
||||
capture.startupUnavailableResponses += 1;
|
||||
}
|
||||
const payload = frame.payload as Record<string, unknown> | undefined;
|
||||
if (payload?.type === "hello-ok" && typeof payload.protocol === "number") {
|
||||
capture.helloProtocols.push(payload.protocol);
|
||||
}
|
||||
}
|
||||
if (downstream.readyState === WebSocket.OPEN) {
|
||||
downstream.send(data);
|
||||
}
|
||||
});
|
||||
const closeDownstream = () => {
|
||||
if (downstream.readyState === WebSocket.OPEN) {
|
||||
downstream.close(1013, "gateway unavailable");
|
||||
}
|
||||
};
|
||||
upstream.on("error", closeDownstream);
|
||||
upstream.on("close", (code, reason) => {
|
||||
if (downstream.readyState === WebSocket.OPEN) {
|
||||
downstream.close(code, reason.toString());
|
||||
}
|
||||
});
|
||||
downstream.on("close", () => upstream.close());
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => resolve());
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("gateway frame proxy did not bind a TCP port");
|
||||
}
|
||||
return {
|
||||
capture,
|
||||
url: `ws://127.0.0.1:${address.port}`,
|
||||
async stop() {
|
||||
for (const client of wss.clients) {
|
||||
client.terminate();
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
wss.close(() => resolve());
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function connectChannelMcpClient(params: {
|
||||
gatewayUrl: string;
|
||||
gatewayToken: string;
|
||||
repoRoot: string;
|
||||
}): Promise<McpClientHandle> {
|
||||
const tempState = createMcpClientTempState({ gatewayToken: params.gatewayToken });
|
||||
const mcpInvocation = resolveChannelMcpInvocation({
|
||||
gatewayToken: params.gatewayToken,
|
||||
gatewayUrl: params.gatewayUrl,
|
||||
repoRoot: params.repoRoot,
|
||||
tokenFile: tempState.tokenFile,
|
||||
});
|
||||
const stderrChunks: Buffer[] = [];
|
||||
const transport = new StdioClientTransport({
|
||||
command: mcpInvocation.command,
|
||||
args: mcpInvocation.args,
|
||||
cwd: mcpInvocation.cwd,
|
||||
env: {
|
||||
...process.env,
|
||||
...mcpInvocation.envPatch,
|
||||
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1",
|
||||
OPENCLAW_LOG_LEVEL: "debug",
|
||||
OPENCLAW_STATE_DIR: tempState.stateDir,
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
|
||||
const client = new Client({ name: "qa-gateway-mcp-client", version: "1.0.0" });
|
||||
let connectTimeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
client.connect(transport),
|
||||
new Promise<never>((_, reject) => {
|
||||
connectTimeout = setTimeout(
|
||||
() => reject(new Error("MCP channel client connect timed out")),
|
||||
MCP_CONNECT_TIMEOUT_MS,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
return {
|
||||
client,
|
||||
cleanup: tempState.cleanup,
|
||||
transport,
|
||||
stderr: () => Buffer.concat(stderrChunks).toString("utf8"),
|
||||
};
|
||||
} catch (error) {
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
tempState.cleanup();
|
||||
throw error;
|
||||
} finally {
|
||||
if (connectTimeout) {
|
||||
clearTimeout(connectTimeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function closeMcpClient(handle: McpClientHandle | undefined) {
|
||||
if (!handle) {
|
||||
return;
|
||||
}
|
||||
await Promise.allSettled([handle.client.close(), handle.transport.close()]);
|
||||
handle.cleanup();
|
||||
}
|
||||
|
||||
async function approvePendingMcpPairing(gateway: Awaited<ReturnType<typeof startQaGatewayChild>>) {
|
||||
const pairing = (await gateway.call("device.pair.list", {})) as {
|
||||
pending?: Array<{ requestId?: string; role?: string }>;
|
||||
};
|
||||
const pending = pairing.pending?.find((entry) => entry.role === "operator");
|
||||
if (!pending?.requestId) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await gateway.call("device.pair.approve", { requestId: pending.requestId });
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (formatErrorMessage(error).includes("unknown requestId")) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function runGatewaySmokeProof(options: ProducerOptions): Promise<string> {
|
||||
const gateway = await startQaGatewayChild({
|
||||
repoRoot: options.repoRoot,
|
||||
command: resolveOpenClawCliInvocation(options.repoRoot).gatewayCommand,
|
||||
transport: emptyTransport(),
|
||||
transportBaseUrl: "http://127.0.0.1",
|
||||
controlUiEnabled: false,
|
||||
});
|
||||
const tempRoot = gateway.tempRoot;
|
||||
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1";
|
||||
let details = "";
|
||||
try {
|
||||
const stdout: string[] = [];
|
||||
const stderr: string[] = [];
|
||||
const exitCode = await runGatewaySmoke(
|
||||
{ token: gateway.token, urlRaw: gateway.wsUrl },
|
||||
{
|
||||
stdout: (message) => stdout.push(message),
|
||||
stderr: (message) => stderr.push(message),
|
||||
},
|
||||
);
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(`gateway smoke exited ${exitCode}: ${stderr.join("\n")}`);
|
||||
}
|
||||
const health = (await gateway.call("health", {})) as { ok?: boolean };
|
||||
if (health.ok !== true) {
|
||||
throw new Error(`gateway health RPC returned ${JSON.stringify(health)}`);
|
||||
}
|
||||
details = `real Gateway pid=${gateway.pid ?? "unknown"}; ${stdout.join("; ")}; health.ok=true`;
|
||||
} finally {
|
||||
await gateway.stop();
|
||||
}
|
||||
if (!keepTemp && existsSync(tempRoot)) {
|
||||
throw new Error(`Gateway temp root was not cleaned up: ${tempRoot}`);
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
async function runMcpGatewayStartupRetryProof(options: ProducerOptions): Promise<string> {
|
||||
const fixture = await createFixturePlugin();
|
||||
let proxy: GatewayProxy | undefined;
|
||||
let mcp: McpClientHandle | undefined;
|
||||
let gateway: Awaited<ReturnType<typeof startQaGatewayChild>> | undefined;
|
||||
let beforeSpawnAt = 0;
|
||||
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1";
|
||||
let details = "";
|
||||
let proofError: Error | undefined;
|
||||
try {
|
||||
const onListening = async (context: QaGatewayChildListeningContext) => {
|
||||
await closeMcpClient(mcp);
|
||||
await proxy?.stop();
|
||||
proxy = await startGatewayProxy(context.wsUrl, () => {
|
||||
void fs.writeFile(fixture.startupGatePath, "observed\n", "utf8");
|
||||
});
|
||||
beforeSpawnAt = Date.now();
|
||||
mcp = await connectChannelMcpClient({
|
||||
gatewayUrl: proxy.url,
|
||||
gatewayToken: context.token,
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
};
|
||||
gateway = await startQaGatewayChild({
|
||||
repoRoot: options.repoRoot,
|
||||
command: resolveOpenClawCliInvocation(options.repoRoot).gatewayCommand,
|
||||
transport: emptyTransport(),
|
||||
transportBaseUrl: "http://127.0.0.1",
|
||||
controlUiEnabled: false,
|
||||
onListening,
|
||||
mutateConfig: (config) => withFixturePlugin(config, fixture.pluginDir),
|
||||
});
|
||||
if (!proxy || !mcp) {
|
||||
throw new Error("MCP client was not started by the Gateway before-spawn hook");
|
||||
}
|
||||
const gatewayReadyAt = Date.now();
|
||||
if (beforeSpawnAt >= gatewayReadyAt) {
|
||||
throw new Error("MCP client did not start before Gateway readiness");
|
||||
}
|
||||
if (await approvePendingMcpPairing(gateway)) {
|
||||
await closeMcpClient(mcp);
|
||||
mcp = await connectChannelMcpClient({
|
||||
gatewayUrl: proxy.url,
|
||||
gatewayToken: gateway.token,
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
}
|
||||
const tools = await mcp.client.listTools();
|
||||
if (!tools.tools.some((tool) => tool.name === "conversations_list")) {
|
||||
throw new Error("real MCP channel server did not expose conversations_list");
|
||||
}
|
||||
const conversations = await mcp.client.callTool({
|
||||
name: "conversations_list",
|
||||
arguments: { limit: 1 },
|
||||
});
|
||||
if (conversations.isError) {
|
||||
throw new Error(`conversations_list failed: ${JSON.stringify(conversations.content)}`);
|
||||
}
|
||||
const capture = proxy.capture;
|
||||
if (capture.startupUnavailableResponses < 1) {
|
||||
throw new Error(
|
||||
`expected a retryable startup-unavailable response; captured=${JSON.stringify(capture)}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
!capture.connectFrames.some(
|
||||
(frame) =>
|
||||
frame.minProtocol === MIN_CLIENT_PROTOCOL_VERSION &&
|
||||
frame.maxProtocol === PROTOCOL_VERSION,
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`MCP Gateway connect frame used unexpected protocol range: ${JSON.stringify(capture)}`,
|
||||
);
|
||||
}
|
||||
if (!capture.helloProtocols.includes(PROTOCOL_VERSION)) {
|
||||
throw new Error(`MCP Gateway negotiation did not select protocol ${PROTOCOL_VERSION}`);
|
||||
}
|
||||
details = [
|
||||
`MCP started ${gatewayReadyAt - beforeSpawnAt}ms before Gateway readiness`,
|
||||
`startup retries=${capture.startupUnavailableResponses}`,
|
||||
`connect frames=${capture.connectFrames.length}`,
|
||||
`negotiated protocol=${PROTOCOL_VERSION}`,
|
||||
].join("; ");
|
||||
} catch (error) {
|
||||
const diagnostics = [
|
||||
mcp?.stderr(),
|
||||
proxy ? `captured Gateway frames: ${JSON.stringify(proxy.capture)}` : undefined,
|
||||
gateway?.logs(),
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join("\n");
|
||||
proofError = new Error(`${formatErrorMessage(error)}${diagnostics ? `\n${diagnostics}` : ""}`, {
|
||||
cause: error,
|
||||
});
|
||||
} finally {
|
||||
await closeMcpClient(mcp);
|
||||
await proxy?.stop().catch(() => undefined);
|
||||
const tempRoot = gateway?.tempRoot;
|
||||
await gateway?.stop().catch(() => undefined);
|
||||
await fixture.cleanup();
|
||||
if (!keepTemp && tempRoot && existsSync(tempRoot) && !proofError) {
|
||||
proofError = new Error(`Gateway temp root was not cleaned up: ${tempRoot}`);
|
||||
}
|
||||
}
|
||||
if (proofError) {
|
||||
throw proofError;
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
async function writePluginToolsConfig(root: string, pluginDir: string) {
|
||||
const configPath = path.join(root, "openclaw.json");
|
||||
const config = withFixturePlugin({} as OpenClawConfig, pluginDir);
|
||||
await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
|
||||
return configPath;
|
||||
}
|
||||
|
||||
async function runMcpPluginToolsProof(options: ProducerOptions): Promise<string> {
|
||||
const fixture = await createFixturePlugin();
|
||||
// openclaw-temp-dir: allow standalone producer cleans and verifies this root in its finally block
|
||||
const runtimeRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-plugin-tools-mcp-"));
|
||||
const stateDir = path.join(runtimeRoot, "state");
|
||||
const homeDir = path.join(runtimeRoot, "home");
|
||||
await Promise.all([
|
||||
fs.mkdir(stateDir, { recursive: true }),
|
||||
fs.mkdir(homeDir, { recursive: true }),
|
||||
]);
|
||||
const configPath = await writePluginToolsConfig(runtimeRoot, fixture.pluginDir);
|
||||
const stderrChunks: Buffer[] = [];
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [
|
||||
"--import",
|
||||
"tsx",
|
||||
"--eval",
|
||||
`import(${JSON.stringify(pathToFileURL(path.join(options.repoRoot, "src/mcp/plugin-tools-serve.ts")).href)}).then((module) => module.servePluginToolsMcp())`,
|
||||
],
|
||||
cwd: options.repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: homeDir,
|
||||
OPENCLAW_CONFIG_PATH: configPath,
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
|
||||
const client = new Client({ name: "qa-plugin-tools-client", version: "1.0.0" });
|
||||
let details = "";
|
||||
let proofError: Error | undefined;
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const listed = await client.listTools();
|
||||
if (!listed.tools.some((tool) => tool.name === FIXTURE_TOOL_NAME)) {
|
||||
throw new Error(
|
||||
`fixture plugin tool was not listed: ${listed.tools.map((tool) => tool.name).join(", ")}`,
|
||||
);
|
||||
}
|
||||
const result = await client.callTool({
|
||||
name: FIXTURE_TOOL_NAME,
|
||||
arguments: { query: "ORBIT-9 codename", maxResults: 3 },
|
||||
});
|
||||
if (result.isError || !JSON.stringify(result.content).includes(FIXTURE_FACT)) {
|
||||
throw new Error(`fixture plugin tool returned unexpected payload: ${JSON.stringify(result)}`);
|
||||
}
|
||||
details = `real plugin-tools pid=${transport.pid ?? "unknown"}; listed and called ${FIXTURE_TOOL_NAME}; received ORBIT-9`;
|
||||
} catch (error) {
|
||||
const stderr = Buffer.concat(stderrChunks).toString("utf8");
|
||||
proofError = new Error(
|
||||
`${formatErrorMessage(error)}${stderr ? `\nplugin-tools stderr:\n${stderr}` : ""}`,
|
||||
{
|
||||
cause: error,
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
await Promise.all([fixture.cleanup(), fs.rm(runtimeRoot, { force: true, recursive: true })]);
|
||||
if (existsSync(runtimeRoot) && !proofError) {
|
||||
proofError = new Error(`plugin-tools runtime root was not cleaned up: ${runtimeRoot}`);
|
||||
}
|
||||
}
|
||||
if (proofError) {
|
||||
throw proofError;
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
async function produceProof(options: ProducerOptions): Promise<ProofResult> {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const details =
|
||||
options.scenarioId === "gateway-smoke"
|
||||
? await runGatewaySmokeProof(options)
|
||||
: options.scenarioId === "mcp-gateway-connect-startup-retry"
|
||||
? await runMcpGatewayStartupRetryProof(options)
|
||||
: await runMcpPluginToolsProof(options);
|
||||
return { details, durationMs: Math.max(1, Date.now() - startedAt), status: "pass" };
|
||||
} catch (error) {
|
||||
return {
|
||||
details: formatErrorMessage(error),
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "fail",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function runGatewayMcpRealTransportProducer(
|
||||
options: ProducerOptions,
|
||||
): Promise<QaEvidenceSummaryJson> {
|
||||
const scenario = SCENARIOS[options.scenarioId];
|
||||
const writer = createQaScriptEvidenceWriter({
|
||||
artifactBase: options.artifactBase,
|
||||
logFileName: `${options.scenarioId}.log`,
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: options.repoRoot,
|
||||
target: {
|
||||
id: options.scenarioId,
|
||||
title: scenario.title,
|
||||
sourcePath: scenario.sourcePath,
|
||||
primaryCoverageIds: scenario.primaryCoverageIds,
|
||||
docsRefs: scenario.docsRefs,
|
||||
codeRefs: scenario.codeRefs,
|
||||
},
|
||||
});
|
||||
const result = await produceProof(options);
|
||||
writer.appendLog(`${result.status}: ${result.details ?? "no details"}\n`);
|
||||
return await writer.write(result);
|
||||
}
|
||||
|
||||
async function main(argv: readonly string[]) {
|
||||
const options = parseOptions(argv);
|
||||
const evidence = await runGatewayMcpRealTransportProducer(options);
|
||||
const status = evidence.entries[0]?.result.status;
|
||||
console.log(`Gateway/MCP real transport evidence: ${QA_EVIDENCE_FILENAME}`);
|
||||
console.log(`Gateway/MCP real transport status: ${status}`);
|
||||
return status === "pass" ? 0 : 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
main(process.argv.slice(2))
|
||||
.then((exitCode) => {
|
||||
process.exit(exitCode);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
console.error(formatErrorMessage(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
resolveChannelMcpInvocation,
|
||||
resolveOpenClawCliInvocation,
|
||||
};
|
||||
424
test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
Normal file
424
test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
Normal file
@@ -0,0 +1,424 @@
|
||||
// Gateway Smoke tests cover QA Lab gateway smoke evidence.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { WebSocket, WebSocketServer } from "ws";
|
||||
import { runGatewaySmoke } from "../../../../scripts/dev/gateway-smoke.js";
|
||||
|
||||
let server: Server | undefined;
|
||||
let wss: WebSocketServer | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
await new Promise<void>((resolve) => {
|
||||
wss?.close(() => resolve());
|
||||
if (!wss) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
wss = undefined;
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server?.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
if (!server) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
server = undefined;
|
||||
});
|
||||
|
||||
describe("gateway-smoke", () => {
|
||||
function healthResponse() {
|
||||
return {
|
||||
ok: true,
|
||||
payload: {
|
||||
agents: [],
|
||||
channelOrder: [],
|
||||
channels: {},
|
||||
defaultAgentId: "codex",
|
||||
durationMs: 3,
|
||||
ok: true,
|
||||
sessions: { count: 0, path: "/state/sessions", recent: [] },
|
||||
ts: Date.now(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function connectHelloResponse(scopes: string[] = []) {
|
||||
return {
|
||||
ok: true,
|
||||
payload: {
|
||||
auth: { role: "operator", scopes },
|
||||
features: { events: [], methods: ["health"] },
|
||||
policy: {
|
||||
maxBufferedBytes: 1024 * 1024,
|
||||
maxPayload: 256 * 1024,
|
||||
tickIntervalMs: 1000,
|
||||
},
|
||||
protocol: 1,
|
||||
server: { connId: "test-conn", version: "dev" },
|
||||
snapshot: {},
|
||||
type: "hello-ok",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function listenGatewaySmokeServer() {
|
||||
const requests: Array<{ method: string; params?: unknown; timeout?: number }> = [];
|
||||
server = createServer();
|
||||
wss = new WebSocketServer({ server });
|
||||
wss.on("connection", (ws: WebSocket) => {
|
||||
ws.on("message", (data) => {
|
||||
const frame = JSON.parse(data.toString()) as {
|
||||
id: string;
|
||||
method: string;
|
||||
params?: unknown;
|
||||
type: string;
|
||||
};
|
||||
requests.push({ method: frame.method, params: frame.params });
|
||||
if (frame.method === "connect") {
|
||||
ws.send(JSON.stringify({ id: frame.id, type: "res", ...connectHelloResponse() }));
|
||||
return;
|
||||
}
|
||||
if (frame.method === "health") {
|
||||
ws.send(JSON.stringify({ id: frame.id, type: "res", ...healthResponse() }));
|
||||
return;
|
||||
}
|
||||
if (frame.method === "chat.history") {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
error: "missing scope: operator.read",
|
||||
id: frame.id,
|
||||
ok: false,
|
||||
type: "res",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
error: `unexpected method ${frame.method}`,
|
||||
id: frame.id,
|
||||
ok: false,
|
||||
type: "res",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server?.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test gateway smoke server did not get a TCP address");
|
||||
}
|
||||
return { requests, url: `ws://127.0.0.1:${address.port}` };
|
||||
}
|
||||
|
||||
function createSmokeDeps(
|
||||
responses: Record<string, { error?: string; ok: boolean } & Record<string, unknown>>,
|
||||
calls: Array<{ method: string; timeout?: number }> = [],
|
||||
) {
|
||||
const stdout: string[] = [];
|
||||
const stderr: string[] = [];
|
||||
let closed = 0;
|
||||
|
||||
return {
|
||||
calls,
|
||||
get closed() {
|
||||
return closed;
|
||||
},
|
||||
stderr,
|
||||
stdout,
|
||||
deps: {
|
||||
createClient: () =>
|
||||
({
|
||||
close: () => {
|
||||
closed += 1;
|
||||
},
|
||||
request: async (method: string, _params?: unknown, timeout?: number) => {
|
||||
calls.push({ method, timeout });
|
||||
const response = responses[method];
|
||||
return {
|
||||
id: method,
|
||||
...response,
|
||||
ok: response?.ok ?? false,
|
||||
error: response?.error,
|
||||
type: "res",
|
||||
};
|
||||
},
|
||||
waitOpen: async () => {},
|
||||
}) as never,
|
||||
stderr: (message: string) => {
|
||||
stderr.push(message);
|
||||
},
|
||||
stdout: (message: string) => {
|
||||
stdout.push(message);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("prints CLI help without connecting", () => {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/gateway-smoke.ts", "--help"],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain("Usage: bun scripts/dev/gateway-smoke.ts");
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("rejects unknown CLI args before connecting", () => {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"scripts/dev/gateway-smoke.ts",
|
||||
"--url",
|
||||
"ws://127.0.0.1:9",
|
||||
"--token",
|
||||
"token",
|
||||
"--wat",
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe("Unknown argument: --wat");
|
||||
});
|
||||
|
||||
it("rejects option-looking CLI values before connecting", () => {
|
||||
for (const [flag, args] of [
|
||||
["--url", ["--url", "-h", "--token", "token"]],
|
||||
["--token", ["--url", "ws://127.0.0.1:9", "--token", "-h"]],
|
||||
] as const) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/gateway-smoke.ts", ...args],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe(`${flag} requires a value`);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects duplicate CLI args before connecting", () => {
|
||||
for (const [flag, args] of [
|
||||
[
|
||||
"--url",
|
||||
["--url", "ws://127.0.0.1:9", "--url", "ws://127.0.0.1:10", "--token", "token"],
|
||||
],
|
||||
[
|
||||
"--token",
|
||||
["--url", "ws://127.0.0.1:9", "--token", "one", "--token", "two"],
|
||||
],
|
||||
["--help", ["--help", "--help"]],
|
||||
] as const) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/gateway-smoke.ts", ...args],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe(`${flag} was provided more than once`);
|
||||
}
|
||||
});
|
||||
|
||||
it("passes against a loopback gateway websocket using the real client", async () => {
|
||||
const stdout: string[] = [];
|
||||
const stderr: string[] = [];
|
||||
const loopback = await listenGatewaySmokeServer();
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: loopback.url },
|
||||
{
|
||||
stderr: (message) => {
|
||||
stderr.push(message);
|
||||
},
|
||||
stdout: (message) => {
|
||||
stdout.push(message);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(code).toBe(0);
|
||||
expect(loopback.requests.map((request) => request.method)).toEqual(["connect", "health"]);
|
||||
expect(loopback.requests[0]?.params).toMatchObject({
|
||||
auth: { token: "secret-token" },
|
||||
client: { id: "openclaw-ios" },
|
||||
role: "operator",
|
||||
scopes: ["operator.read", "operator.write", "operator.admin"],
|
||||
});
|
||||
expect(stdout).toEqual(["ok: connected + health"]);
|
||||
expect(stderr).toEqual([]);
|
||||
});
|
||||
|
||||
it("closes the websocket client when connect fails", async () => {
|
||||
const stderr: string[] = [];
|
||||
const methods: string[] = [];
|
||||
let closed = 0;
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
{
|
||||
createClient: () =>
|
||||
({
|
||||
close: () => {
|
||||
closed += 1;
|
||||
},
|
||||
request: async (method: string) => {
|
||||
methods.push(method);
|
||||
return { error: "bad token", id: "connect", ok: false, type: "res" };
|
||||
},
|
||||
waitOpen: async () => {},
|
||||
}) as never,
|
||||
stderr: (message) => {
|
||||
stderr.push(message);
|
||||
},
|
||||
stdout: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
expect(code).toBe(2);
|
||||
expect(closed).toBe(1);
|
||||
expect(methods).toEqual(["connect"]);
|
||||
expect(stderr).toEqual(["connect failed: bad token"]);
|
||||
});
|
||||
|
||||
it("requires connect and health in order", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: healthResponse(),
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(0);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([
|
||||
{ method: "connect", timeout: undefined },
|
||||
{ method: "health", timeout: undefined },
|
||||
]);
|
||||
expect(fake.stdout).toEqual(["ok: connected + health"]);
|
||||
expect(fake.stderr).toEqual([]);
|
||||
});
|
||||
|
||||
it("fails when connect success is missing hello evidence", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: { ok: true },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(2);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([{ method: "connect", timeout: undefined }]);
|
||||
expect(fake.stdout).toEqual([]);
|
||||
expect(fake.stderr).toEqual(["connect failed: missing hello-ok payload"]);
|
||||
});
|
||||
|
||||
it("fails when the unpaired iOS-shaped connect keeps operator scopes", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(["operator.read"]),
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(2);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([{ method: "connect", timeout: undefined }]);
|
||||
expect(fake.stderr).toEqual([
|
||||
"connect failed: unpaired iOS smoke unexpectedly received operator scopes",
|
||||
]);
|
||||
});
|
||||
|
||||
it("fails after connect when health is unavailable", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: { ok: false, error: "not healthy" },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(3);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls.map((call) => call.method)).toEqual(["connect", "health"]);
|
||||
expect(fake.stderr).toEqual(["health failed: not healthy"]);
|
||||
});
|
||||
|
||||
it("fails when health success is missing summary evidence", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: { ok: true },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(3);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls.map((call) => call.method)).toEqual(["connect", "health"]);
|
||||
expect(fake.stderr).toEqual(["health failed: missing health summary payload"]);
|
||||
});
|
||||
|
||||
it("does not call scoped chat history for an unpaired iOS-shaped client", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: healthResponse(),
|
||||
"chat.history": { ok: false, error: "session store unavailable" },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(0);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([
|
||||
{ method: "connect", timeout: undefined },
|
||||
{ method: "health", timeout: undefined },
|
||||
]);
|
||||
expect(fake.stderr).toEqual([]);
|
||||
});
|
||||
});
|
||||
464
test/e2e/qa-lab/runtime/mcp-channels-docker-client.ts
Normal file
464
test/e2e/qa-lab/runtime/mcp-channels-docker-client.ts
Normal file
@@ -0,0 +1,464 @@
|
||||
// MCP channels Docker client drives the QA-owned channel bridge smoke.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import {
|
||||
assert,
|
||||
ClaudeChannelNotificationSchema,
|
||||
ClaudePermissionNotificationSchema,
|
||||
connectGateway,
|
||||
connectMcpClient,
|
||||
extractTextFromGatewayPayload,
|
||||
type ClaudeChannelNotification,
|
||||
type GatewayRpcClient,
|
||||
maybeApprovePendingBridgePairing,
|
||||
waitFor,
|
||||
} from "./mcp-channels.fixture.ts";
|
||||
import {
|
||||
connectMcpClientWithPairingReconnect,
|
||||
createMcpClientTempState,
|
||||
} from "./mcp-client-temp-state.fixture.ts";
|
||||
|
||||
function summarizeSessionRows(rows: Array<Record<string, unknown>> | undefined) {
|
||||
return (rows ?? []).map((entry) => ({
|
||||
key: entry.key,
|
||||
channel: entry.channel,
|
||||
deliveryContext: entry.deliveryContext,
|
||||
lastChannel: entry.lastChannel,
|
||||
lastTo: entry.lastTo,
|
||||
lastAccountId: entry.lastAccountId,
|
||||
lastThreadId: entry.lastThreadId,
|
||||
}));
|
||||
}
|
||||
|
||||
function findEventByText(events: Array<Record<string, unknown>> | undefined, text: string) {
|
||||
return (events ?? []).find((entry) => entry.text === text);
|
||||
}
|
||||
|
||||
const NON_OWNER_PERMISSION_QUIET_WINDOW_MS = 1_000;
|
||||
|
||||
function formatUnknownError(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
if (error === undefined || error === null) {
|
||||
return "";
|
||||
}
|
||||
if (typeof error === "string") {
|
||||
return error;
|
||||
}
|
||||
if (typeof error === "number" || typeof error === "boolean" || typeof error === "bigint") {
|
||||
return `${error}`;
|
||||
}
|
||||
if (typeof error === "symbol") {
|
||||
return error.description ?? "symbol";
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(error) ?? "";
|
||||
} catch {
|
||||
return Object.prototype.toString.call(error);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForGatewaySeededConversation(gateway: GatewayRpcClient) {
|
||||
let lastList: { sessions?: Array<Record<string, unknown>> } | undefined;
|
||||
let lastError: unknown;
|
||||
try {
|
||||
return await waitFor(
|
||||
"seeded conversation in gateway sessions.list",
|
||||
async () => {
|
||||
try {
|
||||
lastList = await gateway.request<{ sessions?: Array<Record<string, unknown>> }>(
|
||||
"sessions.list",
|
||||
{ limit: 50, includeDerivedTitles: false, includeLastMessage: false },
|
||||
);
|
||||
lastError = undefined;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
return undefined;
|
||||
}
|
||||
return lastList.sessions?.find((entry) => entry.key === "agent:main:main");
|
||||
},
|
||||
180_000,
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`gateway sessions.list did not include seeded conversation: ${JSON.stringify(
|
||||
{
|
||||
count: lastList?.sessions?.length ?? 0,
|
||||
sessions: summarizeSessionRows(lastList?.sessions),
|
||||
lastError: formatUnknownError(lastError),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const gatewayUrl = process.env.GW_URL?.trim();
|
||||
const gatewayToken = process.env.GW_TOKEN?.trim();
|
||||
assert(gatewayUrl, "missing GW_URL");
|
||||
assert(gatewayToken, "missing GW_TOKEN");
|
||||
|
||||
const gateway = await connectGateway({ url: gatewayUrl, token: gatewayToken });
|
||||
let nonOwnerGateway: GatewayRpcClient | undefined;
|
||||
let mcpHandle: Awaited<ReturnType<typeof connectMcpClient>> | undefined;
|
||||
const mcpTempState = createMcpClientTempState({ gatewayToken });
|
||||
|
||||
try {
|
||||
const gatewayConversation = await waitForGatewaySeededConversation(gateway);
|
||||
assert(
|
||||
(gatewayConversation.deliveryContext as { channel?: unknown } | undefined)?.channel ===
|
||||
"imessage",
|
||||
"expected seeded gateway deliveryContext channel",
|
||||
);
|
||||
assert(
|
||||
(gatewayConversation.deliveryContext as { to?: unknown } | undefined)?.to === "+15551234567",
|
||||
"expected seeded gateway deliveryContext target",
|
||||
);
|
||||
|
||||
mcpHandle = await connectMcpClientWithPairingReconnect({
|
||||
tempState: mcpTempState,
|
||||
connect: (tempState) =>
|
||||
connectMcpClient({
|
||||
gatewayUrl,
|
||||
gatewayToken,
|
||||
tempState,
|
||||
}),
|
||||
maybeApprovePairing: () => maybeApprovePendingBridgePairing(gateway),
|
||||
});
|
||||
const mcp = mcpHandle.client;
|
||||
const callTool = <T>(params: Parameters<typeof mcp.callTool>[0]) =>
|
||||
mcp.callTool(params, undefined, { timeout: 240_000 }) as Promise<T>;
|
||||
|
||||
let lastMcpConversationList: unknown;
|
||||
const conversation = await waitFor(
|
||||
"seeded conversation in conversations_list",
|
||||
async () => {
|
||||
const listed = await callTool<{
|
||||
structuredContent?: { conversations?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "conversations_list",
|
||||
arguments: {
|
||||
includeDerivedTitles: false,
|
||||
includeLastMessage: false,
|
||||
},
|
||||
});
|
||||
lastMcpConversationList = listed;
|
||||
return listed.structuredContent?.conversations?.find(
|
||||
(entry) => entry.sessionKey === "agent:main:main",
|
||||
);
|
||||
},
|
||||
240_000,
|
||||
).catch((error: unknown) => {
|
||||
throw new Error(
|
||||
`timeout waiting for seeded MCP conversation: ${JSON.stringify(
|
||||
lastMcpConversationList,
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
});
|
||||
assert(conversation.channel === "imessage", "expected seeded channel");
|
||||
assert(conversation.to === "+15551234567", "expected seeded target");
|
||||
|
||||
const fetched = await callTool<{
|
||||
structuredContent?: { conversation?: Record<string, unknown> };
|
||||
isError?: boolean;
|
||||
}>({
|
||||
name: "conversation_get",
|
||||
arguments: { session_key: "agent:main:main" },
|
||||
});
|
||||
assert(!fetched.isError, "conversation_get should succeed");
|
||||
assert(
|
||||
fetched.structuredContent?.conversation?.sessionKey === "agent:main:main",
|
||||
"conversation_get returned wrong session",
|
||||
);
|
||||
|
||||
let lastHistory: unknown;
|
||||
const messages = await waitFor(
|
||||
"seeded transcript messages",
|
||||
async () => {
|
||||
const history = await callTool<{
|
||||
structuredContent?: { messages?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "messages_read",
|
||||
arguments: { session_key: "agent:main:main", limit: 10 },
|
||||
});
|
||||
lastHistory = history;
|
||||
const currentMessages = history.structuredContent?.messages ?? [];
|
||||
return currentMessages.length >= 2 ? currentMessages : undefined;
|
||||
},
|
||||
240_000,
|
||||
).catch((error: unknown) => {
|
||||
throw new Error(
|
||||
`timeout waiting for seeded transcript messages: ${JSON.stringify(lastHistory, null, 2)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
});
|
||||
await waitFor(
|
||||
"seeded attachment message",
|
||||
() =>
|
||||
messages.find((entry) => {
|
||||
const raw = entry["__openclaw"];
|
||||
return (
|
||||
raw && typeof raw === "object" && (raw as { id?: unknown }).id === "msg-attachment"
|
||||
);
|
||||
}),
|
||||
240_000,
|
||||
);
|
||||
|
||||
const attachments = await callTool<{
|
||||
structuredContent?: { attachments?: Array<Record<string, unknown>> };
|
||||
isError?: boolean;
|
||||
}>({
|
||||
name: "attachments_fetch",
|
||||
arguments: { session_key: "agent:main:main", message_id: "msg-attachment" },
|
||||
});
|
||||
assert(!attachments.isError, "attachments_fetch should succeed");
|
||||
assert(
|
||||
(attachments.structuredContent?.attachments?.length ?? 0) === 1,
|
||||
"expected one seeded attachment",
|
||||
);
|
||||
|
||||
const waited = (await Promise.all([
|
||||
callTool<{
|
||||
structuredContent?: { event?: Record<string, unknown> };
|
||||
}>({
|
||||
name: "events_wait",
|
||||
arguments: {
|
||||
session_key: "agent:main:main",
|
||||
after_cursor: 0,
|
||||
timeout_ms: 10_000,
|
||||
},
|
||||
}),
|
||||
gateway.request("chat.inject", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: "assistant live event",
|
||||
}),
|
||||
]).then(([result]) => result)) as {
|
||||
structuredContent?: { event?: Record<string, unknown> };
|
||||
};
|
||||
const assistantEvent = waited.structuredContent?.event;
|
||||
assert(assistantEvent, "expected events_wait result");
|
||||
assert(assistantEvent.type === "message", "expected message event");
|
||||
assert(assistantEvent.role === "assistant", "expected assistant event role");
|
||||
assert(assistantEvent.text === "assistant live event", "expected assistant event text");
|
||||
const assistantCursor = typeof assistantEvent.cursor === "number" ? assistantEvent.cursor : 0;
|
||||
|
||||
const polled = await callTool<{
|
||||
structuredContent?: { events?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "events_poll",
|
||||
arguments: { session_key: "agent:main:main", after_cursor: 0, limit: 10 },
|
||||
});
|
||||
assert(
|
||||
(polled.structuredContent?.events ?? []).some(
|
||||
(entry) => entry.text === "assistant live event",
|
||||
),
|
||||
"expected assistant event in events_poll",
|
||||
);
|
||||
|
||||
const channelMessage = `hello from docker ${randomUUID()}`;
|
||||
await gateway.request("chat.send", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: channelMessage,
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
const rawGatewayUserMessage = await waitFor(
|
||||
"raw gateway user session.message",
|
||||
() =>
|
||||
gateway.events.find(
|
||||
(entry) =>
|
||||
entry.event === "session.message" &&
|
||||
entry.payload.sessionKey === "agent:main:main" &&
|
||||
extractTextFromGatewayPayload(entry.payload) === channelMessage,
|
||||
),
|
||||
10_000,
|
||||
).catch(() => undefined);
|
||||
let userEvent = await waitFor(
|
||||
"MCP user session.message event",
|
||||
async () => {
|
||||
const polledValue = await callTool<{
|
||||
structuredContent?: { events?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "events_poll",
|
||||
arguments: { session_key: "agent:main:main", after_cursor: assistantCursor, limit: 50 },
|
||||
});
|
||||
return findEventByText(polledValue.structuredContent?.events, channelMessage);
|
||||
},
|
||||
60_000,
|
||||
).catch(() => undefined);
|
||||
let finalPolledEvents: Array<Record<string, unknown>> | undefined;
|
||||
if (userEvent?.text !== channelMessage) {
|
||||
const polledLocal = await callTool<{
|
||||
structuredContent?: { events?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "events_poll",
|
||||
arguments: { session_key: "agent:main:main", after_cursor: assistantCursor, limit: 50 },
|
||||
});
|
||||
finalPolledEvents = polledLocal.structuredContent?.events ?? [];
|
||||
const finalUserEvent = findEventByText(finalPolledEvents, channelMessage);
|
||||
if (finalUserEvent?.text === channelMessage) {
|
||||
userEvent = finalUserEvent;
|
||||
}
|
||||
}
|
||||
if (userEvent?.text !== channelMessage) {
|
||||
throw new Error(
|
||||
`expected user event after chat.send: ${JSON.stringify(
|
||||
{
|
||||
userEvent: userEvent ?? null,
|
||||
rawGatewayUserMessage: rawGatewayUserMessage ?? null,
|
||||
mcpEventsAfterAssistant: finalPolledEvents ?? [],
|
||||
recentGatewayEvents: gateway.events.slice(-10).map((entry) => ({
|
||||
event: entry.event,
|
||||
sessionKey: entry.payload.sessionKey,
|
||||
text: extractTextFromGatewayPayload(entry.payload),
|
||||
})),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
let helpNotification: ClaudeChannelNotification;
|
||||
try {
|
||||
helpNotification = await waitFor(
|
||||
"Claude channel notification",
|
||||
() =>
|
||||
mcpHandle.rawMessages
|
||||
.map((entry) => ClaudeChannelNotificationSchema.safeParse(entry))
|
||||
.find(
|
||||
(entry) =>
|
||||
entry.success &&
|
||||
entry.data.params.meta.session_key === "agent:main:main" &&
|
||||
entry.data.params.content === channelMessage,
|
||||
)?.data.params,
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`timeout waiting for Claude channel notification: ${JSON.stringify(
|
||||
{
|
||||
rawMessages: mcpHandle.rawMessages.slice(-10),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
assert(helpNotification.content === channelMessage, "expected Claude channel content");
|
||||
|
||||
await mcp.notification({
|
||||
method: "notifications/claude/channel/permission_request",
|
||||
params: {
|
||||
request_id: "abcde",
|
||||
tool_name: "Bash",
|
||||
description: "run npm test",
|
||||
input_preview: '{"cmd":"npm test"}',
|
||||
},
|
||||
});
|
||||
|
||||
nonOwnerGateway = await connectGateway({
|
||||
url: gatewayUrl,
|
||||
token: gatewayToken,
|
||||
scopes: ["operator.read", "operator.write"],
|
||||
});
|
||||
await nonOwnerGateway.request("chat.send", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: "yes abcde",
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
await waitFor(
|
||||
"non-owner reply forwarded as an ordinary Claude channel message",
|
||||
() =>
|
||||
mcpHandle.rawMessages
|
||||
.map((entry) => ClaudeChannelNotificationSchema.safeParse(entry))
|
||||
.find(
|
||||
(entry) =>
|
||||
entry.success &&
|
||||
entry.data.params.meta.session_key === "agent:main:main" &&
|
||||
entry.data.params.content === "yes abcde",
|
||||
)?.data.params,
|
||||
60_000,
|
||||
);
|
||||
await delay(NON_OWNER_PERMISSION_QUIET_WINDOW_MS);
|
||||
const nonOwnerPermission = mcpHandle.rawMessages
|
||||
.map((entry) => ClaudePermissionNotificationSchema.safeParse(entry))
|
||||
.find((entry) => entry.success && entry.data.params.request_id === "abcde");
|
||||
assert(!nonOwnerPermission, "non-owner reply must not resolve the Claude permission");
|
||||
|
||||
const ownerNotificationStart = mcpHandle.rawMessages.length;
|
||||
await gateway.request("chat.send", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: "yes abcde",
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
let permission: { request_id: string; behavior: "allow" | "deny" };
|
||||
try {
|
||||
permission = await waitFor(
|
||||
"Claude permission notification",
|
||||
() =>
|
||||
mcpHandle.rawMessages
|
||||
.slice(ownerNotificationStart)
|
||||
.map((entry) => ClaudePermissionNotificationSchema.safeParse(entry))
|
||||
.find((entry) => entry.success && entry.data.params.request_id === "abcde")?.data
|
||||
.params,
|
||||
60_000,
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`timeout waiting for Claude permission notification: ${JSON.stringify(
|
||||
{
|
||||
rawMessages: mcpHandle.rawMessages.slice(-10),
|
||||
recentGatewayEvents: gateway.events.slice(-10).map((entry) => ({
|
||||
event: entry.event,
|
||||
sessionKey: entry.payload.sessionKey,
|
||||
text: extractTextFromGatewayPayload(entry.payload),
|
||||
})),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
assert(permission.behavior === "allow", "expected allow permission reply");
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: true,
|
||||
sessionKey: "agent:main:main",
|
||||
nonOwnerReplyForwarded: true,
|
||||
nonOwnerPermissionBlocked: true,
|
||||
ownerPermissionAllowed: permission.behavior === "allow",
|
||||
rawNotifications: mcpHandle.rawMessages.filter(
|
||||
(entry) =>
|
||||
ClaudeChannelNotificationSchema.safeParse(entry).success ||
|
||||
ClaudePermissionNotificationSchema.safeParse(entry).success,
|
||||
).length,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
);
|
||||
} finally {
|
||||
const closeTasks: Array<Promise<unknown>> = [gateway.close()];
|
||||
if (nonOwnerGateway) {
|
||||
closeTasks.push(nonOwnerGateway.close());
|
||||
}
|
||||
if (mcpHandle) {
|
||||
closeTasks.push(mcpHandle.client.close(), mcpHandle.transport.close());
|
||||
}
|
||||
await Promise.allSettled(closeTasks);
|
||||
mcpHandle?.cleanup();
|
||||
mcpTempState.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
325
test/e2e/qa-lab/runtime/mcp-channels.fixture.ts
Normal file
325
test/e2e/qa-lab/runtime/mcp-channels.fixture.ts
Normal file
@@ -0,0 +1,325 @@
|
||||
// Shared MCP-channel QA/Docker E2E fixture helpers.
|
||||
// The mounted test harness imports packaged dist modules so bridge assertions run
|
||||
// against the OpenClaw npm tarball installed in the functional image.
|
||||
import process from "node:process";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
import { z } from "zod";
|
||||
import { PROTOCOL_VERSION } from "../../../../dist/gateway/protocol/index.js";
|
||||
import { formatErrorMessage } from "../../../../dist/infra/errors.js";
|
||||
import { readStringValue } from "../../../../dist/normalization-core/string-coerce.js";
|
||||
import { resolveGatewaySuccessPayload } from "../../../../scripts/e2e/lib/gateway-frame-payload.mjs";
|
||||
import { readMcpChannelLimits } from "../../../../scripts/e2e/mcp-channel-limits.ts";
|
||||
import {
|
||||
createGatewayWsClient,
|
||||
type GatewayEventFrame,
|
||||
} from "../../../../scripts/lib/gateway-ws-client.ts";
|
||||
import {
|
||||
connectMcpWithTimeout,
|
||||
createMcpClientTempState,
|
||||
type McpClientTempState,
|
||||
} from "./mcp-client-temp-state.fixture.ts";
|
||||
|
||||
export const ClaudeChannelNotificationSchema = z.object({
|
||||
method: z.literal("notifications/claude/channel"),
|
||||
params: z.object({
|
||||
content: z.string(),
|
||||
meta: z.record(z.string(), z.string()),
|
||||
}),
|
||||
});
|
||||
|
||||
export const ClaudePermissionNotificationSchema = z.object({
|
||||
method: z.literal("notifications/claude/channel/permission"),
|
||||
params: z.object({
|
||||
request_id: z.string(),
|
||||
behavior: z.enum(["allow", "deny"]),
|
||||
}),
|
||||
});
|
||||
|
||||
export type ClaudeChannelNotification = z.infer<typeof ClaudeChannelNotificationSchema>["params"];
|
||||
|
||||
export type GatewayRpcClient = {
|
||||
request<T>(method: string, params?: unknown, opts?: { timeoutMs?: number }): Promise<T>;
|
||||
events: Array<{ event: string; payload: Record<string, unknown> }>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export type McpClientHandle = {
|
||||
client: Client;
|
||||
cleanup(): void;
|
||||
transport: StdioClientTransport;
|
||||
rawMessages: unknown[];
|
||||
};
|
||||
|
||||
const GATEWAY_WS_OPEN_TIMEOUT_MS = 45_000;
|
||||
const GATEWAY_RPC_TIMEOUT_MS = 60_000;
|
||||
const GATEWAY_REQUEST_TIMEOUT_MS = 45_000;
|
||||
const GATEWAY_CONNECT_RETRY_WINDOW_MS = 420_000;
|
||||
const MCP_CHANNEL_LIMITS = readMcpChannelLimits();
|
||||
const MCP_CONNECT_TIMEOUT_MS = MCP_CHANNEL_LIMITS.connectTimeoutMs;
|
||||
const GATEWAY_EVENT_RETAIN_LIMIT = MCP_CHANNEL_LIMITS.gatewayEventRetainLimit;
|
||||
const MCP_RAW_MESSAGE_RETAIN_LIMIT = MCP_CHANNEL_LIMITS.rawMessageRetainLimit;
|
||||
|
||||
export function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function pushBounded<T>(items: T[], item: T, limit: number): void {
|
||||
items.push(item);
|
||||
if (items.length > limit) {
|
||||
items.splice(0, items.length - limit);
|
||||
}
|
||||
}
|
||||
|
||||
export function extractTextFromGatewayPayload(
|
||||
payload: Record<string, unknown> | undefined,
|
||||
): string | undefined {
|
||||
const message = payload?.message;
|
||||
if (!message || typeof message !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (typeof content === "string" && content.trim().length > 0) {
|
||||
return content;
|
||||
}
|
||||
if (!Array.isArray(content)) {
|
||||
return undefined;
|
||||
}
|
||||
const first = content[0];
|
||||
if (!first || typeof first !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return readStringValue((first as { text?: unknown }).text);
|
||||
}
|
||||
|
||||
export async function waitFor<T>(
|
||||
label: string,
|
||||
predicate: () => Promise<T | undefined> | T | undefined,
|
||||
timeoutMs = 10_000,
|
||||
): Promise<T> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
const value = await predicate();
|
||||
if (value !== undefined) {
|
||||
return value;
|
||||
}
|
||||
await delay(50);
|
||||
}
|
||||
throw new Error(`timeout waiting for ${label}`);
|
||||
}
|
||||
|
||||
export async function connectGateway(params: {
|
||||
url: string;
|
||||
token: string;
|
||||
scopes?: readonly string[];
|
||||
}): Promise<GatewayRpcClient> {
|
||||
const startedAt = Date.now();
|
||||
let attempt = 0;
|
||||
let lastError: Error | null = null;
|
||||
|
||||
while (Date.now() - startedAt < GATEWAY_CONNECT_RETRY_WINDOW_MS) {
|
||||
attempt += 1;
|
||||
try {
|
||||
return await connectGatewayOnce(params);
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
if (!isRetryableGatewayConnectError(lastError)) {
|
||||
throw lastError;
|
||||
}
|
||||
await delay(Math.min(500 * attempt, 2_000));
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError ?? new Error("gateway ws open timeout");
|
||||
}
|
||||
|
||||
async function connectGatewayOnce(params: {
|
||||
url: string;
|
||||
token: string;
|
||||
scopes?: readonly string[];
|
||||
}): Promise<GatewayRpcClient> {
|
||||
const requestedScopes = params.scopes ?? [
|
||||
"operator.read",
|
||||
"operator.write",
|
||||
"operator.pairing",
|
||||
"operator.admin",
|
||||
];
|
||||
const events: Array<{ event: string; payload: Record<string, unknown> }> = [];
|
||||
const gatewayClient = createGatewayWsClient({
|
||||
handshakeTimeoutMs: GATEWAY_WS_OPEN_TIMEOUT_MS,
|
||||
onEvent(event: GatewayEventFrame) {
|
||||
pushBounded(
|
||||
events,
|
||||
{
|
||||
event: event.event,
|
||||
payload:
|
||||
event.payload && typeof event.payload === "object"
|
||||
? (event.payload as Record<string, unknown>)
|
||||
: {},
|
||||
},
|
||||
GATEWAY_EVENT_RETAIN_LIMIT,
|
||||
);
|
||||
},
|
||||
openTimeoutMs: GATEWAY_WS_OPEN_TIMEOUT_MS,
|
||||
openTimeoutMessage: "gateway ws open timeout",
|
||||
url: params.url,
|
||||
});
|
||||
await gatewayClient.waitOpen();
|
||||
|
||||
const sendGatewayRequest = <T = unknown>(
|
||||
method: string,
|
||||
requestParams: unknown,
|
||||
timeoutMs: number,
|
||||
): Promise<T> => {
|
||||
return gatewayClient.request(method, requestParams ?? {}, timeoutMs).then((response) => {
|
||||
if (response.ok) {
|
||||
return resolveGatewaySuccessPayload(response) as T;
|
||||
}
|
||||
throw new Error(
|
||||
response.error && typeof response.error === "object" && "message" in response.error
|
||||
? String(response.error.message)
|
||||
: "gateway request failed",
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
await sendGatewayRequest(
|
||||
"connect",
|
||||
{
|
||||
minProtocol: PROTOCOL_VERSION,
|
||||
maxProtocol: PROTOCOL_VERSION,
|
||||
client: {
|
||||
id: "openclaw-tui",
|
||||
displayName: "docker-mcp-channels",
|
||||
version: "1.0.0",
|
||||
platform: process.platform,
|
||||
mode: "ui",
|
||||
},
|
||||
role: "operator",
|
||||
scopes: requestedScopes,
|
||||
caps: [],
|
||||
auth: { token: params.token },
|
||||
},
|
||||
GATEWAY_RPC_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
await sendGatewayRequest("sessions.subscribe", {}, GATEWAY_RPC_TIMEOUT_MS);
|
||||
|
||||
return {
|
||||
request(method, requestParams, opts) {
|
||||
return sendGatewayRequest(
|
||||
method,
|
||||
requestParams,
|
||||
opts?.timeoutMs ?? GATEWAY_REQUEST_TIMEOUT_MS,
|
||||
);
|
||||
},
|
||||
events,
|
||||
async close() {
|
||||
gatewayClient.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function isRetryableGatewayConnectError(error: Error): boolean {
|
||||
const message = error.message.toLowerCase();
|
||||
return (
|
||||
message.includes("gateway ws open timeout") ||
|
||||
message.includes("gateway connect timeout") ||
|
||||
message.includes("closed before open") ||
|
||||
message.includes("gateway closed") ||
|
||||
message.includes("gateway websocket closed") ||
|
||||
message.includes("econnrefused") ||
|
||||
message.includes("socket hang up")
|
||||
);
|
||||
}
|
||||
|
||||
export async function connectMcpClient(params: {
|
||||
gatewayUrl: string;
|
||||
gatewayToken: string;
|
||||
tempState?: McpClientTempState;
|
||||
}): Promise<McpClientHandle> {
|
||||
const ownsTempState = !params.tempState;
|
||||
const tempState =
|
||||
params.tempState ?? createMcpClientTempState({ gatewayToken: params.gatewayToken });
|
||||
const transport = new StdioClientTransport({
|
||||
command: "node",
|
||||
args: [
|
||||
"/app/openclaw.mjs",
|
||||
"mcp",
|
||||
"serve",
|
||||
"--url",
|
||||
params.gatewayUrl,
|
||||
"--token-file",
|
||||
tempState.tokenFile,
|
||||
"--claude-channel-mode",
|
||||
"on",
|
||||
],
|
||||
cwd: "/app",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1",
|
||||
OPENCLAW_STATE_DIR: tempState.stateDir,
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", (chunk) => {
|
||||
process.stderr.write(`[openclaw mcp] ${String(chunk)}`);
|
||||
});
|
||||
const rawMessages: unknown[] = [];
|
||||
Reflect.set(transport, "onmessage", (message: unknown) => {
|
||||
pushBounded(rawMessages, message, MCP_RAW_MESSAGE_RETAIN_LIMIT);
|
||||
});
|
||||
|
||||
const client = new Client({ name: "docker-mcp-channels", version: "1.0.0" });
|
||||
try {
|
||||
await connectMcpWithTimeout(client, transport, MCP_CONNECT_TIMEOUT_MS);
|
||||
return {
|
||||
client,
|
||||
cleanup: ownsTempState ? tempState.cleanup : () => {},
|
||||
transport,
|
||||
rawMessages,
|
||||
};
|
||||
} catch (error) {
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
if (ownsTempState) {
|
||||
tempState.cleanup();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function maybeApprovePendingBridgePairing(
|
||||
gateway: GatewayRpcClient,
|
||||
): Promise<boolean> {
|
||||
let pairingState:
|
||||
| {
|
||||
pending?: Array<{ requestId?: string; role?: string }>;
|
||||
}
|
||||
| undefined;
|
||||
try {
|
||||
pairingState = await gateway.request<{
|
||||
pending?: Array<{ requestId?: string; role?: string }>;
|
||||
}>("device.pair.list", {});
|
||||
} catch (error) {
|
||||
const message = formatErrorMessage(error);
|
||||
if (
|
||||
message.includes("missing scope: operator.pairing") ||
|
||||
message.includes("device.pair.list")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!pairingState) {
|
||||
return false;
|
||||
}
|
||||
const pendingRequest = pairingState.pending?.find((entry) => entry.role === "operator");
|
||||
if (!pendingRequest?.requestId) {
|
||||
return false;
|
||||
}
|
||||
await gateway.request("device.pair.approve", { requestId: pendingRequest.requestId });
|
||||
return true;
|
||||
}
|
||||
116
test/e2e/qa-lab/runtime/mcp-client-temp-state.fixture.ts
Normal file
116
test/e2e/qa-lab/runtime/mcp-client-temp-state.fixture.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
// MCP client temp-state helpers used by QA-owned MCP E2E fixtures.
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
export type McpClientTempState = {
|
||||
cleanup: () => void;
|
||||
root: string;
|
||||
stateDir: string;
|
||||
tokenFile: string;
|
||||
};
|
||||
|
||||
export type ReconnectableMcpClientHandle = {
|
||||
cleanup: () => void;
|
||||
client: { close: () => Promise<unknown> };
|
||||
transport: { close: () => Promise<unknown> };
|
||||
};
|
||||
|
||||
type McpConnectTransport = {
|
||||
close?(): Promise<void> | void;
|
||||
};
|
||||
|
||||
const MCP_TIMEOUT_CLOSE_GRACE_MS = 5_000;
|
||||
|
||||
export function createMcpClientTempState(params: {
|
||||
gatewayToken: string;
|
||||
tempRoot?: string;
|
||||
}): McpClientTempState {
|
||||
const root = mkdtempSync(path.join(params.tempRoot ?? tmpdir(), "openclaw-mcp-client-"));
|
||||
const stateDir = path.join(root, "state");
|
||||
const tokenFile = path.join(root, "gateway.token");
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
writeFileSync(tokenFile, `${params.gatewayToken}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
return {
|
||||
cleanup: () => {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
},
|
||||
root,
|
||||
stateDir,
|
||||
tokenFile,
|
||||
};
|
||||
}
|
||||
|
||||
export async function connectMcpWithTimeout<TTransport extends McpConnectTransport>(
|
||||
client: { connect(transport: TTransport): Promise<void> },
|
||||
transport: TTransport,
|
||||
timeoutMs: number,
|
||||
): Promise<void> {
|
||||
let timedOut = false;
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
reject(new Error(`MCP stdio connect timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
timeout.unref?.();
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.race([client.connect(transport), timeoutPromise]);
|
||||
} catch (error) {
|
||||
if (timedOut) {
|
||||
await closeTimedOutTransport(transport);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function closeTimedOutTransport(transport: McpConnectTransport): Promise<void> {
|
||||
if (!transport.close) {
|
||||
return;
|
||||
}
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
Promise.resolve(transport.close()).catch(() => undefined),
|
||||
new Promise<void>((resolve) => {
|
||||
timer = setTimeout(resolve, MCP_TIMEOUT_CLOSE_GRACE_MS);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function connectMcpClientWithPairingReconnect<
|
||||
T extends ReconnectableMcpClientHandle,
|
||||
>(params: {
|
||||
connect: (tempState: McpClientTempState) => Promise<T>;
|
||||
maybeApprovePairing: () => Promise<boolean>;
|
||||
tempState: McpClientTempState;
|
||||
}): Promise<T> {
|
||||
let handle = await params.connect(params.tempState);
|
||||
let shouldReconnect: boolean;
|
||||
try {
|
||||
shouldReconnect = await params.maybeApprovePairing();
|
||||
} catch (error) {
|
||||
await Promise.allSettled([handle.client.close(), handle.transport.close()]);
|
||||
handle.cleanup();
|
||||
throw error;
|
||||
}
|
||||
if (!shouldReconnect) {
|
||||
return handle;
|
||||
}
|
||||
await Promise.allSettled([handle.client.close(), handle.transport.close()]);
|
||||
handle.cleanup();
|
||||
handle = await params.connect(params.tempState);
|
||||
return handle;
|
||||
}
|
||||
185
test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts
Normal file
185
test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
// QA Lab MCP gateway transport tests cover script-backed MCP client state.
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
connectMcpClientWithPairingReconnect,
|
||||
connectMcpWithTimeout,
|
||||
createMcpClientTempState,
|
||||
type McpClientTempState,
|
||||
} from "./mcp-client-temp-state.fixture.ts";
|
||||
|
||||
describe("MCP gateway transport fixture", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("creates unique client temp state and removes token files on cleanup", () => {
|
||||
const tempRoot = mkdtempSync(path.join(tmpdir(), "openclaw-mcp-harness-test-"));
|
||||
try {
|
||||
const first = createMcpClientTempState({ gatewayToken: "first-token", tempRoot });
|
||||
const second = createMcpClientTempState({ gatewayToken: "second-token", tempRoot });
|
||||
|
||||
expect(first.root).not.toBe(second.root);
|
||||
expect(first.stateDir).toBe(path.join(first.root, "state"));
|
||||
expect(readFileSync(first.tokenFile, "utf8")).toBe("first-token\n");
|
||||
expect(statSync(first.tokenFile).mode & 0o777).toBe(0o600);
|
||||
expect(readFileSync(second.tokenFile, "utf8")).toBe("second-token\n");
|
||||
|
||||
first.cleanup();
|
||||
second.cleanup();
|
||||
|
||||
expect(existsSync(first.root)).toBe(false);
|
||||
expect(existsSync(second.root)).toBe(false);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses one MCP temp state across the pairing reconnect path", async () => {
|
||||
const tempState = createMcpClientTempState({ gatewayToken: "pairing-token" });
|
||||
const firstHandle = {
|
||||
cleanup: vi.fn(),
|
||||
client: { close: vi.fn(async () => undefined) },
|
||||
transport: { close: vi.fn(async () => undefined) },
|
||||
};
|
||||
const secondHandle = {
|
||||
cleanup: vi.fn(),
|
||||
client: { close: vi.fn(async () => undefined) },
|
||||
transport: { close: vi.fn(async () => undefined) },
|
||||
};
|
||||
const connectCalls: McpClientTempState[] = [];
|
||||
const connect = vi.fn(async (state: McpClientTempState) => {
|
||||
connectCalls.push(state);
|
||||
return connectCalls.length === 1 ? firstHandle : secondHandle;
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
connectMcpClientWithPairingReconnect({
|
||||
connect,
|
||||
maybeApprovePairing: async () => true,
|
||||
tempState,
|
||||
}),
|
||||
).resolves.toBe(secondHandle);
|
||||
|
||||
expect(connect).toHaveBeenCalledTimes(2);
|
||||
expect(connectCalls).toEqual([tempState, tempState]);
|
||||
expect(firstHandle.client.close).toHaveBeenCalledOnce();
|
||||
expect(firstHandle.transport.close).toHaveBeenCalledOnce();
|
||||
expect(firstHandle.cleanup).toHaveBeenCalledOnce();
|
||||
expect(secondHandle.cleanup).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
tempState.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans up the first MCP client when pairing approval fails", async () => {
|
||||
const tempState = createMcpClientTempState({ gatewayToken: "pairing-token" });
|
||||
const handle = {
|
||||
cleanup: vi.fn(),
|
||||
client: { close: vi.fn(async () => undefined) },
|
||||
transport: { close: vi.fn(async () => undefined) },
|
||||
};
|
||||
const failure = new Error("pairing approval failed");
|
||||
|
||||
try {
|
||||
await expect(
|
||||
connectMcpClientWithPairingReconnect({
|
||||
connect: async () => handle,
|
||||
maybeApprovePairing: async () => {
|
||||
throw failure;
|
||||
},
|
||||
tempState,
|
||||
}),
|
||||
).rejects.toBe(failure);
|
||||
|
||||
expect(handle.client.close).toHaveBeenCalledOnce();
|
||||
expect(handle.transport.close).toHaveBeenCalledOnce();
|
||||
expect(handle.cleanup).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
tempState.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves when the MCP client connects before the timeout", async () => {
|
||||
const client = {
|
||||
connect: vi.fn(async () => undefined),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
await expect(connectMcpWithTimeout(client, transport, 1000)).resolves.toBeUndefined();
|
||||
|
||||
expect(client.connect).toHaveBeenCalledWith(transport);
|
||||
expect(transport.close).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("closes the transport when MCP initialize hangs", async () => {
|
||||
vi.useFakeTimers();
|
||||
const client = {
|
||||
connect: vi.fn(() => new Promise<void>(() => {})),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
const result = connectMcpWithTimeout(client, transport, 100);
|
||||
const rejection = expect(result).rejects.toThrow("MCP stdio connect timed out after 100ms");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await rejection;
|
||||
expect(transport.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("waits for timed-out transport cleanup before rejecting", async () => {
|
||||
vi.useFakeTimers();
|
||||
let closeSettled = false;
|
||||
const client = {
|
||||
connect: vi.fn(() => new Promise<void>(() => {})),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
setTimeout(() => {
|
||||
closeSettled = true;
|
||||
resolve();
|
||||
}, 25);
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
const result = connectMcpWithTimeout(client, transport, 100);
|
||||
const rejection = expect(result).rejects.toThrow("MCP stdio connect timed out after 100ms");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(transport.close).toHaveBeenCalledOnce();
|
||||
expect(closeSettled).toBe(false);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
await rejection;
|
||||
expect(closeSettled).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the original timeout error when cleanup rejects", async () => {
|
||||
vi.useFakeTimers();
|
||||
const client = {
|
||||
connect: vi.fn(() => new Promise<void>(() => {})),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(async () => {
|
||||
throw new Error("close failed");
|
||||
}),
|
||||
};
|
||||
|
||||
const result = connectMcpWithTimeout(client, transport, 100);
|
||||
const rejection = expect(result).rejects.toThrow("MCP stdio connect timed out after 100ms");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await rejection;
|
||||
expect(transport.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
428
test/e2e/qa-lab/runtime/openai-compatible-chat-tools.e2e.test.ts
Normal file
428
test/e2e/qa-lab/runtime/openai-compatible-chat-tools.e2e.test.ts
Normal file
@@ -0,0 +1,428 @@
|
||||
// OpenAI-compatible chat tools tests cover QA Lab HTTP tool-call evidence.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import { createBoundedChildOutput } from "../../../helpers/bounded-child-output.js";
|
||||
import { cleanupTempDirs, makeTempDir } from "../../../helpers/temp-dir.js";
|
||||
|
||||
const clientPath = path.resolve("scripts/e2e/lib/openai-chat-tools/client.mjs");
|
||||
const dockerRunnerPath = path.resolve("scripts/e2e/openai-chat-tools-docker.sh");
|
||||
const writeConfigPath = path.resolve("scripts/e2e/lib/openai-chat-tools/write-config.mjs");
|
||||
|
||||
interface ClientResult {
|
||||
error?: Error;
|
||||
status: number | null;
|
||||
stderr: string;
|
||||
stdout: string;
|
||||
}
|
||||
|
||||
async function listen(server: Server): Promise<number> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not expose a TCP port");
|
||||
}
|
||||
return address.port;
|
||||
}
|
||||
|
||||
function runClient(
|
||||
port: number | string,
|
||||
env: Record<string, string> = {},
|
||||
timeout = 5_000,
|
||||
): Promise<ClientResult> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(process.execPath, [clientPath], {
|
||||
env: {
|
||||
...process.env,
|
||||
MODEL_REF: "openai/gpt-5.4-mini",
|
||||
OPENCLAW_GATEWAY_TOKEN: "test-token",
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "1",
|
||||
PORT: String(port),
|
||||
...env,
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = createBoundedChildOutput();
|
||||
const stderr = createBoundedChildOutput();
|
||||
let timedOut = false;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr.append(chunk);
|
||||
});
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGKILL");
|
||||
}, timeout);
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ error, status: null, stderr: stderr.text(), stdout: stdout.text() });
|
||||
});
|
||||
child.on("exit", (status) => {
|
||||
clearTimeout(timer);
|
||||
resolve({
|
||||
error: timedOut ? new Error(`client timed out after ${timeout}ms`) : undefined,
|
||||
status,
|
||||
stderr: stderr.text(),
|
||||
stdout: stdout.text(),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function runWriteConfig(root: string, env: Record<string, string> = {}) {
|
||||
return spawnSync(process.execPath, [writeConfigPath], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_CONFIG_PATH: path.join(root, "openclaw.json"),
|
||||
OPENCLAW_GATEWAY_TOKEN: "test-token",
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_MODEL: "openai/gpt-5.5",
|
||||
OPENCLAW_STATE_DIR: path.join(root, "state"),
|
||||
OPENCLAW_TEST_WORKSPACE_DIR: path.join(root, "workspace"),
|
||||
PORT: "18789",
|
||||
...env,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function runDockerRunnerAuthPreflight(root: string, env: Record<string, string> = {}) {
|
||||
return spawnSync("bash", [dockerRunnerPath], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: root,
|
||||
OPENAI_API_KEY: "",
|
||||
OPENAI_BASE_URL: "",
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_PROFILE_FILE: path.join(root, "missing.profile"),
|
||||
...env,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function toolCallResponse(messageOverrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
choices: [
|
||||
{
|
||||
finish_reason: "tool_calls",
|
||||
message: {
|
||||
...messageOverrides,
|
||||
tool_calls: [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "get_weather",
|
||||
arguments: JSON.stringify({ city: "Paris, France" }),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("scripts/e2e/lib/openai-chat-tools/client.mjs", () => {
|
||||
let bodyReadTimeoutProbe: {
|
||||
elapsedMs: number;
|
||||
responseClosed: boolean;
|
||||
result: ClientResult;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
let responseClosed = false;
|
||||
const server = createServer((_request, response) => {
|
||||
response.on("close", () => {
|
||||
responseClosed = true;
|
||||
});
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.write('{"choices":');
|
||||
});
|
||||
const port = await listen(server);
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
bodyReadTimeoutProbe = {
|
||||
result: await runClient(port, {}, 4_000),
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
responseClosed,
|
||||
};
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps full profile exports out of the Docker build phase", () => {
|
||||
const runner = readFileSync(dockerRunnerPath, "utf8");
|
||||
const preflightSourceIndex = runner.indexOf('source "$profile_file"');
|
||||
const buildIndex = runner.indexOf("docker_e2e_build_or_reuse");
|
||||
const fullProfileSourceIndex = runner.indexOf('source "$PROFILE_FILE"', buildIndex);
|
||||
|
||||
expect(preflightSourceIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(buildIndex).toBeGreaterThan(preflightSourceIndex);
|
||||
expect(fullProfileSourceIndex).toBeGreaterThan(buildIndex);
|
||||
});
|
||||
|
||||
it("fails auth preflight before Docker build work starts", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runDockerRunnerAuthPreflight(root);
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(output).toContain("OPENAI_API_KEY was not available");
|
||||
expect(output).not.toContain("Building Docker image:");
|
||||
expect(output).not.toContain("Reusing Docker image:");
|
||||
expect(output).not.toContain("Running OpenAI Chat Completions tools Docker E2E");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("treats placeholder profile auth as missing before Docker build work starts", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const profile = path.join(root, "profile");
|
||||
writeFileSync(profile, "OPENAI_API_KEY=undefined\n");
|
||||
const result = runDockerRunnerAuthPreflight(root, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_PROFILE_FILE: profile,
|
||||
});
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(output).toContain("OPENAI_API_KEY was not available");
|
||||
expect(output).not.toContain("Building Docker image:");
|
||||
expect(output).not.toContain("Reusing Docker image:");
|
||||
expect(output).not.toContain("Running OpenAI Chat Completions tools Docker E2E");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["timeout", "OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS", "1e3"],
|
||||
["body limit", "OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES", "64bytes"],
|
||||
])(
|
||||
"rejects invalid Docker runner %s before auth or Docker build work starts",
|
||||
(_label, envName, value) => {
|
||||
const tempDirs: string[] = [];
|
||||
const root = makeTempDir(tempDirs, "openclaw-openai-chat-tools-");
|
||||
try {
|
||||
const result = runDockerRunnerAuthPreflight(root, { [envName]: value });
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
|
||||
expect(result.status).toBe(2);
|
||||
expect(output).toContain(`invalid ${envName}: ${value}`);
|
||||
expect(output).not.toContain("OPENAI_API_KEY was not available");
|
||||
expect(output).not.toContain("Building Docker image:");
|
||||
expect(output).not.toContain("Reusing Docker image:");
|
||||
expect(output).not.toContain("Running OpenAI Chat Completions tools Docker E2E");
|
||||
} finally {
|
||||
cleanupTempDirs(tempDirs);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("passes normalized timeout and body limits into the Docker runner", () => {
|
||||
const runner = readFileSync(dockerRunnerPath, "utf8");
|
||||
|
||||
expect(runner).toContain(
|
||||
"docker_e2e_read_positive_int_env OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS 180",
|
||||
);
|
||||
expect(runner).toContain(
|
||||
"docker_e2e_read_positive_int_env OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES 1048576",
|
||||
);
|
||||
expect(runner).toContain('-e "OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS=$TIMEOUT_SECONDS"');
|
||||
expect(runner).toContain('-e "OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES=$MAX_BODY_BYTES"');
|
||||
});
|
||||
|
||||
it("rejects loose timeout env values instead of parsing numeric prefixes", async () => {
|
||||
const result = await runClient(1, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "1e3",
|
||||
});
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: 1e3");
|
||||
});
|
||||
|
||||
it("rejects loose body limit env values instead of parsing numeric prefixes", async () => {
|
||||
const result = await runClient(1, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64bytes",
|
||||
});
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: 64bytes");
|
||||
});
|
||||
|
||||
it("rejects out-of-range client gateway ports", async () => {
|
||||
const result = await runClient("65536");
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid PORT: 65536");
|
||||
});
|
||||
|
||||
it("rejects loose write-config timeout env values", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runWriteConfig(root, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "1e3",
|
||||
});
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: 1e3");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects out-of-range write-config gateway ports", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runWriteConfig(root, { PORT: "65536" });
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid PORT: 65536");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("writes strict positive timeout and port values into generated config", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runWriteConfig(root, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "240",
|
||||
PORT: "19001",
|
||||
});
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
const config = JSON.parse(readFileSync(path.join(root, "openclaw.json"), "utf8"));
|
||||
expect(config.gateway.port).toBe(19001);
|
||||
expect(config.models.providers.openai.timeoutSeconds).toBe(240);
|
||||
expect(config.agents.defaults.timeoutSeconds).toBe(240);
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts a matching chat completions tool call response", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
expect(request.method).toBe("POST");
|
||||
expect(request.url).toBe("/v1/chat/completions");
|
||||
expect(request.headers.authorization).toBe("Bearer test-token");
|
||||
expect(request.headers["x-openclaw-model"]).toBe("openai/gpt-5.4-mini");
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify(toolCallResponse()));
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const result = await runClient(port);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
args: { city: "Paris, France" },
|
||||
finishReason: "tool_calls",
|
||||
ok: true,
|
||||
toolName: "get_weather",
|
||||
});
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects chat completions responses that include content beside the tool call", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify(toolCallResponse({ content: "I will call the tool now." })));
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const result = await runClient(port);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("expected tool call only response");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the request timeout active while reading the response body", async () => {
|
||||
expect(bodyReadTimeoutProbe.result.error).toBeUndefined();
|
||||
expect(bodyReadTimeoutProbe.result.status).not.toBe(0);
|
||||
expect(bodyReadTimeoutProbe.result.stderr).toMatch(/timed out|aborted|AbortError/iu);
|
||||
expect(bodyReadTimeoutProbe.elapsedMs).toBeLessThan(3_500);
|
||||
expect(bodyReadTimeoutProbe.responseClosed).toBe(true);
|
||||
});
|
||||
|
||||
it("caps chat completion response bodies before JSON parsing", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end("x".repeat(256));
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const result = await runClient(port, { OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64" });
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat completions response body exceeded 64 bytes");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects declared oversized chat completion bodies before waiting on the stream", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, {
|
||||
"content-length": "65",
|
||||
"content-type": "application/json",
|
||||
});
|
||||
response.flushHeaders();
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
const result = await runClient(port, { OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64" });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat completions response body exceeded 64 bytes");
|
||||
expect(Date.now() - startedAt).toBeLessThan(3_500);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects unsafe declared chat completion body lengths before waiting on the stream", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, {
|
||||
"content-length": "9007199254740993",
|
||||
"content-type": "application/json",
|
||||
});
|
||||
response.flushHeaders();
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
const result = await runClient(port, { OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64" });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat completions response body exceeded 64 bytes");
|
||||
expect(result.stderr).not.toContain("timed out");
|
||||
expect(Date.now() - startedAt).toBeLessThan(3_500);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
255
test/e2e/qa-lab/runtime/openai-image-auth-docker-client.ts
Normal file
255
test/e2e/qa-lab/runtime/openai-image-auth-docker-client.ts
Normal file
@@ -0,0 +1,255 @@
|
||||
// Openai Image Auth Docker Client script supports OpenClaw repository automation.
|
||||
import http from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
isRequestBodyTooLargeError,
|
||||
readBody,
|
||||
} from "../../../../scripts/e2e/lib/mock-openai-http.mjs";
|
||||
|
||||
const DIRECT_IMAGE_BYTES = Buffer.from("docker-direct-image");
|
||||
const CODEX_IMAGE_BYTES = Buffer.from("docker-codex-image");
|
||||
const DIRECT_TOKEN = "sk-openclaw-image-auth-e2e";
|
||||
const CODEX_TOKEN = "docker-codex-oauth-token";
|
||||
|
||||
export type RequestRecord = {
|
||||
method?: string;
|
||||
url?: string;
|
||||
authorization?: string;
|
||||
accept?: string;
|
||||
contentType?: string;
|
||||
body: string;
|
||||
};
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function writeJson(res: http.ServerResponse, status: number, body: unknown): void {
|
||||
res.writeHead(status, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
function writeCodexSse(res: http.ServerResponse): void {
|
||||
const events = [
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "image_generation_call",
|
||||
result: CODEX_IMAGE_BYTES.toString("base64"),
|
||||
revised_prompt: "docker codex revised prompt",
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
usage: { input_tokens: 1, output_tokens: 2, total_tokens: 3 },
|
||||
tool_usage: { image_gen: { total_tokens: 3 } },
|
||||
},
|
||||
},
|
||||
];
|
||||
res.writeHead(200, { "content-type": "text/event-stream" });
|
||||
for (const event of events) {
|
||||
res.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
res.end("data: [DONE]\n\n");
|
||||
}
|
||||
|
||||
export async function startMockServer(records: RequestRecord[]): Promise<{
|
||||
baseUrl: string;
|
||||
close: () => Promise<void>;
|
||||
}> {
|
||||
const server = http.createServer((req, res) => {
|
||||
void (async () => {
|
||||
try {
|
||||
let body: string;
|
||||
try {
|
||||
body = await readBody(req);
|
||||
} catch (error) {
|
||||
if (isRequestBodyTooLargeError(error)) {
|
||||
writeJson(res, 413, { error: { message: error.message } });
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
records.push({
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
authorization: req.headers.authorization,
|
||||
accept: req.headers.accept,
|
||||
contentType: req.headers["content-type"],
|
||||
body,
|
||||
});
|
||||
|
||||
if (req.method === "POST" && req.url === "/v1/images/generations") {
|
||||
assert(
|
||||
req.headers.authorization === `Bearer ${DIRECT_TOKEN}`,
|
||||
`direct image route used wrong auth: ${req.headers.authorization}`,
|
||||
);
|
||||
const parsed = JSON.parse(body) as { model?: string; prompt?: string; size?: string };
|
||||
assert(parsed.model === "gpt-image-2", `direct route model mismatch: ${body}`);
|
||||
assert(
|
||||
parsed.prompt === "docker direct image auth",
|
||||
`direct route prompt mismatch: ${body}`,
|
||||
);
|
||||
assert(parsed.size === "1024x1024", `direct route size mismatch: ${body}`);
|
||||
writeJson(res, 200, {
|
||||
data: [
|
||||
{
|
||||
b64_json: DIRECT_IMAGE_BYTES.toString("base64"),
|
||||
revised_prompt: "docker direct revised prompt",
|
||||
},
|
||||
],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === "POST" && req.url === "/backend-api/codex/responses") {
|
||||
assert(
|
||||
req.headers.authorization === `Bearer ${CODEX_TOKEN}`,
|
||||
`codex image route used wrong auth: ${req.headers.authorization}`,
|
||||
);
|
||||
const parsed = JSON.parse(body) as {
|
||||
tools?: Array<{ type?: string; model?: string; size?: string }>;
|
||||
input?: Array<{ content?: Array<{ type?: string; text?: string }> }>;
|
||||
};
|
||||
assert(
|
||||
parsed.tools?.[0]?.type === "image_generation" &&
|
||||
parsed.tools[0].model === "gpt-image-2" &&
|
||||
parsed.tools[0].size === "1024x1024",
|
||||
`codex image tool mismatch: ${body}`,
|
||||
);
|
||||
assert(
|
||||
parsed.input?.[0]?.content?.some(
|
||||
(entry) =>
|
||||
entry.type === "input_text" && entry.text === "docker codex oauth image auth",
|
||||
),
|
||||
`codex prompt missing: ${body}`,
|
||||
);
|
||||
writeCodexSse(res);
|
||||
return;
|
||||
}
|
||||
|
||||
writeJson(res, 404, { error: `unexpected ${req.method} ${req.url}` });
|
||||
} catch (error) {
|
||||
writeJson(res, 500, { error: String(error instanceof Error ? error.message : error) });
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address() as AddressInfo;
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${address.port}`,
|
||||
close: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function createCodexOAuthStore() {
|
||||
return {
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:chatgpt": {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: CODEX_TOKEN,
|
||||
refresh: "docker-codex-refresh-token",
|
||||
expires: Date.now() + 60 * 60 * 1000,
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
}
|
||||
|
||||
export async function main() {
|
||||
assert(
|
||||
process.env.OPENAI_API_KEY === DIRECT_TOKEN,
|
||||
"Docker lane must expose the direct OpenAI API key",
|
||||
);
|
||||
const records: RequestRecord[] = [];
|
||||
const mock = await startMockServer(records);
|
||||
try {
|
||||
const { buildOpenAIImageGenerationProvider } =
|
||||
await import("../../../../dist/extensions/openai/image-generation-provider.js");
|
||||
const provider = buildOpenAIImageGenerationProvider();
|
||||
|
||||
const directResult = await provider.generateImage({
|
||||
provider: "openai",
|
||||
model: "gpt-image-2",
|
||||
prompt: "docker direct image auth",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: `${mock.baseUrl}/v1`,
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
assert(
|
||||
directResult.images?.[0]?.buffer?.equals(DIRECT_IMAGE_BYTES),
|
||||
"direct image route did not return expected bytes",
|
||||
);
|
||||
assert(
|
||||
records.some((entry) => entry.url === "/v1/images/generations"),
|
||||
"direct image route was not called",
|
||||
);
|
||||
|
||||
records.length = 0;
|
||||
const codexResult = await provider.generateImage({
|
||||
provider: "openai",
|
||||
model: "gpt-image-2",
|
||||
prompt: "docker codex oauth image auth",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: `${mock.baseUrl}/backend-api/codex`,
|
||||
api: "openai-chatgpt-responses",
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
authStore: createCodexOAuthStore(),
|
||||
});
|
||||
assert(
|
||||
codexResult.images?.[0]?.buffer?.equals(CODEX_IMAGE_BYTES),
|
||||
"Codex OAuth image route did not return expected bytes",
|
||||
);
|
||||
assert(
|
||||
records.some((entry) => entry.url === "/backend-api/codex/responses"),
|
||||
"Codex OAuth image route was not called",
|
||||
);
|
||||
assert(
|
||||
!records.some((entry) => entry.url === "/v1/images/generations"),
|
||||
"Codex OAuth image route fell back to the direct OpenAI API key",
|
||||
);
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify({
|
||||
ok: true,
|
||||
routes: records.map((entry) => entry.url),
|
||||
directBytes: directResult.images[0]?.buffer.length,
|
||||
codexBytes: codexResult.images[0]?.buffer.length,
|
||||
}) + "\n",
|
||||
);
|
||||
} finally {
|
||||
await mock.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// OpenAI web-search minimal assertion tests cover QA Lab native web_search evidence.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const ASSERTIONS_SCRIPT = "scripts/e2e/lib/openai-web-search-minimal/assertions.mjs";
|
||||
|
||||
function runAssertSuccessRequest(logPath: string) {
|
||||
return spawnSync(process.execPath, [ASSERTIONS_SCRIPT, "assert-success-request", logPath], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
}
|
||||
|
||||
describe("openai web-search minimal assertions", () => {
|
||||
it("accepts a success request with web_search and non-minimal reasoning", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
input: "OPENCLAW_SCHEMA_E2E_OK",
|
||||
reasoning: { effort: "low" },
|
||||
tools: [{ type: "web_search" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
expect(runAssertSuccessRequest(logPath).status).toBe(0);
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("finds success requests split across large scan chunks", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({ path: "/health", body: { pad: "x".repeat(70 * 1024) } })}\n${JSON.stringify(
|
||||
{
|
||||
body: {
|
||||
input: "OPENCLAW_SCHEMA_E2E_OK",
|
||||
reasoning: { effort: "low" },
|
||||
tools: [{ type: "web_search" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
},
|
||||
)}\n`,
|
||||
);
|
||||
|
||||
expect(runAssertSuccessRequest(logPath).status).toBe(0);
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds diagnostics when the OpenAI responses endpoint was not used", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
old: `DO_NOT_DUMP_OLD_REQUESTS${"x".repeat(70 * 1024)}`,
|
||||
},
|
||||
path: "/health",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
const result = runAssertSuccessRequest(logPath);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Request log tail:");
|
||||
expect(result.stderr).not.toContain("DO_NOT_DUMP_OLD_REQUESTS");
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds diagnostics when no success response is present", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
input: `DO_NOT_DUMP_OLD_RESPONSE${"x".repeat(70 * 1024)}recent response tail`,
|
||||
tools: [{ type: "web_search" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
const result = runAssertSuccessRequest(logPath);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Recent /v1/responses:");
|
||||
expect(result.stderr).toContain("recent response tail");
|
||||
expect(result.stderr).not.toContain("DO_NOT_DUMP_OLD_RESPONSE");
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects function-shaped web_search as native Responses proof", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
input: "OPENCLAW_SCHEMA_E2E_OK",
|
||||
reasoning: { effort: "low" },
|
||||
tools: [{ name: "web_search", type: "function" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
const result = runAssertSuccessRequest(logPath);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("success request did not include native web_search");
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
119
test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts
Normal file
119
test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
// OpenAI web-search minimal tests cover QA Lab hosted provider schema evidence.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { testing } from "../../../../scripts/e2e/lib/openai-web-search-minimal/client.mjs";
|
||||
|
||||
describe("scripts/e2e/lib/openai-web-search-minimal/client.mjs", () => {
|
||||
it("accepts only the expected raw schema rejection in reject mode", () => {
|
||||
expect(
|
||||
testing.validateRejectResult({
|
||||
ok: false,
|
||||
error: new Error(`gateway failed: ${testing.DEFAULT_RAW_SCHEMA_ERROR}`),
|
||||
}),
|
||||
).toContain(testing.DEFAULT_RAW_SCHEMA_ERROR);
|
||||
});
|
||||
|
||||
it("accepts the gateway schema rejection wrapper in reject mode", () => {
|
||||
expect(
|
||||
testing.validateRejectResult({
|
||||
ok: false,
|
||||
error: new Error(
|
||||
`GatewayClientRequestError: FailoverError: ${testing.DEFAULT_GATEWAY_SCHEMA_ERROR}.`,
|
||||
),
|
||||
}),
|
||||
).toContain(testing.DEFAULT_GATEWAY_SCHEMA_ERROR);
|
||||
});
|
||||
|
||||
it("fails reject mode when the agent run unexpectedly succeeds", () => {
|
||||
expect(() =>
|
||||
testing.validateRejectResult({
|
||||
ok: true,
|
||||
value: { status: "ok" },
|
||||
}),
|
||||
).toThrow(/reject mode unexpectedly completed/u);
|
||||
});
|
||||
|
||||
it("fails reject mode on unrelated transport errors", () => {
|
||||
expect(() =>
|
||||
testing.validateRejectResult({
|
||||
ok: false,
|
||||
error: new Error("connect ECONNREFUSED 127.0.0.1:9"),
|
||||
}),
|
||||
).toThrow(/reject mode failed for an unexpected reason/u);
|
||||
});
|
||||
|
||||
it("rejects out-of-range gateway ports before connecting", () => {
|
||||
expect(() => testing.resolveGatewayPort({ PORT: "65536" })).toThrow("invalid PORT: 65536");
|
||||
});
|
||||
|
||||
it("accepts success mode only when the final assistant reply contains the marker", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
meta: { finalAssistantVisibleText: `done: ${testing.SUCCESS_MARKER}` },
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts success markers from non-error reply payload text", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
payloads: [{ text: testing.SUCCESS_MARKER }],
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts success markers from the gateway agent result envelope", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
result: {
|
||||
meta: { finalAssistantVisibleText: testing.SUCCESS_MARKER },
|
||||
payloads: [{ text: "secondary reply" }],
|
||||
},
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("fails success mode when the agent run completes without the marker", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: { status: "ok" },
|
||||
}),
|
||||
).toThrow(/completed without success marker/u);
|
||||
});
|
||||
|
||||
it("does not accept success markers from error payload text", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
payloads: [{ isError: true, text: testing.SUCCESS_MARKER }],
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).toThrow(/completed without success marker/u);
|
||||
});
|
||||
|
||||
it("keeps non-ok success mode failures distinct from marker failures", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
meta: { finalAssistantVisibleText: testing.SUCCESS_MARKER },
|
||||
status: "blocked",
|
||||
},
|
||||
}),
|
||||
).toThrow(/agent run did not complete successfully/u);
|
||||
});
|
||||
});
|
||||
497
test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts
Normal file
497
test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts
Normal file
@@ -0,0 +1,497 @@
|
||||
// OpenWebUI probe tests cover QA Lab OpenAI-compatible API evidence.
|
||||
import { spawn } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createServer, type IncomingMessage, type Server as HttpServer } from "node:http";
|
||||
import { createServer as createTcpServer, type Server as TcpServer, type Socket } from "node:net";
|
||||
import path from "node:path";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createBoundedChildOutput } from "../../../helpers/bounded-child-output.js";
|
||||
|
||||
const probePath = path.resolve("scripts/e2e/openwebui-probe.mjs");
|
||||
|
||||
interface ProbeResult {
|
||||
error?: Error;
|
||||
status: number | null;
|
||||
stderr: string;
|
||||
stdout: string;
|
||||
}
|
||||
|
||||
async function listen(server: HttpServer | TcpServer): Promise<string> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not expose a TCP port");
|
||||
}
|
||||
return `http://127.0.0.1:${address.port}`;
|
||||
}
|
||||
|
||||
function runProbe(baseUrl: string, env: Record<string, string> = {}, timeout = 3_000) {
|
||||
return new Promise<ProbeResult>((resolve) => {
|
||||
const child = spawn(process.execPath, [probePath], {
|
||||
env: {
|
||||
...process.env,
|
||||
OPENWEBUI_ADMIN_EMAIL: "openwebui-e2e@example.com",
|
||||
OPENWEBUI_ADMIN_PASSWORD: "test-password",
|
||||
OPENWEBUI_BASE_URL: baseUrl,
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "250",
|
||||
OPENWEBUI_EXPECTED_NONCE: "nonce-123",
|
||||
OPENWEBUI_MODEL_ATTEMPTS: "1",
|
||||
OPENWEBUI_MODEL_RETRY_MS: "0",
|
||||
OPENWEBUI_PROMPT: "reply with nonce-123",
|
||||
OPENWEBUI_SMOKE_MODE: "models",
|
||||
...env,
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = createBoundedChildOutput();
|
||||
const stderr = createBoundedChildOutput();
|
||||
let timedOut = false;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr.append(chunk);
|
||||
});
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGKILL");
|
||||
}, timeout);
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ error, status: null, stderr: stderr.text(), stdout: stdout.text() });
|
||||
});
|
||||
child.on("exit", (status) => {
|
||||
clearTimeout(timer);
|
||||
resolve({
|
||||
error: timedOut ? new Error(`probe timed out after ${timeout}ms`) : undefined,
|
||||
status,
|
||||
stderr: stderr.text(),
|
||||
stdout: stdout.text(),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function readRequestBody(request: IncomingMessage): Promise<string> {
|
||||
let body = "";
|
||||
request.setEncoding("utf8");
|
||||
for await (const chunk of request) {
|
||||
body += chunk;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
describe("scripts/e2e/openwebui-probe.mjs", () => {
|
||||
it("rejects loose numeric timeout env values instead of parsing prefixes", async () => {
|
||||
const result = await runProbe("http://127.0.0.1:9", {
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "25ms",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain(
|
||||
"OPENWEBUI_CONTROL_TIMEOUT_MS must be a positive integer; got: 25ms",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects zero where positive retry counts are required", async () => {
|
||||
const result = await runProbe("http://127.0.0.1:9", {
|
||||
OPENWEBUI_MODEL_ATTEMPTS: "0",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("OPENWEBUI_MODEL_ATTEMPTS must be a positive integer; got: 0");
|
||||
});
|
||||
|
||||
it("rejects loose response body cap env values instead of parsing prefixes", async () => {
|
||||
const result = await runProbe("http://127.0.0.1:9", {
|
||||
OPENWEBUI_RESPONSE_BODY_MAX_BYTES: "1mb",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain(
|
||||
"OPENWEBUI_RESPONSE_BODY_MAX_BYTES must be a positive integer; got: 1mb",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses a short control-plane timeout for stalled sign-in requests", async () => {
|
||||
const sockets = new Set<Socket>();
|
||||
const server = createTcpServer((socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on("close", () => sockets.delete(socket));
|
||||
socket.on("data", () => {});
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const result = await runProbe(
|
||||
baseUrl,
|
||||
{
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "25",
|
||||
OPENWEBUI_FETCH_TIMEOUT_MS: "5000",
|
||||
},
|
||||
2_000,
|
||||
);
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Open WebUI signin timed out after 25ms");
|
||||
expect(elapsedMs).toBeLessThan(1500);
|
||||
} finally {
|
||||
for (const socket of sockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the control-plane timeout active while reading sign-in bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.flushHeaders();
|
||||
response.write("{");
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(
|
||||
baseUrl,
|
||||
{
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "25",
|
||||
OPENWEBUI_FETCH_TIMEOUT_MS: "5000",
|
||||
},
|
||||
2_000,
|
||||
);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Open WebUI signin timed out after 25ms");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("passes Open WebUI request timeouts into bounded body reads", () => {
|
||||
const script = readFileSync(probePath, "utf8");
|
||||
|
||||
expect(script).toContain("run(controller.signal, timeoutPromise)");
|
||||
expect(script).toMatch(
|
||||
/readBoundedResponseTextWithLimit\(\s*response,\s*label,\s*responseBodyMaxBytes,\s*timeoutPromise,/u,
|
||||
);
|
||||
expect(script.match(/async \(signal, timeoutPromise\)/gu)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("bounds sign-in error response bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(500, { "content-type": "text/plain" });
|
||||
response.end("x".repeat(64));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, {
|
||||
OPENWEBUI_RESPONSE_BODY_MAX_BYTES: "16",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Open WebUI signin response body exceeded 16 bytes");
|
||||
expect(result.stderr).not.toContain("x".repeat(64));
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("redacts admin credentials from sign-in error bodies", async () => {
|
||||
const adminEmail = "openwebui-e2e" + "@example.com";
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(401, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
error: "invalid credentials",
|
||||
email: adminEmail,
|
||||
password: 'pa"ss',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, { OPENWEBUI_ADMIN_PASSWORD: 'pa"ss' });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("signin failed: HTTP 401");
|
||||
expect(result.stderr).toContain("<redacted>");
|
||||
expect(result.stderr).not.toContain(adminEmail);
|
||||
expect(result.stderr).not.toContain('pa"ss');
|
||||
expect(result.stderr).not.toContain('pa\\"ss');
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds model-list error response bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=chat=secret=cookie; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "chat-secret-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(502, { "content-type": "text/plain" });
|
||||
response.end("y".repeat(96));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, {
|
||||
OPENWEBUI_RESPONSE_BODY_MAX_BYTES: "32",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain(
|
||||
"Open WebUI models attempt 1 response body exceeded 32 bytes",
|
||||
);
|
||||
expect(result.stderr).not.toContain("y".repeat(96));
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("redacts auth material from model-list error bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=model=secret=cookie; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "model-secret-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(502, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
error: "upstream rejected Authorization Bearer model-secret-token",
|
||||
cookieValue: "model=secret=cookie",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("HTTP 502");
|
||||
expect(result.stderr).toContain("<redacted>");
|
||||
expect(result.stderr).not.toContain("model-secret-token");
|
||||
expect(result.stderr).not.toContain("model=secret=cookie");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not sleep after the final model-list attempt", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=chat=secret=cookie; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "chat-secret-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "other-model" }] }));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(
|
||||
baseUrl,
|
||||
{
|
||||
OPENWEBUI_MODEL_ATTEMPTS: "1",
|
||||
OPENWEBUI_MODEL_RETRY_MS: "1500",
|
||||
},
|
||||
1_000,
|
||||
);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("openclaw model missing from Open WebUI model list");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("passes in models mode when Open WebUI exposes the OpenClaw model", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
setTimeout(() => {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=test; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "test-token" }));
|
||||
}, 25);
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
expect(request.headers.authorization).toBe("Bearer test-token");
|
||||
expect(request.headers.cookie).toContain("openwebui-session=test");
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] }));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, {
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: String(MAX_TIMER_TIMEOUT_MS + 1),
|
||||
OPENWEBUI_FETCH_TIMEOUT_MS: String(MAX_TIMER_TIMEOUT_MS + 1),
|
||||
});
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
mode: "models",
|
||||
model: "openclaw/default",
|
||||
ok: true,
|
||||
});
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("runs chat mode through Open WebUI chat completions and validates the nonce", async () => {
|
||||
const chatRequests: unknown[] = [];
|
||||
const server = createServer(async (request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=test; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "test-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/chat/completions") {
|
||||
expect(request.headers.authorization).toBe("Bearer test-token");
|
||||
expect(request.headers.cookie).toContain("openwebui-session=test");
|
||||
chatRequests.push(JSON.parse(await readRequestBody(request)));
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
choices: [{ message: { content: "OpenClaw replied with nonce-123" } }],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, { OPENWEBUI_SMOKE_MODE: "chat" });
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
model: "openclaw/default",
|
||||
ok: true,
|
||||
reply: "OpenClaw replied with nonce-123",
|
||||
});
|
||||
expect(chatRequests).toEqual([
|
||||
{
|
||||
messages: [{ content: "reply with nonce-123", role: "user" }],
|
||||
model: "openclaw/default",
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails chat mode when the Open WebUI reply omits the expected nonce", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=test; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "test-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/chat/completions") {
|
||||
const authorization = request.headers.authorization ?? "";
|
||||
const cookie = request.headers.cookie ?? "";
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
message: {
|
||||
content: `missing the marker with ${authorization} and ${cookie}`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, { OPENWEBUI_SMOKE_MODE: "chat" });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat reply missing nonce");
|
||||
expect(result.stderr).toContain("<redacted>");
|
||||
expect(result.stderr).not.toContain("chat-secret-token");
|
||||
expect(result.stderr).not.toContain("chat=secret=cookie");
|
||||
expect(result.stderr).not.toContain("openwebui-session=chat");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
646
test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts
Normal file
646
test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts
Normal file
@@ -0,0 +1,646 @@
|
||||
// Package OpenClaw For Docker tests cover QA Lab package artifact evidence.
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildPackageArtifacts,
|
||||
packOpenClawPackageForDocker,
|
||||
parseArgs,
|
||||
prepareBundledAiRuntimePackage,
|
||||
runCommandForTest,
|
||||
} from "../../../../scripts/package-openclaw-for-docker.mjs";
|
||||
|
||||
const skipBundledAiRuntime = async (): Promise<() => Promise<void>> => async () => {};
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
if (!Number.isSafeInteger(pid) || pid <= 0) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
async function readPid(filePath: string, timeoutMs: number): Promise<number> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (fs.existsSync(filePath)) {
|
||||
const pid = Number(fs.readFileSync(filePath, "utf8").trim());
|
||||
if (Number.isSafeInteger(pid) && pid > 0) {
|
||||
return pid;
|
||||
}
|
||||
}
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error(`timeout waiting for a positive pid in ${filePath}`);
|
||||
}
|
||||
|
||||
async function waitForDead(pid: number, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (!isProcessAlive(pid)) {
|
||||
return;
|
||||
}
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error(`process still alive: ${pid}`);
|
||||
}
|
||||
|
||||
async function waitForExit(
|
||||
child: ReturnType<typeof spawn>,
|
||||
timeoutMs: number,
|
||||
): Promise<{ signal: NodeJS.Signals | null; status: number | null }> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(
|
||||
() => reject(new Error("timeout waiting for child exit")),
|
||||
timeoutMs,
|
||||
);
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
resolve({ signal, status });
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe("package-openclaw-for-docker", () => {
|
||||
it("parses package artifact output options", () => {
|
||||
expect(
|
||||
parseArgs([
|
||||
"--output-dir",
|
||||
".artifacts/docker",
|
||||
"--output-name=openclaw-current.tgz",
|
||||
"--source-dir",
|
||||
"/repo",
|
||||
"--skip-build",
|
||||
]),
|
||||
).toEqual({
|
||||
outputDir: ".artifacts/docker",
|
||||
outputName: "openclaw-current.tgz",
|
||||
skipBuild: true,
|
||||
sourceDir: "/repo",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects missing package artifact option values", () => {
|
||||
for (const flag of ["--output-dir", "--output-name", "--source-dir"]) {
|
||||
expect(() => parseArgs([flag])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([flag, "--skip-build"])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([flag, "-h"])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([`${flag}=`])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([`${flag}=-h`])).toThrow(`${flag} requires a value`);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects duplicate package artifact CLI options", () => {
|
||||
const duplicateCases = [
|
||||
["--output-dir", ["--output-dir", "one", "--output-dir=two"]],
|
||||
["--output-name", ["--output-name", "one.tgz", "--output-name=two.tgz"]],
|
||||
["--source-dir", ["--source-dir", "/repo-a", "--source-dir=/repo-b"]],
|
||||
["--skip-build", ["--skip-build", "--skip-build"]],
|
||||
] satisfies Array<[string, string[]]>;
|
||||
|
||||
for (const [flag, args] of duplicateCases) {
|
||||
expect(() => parseArgs(args), flag).toThrow(`${flag} was provided more than once`);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects package artifact output names that escape the output directory", () => {
|
||||
for (const outputName of [
|
||||
"../openclaw-current.tgz",
|
||||
"nested/openclaw-current.tgz",
|
||||
"openclaw-current.zip",
|
||||
".openclaw-current.tgz",
|
||||
]) {
|
||||
expect(() => parseArgs(["--output-name", outputName])).toThrow(
|
||||
`--output-name must be a tarball filename, not a path: ${outputName}`,
|
||||
);
|
||||
}
|
||||
|
||||
expect(parseArgs(["--output-name", "openclaw-current.tar.gz"]).outputName).toBe(
|
||||
"openclaw-current.tar.gz",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses build-all with declaration generation for package artifacts", async () => {
|
||||
const calls: Array<{
|
||||
command: string;
|
||||
args: string[];
|
||||
cwd: string;
|
||||
noPnpm: string | undefined;
|
||||
skipDts: string | undefined;
|
||||
timeoutMs: number | undefined;
|
||||
}> = [];
|
||||
const previousTimeout = process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
const previousSkipDts = process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD;
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = "1234";
|
||||
process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD = "1";
|
||||
|
||||
try {
|
||||
await buildPackageArtifacts("/repo", {
|
||||
runImpl: async (
|
||||
command: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
options: { env?: NodeJS.ProcessEnv; timeoutMs?: number },
|
||||
) => {
|
||||
calls.push({
|
||||
command,
|
||||
args,
|
||||
cwd,
|
||||
noPnpm: options.env?.OPENCLAW_BUILD_ALL_NO_PNPM,
|
||||
skipDts: options.env?.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD,
|
||||
timeoutMs: options.timeoutMs,
|
||||
});
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
if (previousTimeout === undefined) {
|
||||
delete process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
} else {
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = previousTimeout;
|
||||
}
|
||||
if (previousSkipDts === undefined) {
|
||||
delete process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD;
|
||||
} else {
|
||||
process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD = previousSkipDts;
|
||||
}
|
||||
}
|
||||
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
command: "node",
|
||||
args: ["scripts/build-all.mjs", "ciArtifacts"],
|
||||
cwd: "/repo",
|
||||
noPnpm: "1",
|
||||
skipDts: "0",
|
||||
timeoutMs: 1234,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects loose package artifact timeout env values", async () => {
|
||||
const previousTimeout = process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
try {
|
||||
for (const value of ["1e3", "123.9", "9007199254740993", "0"]) {
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = value;
|
||||
|
||||
await expect(
|
||||
buildPackageArtifacts("/repo", {
|
||||
runImpl: async () => undefined,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS must be a positive timeout in milliseconds",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if (previousTimeout === undefined) {
|
||||
delete process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
} else {
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = previousTimeout;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("bundles and restores the separately packed AI runtime", async () => {
|
||||
const sourceDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-ai-source-"));
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-ai-output-"));
|
||||
const packageJsonPath = path.join(sourceDir, "package.json");
|
||||
const originalPackageJson = `${JSON.stringify(
|
||||
{
|
||||
dependencies: { "@openclaw/ai": "workspace:*", "dep-a": "1.2.3" },
|
||||
files: ["dist"],
|
||||
name: "openclaw",
|
||||
version: "2026.6.17",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`;
|
||||
const installedAiPath = path.join(sourceDir, "node_modules", "@openclaw", "ai");
|
||||
fs.mkdirSync(path.join(sourceDir, "packages", "ai"), { recursive: true });
|
||||
fs.mkdirSync(installedAiPath, { recursive: true });
|
||||
fs.writeFileSync(path.join(installedAiPath, "original-marker"), "workspace package");
|
||||
fs.writeFileSync(packageJsonPath, originalPackageJson);
|
||||
|
||||
try {
|
||||
const cleanup = await prepareBundledAiRuntimePackage(
|
||||
sourceDir,
|
||||
outputDir,
|
||||
async (command: string, args: string[], cwd: string) => {
|
||||
expect({ args, command, cwd }).toEqual({
|
||||
args: ["--dir", "packages/ai", "pack", "--silent", "--pack-destination", outputDir],
|
||||
command: "pnpm",
|
||||
cwd: sourceDir,
|
||||
});
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-ai-2026.6.17.tgz"), "ai package");
|
||||
return "";
|
||||
},
|
||||
{
|
||||
extractAiRuntime: async (_tarballPath: string, destination: string) => {
|
||||
fs.writeFileSync(
|
||||
path.join(destination, "package.json"),
|
||||
`${JSON.stringify({
|
||||
dependencies: { "dep-a": "1.2.3" },
|
||||
name: "@openclaw/ai",
|
||||
version: "2026.6.17",
|
||||
})}\n`,
|
||||
);
|
||||
fs.writeFileSync(path.join(destination, "runtime.js"), "export {};\n");
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as {
|
||||
bundleDependencies: string[];
|
||||
dependencies: Record<string, string>;
|
||||
};
|
||||
expect(packageJson.dependencies["@openclaw/ai"]).toBe("2026.6.17");
|
||||
expect(packageJson.bundleDependencies).toContain("@openclaw/ai");
|
||||
expect(fs.existsSync(path.join(installedAiPath, "original-marker"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(installedAiPath, "runtime.js"))).toBe(true);
|
||||
const stagedAiPackageJson = JSON.parse(
|
||||
fs.readFileSync(path.join(installedAiPath, "package.json"), "utf8"),
|
||||
) as { dependencies?: Record<string, string> };
|
||||
expect(stagedAiPackageJson.dependencies).toBeUndefined();
|
||||
|
||||
await cleanup();
|
||||
expect(fs.readFileSync(packageJsonPath, "utf8")).toBe(originalPackageJson);
|
||||
expect(fs.readFileSync(path.join(installedAiPath, "original-marker"), "utf8")).toBe(
|
||||
"workspace package",
|
||||
);
|
||||
expect(fs.existsSync(path.join(outputDir, "openclaw-ai-2026.6.17.tgz"))).toBe(false);
|
||||
} finally {
|
||||
fs.rmSync(sourceDir, { recursive: true, force: true });
|
||||
fs.rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("trims and restores the changelog around ignore-scripts package artifacts", async () => {
|
||||
const calls: string[] = [];
|
||||
const tarball = await packOpenClawPackageForDocker("/repo", "/out", {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async (cwd: string) => {
|
||||
calls.push(`prepare:${cwd}`);
|
||||
},
|
||||
restoreChangelog: async (cwd: string) => {
|
||||
calls.push(`restore:${cwd}`);
|
||||
},
|
||||
runCaptureImpl: async (
|
||||
command: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
options: { deferForwardedSignalExit?: boolean },
|
||||
) => {
|
||||
calls.push(`${command}:${args.join(" ")}:${cwd}`);
|
||||
expect(options.deferForwardedSignalExit).toBe(true);
|
||||
return "openclaw-2026.5.28.tgz\n";
|
||||
},
|
||||
});
|
||||
|
||||
expect(tarball).toBe(path.join("/out", "openclaw-2026.5.28.tgz"));
|
||||
expect(calls).toEqual([
|
||||
"prepare:/repo",
|
||||
"npm:pack --silent --ignore-scripts --pack-destination /out:/repo",
|
||||
"restore:/repo",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects path-like npm pack stdout before resolving Docker package tarballs", async () => {
|
||||
for (const filename of [
|
||||
"../openclaw-2026.6.17.tgz",
|
||||
"/tmp/openclaw-2026.6.17.tgz",
|
||||
String.raw`C:\temp\openclaw-2026.6.17.tgz`,
|
||||
"openclaw-nested/evil.tgz",
|
||||
String.raw`openclaw-nested\evil.tgz`,
|
||||
"openclaw-C:evil.tgz",
|
||||
]) {
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", "/out", {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => `${filename}\n`,
|
||||
}),
|
||||
).rejects.toThrow("npm pack reported unsafe OpenClaw tarball filename");
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores unsafe output directory tarball names when npm stdout is not usable", async () => {
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-"));
|
||||
try {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-C:evil.tgz"), "");
|
||||
fs.writeFileSync(path.join(outputDir, String.raw`openclaw-nested\evil.tgz`), "");
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => "npm notice\n",
|
||||
}),
|
||||
).rejects.toThrow("missing packed OpenClaw tarball");
|
||||
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-2026.6.17.tgz"), "");
|
||||
return "npm notice\n";
|
||||
},
|
||||
}),
|
||||
).resolves.toBe(path.join(outputDir, "openclaw-2026.6.17.tgz"));
|
||||
} finally {
|
||||
fs.rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores stale package tarballs before fallback scanning npm output", async () => {
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-stale-"));
|
||||
try {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-9999.1.1.tgz"), "stale");
|
||||
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-2026.6.17.tgz"), "current");
|
||||
return "npm notice\n";
|
||||
},
|
||||
}),
|
||||
).resolves.toBe(path.join(outputDir, "openclaw-2026.6.17.tgz"));
|
||||
|
||||
expect(fs.existsSync(path.join(outputDir, "openclaw-9999.1.1.tgz"))).toBe(false);
|
||||
expect(fs.readFileSync(path.join(outputDir, "openclaw-2026.6.17.tgz"), "utf8")).toBe(
|
||||
"current",
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("restores the changelog when ignore-scripts packaging fails", async () => {
|
||||
const calls: string[] = [];
|
||||
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", "/out", {
|
||||
prepareBundledAiRuntime: async () => {
|
||||
calls.push("embed");
|
||||
return async () => {
|
||||
calls.push("cleanup");
|
||||
};
|
||||
},
|
||||
prepareChangelog: async (cwd: string) => {
|
||||
calls.push(`prepare:${cwd}`);
|
||||
},
|
||||
restoreChangelog: async (cwd: string) => {
|
||||
calls.push(`restore:${cwd}`);
|
||||
},
|
||||
runCaptureImpl: async () => {
|
||||
calls.push("pack");
|
||||
throw new Error("pack failed");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("pack failed");
|
||||
|
||||
expect(calls).toEqual(["prepare:/repo", "embed", "pack", "cleanup", "restore:/repo"]);
|
||||
});
|
||||
|
||||
it("clamps oversized command timers before scheduling", async () => {
|
||||
await expect(
|
||||
runCommandForTest(
|
||||
process.execPath,
|
||||
["-e", "setTimeout(() => process.exit(0), 25);"],
|
||||
process.cwd(),
|
||||
{
|
||||
killAfterMs: MAX_TIMER_TIMEOUT_MS + 1,
|
||||
timeoutMs: MAX_TIMER_TIMEOUT_MS + 1,
|
||||
},
|
||||
),
|
||||
).resolves.toBe("");
|
||||
});
|
||||
|
||||
it("kills timed-out child process groups", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-timeout-"));
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
let childPid;
|
||||
try {
|
||||
const childScript = ["process.on('SIGTERM', () => {});", "setInterval(() => {}, 1000);"].join(
|
||||
"",
|
||||
);
|
||||
const parentScript = [
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"const fs = require('node:fs');",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"fs.writeFileSync(process.env.OPENCLAW_TEST_CHILD_PID, String(child.pid));",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
const runPromise = runCommandForTest(process.execPath, ["-e", parentScript], process.cwd(), {
|
||||
env: { ...process.env, OPENCLAW_TEST_CHILD_PID: childPidPath },
|
||||
killAfterMs: 25,
|
||||
timeoutMs: 500,
|
||||
});
|
||||
const timeoutAssertion = expect(runPromise).rejects.toThrow(/timed out after 500ms/u);
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
await timeoutAssertion;
|
||||
await waitForDead(childPid, 2000);
|
||||
} finally {
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("clamps oversized kill grace before scheduling", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-grace-"));
|
||||
const donePath = path.join(tempDir, "done");
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
let childPid;
|
||||
try {
|
||||
const script = [
|
||||
"const fs = require('node:fs');",
|
||||
`fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`,
|
||||
"process.on('SIGTERM', () => {",
|
||||
` setTimeout(() => { fs.writeFileSync(${JSON.stringify(donePath)}, 'done'); process.exit(0); }, 75);`,
|
||||
"});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n");
|
||||
|
||||
const runPromise = runCommandForTest(process.execPath, ["-e", script], process.cwd(), {
|
||||
killAfterMs: MAX_TIMER_TIMEOUT_MS + 1,
|
||||
timeoutMs: 500,
|
||||
});
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
|
||||
await expect(runPromise).rejects.toThrow(/timed out after 500ms/u);
|
||||
expect(fs.readFileSync(donePath, "utf8")).toBe("done");
|
||||
} finally {
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps fallback SIGKILL armed for descendants after the direct child exits", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-descendant-"));
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
let childPid;
|
||||
try {
|
||||
const childScript = ["process.on('SIGTERM', () => {});", "setInterval(() => {}, 1000);"].join(
|
||||
"",
|
||||
);
|
||||
const parentScript = [
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"const fs = require('node:fs');",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"fs.writeFileSync(process.env.OPENCLAW_TEST_CHILD_PID, String(child.pid));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
await expect(
|
||||
runCommandForTest(process.execPath, ["-e", parentScript], process.cwd(), {
|
||||
env: { ...process.env, OPENCLAW_TEST_CHILD_PID: childPidPath },
|
||||
killAfterMs: 25,
|
||||
timeoutMs: 500,
|
||||
}),
|
||||
).rejects.toThrow(/timed out after 500ms/u);
|
||||
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
await waitForDead(childPid, 2000);
|
||||
} finally {
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not fire delayed SIGKILL after a timed-out child exits during grace", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const killSpy = vi.spyOn(process, "kill");
|
||||
try {
|
||||
const script = [
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
await expect(
|
||||
runCommandForTest(process.execPath, ["-e", script], process.cwd(), {
|
||||
killAfterMs: 100,
|
||||
timeoutMs: 25,
|
||||
}),
|
||||
).rejects.toThrow(/timed out after 25ms/u);
|
||||
|
||||
const sigkillCallsAfterExit = killSpy.mock.calls.filter(
|
||||
([, signal]) => signal === "SIGKILL",
|
||||
).length;
|
||||
await sleep(150);
|
||||
expect(killSpy.mock.calls.filter(([, signal]) => signal === "SIGKILL")).toHaveLength(
|
||||
sigkillCallsAfterExit,
|
||||
);
|
||||
} finally {
|
||||
killSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails captured commands that exceed the stdout limit", async () => {
|
||||
const script = [
|
||||
"process.stdout.write('x'.repeat(2048));",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
await expect(
|
||||
runCommandForTest(process.execPath, ["-e", script], process.cwd(), {
|
||||
captureStdout: true,
|
||||
killAfterMs: 50,
|
||||
maxCapturedStdoutBytes: 1024,
|
||||
timeoutMs: 5000,
|
||||
}),
|
||||
).rejects.toThrow(/exceeded captured stdout limit \(1024 bytes\)/u);
|
||||
});
|
||||
|
||||
it("forwards external termination to active child process groups", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-signal-"));
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
const scriptUrl = pathToFileURL(path.resolve("scripts/package-openclaw-for-docker.mjs")).href;
|
||||
let childPid = 0;
|
||||
let runnerPid;
|
||||
try {
|
||||
const childScript = "setInterval(() => {}, 1000);";
|
||||
const parentScript = [
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"const fs = require('node:fs');",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"fs.writeFileSync(process.env.OPENCLAW_TEST_CHILD_PID, String(child.pid));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
const runnerScript = [
|
||||
`import { runCommandForTest } from ${JSON.stringify(scriptUrl)};`,
|
||||
`await runCommandForTest(process.execPath, ['-e', ${JSON.stringify(parentScript)}], process.cwd(), { timeoutMs: 60000 });`,
|
||||
].join("\n");
|
||||
const runner = spawn(process.execPath, ["--input-type=module", "-e", runnerScript], {
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, OPENCLAW_TEST_CHILD_PID: childPidPath },
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
});
|
||||
runnerPid = runner.pid ?? 0;
|
||||
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
runner.kill("SIGTERM");
|
||||
const result = await waitForExit(runner, 5000);
|
||||
|
||||
expect(result).toEqual({ signal: null, status: 143 });
|
||||
await waitForDead(childPid, 2000);
|
||||
} finally {
|
||||
if (runnerPid && isProcessAlive(runnerPid)) {
|
||||
process.kill(runnerPid, "SIGKILL");
|
||||
}
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
2063
test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts
Normal file
2063
test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts
Normal file
File diff suppressed because it is too large
Load Diff
867
test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts
Normal file
867
test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts
Normal file
@@ -0,0 +1,867 @@
|
||||
// QA OTEL Smoke tests cover QA Lab telemetry evidence.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { createConnection as createNetConnection } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { resolveWindowsTaskkillPath } from "../../../../scripts/lib/windows-taskkill.mjs";
|
||||
import { testing } from "./qa-otel-smoke-runtime.js";
|
||||
|
||||
function expectedTaskkillPath(): string {
|
||||
return resolveWindowsTaskkillPath();
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("qa-otel-smoke receiver bounds", () => {
|
||||
let configuredBodyLimitLoad: ReturnType<typeof spawnSync>;
|
||||
|
||||
beforeAll(() => {
|
||||
configuredBodyLimitLoad = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
'await import("./test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts");',
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_QA_OTEL_MAX_CAPTURED_BODY_TEXT_BYTES: "1024",
|
||||
OPENCLAW_QA_OTEL_MAX_COMPRESSED_BODY_BYTES: "2048",
|
||||
OPENCLAW_QA_OTEL_MAX_DECODED_BODY_BYTES: "4096",
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
function makePassingSmokeAssertionInput(): Parameters<typeof testing.assertSmoke>[0] {
|
||||
return {
|
||||
bodyText: {
|
||||
logs: ["diagnostics-otel: logs exporter enabled"],
|
||||
},
|
||||
childExitCode: 0,
|
||||
disallowedBodyNeedles: ["OTEL-QA-SECRET"],
|
||||
logsExporter: "otlp",
|
||||
logRecords: [
|
||||
{
|
||||
body: "diagnostics-otel: logs exporter enabled",
|
||||
traceId: "trace",
|
||||
spanId: "span",
|
||||
},
|
||||
],
|
||||
metrics: [{ name: "openclaw.harness.duration_ms" }],
|
||||
requests: [
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 16,
|
||||
contentEncoding: undefined,
|
||||
status: 200,
|
||||
spanCount: 5,
|
||||
metricCount: 0,
|
||||
logCount: 0,
|
||||
},
|
||||
{
|
||||
path: "/v1/metrics",
|
||||
signal: "metrics",
|
||||
bytes: 16,
|
||||
contentEncoding: undefined,
|
||||
status: 200,
|
||||
spanCount: 0,
|
||||
metricCount: 1,
|
||||
logCount: 0,
|
||||
},
|
||||
{
|
||||
path: "/v1/logs",
|
||||
signal: "logs",
|
||||
bytes: 16,
|
||||
contentEncoding: undefined,
|
||||
status: 200,
|
||||
spanCount: 0,
|
||||
metricCount: 0,
|
||||
logCount: 1,
|
||||
},
|
||||
],
|
||||
stdoutLogLines: [],
|
||||
stdoutLogRecords: [],
|
||||
spans: [
|
||||
{ name: "openclaw.run", parent: false, attributes: {} },
|
||||
{ name: "openclaw.harness.run", parent: true, attributes: {} },
|
||||
{ name: "openclaw.context.assembled", parent: true, attributes: {} },
|
||||
{ name: "openclaw.message.delivery", parent: true, attributes: {} },
|
||||
{
|
||||
name: "chat gpt-5.5",
|
||||
parent: true,
|
||||
attributes: {
|
||||
"gen_ai.operation.name": "chat",
|
||||
"gen_ai.request.model": "gpt-5.5",
|
||||
"openclaw.model": "gpt-5.5",
|
||||
"openclaw.provider": "openai",
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
it("accepts package-manager forwarded arguments", () => {
|
||||
expect(
|
||||
testing.parseArgs([
|
||||
"--",
|
||||
"--collector",
|
||||
"docker",
|
||||
"--provider-mode",
|
||||
"mock-openai",
|
||||
"--scenario",
|
||||
"otel-stdout-log-smoke",
|
||||
"--logs-exporter",
|
||||
"stdout",
|
||||
]),
|
||||
).toMatchObject({
|
||||
collectorMode: "docker",
|
||||
logsExporter: "stdout",
|
||||
providerMode: "mock-openai",
|
||||
scenarioId: "otel-stdout-log-smoke",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["--collector", ["--collector", "--logs-exporter"]],
|
||||
["--logs-exporter", ["--logs-exporter", "--collector"]],
|
||||
["--output-dir", ["--output-dir", "--collector"]],
|
||||
["--provider-mode", ["--provider-mode", "--collector"]],
|
||||
["--scenario", ["--scenario", "--collector"]],
|
||||
["--model", ["--model", "--collector"]],
|
||||
["--alt-model", ["--alt-model", "--collector"]],
|
||||
])("rejects missing values for %s before shifting parser state", (flag, args) => {
|
||||
expect(() => testing.parseArgs(args)).toThrow(`${flag} requires a value`);
|
||||
});
|
||||
|
||||
it("rejects duplicate OTEL smoke CLI options", () => {
|
||||
const duplicateCases = [
|
||||
["--collector", ["--collector", "local", "--collector", "docker"]],
|
||||
["--logs-exporter", ["--logs-exporter", "otlp", "--logs-exporter", "stdout"]],
|
||||
["--output-dir", ["--output-dir", ".artifacts/one", "--output-dir", ".artifacts/two"]],
|
||||
["--provider-mode", ["--provider-mode", "mock-openai", "--provider-mode", "live-frontier"]],
|
||||
["--scenario", ["--scenario", "custom-one", "--scenario", "custom-two"]],
|
||||
["--model", ["--model", "openai/gpt-5.5", "--model", "openai/gpt-5.4"]],
|
||||
["--alt-model", ["--alt-model", "openai/gpt-5.5", "--alt-model", "openai/gpt-5.4"]],
|
||||
] satisfies Array<[string, string[]]>;
|
||||
|
||||
for (const [flag, args] of duplicateCases) {
|
||||
expect(() => testing.parseArgs(args), flag).toThrow(`${flag} was provided more than once`);
|
||||
}
|
||||
});
|
||||
|
||||
it("selects the matching scenario for the requested log exporter", () => {
|
||||
expect(testing.parseArgs(["--logs-exporter", "otlp"]).scenarioId).toBe("otel-trace-smoke");
|
||||
expect(testing.parseArgs(["--logs-exporter", "stdout"]).scenarioId).toBe(
|
||||
"otel-stdout-log-smoke",
|
||||
);
|
||||
expect(testing.parseArgs(["--logs-exporter", "both"]).scenarioId).toBe("otel-both-log-smoke");
|
||||
});
|
||||
|
||||
it("rejects explicit scenarios that do not match the log exporter", () => {
|
||||
expect(() =>
|
||||
testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "otel-trace-smoke"]),
|
||||
).toThrow("--logs-exporter stdout requires --scenario otel-stdout-log-smoke");
|
||||
});
|
||||
|
||||
it("allows explicit custom scenarios to own their exporter config", () => {
|
||||
expect(testing.parseArgs(["--scenario", "custom-otel-smoke"]).scenarioId).toBe(
|
||||
"custom-otel-smoke",
|
||||
);
|
||||
expect(
|
||||
testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "custom-stdout-smoke"])
|
||||
.scenarioId,
|
||||
).toBe("custom-stdout-smoke");
|
||||
});
|
||||
|
||||
it("uses unique default output dirs", () => {
|
||||
const firstOutputDir = testing.parseArgs([]).outputDir;
|
||||
const secondOutputDir = testing.parseArgs([]).outputDir;
|
||||
|
||||
expect(path.dirname(firstOutputDir)).toBe(path.join(".artifacts", "qa-e2e"));
|
||||
expect(path.basename(firstOutputDir)).toMatch(/^otel-smoke-[a-z0-9]+-[a-f0-9]{8}$/u);
|
||||
expect(secondOutputDir).not.toBe(firstOutputDir);
|
||||
expect(testing.parseArgs(["--output-dir", ".artifacts/custom"]).outputDir).toBe(
|
||||
".artifacts/custom",
|
||||
);
|
||||
});
|
||||
|
||||
it("passes a repo-relative output dir to the child QA suite", () => {
|
||||
const repoRoot = path.join(path.sep, "repo");
|
||||
const options = testing.parseArgs([
|
||||
"--output-dir",
|
||||
path.join(repoRoot, ".artifacts", "qa-e2e", "otel-smoke"),
|
||||
]);
|
||||
|
||||
expect(testing.buildQaArgs(options, repoRoot)).toContain(".artifacts/qa-e2e/otel-smoke");
|
||||
const repoRootArgs = testing.buildQaArgs(
|
||||
testing.parseArgs(["--output-dir", repoRoot]),
|
||||
repoRoot,
|
||||
);
|
||||
expect(repoRootArgs[repoRootArgs.indexOf("--output-dir") + 1]).toBe(".");
|
||||
expect(() =>
|
||||
testing.buildQaArgs(
|
||||
testing.parseArgs(["--output-dir", path.join(path.sep, "outside", "otel-smoke")]),
|
||||
repoRoot,
|
||||
),
|
||||
).toThrow("--output-dir must stay within the repo root");
|
||||
});
|
||||
|
||||
it("parses body-size limit env values as strict positive integers", () => {
|
||||
expect(testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, {})).toBe(64);
|
||||
expect(
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: " 128 " }),
|
||||
).toBe(128);
|
||||
|
||||
expect(() =>
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: "1e3" }),
|
||||
).toThrow("OTEL_TEST_LIMIT must be a positive integer");
|
||||
expect(() =>
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: "1024bytes" }),
|
||||
).toThrow("OTEL_TEST_LIMIT must be a positive integer");
|
||||
expect(() =>
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: "0" }),
|
||||
).toThrow("OTEL_TEST_LIMIT must be a positive integer");
|
||||
});
|
||||
|
||||
it("loads with configured body-size limit env values", () => {
|
||||
expect(configuredBodyLimitLoad.status).toBe(0);
|
||||
expect(configuredBodyLimitLoad.stderr).not.toContain("ReferenceError");
|
||||
});
|
||||
|
||||
it("scrubs inherited OpenTelemetry exporter env before running the QA suite", () => {
|
||||
vi.stubEnv("OTEL_SDK_DISABLED", "true");
|
||||
vi.stubEnv("OTEL_TRACES_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_METRICS_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_LOGS_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.example.test:4318");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=secret");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "authorization=logs-secret");
|
||||
vi.stubEnv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=developer-laptop");
|
||||
|
||||
const env = testing.buildQaEnv(4318);
|
||||
|
||||
expect(env.OTEL_SDK_DISABLED).toBeUndefined();
|
||||
expect(env.OTEL_TRACES_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_METRICS_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_LOGS_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_ENDPOINT).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_METRICS_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_HEADERS).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toBeUndefined();
|
||||
expect(env.OTEL_RESOURCE_ATTRIBUTES).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).toBe("http://127.0.0.1:4318/v1/traces");
|
||||
expect(env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT).toBe("http://127.0.0.1:4318/v1/metrics");
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT).toBe("http://127.0.0.1:4318/v1/logs");
|
||||
expect(env.OTEL_SERVICE_NAME).toBe("openclaw-qa-lab-otel-smoke");
|
||||
});
|
||||
|
||||
it("rejects identity OTLP bodies above the decoded byte ceiling", () => {
|
||||
expect(() => testing.decodeRequestBody(Buffer.alloc(65), undefined, 64)).toThrow(
|
||||
"OTLP request body exceeded 64 bytes: 65 bytes",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects gzip OTLP bodies above the decoded byte ceiling", () => {
|
||||
const compressed = gzipSync(Buffer.alloc(256, "a"));
|
||||
|
||||
expect(() => testing.decodeRequestBody(compressed, "gzip", 64)).toThrow(
|
||||
"decoded OTLP request body exceeded 64 bytes",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps captured OTLP body text bounded per signal", () => {
|
||||
const captured: { traces?: string[] } = {};
|
||||
|
||||
testing.appendCapturedBodyText(captured, "traces", Buffer.from("a".repeat(20)), 16, [
|
||||
"OTEL-QA-SECRET",
|
||||
]);
|
||||
testing.appendCapturedBodyText(captured, "traces", Buffer.from("b".repeat(20)), 16);
|
||||
|
||||
expect(captured.traces).toHaveLength(1);
|
||||
expect(captured.traces?.[0]).toContain("[captured body text truncated to last 16 bytes]");
|
||||
expect(captured.traces?.[0]).toContain("b".repeat(16));
|
||||
expect(captured.traces?.[0]).not.toContain("a".repeat(20));
|
||||
});
|
||||
|
||||
it("returns a bounded failure for malformed local OTLP protobuf", async () => {
|
||||
const receiver = testing.startLocalOtlpReceiver(["OTEL-QA-SECRET"]);
|
||||
const port = await receiver.listen();
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/v1/traces`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-protobuf" },
|
||||
body: Buffer.concat([Buffer.from([0x0a]), Buffer.from("OTEL-QA-SECRET")]),
|
||||
});
|
||||
const text = await response.text();
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(text).toContain("truncated protobuf");
|
||||
expect(receiver.capturedRequests).toEqual([
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 15,
|
||||
contentEncoding: undefined,
|
||||
status: 400,
|
||||
spanCount: 0,
|
||||
metricCount: 0,
|
||||
logCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(receiver.capturedBodyText.traces).toEqual([
|
||||
"[detected leak needle] OTEL-QA-SECRET",
|
||||
"\nOTEL-QA-SECRET",
|
||||
]);
|
||||
} finally {
|
||||
await receiver.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects truncated unknown fixed-width protobuf fields", async () => {
|
||||
const receiver = testing.startLocalOtlpReceiver();
|
||||
const port = await receiver.listen();
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/v1/traces`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-protobuf" },
|
||||
body: Buffer.from([0x09]),
|
||||
});
|
||||
const text = await response.text();
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(text).toContain("truncated protobuf fixed64");
|
||||
expect(receiver.capturedRequests).toMatchObject([
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 1,
|
||||
status: 400,
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
await receiver.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("closes active local OTLP receiver sockets during cleanup", async () => {
|
||||
const receiver = testing.startLocalOtlpReceiver();
|
||||
const port = await receiver.listen();
|
||||
const socket = createNetConnection(port, "127.0.0.1");
|
||||
const socketClosed = new Promise<void>((resolve) => {
|
||||
socket.once("close", resolve);
|
||||
});
|
||||
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.once("connect", resolve);
|
||||
socket.once("error", reject);
|
||||
});
|
||||
socket.write(
|
||||
[
|
||||
"POST /v1/traces HTTP/1.1",
|
||||
"Host: 127.0.0.1",
|
||||
"Content-Type: application/x-protobuf",
|
||||
"Content-Length: 1048576",
|
||||
"",
|
||||
"x",
|
||||
].join("\r\n"),
|
||||
);
|
||||
|
||||
await Promise.race([
|
||||
receiver.close(),
|
||||
delay(1_000).then(() => {
|
||||
throw new Error("receiver close timed out");
|
||||
}),
|
||||
]);
|
||||
await Promise.race([
|
||||
socketClosed,
|
||||
delay(1_000).then(() => {
|
||||
throw new Error("socket close timed out");
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
socket.destroy();
|
||||
await receiver.close().catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
it("fails smoke assertions for captured non-2xx OTLP requests", () => {
|
||||
const assertion = testing.assertSmoke({
|
||||
bodyText: {},
|
||||
childExitCode: 0,
|
||||
disallowedBodyNeedles: [],
|
||||
logsExporter: "otlp",
|
||||
logRecords: [],
|
||||
metrics: [],
|
||||
requests: [
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 15,
|
||||
contentEncoding: undefined,
|
||||
status: 400,
|
||||
spanCount: 0,
|
||||
metricCount: 0,
|
||||
logCount: 0,
|
||||
},
|
||||
],
|
||||
stdoutLogLines: [],
|
||||
stdoutLogRecords: [],
|
||||
spans: [],
|
||||
});
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("OTLP traces request /v1/traces returned status 400");
|
||||
});
|
||||
|
||||
it("allows safe operational OTLP log bodies while leak checks inspect raw payloads", () => {
|
||||
const assertion = testing.assertSmoke(makePassingSmokeAssertionInput());
|
||||
|
||||
expect(assertion.passed).toBe(true);
|
||||
expect(assertion.failures).toEqual([]);
|
||||
});
|
||||
|
||||
it("allows stdout diagnostic logs without OTLP log requests", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.bodyText = {};
|
||||
input.logRecords = [];
|
||||
input.requests = input.requests.filter((request) => request.signal !== "logs");
|
||||
input.stdoutLogRecords = [
|
||||
{
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
signal: "openclaw.diagnostic.log",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "log",
|
||||
attributes: {
|
||||
"openclaw.log.level": "INFO",
|
||||
},
|
||||
},
|
||||
];
|
||||
input.stdoutLogLines = [JSON.stringify(input.stdoutLogRecords[0])];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(true);
|
||||
expect(assertion.failures).toEqual([]);
|
||||
expect(assertion.signalRequestCounts.logs).toBe(0);
|
||||
expect(assertion.stdoutLogRecordCount).toBe(1);
|
||||
});
|
||||
|
||||
it("fails stdout diagnostic mode when no stdout log records are captured", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.bodyText = {};
|
||||
input.logRecords = [];
|
||||
input.requests = input.requests.filter((request) => request.signal !== "logs");
|
||||
input.stdoutLogRecords = [];
|
||||
input.stdoutLogLines = [];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("no stdout diagnostic log records were captured");
|
||||
expect(assertion.signalRequestCounts.logs).toBe(0);
|
||||
expect(assertion.stdoutLogRecordCount).toBe(0);
|
||||
});
|
||||
|
||||
it("fails stdout diagnostic mode when OTLP log requests are still emitted", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.logRecords = [];
|
||||
input.stdoutLogRecords = [
|
||||
{
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
signal: "openclaw.diagnostic.log",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "log",
|
||||
attributes: {},
|
||||
},
|
||||
];
|
||||
input.stdoutLogLines = [JSON.stringify(input.stdoutLogRecords[0])];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain(
|
||||
"OTLP logs requests were received for stdout logs exporter",
|
||||
);
|
||||
});
|
||||
|
||||
it("still fails when OTLP log payload text leaks scenario content", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.bodyText = {
|
||||
logs: ["diagnostics-otel: log payload contains OTEL-QA-SECRET"],
|
||||
};
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("OTLP logs payload leaked content: OTEL-QA-SECRET");
|
||||
expect(assertion.leakContexts.logs?.[0]).toContain("[needle]");
|
||||
});
|
||||
|
||||
it("still fails when stdout diagnostic log payload text leaks scenario content", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.bodyText = {};
|
||||
input.logRecords = [];
|
||||
input.requests = input.requests.filter((request) => request.signal !== "logs");
|
||||
input.stdoutLogRecords = [
|
||||
{
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
signal: "openclaw.diagnostic.log",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "log",
|
||||
attributes: {},
|
||||
},
|
||||
];
|
||||
input.stdoutLogLines = [
|
||||
JSON.stringify({
|
||||
...input.stdoutLogRecords[0],
|
||||
body: "diagnostics-otel: log payload contains OTEL-QA-SECRET",
|
||||
}),
|
||||
];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain(
|
||||
"stdout diagnostic log payload leaked content: OTEL-QA-SECRET",
|
||||
);
|
||||
expect(assertion.leakContexts.logs?.[0]).toContain("[needle]");
|
||||
});
|
||||
|
||||
it("still requires OTLP log records to carry trace correlation", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logRecords = [
|
||||
{
|
||||
body: "diagnostics-otel: logs exporter enabled",
|
||||
traceId: "",
|
||||
spanId: "",
|
||||
},
|
||||
];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("no OTLP log records included trace/span correlation ids");
|
||||
});
|
||||
|
||||
it("preserves leak markers even when later body text is truncated", () => {
|
||||
const captured: { traces?: string[] } = {};
|
||||
|
||||
testing.appendCapturedBodyText(
|
||||
captured,
|
||||
"traces",
|
||||
Buffer.from(`prefix OTEL-QA-SECRET ${"a".repeat(20)}`),
|
||||
16,
|
||||
["OTEL-QA-SECRET"],
|
||||
);
|
||||
testing.appendCapturedBodyText(captured, "traces", Buffer.from("b".repeat(128)), 16, [
|
||||
"OTEL-QA-SECRET",
|
||||
]);
|
||||
|
||||
expect(captured.traces?.join("\n")).toContain("OTEL-QA-SECRET");
|
||||
expect(captured.traces?.join("\n")).toContain("[captured body text truncated");
|
||||
});
|
||||
|
||||
it("keeps collector output tails bounded without retaining earlier chunks", () => {
|
||||
const output = testing.createBoundedTextAccumulator(64);
|
||||
|
||||
output.append("DO_NOT_RETAIN_COLLECTOR_PREFIX\n");
|
||||
output.append(Buffer.alloc(2048, "x"));
|
||||
output.append("\nCOLLECTOR_TAIL_MARKER\n");
|
||||
|
||||
expect(output.byteLength()).toBeLessThanOrEqual(64);
|
||||
expect(output.text()).toContain("COLLECTOR_TAIL_MARKER");
|
||||
expect(output.text()).toContain("...");
|
||||
expect(output.text()).not.toContain("DO_NOT_RETAIN_COLLECTOR_PREFIX");
|
||||
});
|
||||
|
||||
it("streams gateway stdout artifact records without requiring them in the tail", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-stdout-stream-"));
|
||||
const logPath = path.join(tempRoot, "gateway.stdout.log");
|
||||
const capture = testing.createStdoutDiagnosticLogCapture();
|
||||
const record = {
|
||||
signal: "openclaw.diagnostic.log",
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "early log",
|
||||
attributes: {},
|
||||
};
|
||||
try {
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify(record)}\n${"x".repeat(256 * 1024)}\nGATEWAY_STDOUT_TAIL\n`,
|
||||
);
|
||||
|
||||
await testing.appendUtf8FileToStdoutDiagnosticCapture(logPath, capture);
|
||||
capture.flush();
|
||||
|
||||
expect(capture.records).toEqual([record]);
|
||||
expect(capture.lines).toHaveLength(1);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps gateway stdout artifact fallback parsing bounded", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-stdout-artifact-"));
|
||||
const outputDir = path.join(tempRoot, "output");
|
||||
const artifactDir = path.join(outputDir, "artifacts", "gateway-runtime");
|
||||
const record = {
|
||||
signal: "openclaw.diagnostic.log",
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "tail log",
|
||||
attributes: {},
|
||||
};
|
||||
try {
|
||||
mkdirSync(artifactDir, { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(artifactDir, "gateway.stdout.log"),
|
||||
`${JSON.stringify(record)}\n${"x".repeat(256 * 1024)}\n`,
|
||||
);
|
||||
const capture = testing.createStdoutDiagnosticLogCapture();
|
||||
|
||||
await testing.appendGatewayStdoutArtifactLogs({ capture, outputDir });
|
||||
|
||||
expect(capture.records).toEqual([record]);
|
||||
expect(capture.lines).toHaveLength(1);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("times out and kills a wedged QA suite child with a detached gateway", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-child-"));
|
||||
const markerPath = path.join(tempDir, "marker.txt");
|
||||
try {
|
||||
const gatewayScript = [
|
||||
"import fs from 'node:fs';",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
`setInterval(() => fs.appendFileSync(${JSON.stringify(markerPath)}, "x"), 20);`,
|
||||
].join("\n");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
[
|
||||
"import childProcess from 'node:child_process';",
|
||||
`childProcess.spawn(process.execPath, ["--input-type=module", "--eval", ${JSON.stringify(
|
||||
gatewayScript,
|
||||
)}], { detached: true, stdio: "ignore" });`,
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n"),
|
||||
],
|
||||
{
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
},
|
||||
);
|
||||
|
||||
await expect(testing.waitForChild(child, 100, 100)).rejects.toThrow(
|
||||
"openclaw qa suite timed out after 100ms",
|
||||
);
|
||||
const sizeAfterReturn = existsSync(markerPath) ? statSync(markerPath).size : 0;
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 150);
|
||||
});
|
||||
const sizeAfterWait = existsSync(markerPath) ? statSync(markerPath).size : 0;
|
||||
expect(sizeAfterWait).toBe(sizeAfterReturn);
|
||||
} finally {
|
||||
rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("clamps oversized QA suite child timers before scheduling", async () => {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
["--input-type=module", "--eval", "setTimeout(() => process.exit(0), 25);"],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
|
||||
await expect(testing.waitForChild(child, MAX_TIMER_TIMEOUT_MS + 1, 100)).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it("uses taskkill for Windows QA suite timeout cleanup", () => {
|
||||
const kill = vi.fn();
|
||||
const runTaskkill = vi.fn(() => ({ status: 0 }));
|
||||
|
||||
testing.terminateChildTree(
|
||||
{ kill, pid: 1234 } as never,
|
||||
"SIGTERM",
|
||||
[],
|
||||
"win32",
|
||||
runTaskkill as never,
|
||||
);
|
||||
|
||||
expect(runTaskkill).toHaveBeenCalledWith(expectedTaskkillPath(), ["/PID", "1234", "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
expect(kill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves Docker collector telemetry off the default host port", async () => {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
stdout: EventEmitter;
|
||||
};
|
||||
child.stderr = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
let writtenConfig = "";
|
||||
const stopDockerContainer = vi.fn(async () => {});
|
||||
const removePath = vi.fn(async () => {});
|
||||
const ports = [4318, 4318, 45679];
|
||||
|
||||
const collector = await testing.startDockerOtelCollector(4317, {
|
||||
mkdtemp: async () => "/tmp/openclaw-otel-collector-test",
|
||||
platform: "linux",
|
||||
randomUUID: () => "00000000-0000-4000-8000-000000000000",
|
||||
reserveLocalPort: async () => ports.shift() ?? 49999,
|
||||
rm: removePath as never,
|
||||
spawn: vi.fn(() => child) as never,
|
||||
stopDockerContainer,
|
||||
waitForLocalPort: async () => {},
|
||||
writeFile: async (_path, config) => {
|
||||
writtenConfig = String(config);
|
||||
},
|
||||
});
|
||||
|
||||
expect(writtenConfig).toContain("endpoint: 127.0.0.1:4318");
|
||||
expect(writtenConfig).toContain("telemetry:");
|
||||
expect(writtenConfig).toContain("address: 127.0.0.1:45679");
|
||||
expect(writtenConfig).not.toContain("address: :8888");
|
||||
|
||||
await collector.close();
|
||||
expect(stopDockerContainer).toHaveBeenCalledWith(
|
||||
"openclaw-otel-smoke-00000000-0000-4000-8000-000000000000",
|
||||
);
|
||||
expect(removePath).toHaveBeenCalledWith("/tmp/openclaw-otel-collector-test", {
|
||||
force: true,
|
||||
recursive: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("cleans Docker collector containers and temp config after readiness failures", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-collector-"));
|
||||
const collectorDir = path.join(tempRoot, "collector");
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
stdout: EventEmitter;
|
||||
};
|
||||
child.stderr = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
const stopDockerContainer = vi.fn(async () => {});
|
||||
const ports = [4318, 45679];
|
||||
|
||||
try {
|
||||
await expect(
|
||||
testing.startDockerOtelCollector(4317, {
|
||||
mkdtemp: async () => {
|
||||
mkdirSync(collectorDir);
|
||||
return collectorDir;
|
||||
},
|
||||
randomUUID: () => "00000000-0000-4000-8000-000000000000",
|
||||
reserveLocalPort: async () => ports.shift() ?? 49999,
|
||||
spawn: vi.fn(() => child) as never,
|
||||
stopDockerContainer,
|
||||
waitForLocalPort: async () => {
|
||||
throw new Error("collector never became ready");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("collector never became ready");
|
||||
|
||||
expect(stopDockerContainer).toHaveBeenCalledWith(
|
||||
"openclaw-otel-smoke-00000000-0000-4000-8000-000000000000",
|
||||
);
|
||||
expect(existsSync(collectorDir)).toBe(false);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reports bounded Docker collector output when readiness exits", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-collector-output-"));
|
||||
const collectorDir = path.join(tempRoot, "collector");
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
stdout: EventEmitter;
|
||||
};
|
||||
child.stderr = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
const ports = [4318, 45679];
|
||||
|
||||
try {
|
||||
let thrown: unknown;
|
||||
await testing
|
||||
.startDockerOtelCollector(4317, {
|
||||
mkdtemp: async () => {
|
||||
mkdirSync(collectorDir);
|
||||
return collectorDir;
|
||||
},
|
||||
randomUUID: () => "00000000-0000-4000-8000-000000000000",
|
||||
reserveLocalPort: async () => ports.shift() ?? 49999,
|
||||
spawn: vi.fn(() => child) as never,
|
||||
stopDockerContainer: vi.fn(async () => {}),
|
||||
waitForLocalPort: async (_port, _timeout, readFailure) => {
|
||||
child.stdout.emit("data", "DO_NOT_DUMP_COLLECTOR_PREFIX\n");
|
||||
child.stderr.emit("data", Buffer.alloc(64 * 1024, "x"));
|
||||
child.stderr.emit("data", "\nCOLLECTOR_TAIL_MARKER\n");
|
||||
child.emit("close", 1);
|
||||
await delay(0);
|
||||
throw new Error(readFailure());
|
||||
},
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
thrown = error;
|
||||
});
|
||||
|
||||
expect(thrown).toBeInstanceOf(Error);
|
||||
const message = thrown instanceof Error ? thrown.message : String(thrown);
|
||||
expect(message).toContain("COLLECTOR_TAIL_MARKER");
|
||||
expect(message).not.toContain("DO_NOT_DUMP_COLLECTOR_PREFIX");
|
||||
expect(message.length).toBeLessThan(24 * 1024);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
199
test/e2e/qa-lab/runtime/script-evidence.test.ts
Normal file
199
test/e2e/qa-lab/runtime/script-evidence.test.ts
Normal file
@@ -0,0 +1,199 @@
|
||||
// QA script evidence writer tests cover status, bounded logs, and artifact paths.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { validateQaEvidenceSummaryJson } from "../../../../extensions/qa-lab/api.js";
|
||||
import {
|
||||
createQaScriptBlockedStatusTracker,
|
||||
createQaScriptEvidenceWriter,
|
||||
} from "./script-evidence.js";
|
||||
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
async function makeWriter(
|
||||
params: { maxDetailsBytes?: number; maxLogBytes?: number } = {},
|
||||
) {
|
||||
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-script-evidence-"));
|
||||
tempRoots.push(repoRoot);
|
||||
return {
|
||||
artifactBase: path.join(repoRoot, ".artifacts", "qa-e2e", "script"),
|
||||
repoRoot,
|
||||
writer: createQaScriptEvidenceWriter({
|
||||
artifactBase: path.join(repoRoot, ".artifacts", "qa-e2e", "script"),
|
||||
logFileName: "producer.log",
|
||||
maxDetailsBytes: params.maxDetailsBytes,
|
||||
maxLogBytes: params.maxLogBytes ?? 64,
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot,
|
||||
target: {
|
||||
id: "script-evidence-test",
|
||||
title: "Script evidence test",
|
||||
sourcePath: "test/e2e/qa-lab/runtime/script-evidence.test.ts",
|
||||
primaryCoverageIds: ["qa.script-evidence"],
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await Promise.all(
|
||||
tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe("QA script evidence writer", () => {
|
||||
for (const status of ["pass", "fail", "blocked"] as const) {
|
||||
it(`writes ${status} evidence with normalized artifact paths`, async () => {
|
||||
const { artifactBase, writer } = await makeWriter();
|
||||
const summaryPath = path.join(artifactBase, "nested", "summary.json");
|
||||
await fs.mkdir(path.dirname(summaryPath), { recursive: true });
|
||||
await fs.writeFile(summaryPath, "{}\n", "utf8");
|
||||
writer.appendLog("producer output\n");
|
||||
|
||||
const evidence = await writer.write({
|
||||
artifacts: [{ kind: "summary", filePath: summaryPath }],
|
||||
details: `${status} details`,
|
||||
durationMs: 25,
|
||||
status,
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
execution: {
|
||||
artifacts: [
|
||||
{ kind: "log", path: "producer.log", source: "script" },
|
||||
{ kind: "summary", path: path.join("nested", "summary.json"), source: "script" },
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status,
|
||||
timing: { wallMs: 25 },
|
||||
},
|
||||
});
|
||||
const diskEvidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(path.join(artifactBase, "qa-evidence.json"), "utf8")),
|
||||
);
|
||||
expect(diskEvidence).toEqual(evidence);
|
||||
expect(
|
||||
JSON.parse(await fs.readFile(path.join(artifactBase, "latest-run.json"), "utf8")),
|
||||
).toEqual({ qaEvidence: "qa-evidence.json" });
|
||||
});
|
||||
}
|
||||
|
||||
it("keeps only the bounded log tail", async () => {
|
||||
const { artifactBase, writer } = await makeWriter({ maxLogBytes: 24 });
|
||||
writer.appendLog(`discard-me-${"x".repeat(64)}`);
|
||||
writer.appendLog("recent-tail");
|
||||
|
||||
await writer.write({ durationMs: 1, status: "pass" });
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
expect(log).toContain("recent-tail");
|
||||
expect(log).not.toContain("discard-me");
|
||||
expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(24);
|
||||
});
|
||||
|
||||
it("keeps only the bounded failure detail tail", async () => {
|
||||
const { writer } = await makeWriter({ maxDetailsBytes: 24 });
|
||||
|
||||
const evidence = writer.build({
|
||||
details: `discard-me-${"x".repeat(64)}recent-reason`,
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
});
|
||||
|
||||
const reason = evidence.entries[0]?.result.failure?.reason ?? "";
|
||||
expect(reason).toContain("recent-reason");
|
||||
expect(reason).not.toContain("discard-me");
|
||||
expect(Buffer.byteLength(reason, "utf8")).toBeLessThanOrEqual(24);
|
||||
});
|
||||
|
||||
it("keeps UTF-8 logs and failure details within byte limits", async () => {
|
||||
const { artifactBase, writer } = await makeWriter({
|
||||
maxDetailsBytes: 9,
|
||||
maxLogBytes: 9,
|
||||
});
|
||||
const diagnostic = `${"🙂".repeat(32)}done`;
|
||||
writer.appendLog(diagnostic);
|
||||
|
||||
const evidence = await writer.write({
|
||||
details: diagnostic,
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
});
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
const reason = evidence.entries[0]?.result.failure?.reason ?? "";
|
||||
expect(log).toContain("done");
|
||||
expect(reason).toContain("done");
|
||||
expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(9);
|
||||
expect(Buffer.byteLength(reason, "utf8")).toBeLessThanOrEqual(9);
|
||||
});
|
||||
|
||||
it("applies built-in and configured redaction before bounding logs and details", async () => {
|
||||
const { artifactBase, repoRoot, writer } = await makeWriter({
|
||||
maxDetailsBytes: 4096,
|
||||
maxLogBytes: 4096,
|
||||
});
|
||||
const configPath = path.join(repoRoot, "openclaw.json");
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
`${JSON.stringify({ logging: { redactPatterns: ["/internal-\\d+/g"] } })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath);
|
||||
writer.appendLog(`${"x".repeat(16_380)}inter`);
|
||||
writer.appendLog("nal-12345 should hide password=s");
|
||||
writer.appendLog("k-split-secret-1234567890");
|
||||
|
||||
const evidence = await writer.write({
|
||||
details: "reason internal-67890 should hide",
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
});
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
expect(log).toContain("*** should hide password=sk-spl…7890");
|
||||
expect(log).not.toContain("internal-12345");
|
||||
expect(evidence.entries[0]?.result.failure?.reason).toBe("reason *** should hide");
|
||||
});
|
||||
|
||||
it("omits oversized raw logs instead of truncating before redaction", async () => {
|
||||
const secretTail = "private-secret-tail-1234567890";
|
||||
const { artifactBase, writer } = await makeWriter({ maxLogBytes: 128 });
|
||||
writer.appendLog(`password=${"x".repeat(40 * 1024)}${secretTail}`);
|
||||
|
||||
await writer.write({ durationMs: 1, status: "fail" });
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
expect(log).toBe("QA evidence log omitted: safe redaction buffer exceeded.\n");
|
||||
expect(log).not.toContain(secretTail);
|
||||
expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(128);
|
||||
});
|
||||
|
||||
it.each(["..", "../outside.log"])(
|
||||
"rejects artifact path %s outside the producer output directory",
|
||||
async (filePath) => {
|
||||
const { writer } = await makeWriter();
|
||||
|
||||
expect(() =>
|
||||
writer.build({
|
||||
artifacts: [{ kind: "log", filePath }],
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
}),
|
||||
).toThrow("QA evidence artifact must be inside artifact base");
|
||||
},
|
||||
);
|
||||
|
||||
it("tracks blocked output before discarded diagnostic tails", () => {
|
||||
const tracker = createQaScriptBlockedStatusTracker([/missing provider auth/i]);
|
||||
|
||||
tracker.append("missing provider ");
|
||||
tracker.append(`auth\n${"x".repeat(4096)}`);
|
||||
|
||||
expect(tracker.status()).toBe("blocked");
|
||||
});
|
||||
});
|
||||
236
test/e2e/qa-lab/runtime/script-evidence.ts
Normal file
236
test/e2e/qa-lab/runtime/script-evidence.ts
Normal file
@@ -0,0 +1,236 @@
|
||||
// Shared QA script evidence writer keeps producer logs and artifact paths bounded.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
buildScriptEvidenceSummary,
|
||||
QA_EVIDENCE_FILENAME,
|
||||
type QaEvidencePackageSource,
|
||||
type QaEvidenceStatus,
|
||||
type QaEvidenceSummaryJson,
|
||||
type QaProviderMode,
|
||||
} from "../../../../extensions/qa-lab/api.js";
|
||||
import { readLoggingConfig } from "../../../../src/logging/config.js";
|
||||
import { withFullContextToolPayloadRedaction } from "../../../../src/logging/redact-internal.js";
|
||||
import { redactToolPayloadTextWithConfig } from "../../../../src/logging/redact.js";
|
||||
import { DEFAULT_CHILD_OUTPUT_TAIL_BYTES } from "../../../helpers/bounded-child-output.js";
|
||||
|
||||
export const DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES = 32 * 1024;
|
||||
const QA_SCRIPT_STATUS_MATCH_CARRY_CHARS = 1024;
|
||||
const QA_SCRIPT_LOG_OVERFLOW_MESSAGE = "QA evidence log omitted: safe redaction buffer exceeded.\n";
|
||||
const QA_SCRIPT_DETAILS_OVERFLOW_MESSAGE =
|
||||
"QA evidence details omitted: safe redaction buffer exceeded.";
|
||||
|
||||
type QaScriptEvidenceArtifactInput = {
|
||||
filePath: string;
|
||||
kind: string;
|
||||
};
|
||||
|
||||
type QaScriptEvidenceTarget = {
|
||||
codeRefs?: readonly string[];
|
||||
docsRefs?: readonly string[];
|
||||
id: string;
|
||||
primaryCoverageIds?: readonly string[];
|
||||
secondaryCoverageIds?: readonly string[];
|
||||
sourcePath: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
export type QaScriptEvidenceStatus = Exclude<QaEvidenceStatus, "skipped">;
|
||||
|
||||
type QaScriptEvidenceResult = {
|
||||
artifacts?: readonly QaScriptEvidenceArtifactInput[];
|
||||
details?: string;
|
||||
durationMs: number;
|
||||
status: QaScriptEvidenceStatus;
|
||||
};
|
||||
|
||||
type QaScriptEvidenceWriterOptions = {
|
||||
artifactBase: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
evidenceMode?: "full" | "slim";
|
||||
logFileName: string;
|
||||
maxDetailsBytes?: number;
|
||||
maxLogBytes?: number;
|
||||
packageSource?: QaEvidencePackageSource;
|
||||
primaryModel: string;
|
||||
providerMode: QaProviderMode;
|
||||
repoRoot: string;
|
||||
target: QaScriptEvidenceTarget;
|
||||
};
|
||||
|
||||
function resolveArtifactPath(artifactBase: string, filePath: string) {
|
||||
const absoluteArtifactBase = path.resolve(artifactBase);
|
||||
const absoluteFilePath = path.resolve(absoluteArtifactBase, filePath);
|
||||
const relativePath = path.relative(absoluteArtifactBase, absoluteFilePath);
|
||||
const escapesArtifactBase =
|
||||
relativePath === ".." ||
|
||||
relativePath.startsWith(`..${path.sep}`) ||
|
||||
path.isAbsolute(relativePath);
|
||||
if (!relativePath || escapesArtifactBase) {
|
||||
throw new Error(`QA evidence artifact must be inside artifact base: ${filePath}`);
|
||||
}
|
||||
return { absoluteFilePath, relativePath };
|
||||
}
|
||||
|
||||
async function writeJson(filePath: string, value: unknown) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function resolveByteLimit(value: number | undefined, fallback: number) {
|
||||
return Number.isInteger(value) && Number(value) > 0 ? Number(value) : fallback;
|
||||
}
|
||||
|
||||
function utf8Tail(text: string, maxBytes: number) {
|
||||
const buffer = Buffer.from(text, "utf8");
|
||||
if (buffer.byteLength <= maxBytes) {
|
||||
return text;
|
||||
}
|
||||
let start = buffer.byteLength - maxBytes;
|
||||
while (start < buffer.byteLength && (buffer[start]! & 0xc0) === 0x80) {
|
||||
start += 1;
|
||||
}
|
||||
return buffer.subarray(start).toString("utf8");
|
||||
}
|
||||
|
||||
function createSafeRedactionBuffer(maxBytes: number) {
|
||||
const chunks: Buffer[] = [];
|
||||
let totalBytes = 0;
|
||||
let overflowed = false;
|
||||
|
||||
return {
|
||||
append(chunk: string) {
|
||||
if (overflowed) {
|
||||
return;
|
||||
}
|
||||
const buffer = Buffer.from(chunk);
|
||||
if (totalBytes + buffer.byteLength > maxBytes) {
|
||||
// Arbitrary configured patterns need the complete raw context. Omit an
|
||||
// oversized log instead of truncating before redaction and leaking a tail.
|
||||
chunks.length = 0;
|
||||
totalBytes = 0;
|
||||
overflowed = true;
|
||||
return;
|
||||
}
|
||||
chunks.push(buffer);
|
||||
totalBytes += buffer.byteLength;
|
||||
},
|
||||
overflowed() {
|
||||
return overflowed;
|
||||
},
|
||||
text() {
|
||||
return Buffer.concat(chunks, totalBytes).toString("utf8");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createQaScriptBlockedStatusTracker(blockedPatterns: readonly RegExp[]) {
|
||||
let blocked = false;
|
||||
let carry = "";
|
||||
|
||||
return {
|
||||
append(chunk: unknown) {
|
||||
if (blocked) {
|
||||
return;
|
||||
}
|
||||
const text = `${carry}${String(chunk)}`;
|
||||
blocked = blockedPatterns.some((pattern) => {
|
||||
pattern.lastIndex = 0;
|
||||
return pattern.test(text);
|
||||
});
|
||||
// Keep enough overlap for a prerequisite phrase split across stream chunks.
|
||||
carry = text.slice(-QA_SCRIPT_STATUS_MATCH_CARRY_CHARS);
|
||||
},
|
||||
status(): QaScriptEvidenceStatus {
|
||||
return blocked ? "blocked" : "fail";
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOptions) {
|
||||
const maxLogBytes = resolveByteLimit(options.maxLogBytes, DEFAULT_CHILD_OUTPUT_TAIL_BYTES);
|
||||
const logFile = resolveArtifactPath(options.artifactBase, options.logFileName);
|
||||
const maxDetailsBytes = resolveByteLimit(
|
||||
options.maxDetailsBytes,
|
||||
DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES,
|
||||
);
|
||||
const log = createSafeRedactionBuffer(maxLogBytes + DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES);
|
||||
const redact = (text: string) =>
|
||||
redactToolPayloadTextWithConfig(text, withFullContextToolPayloadRedaction(readLoggingConfig()));
|
||||
const boundedLogText = () => {
|
||||
if (log.overflowed()) {
|
||||
return utf8Tail(QA_SCRIPT_LOG_OVERFLOW_MESSAGE, maxLogBytes);
|
||||
}
|
||||
return utf8Tail(redact(log.text()), maxLogBytes);
|
||||
};
|
||||
|
||||
const boundedDetails = (details: string | undefined) => {
|
||||
if (!details) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
Buffer.byteLength(details, "utf8") >
|
||||
maxDetailsBytes + DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES
|
||||
) {
|
||||
return utf8Tail(QA_SCRIPT_DETAILS_OVERFLOW_MESSAGE, maxDetailsBytes);
|
||||
}
|
||||
return utf8Tail(redact(details), maxDetailsBytes);
|
||||
};
|
||||
|
||||
const normalizeArtifacts = (artifacts: readonly QaScriptEvidenceArtifactInput[] = []) => {
|
||||
const normalized = [
|
||||
{ kind: "log", path: logFile.relativePath },
|
||||
...artifacts.map((artifact) => ({
|
||||
kind: artifact.kind,
|
||||
path: resolveArtifactPath(options.artifactBase, artifact.filePath).relativePath,
|
||||
})),
|
||||
];
|
||||
return [
|
||||
...new Map(
|
||||
normalized.map((artifact) => [`${artifact.kind}:${artifact.path}`, artifact]),
|
||||
).values(),
|
||||
];
|
||||
};
|
||||
|
||||
const build = (result: QaScriptEvidenceResult): QaEvidenceSummaryJson =>
|
||||
buildScriptEvidenceSummary({
|
||||
artifactPaths: normalizeArtifacts(result.artifacts),
|
||||
evidenceMode: options.evidenceMode ?? "full",
|
||||
env: options.env ?? process.env,
|
||||
generatedAt: new Date().toISOString(),
|
||||
packageSource: options.packageSource,
|
||||
primaryModel: options.primaryModel,
|
||||
providerMode: options.providerMode,
|
||||
repoRoot: options.repoRoot,
|
||||
runner: "script",
|
||||
targets: [options.target],
|
||||
results: [
|
||||
{
|
||||
id: options.target.id,
|
||||
status: result.status,
|
||||
durationMs: result.durationMs,
|
||||
failureMessage: boundedDetails(result.details),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
appendLog(chunk: unknown) {
|
||||
log.append(String(chunk));
|
||||
},
|
||||
build,
|
||||
logText() {
|
||||
return boundedLogText();
|
||||
},
|
||||
async write(result: QaScriptEvidenceResult) {
|
||||
const evidence = build(result);
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
await fs.writeFile(logFile.absoluteFilePath, boundedLogText(), "utf8");
|
||||
await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence);
|
||||
await writeJson(path.join(options.artifactBase, "latest-run.json"), {
|
||||
qaEvidence: QA_EVIDENCE_FILENAME,
|
||||
});
|
||||
return evidence;
|
||||
},
|
||||
};
|
||||
}
|
||||
173
test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts
Normal file
173
test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
// QA Talk E2E tests cover provider session creation and active-run voice controls.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { talkHandlers } from "../../../../src/gateway/server-methods/talk.ts";
|
||||
import { createPluginRecord } from "../../../../src/plugins/loader-records.ts";
|
||||
import { createPluginRegistry } from "../../../../src/plugins/registry.ts";
|
||||
import {
|
||||
resetPluginRuntimeStateForTest,
|
||||
setActivePluginRegistry,
|
||||
} from "../../../../src/plugins/runtime.ts";
|
||||
import { controlRealtimeVoiceAgentRun } from "../../../../src/talk/agent-run-control.ts";
|
||||
import type { TalkEvent } from "../../../../src/talk/talk-events.ts";
|
||||
|
||||
const noopLogger = {
|
||||
info() {},
|
||||
warn() {},
|
||||
error() {},
|
||||
debug() {},
|
||||
};
|
||||
|
||||
function installMockRealtimeProvider() {
|
||||
const registry = createPluginRegistry({
|
||||
logger: noopLogger,
|
||||
runtime: {},
|
||||
activateGlobalSideEffects: false,
|
||||
});
|
||||
const record = createPluginRecord({
|
||||
id: "qa-talk-realtime",
|
||||
name: "QA Talk Realtime",
|
||||
source: "test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts",
|
||||
origin: "global",
|
||||
enabled: true,
|
||||
configSchema: false,
|
||||
});
|
||||
const createBrowserSession = vi.fn(async () => ({
|
||||
provider: "qa-realtime",
|
||||
transport: "provider-websocket" as const,
|
||||
protocol: "google-live-bidi" as const,
|
||||
clientSecret: "auth_tokens/qa-talk",
|
||||
websocketUrl:
|
||||
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained",
|
||||
audio: {
|
||||
inputEncoding: "pcm16" as const,
|
||||
inputSampleRateHz: 16_000,
|
||||
outputEncoding: "pcm16" as const,
|
||||
outputSampleRateHz: 24_000,
|
||||
},
|
||||
}));
|
||||
registry.registerRealtimeVoiceProvider(record, {
|
||||
id: "qa-realtime",
|
||||
label: "QA Realtime",
|
||||
isConfigured: () => true,
|
||||
createBrowserSession,
|
||||
createBridge: vi.fn(),
|
||||
});
|
||||
setActivePluginRegistry(registry.registry);
|
||||
return createBrowserSession;
|
||||
}
|
||||
|
||||
function createControlDeps() {
|
||||
return {
|
||||
abortEmbeddedAgentRun: vi.fn(() => true),
|
||||
queueEmbeddedAgentMessageWithOutcomeAsync: vi.fn(async (sessionId: string) => ({
|
||||
queued: true as const,
|
||||
sessionId,
|
||||
target: "embedded_run" as const,
|
||||
gatewayHealth: "live" as const,
|
||||
enqueuedAtMs: 123,
|
||||
})),
|
||||
getDiagnosticSessionActivitySnapshot: vi.fn(() => ({
|
||||
activeWorkKind: "embedded_run" as const,
|
||||
hasActiveEmbeddedRun: true,
|
||||
})),
|
||||
resolveActiveEmbeddedRunSessionId: vi.fn(() => "session-active"),
|
||||
};
|
||||
}
|
||||
|
||||
describe("QA active Talk agent-run status", () => {
|
||||
afterEach(() => {
|
||||
resetPluginRuntimeStateForTest();
|
||||
});
|
||||
|
||||
it("creates a mock realtime session and controls one active agent run", async () => {
|
||||
const createBrowserSession = installMockRealtimeProvider();
|
||||
const respond = vi.fn();
|
||||
await talkHandlers["talk.client.create"]({
|
||||
req: { type: "req", id: "create", method: "talk.client.create" },
|
||||
params: { sessionKey: "agent:main:main", provider: "qa-realtime" },
|
||||
client: { connId: "qa-talk" },
|
||||
isWebchatConnect: () => true,
|
||||
respond,
|
||||
context: {
|
||||
getRuntimeConfig: () =>
|
||||
({
|
||||
talk: {
|
||||
realtime: {
|
||||
provider: "qa-realtime",
|
||||
providers: { "qa-realtime": {} },
|
||||
},
|
||||
},
|
||||
}) satisfies OpenClawConfig,
|
||||
},
|
||||
} as never);
|
||||
|
||||
expect(createBrowserSession).toHaveBeenCalledTimes(1);
|
||||
expect(createBrowserSession.mock.calls[0]?.[0]).toMatchObject({
|
||||
tools: [
|
||||
expect.objectContaining({ name: "openclaw_agent_consult" }),
|
||||
expect.objectContaining({ name: "openclaw_agent_control" }),
|
||||
],
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({ provider: "qa-realtime", transport: "provider-websocket" }),
|
||||
undefined,
|
||||
);
|
||||
|
||||
const deps = createControlDeps();
|
||||
const recentEvents = [
|
||||
{
|
||||
id: "event-1",
|
||||
type: "tool.progress",
|
||||
sessionId: "talk-1",
|
||||
seq: 1,
|
||||
timestamp: new Date(0).toISOString(),
|
||||
mode: "realtime",
|
||||
transport: "provider-websocket",
|
||||
brain: "agent-consult",
|
||||
payload: { name: "exec_command", phase: "running" },
|
||||
} satisfies TalkEvent,
|
||||
];
|
||||
|
||||
await expect(
|
||||
controlRealtimeVoiceAgentRun(
|
||||
{
|
||||
sessionKey: "agent:main:main",
|
||||
text: "status",
|
||||
mode: "status",
|
||||
recentEvents,
|
||||
},
|
||||
deps,
|
||||
),
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
active: true,
|
||||
message: "OpenClaw is working in exec_command (running).",
|
||||
});
|
||||
|
||||
await expect(
|
||||
controlRealtimeVoiceAgentRun(
|
||||
{ sessionKey: "agent:main:main", text: "use the safer path", mode: "steer" },
|
||||
deps,
|
||||
),
|
||||
).resolves.toMatchObject({ ok: true, mode: "steer", queued: true });
|
||||
await expect(
|
||||
controlRealtimeVoiceAgentRun(
|
||||
{ sessionKey: "agent:main:main", text: "also check migration", mode: "followup" },
|
||||
deps,
|
||||
),
|
||||
).resolves.toMatchObject({ ok: true, mode: "followup", queued: true });
|
||||
expect(deps.queueEmbeddedAgentMessageWithOutcomeAsync.mock.calls[1]?.[1]).toContain(
|
||||
"Spoken follow-up for the current voice call.",
|
||||
);
|
||||
|
||||
await expect(
|
||||
controlRealtimeVoiceAgentRun(
|
||||
{ sessionKey: "agent:main:main", text: "cancel", mode: "cancel" },
|
||||
deps,
|
||||
),
|
||||
).resolves.toMatchObject({ ok: true, mode: "cancel", aborted: true });
|
||||
expect(deps.abortEmbeddedAgentRun).toHaveBeenCalledWith("session-active");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user