Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
211
test/e2e/qa-lab/runtime/agent-bundle-mcp-tools-docker-client.ts
Normal file
211
test/e2e/qa-lab/runtime/agent-bundle-mcp-tools-docker-client.ts
Normal file
@@ -0,0 +1,211 @@
|
||||
// OpenClaw bundle MCP tools Docker harness.
|
||||
// Imports packaged dist modules so tool materialization is verified against the
|
||||
// npm tarball installed in the functional image.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { materializeBundleMcpToolsForRun } from "../../../../dist/agents/agent-bundle-mcp-materialize.js";
|
||||
import {
|
||||
disposeAllSessionMcpRuntimes,
|
||||
getOrCreateSessionMcpRuntime,
|
||||
} from "../../../../dist/agents/agent-bundle-mcp-runtime.js";
|
||||
import {
|
||||
applyFinalEffectiveToolPolicy,
|
||||
resolveConversationCapabilityProfile,
|
||||
} from "../../../../dist/agents/embedded-agent-runner/effective-tool-policy.js";
|
||||
import { splitSdkTools } from "../../../../dist/agents/embedded-agent-runner/tool-split.js";
|
||||
import type { OpenClawConfig } from "../../../../dist/config/types.openclaw.js";
|
||||
import { getPluginToolMeta } from "../../../../dist/plugins/tools.js";
|
||||
import { createE2eStateDir } from "../../../../scripts/e2e/lib/temp-state-dir.ts";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeProbeServer(serverPath: string) {
|
||||
const sdkMcpServerPath = require.resolve("@modelcontextprotocol/sdk/server/mcp.js");
|
||||
const sdkStdioServerPath = require.resolve("@modelcontextprotocol/sdk/server/stdio.js");
|
||||
await fs.writeFile(
|
||||
serverPath,
|
||||
`#!/usr/bin/env node
|
||||
import { McpServer } from ${JSON.stringify(sdkMcpServerPath)};
|
||||
import { StdioServerTransport } from ${JSON.stringify(sdkStdioServerPath)};
|
||||
|
||||
const server = new McpServer({ name: "agent-bundle-mcp-tools-probe", version: "1.0.0" });
|
||||
server.tool("docker_probe", "Docker OpenClaw MCP tool availability probe", async () => ({
|
||||
content: [{ type: "text", text: "agent-bundle-mcp-tools-ok" }],
|
||||
}));
|
||||
|
||||
await server.connect(new StdioServerTransport());
|
||||
`,
|
||||
{ encoding: "utf-8", mode: 0o755 },
|
||||
);
|
||||
}
|
||||
|
||||
function applyPolicy(params: {
|
||||
tools: Awaited<ReturnType<typeof materializeBundleMcpToolsForRun>>["tools"];
|
||||
config: OpenClawConfig;
|
||||
}) {
|
||||
const warnings: string[] = [];
|
||||
return {
|
||||
tools: applyFinalEffectiveToolPolicy({
|
||||
bundledTools: params.tools,
|
||||
config: params.config,
|
||||
conversationCapabilityProfile: resolveConversationCapabilityProfile({
|
||||
config: params.config,
|
||||
sessionKey: "agent:main:docker-agent-bundle-mcp",
|
||||
agentId: "main",
|
||||
}),
|
||||
warn: (message) => {
|
||||
warnings.push(message);
|
||||
},
|
||||
}),
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tempState = await createE2eStateDir("openclaw-agent-bundle-mcp-");
|
||||
tempState.registerExitCleanup();
|
||||
const stateDir = tempState.stateDir;
|
||||
const probeDir = path.join(stateDir, "agent-bundle-mcp-tools");
|
||||
const serverPath = path.join(probeDir, "probe-server.mjs");
|
||||
await fs.mkdir(probeDir, { recursive: true });
|
||||
await writeProbeServer(serverPath);
|
||||
|
||||
const cfg: OpenClawConfig = {
|
||||
tools: {
|
||||
profile: "coding",
|
||||
},
|
||||
mcp: {
|
||||
servers: {
|
||||
dockerProbe: {
|
||||
command: "node",
|
||||
args: [serverPath],
|
||||
cwd: probeDir,
|
||||
connectionTimeoutMs: 5000,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const runtime = await getOrCreateSessionMcpRuntime({
|
||||
sessionId: `docker-agent-bundle-mcp-${randomUUID()}`,
|
||||
sessionKey: "agent:main:docker-agent-bundle-mcp",
|
||||
workspaceDir: probeDir,
|
||||
cfg,
|
||||
});
|
||||
const materialized = await materializeBundleMcpToolsForRun({ runtime });
|
||||
const probeTool = materialized.tools.find((tool) => tool.name === "dockerProbe__docker_probe");
|
||||
assert(probeTool, "expected dockerProbe__docker_probe to materialize");
|
||||
assert(
|
||||
getPluginToolMeta(probeTool)?.pluginId === "bundle-mcp",
|
||||
"expected materialized MCP tool to be tagged as bundle-mcp",
|
||||
);
|
||||
|
||||
const result = await probeTool.execute("docker-mcp-probe", {}, undefined, undefined);
|
||||
assert(
|
||||
result.content.some(
|
||||
(item) => item.type === "text" && item.text === "agent-bundle-mcp-tools-ok",
|
||||
),
|
||||
"expected materialized MCP tool execution result",
|
||||
);
|
||||
|
||||
const coding = applyPolicy({ tools: materialized.tools, config: cfg });
|
||||
assert(
|
||||
coding.tools.some((tool) => tool.name === probeTool.name),
|
||||
"expected coding profile to keep bundle MCP tools",
|
||||
);
|
||||
|
||||
const messaging = applyPolicy({
|
||||
tools: materialized.tools,
|
||||
config: { ...cfg, tools: { profile: "messaging" } },
|
||||
});
|
||||
assert(
|
||||
messaging.tools.some((tool) => tool.name === probeTool.name),
|
||||
"expected messaging profile to keep bundle MCP tools",
|
||||
);
|
||||
|
||||
const minimal = applyPolicy({
|
||||
tools: materialized.tools,
|
||||
config: { ...cfg, tools: { profile: "minimal" } },
|
||||
});
|
||||
assert(minimal.tools.length === 0, "expected minimal profile to filter bundle MCP tools");
|
||||
|
||||
const denied = applyPolicy({
|
||||
tools: materialized.tools,
|
||||
config: { ...cfg, tools: { profile: "coding", deny: ["bundle-mcp"] } },
|
||||
});
|
||||
assert(denied.tools.length === 0, "expected tools.deny bundle-mcp to filter MCP tools");
|
||||
|
||||
// The disputed boundary on #76063 is what reaches the SDK as `customTools`,
|
||||
// since that is the exact value serialized to the outbound provider request.
|
||||
// Prove the live stdio probe survives the materialize -> filter -> split chain
|
||||
// through `splitSdkTools` for the same four profiles already asserted above.
|
||||
const codingCustom = splitSdkTools({ tools: coding.tools, sandboxEnabled: false }).customTools;
|
||||
const messagingCustom = splitSdkTools({
|
||||
tools: messaging.tools,
|
||||
sandboxEnabled: false,
|
||||
}).customTools;
|
||||
const minimalCustom = splitSdkTools({
|
||||
tools: minimal.tools,
|
||||
sandboxEnabled: false,
|
||||
}).customTools;
|
||||
const deniedCustom = splitSdkTools({ tools: denied.tools, sandboxEnabled: false }).customTools;
|
||||
assert(
|
||||
codingCustom.some((tool) => tool.name === probeTool.name),
|
||||
"expected coding profile customTools to include bundle MCP tools",
|
||||
);
|
||||
assert(
|
||||
messagingCustom.some((tool) => tool.name === probeTool.name),
|
||||
"expected messaging profile customTools to include bundle MCP tools",
|
||||
);
|
||||
assert(
|
||||
minimalCustom.length === 0,
|
||||
"expected minimal profile customTools to exclude bundle MCP tools",
|
||||
);
|
||||
assert(
|
||||
deniedCustom.length === 0,
|
||||
"expected tools.deny bundle-mcp customTools to exclude bundle MCP tools",
|
||||
);
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: true,
|
||||
tool: probeTool.name,
|
||||
profileCounts: {
|
||||
coding: coding.tools.length,
|
||||
messaging: messaging.tools.length,
|
||||
minimal: minimal.tools.length,
|
||||
denied: denied.tools.length,
|
||||
},
|
||||
customToolsCounts: {
|
||||
coding: codingCustom.length,
|
||||
messaging: messagingCustom.length,
|
||||
minimal: minimalCustom.length,
|
||||
denied: deniedCustom.length,
|
||||
},
|
||||
customToolNames: {
|
||||
coding: codingCustom.map((tool) => tool.name),
|
||||
messaging: messagingCustom.map((tool) => tool.name),
|
||||
minimal: minimalCustom.map((tool) => tool.name),
|
||||
denied: deniedCustom.map((tool) => tool.name),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
);
|
||||
} finally {
|
||||
await disposeAllSessionMcpRuntimes();
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
187
test/e2e/qa-lab/runtime/crestodian-first-run-docker-client.ts
Normal file
187
test/e2e/qa-lab/runtime/crestodian-first-run-docker-client.ts
Normal file
@@ -0,0 +1,187 @@
|
||||
// Crestodian first-run Docker harness.
|
||||
// Imports packaged dist modules so the Docker lane verifies the npm tarball,
|
||||
// while this small test driver stays mounted from the checkout.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
runCli,
|
||||
shouldStartCrestodianForModernOnboard,
|
||||
shouldStartOnboardingForFreshInstall,
|
||||
} from "../../../../dist/cli/run-main.js";
|
||||
import { clearConfigCache } from "../../../../dist/config/config.js";
|
||||
import type { OpenClawConfig } from "../../../../dist/config/types.openclaw.js";
|
||||
import { runCrestodian } from "../../../../dist/crestodian/crestodian.js";
|
||||
import type { RuntimeEnv } from "../../../../dist/runtime.js";
|
||||
import { createE2eStateDir } from "../../../../scripts/e2e/lib/temp-state-dir.ts";
|
||||
|
||||
type CrestodianFirstRunCommand = {
|
||||
id: string;
|
||||
message: string;
|
||||
expectOutput: string;
|
||||
approve: boolean;
|
||||
};
|
||||
|
||||
type CrestodianFirstRunSpec = {
|
||||
dockerDefaultWorkspace: string;
|
||||
dockerAgentWorkspace: string;
|
||||
agentId: string;
|
||||
model: string;
|
||||
discordEnv: string;
|
||||
discordToken: string;
|
||||
commands: CrestodianFirstRunCommand[];
|
||||
auditOperations: string[];
|
||||
};
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function setEnvValue(key: string, value: string): void {
|
||||
Reflect.set(process.env, key, value);
|
||||
}
|
||||
|
||||
function createRuntime(): { runtime: RuntimeEnv; lines: string[] } {
|
||||
const lines: string[] = [];
|
||||
return {
|
||||
lines,
|
||||
runtime: {
|
||||
log: (...args) => lines.push(args.join(" ")),
|
||||
error: (...args) => lines.push(args.join(" ")),
|
||||
exit: (code) => {
|
||||
throw new Error(`exit ${code}`);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readFirstRunSpec(): Promise<CrestodianFirstRunSpec> {
|
||||
return JSON.parse(
|
||||
await fs.readFile(
|
||||
path.join(process.cwd(), "scripts", "e2e", "crestodian-first-run-spec.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as CrestodianFirstRunSpec;
|
||||
}
|
||||
|
||||
function renderCommandTemplate(template: string, vars: Record<string, string>): string {
|
||||
return template.replace(/\{([A-Za-z0-9_]+)\}/g, (match, key: string) => vars[key] ?? match);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const spec = await readFirstRunSpec();
|
||||
const tempState = await createE2eStateDir("openclaw-crestodian-first-run-");
|
||||
tempState.registerExitCleanup();
|
||||
const stateDir = tempState.stateDir;
|
||||
const configPath = process.env.OPENCLAW_CONFIG_PATH ?? path.join(stateDir, "openclaw.json");
|
||||
setEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
setEnvValue("OPENCLAW_CONFIG_PATH", configPath);
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
await fs.mkdir(stateDir, { recursive: true });
|
||||
clearConfigCache();
|
||||
|
||||
assert(
|
||||
await shouldStartOnboardingForFreshInstall(["node", "openclaw"]),
|
||||
"fresh bare OpenClaw invocation did not route to onboarding",
|
||||
);
|
||||
assert(
|
||||
shouldStartCrestodianForModernOnboard(["node", "openclaw", "onboard", "--modern"]),
|
||||
"modern onboard invocation did not route to Crestodian",
|
||||
);
|
||||
process.exitCode = undefined;
|
||||
await runCli(["node", "openclaw", "onboard", "--modern", "--non-interactive", "--json"]);
|
||||
assert(
|
||||
process.exitCode === undefined || process.exitCode === 0,
|
||||
"modern onboard overview exited nonzero",
|
||||
);
|
||||
|
||||
const overviewRuntime = createRuntime();
|
||||
await runCrestodian({ message: "overview", interactive: false }, overviewRuntime.runtime);
|
||||
const overviewOutput = overviewRuntime.lines.join("\n");
|
||||
assert(
|
||||
overviewOutput.includes("Config: missing"),
|
||||
"fresh overview did not report missing config",
|
||||
);
|
||||
assert(
|
||||
overviewOutput.includes('Next: run "setup" to create a starter config'),
|
||||
"fresh overview did not include setup recommendation",
|
||||
);
|
||||
|
||||
setEnvValue(spec.discordEnv, spec.discordToken);
|
||||
|
||||
const commandVars = {
|
||||
defaultWorkspace: spec.dockerDefaultWorkspace,
|
||||
agentWorkspace: spec.dockerAgentWorkspace,
|
||||
agentId: spec.agentId,
|
||||
model: spec.model,
|
||||
discordEnv: spec.discordEnv,
|
||||
};
|
||||
for (const command of spec.commands) {
|
||||
clearConfigCache();
|
||||
const commandRuntime = createRuntime();
|
||||
await runCrestodian(
|
||||
{
|
||||
message: renderCommandTemplate(command.message, commandVars),
|
||||
yes: command.approve,
|
||||
interactive: false,
|
||||
},
|
||||
commandRuntime.runtime,
|
||||
);
|
||||
const output = commandRuntime.lines.join("\n");
|
||||
assert(
|
||||
output.includes(command.expectOutput),
|
||||
`Crestodian first-run command ${command.id} did not apply: ${output}`,
|
||||
);
|
||||
}
|
||||
|
||||
const config = JSON.parse(await fs.readFile(configPath, "utf8")) as OpenClawConfig;
|
||||
assert(
|
||||
config.agents?.defaults?.workspace === spec.dockerDefaultWorkspace,
|
||||
"first-run setup did not write default workspace",
|
||||
);
|
||||
assert(
|
||||
config.agents?.defaults?.model &&
|
||||
typeof config.agents.defaults.model === "object" &&
|
||||
"primary" in config.agents.defaults.model &&
|
||||
config.agents.defaults.model.primary === spec.model,
|
||||
"first-run setup did not write default model",
|
||||
);
|
||||
const reef = config.agents?.list?.find((agent) => agent.id === spec.agentId);
|
||||
assert(reef, "Crestodian did not create reef agent");
|
||||
assert(reef.workspace === spec.dockerAgentWorkspace, "Crestodian did not write reef workspace");
|
||||
assert(reef.model === spec.model, "Crestodian did not write reef model");
|
||||
assert(config.plugins?.allow?.includes("discord"), "Crestodian did not allow Discord plugin");
|
||||
assert(
|
||||
config.plugins?.entries?.discord?.enabled === true,
|
||||
"Crestodian did not enable Discord plugin entry",
|
||||
);
|
||||
assert(config.channels?.discord?.enabled === true, "Crestodian did not enable Discord");
|
||||
const discordToken = config.channels?.discord?.token;
|
||||
assert(
|
||||
discordToken &&
|
||||
typeof discordToken === "object" &&
|
||||
"source" in discordToken &&
|
||||
discordToken.source === "env" &&
|
||||
"id" in discordToken &&
|
||||
discordToken.id === spec.discordEnv,
|
||||
"Crestodian did not write Discord token SecretRef",
|
||||
);
|
||||
assert(
|
||||
!JSON.stringify(config.channels.discord).includes(spec.discordToken),
|
||||
"Crestodian persisted the raw Discord token",
|
||||
);
|
||||
|
||||
const auditPath = path.join(stateDir, "audit", "crestodian.jsonl");
|
||||
const audit = (await fs.readFile(auditPath, "utf8")).trim();
|
||||
for (const operation of spec.auditOperations) {
|
||||
assert(audit.includes(`"operation":"${operation}"`), `${operation} audit entry missing`);
|
||||
}
|
||||
|
||||
console.log("Crestodian first-run Docker E2E passed");
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
84
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.test.ts
Normal file
84
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.test.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
// Docker E2E lane fixture tests keep QA scenario dispatch policy reusable.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
formatQaDockerE2eLaneUsage,
|
||||
listQaDockerE2eLaneNames,
|
||||
parseQaDockerE2eLaneArgs,
|
||||
resolveQaDockerE2eLane,
|
||||
runQaDockerE2eLane,
|
||||
} from "./docker-e2e-lane.fixture.ts";
|
||||
|
||||
describe("QA Docker E2E lane fixture", () => {
|
||||
it("lists known Docker lanes for scenario wrappers", () => {
|
||||
expect(listQaDockerE2eLaneNames()).toEqual(
|
||||
expect.arrayContaining([
|
||||
"agent-bundle-mcp-tools",
|
||||
"crestodian-first-run",
|
||||
"gateway-network",
|
||||
"release-plugin-marketplace",
|
||||
"update-migration",
|
||||
"update-restart-auth",
|
||||
]),
|
||||
);
|
||||
expect(listQaDockerE2eLaneNames()).toEqual([...listQaDockerE2eLaneNames()].sort());
|
||||
});
|
||||
|
||||
it("parses help, list, and lane arguments", () => {
|
||||
expect(parseQaDockerE2eLaneArgs(["--help"])).toEqual({ kind: "help" });
|
||||
expect(parseQaDockerE2eLaneArgs(["--list"])).toEqual({ kind: "list" });
|
||||
expect(parseQaDockerE2eLaneArgs(["--lane", "gateway-network"])).toEqual({
|
||||
kind: "run",
|
||||
laneName: "gateway-network",
|
||||
});
|
||||
|
||||
expect(() => parseQaDockerE2eLaneArgs([])).toThrow("--lane is required");
|
||||
expect(() => parseQaDockerE2eLaneArgs(["--lane"])).toThrow("--lane requires a value");
|
||||
});
|
||||
|
||||
it("renders usage from the shared lane registry", () => {
|
||||
const usage = formatQaDockerE2eLaneUsage("node qa-docker.js");
|
||||
|
||||
expect(usage).toContain("Usage: node qa-docker.js --lane <name>");
|
||||
expect(usage).toContain(" - gateway-network");
|
||||
expect(usage).toContain(" - update-restart-auth");
|
||||
});
|
||||
|
||||
it("resolves lane-specific environment overlays at run time", () => {
|
||||
const updateMigration = resolveQaDockerE2eLane("update-migration", {
|
||||
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: "openclaw@custom",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: "custom-scenario",
|
||||
});
|
||||
|
||||
expect(updateMigration.script).toBe("scripts/e2e/upgrade-survivor-docker.sh");
|
||||
expect(updateMigration.env.OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE).toBe("1");
|
||||
expect(updateMigration.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC).toBe("openclaw@custom");
|
||||
expect(updateMigration.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO).toBe("custom-scenario");
|
||||
|
||||
const updateRestartAuth = resolveQaDockerE2eLane("update-restart-auth", {});
|
||||
expect(updateRestartAuth.env.OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE).toBe("auto-auth");
|
||||
expect(updateRestartAuth.env.OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT).toBe("1500s");
|
||||
});
|
||||
|
||||
it("dispatches through bash without running Docker in fixture tests", () => {
|
||||
const spawn = vi.fn(() => ({ signal: null, status: 0 }));
|
||||
|
||||
expect(runQaDockerE2eLane("gateway-network", { env: { EXTRA: "1" }, spawn })).toEqual({
|
||||
signal: null,
|
||||
status: 0,
|
||||
});
|
||||
|
||||
expect(spawn).toHaveBeenCalledWith("bash", ["scripts/e2e/gateway-network-docker.sh"], {
|
||||
env: { EXTRA: "1" },
|
||||
stdio: "inherit",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects unknown lanes before spawning", () => {
|
||||
const spawn = vi.fn(() => ({ signal: null, status: 0 }));
|
||||
|
||||
expect(() => runQaDockerE2eLane("missing-lane", { spawn })).toThrow(
|
||||
"unknown Docker E2E lane: missing-lane",
|
||||
);
|
||||
expect(spawn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
169
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.ts
Normal file
169
test/e2e/qa-lab/runtime/docker-e2e-lane.fixture.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
// Shared QA Lab fixture for dispatching existing Docker E2E lanes.
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
export type QaDockerE2eLaneDefinition = {
|
||||
env?: (env: NodeJS.ProcessEnv) => Record<string, string>;
|
||||
script: string;
|
||||
};
|
||||
|
||||
type QaDockerE2eLaneRunResult = {
|
||||
error?: Error;
|
||||
signal: NodeJS.Signals | null;
|
||||
status: number | null;
|
||||
};
|
||||
|
||||
type SpawnQaDockerE2eLane = (
|
||||
command: string,
|
||||
args: string[],
|
||||
options: { env: NodeJS.ProcessEnv; stdio: "inherit" },
|
||||
) => QaDockerE2eLaneRunResult;
|
||||
|
||||
export const QA_DOCKER_E2E_LANES = {
|
||||
"agent-bundle-mcp-tools": {
|
||||
script: "scripts/e2e/agent-bundle-mcp-tools-docker.sh",
|
||||
},
|
||||
"agents-delete-shared-workspace": {
|
||||
script: "scripts/e2e/agents-delete-shared-workspace-docker.sh",
|
||||
},
|
||||
"bundled-plugin-install-uninstall": {
|
||||
script: "scripts/e2e/bundled-plugin-install-uninstall-docker.sh",
|
||||
},
|
||||
"crestodian-first-run": {
|
||||
script: "scripts/e2e/crestodian-first-run-docker.sh",
|
||||
},
|
||||
"docker-build-image": {
|
||||
script: "scripts/e2e/build-image.sh",
|
||||
},
|
||||
"gateway-network": {
|
||||
script: "scripts/e2e/gateway-network-docker.sh",
|
||||
},
|
||||
"npm-onboard-channel-agent": {
|
||||
script: "scripts/e2e/npm-onboard-channel-agent-docker.sh",
|
||||
},
|
||||
"openai-chat-tools": {
|
||||
script: "scripts/e2e/openai-chat-tools-docker.sh",
|
||||
},
|
||||
"openai-web-search-minimal": {
|
||||
script: "scripts/e2e/openai-web-search-minimal-docker.sh",
|
||||
},
|
||||
openwebui: {
|
||||
script: "scripts/e2e/openwebui-docker.sh",
|
||||
},
|
||||
"plugin-lifecycle-matrix": {
|
||||
script: "scripts/e2e/plugin-lifecycle-matrix-docker.sh",
|
||||
},
|
||||
"release-plugin-marketplace": {
|
||||
script: "scripts/e2e/release-plugin-marketplace-docker.sh",
|
||||
},
|
||||
"release-upgrade-user-journey": {
|
||||
script: "scripts/e2e/release-upgrade-user-journey-docker.sh",
|
||||
},
|
||||
"release-user-journey": {
|
||||
script: "scripts/e2e/release-user-journey-docker.sh",
|
||||
},
|
||||
"update-channel-switch": {
|
||||
script: "scripts/e2e/update-channel-switch-docker.sh",
|
||||
},
|
||||
"update-migration": {
|
||||
env: (env) => ({
|
||||
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC:
|
||||
env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC ?? "openclaw@2026.4.23",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "1",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO:
|
||||
env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO ?? "plugin-deps-cleanup",
|
||||
}),
|
||||
script: "scripts/e2e/upgrade-survivor-docker.sh",
|
||||
},
|
||||
"update-restart-auth": {
|
||||
env: (env) => ({
|
||||
OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT:
|
||||
env.OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT ?? "1500s",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE: "auto-auth",
|
||||
}),
|
||||
script: "scripts/e2e/upgrade-survivor-docker.sh",
|
||||
},
|
||||
"upgrade-survivor": {
|
||||
script: "scripts/e2e/upgrade-survivor-docker.sh",
|
||||
},
|
||||
} satisfies Record<string, QaDockerE2eLaneDefinition>;
|
||||
|
||||
export type QaDockerE2eLaneName = keyof typeof QA_DOCKER_E2E_LANES;
|
||||
|
||||
export type QaDockerE2eLaneArgs =
|
||||
| { kind: "help" }
|
||||
| { kind: "list" }
|
||||
| { kind: "run"; laneName: string };
|
||||
|
||||
export type ResolvedQaDockerE2eLane = {
|
||||
env: NodeJS.ProcessEnv;
|
||||
name: QaDockerE2eLaneName;
|
||||
script: string;
|
||||
};
|
||||
|
||||
export function listQaDockerE2eLaneNames(): QaDockerE2eLaneName[] {
|
||||
return Object.keys(QA_DOCKER_E2E_LANES).toSorted() as QaDockerE2eLaneName[];
|
||||
}
|
||||
|
||||
export function formatQaDockerE2eLaneUsage(
|
||||
entrypoint = "node --import tsx test/e2e/qa-lab/runtime/docker-e2e-lane.ts",
|
||||
): string {
|
||||
return [
|
||||
`Usage: ${entrypoint} --lane <name>`,
|
||||
"",
|
||||
"Known lanes:",
|
||||
...listQaDockerE2eLaneNames().map((lane) => ` - ${lane}`),
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function parseQaDockerE2eLaneArgs(argv: string[]): QaDockerE2eLaneArgs {
|
||||
if (argv.includes("--help") || argv.includes("-h")) {
|
||||
return { kind: "help" };
|
||||
}
|
||||
if (argv.includes("--list")) {
|
||||
return { kind: "list" };
|
||||
}
|
||||
const index = argv.indexOf("--lane");
|
||||
if (index === -1) {
|
||||
throw new Error("--lane is required");
|
||||
}
|
||||
const laneName = argv[index + 1];
|
||||
if (!laneName || laneName.startsWith("-")) {
|
||||
throw new Error("--lane requires a value");
|
||||
}
|
||||
return { kind: "run", laneName };
|
||||
}
|
||||
|
||||
export function resolveQaDockerE2eLane(
|
||||
laneName: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): ResolvedQaDockerE2eLane {
|
||||
if (!isQaDockerE2eLaneName(laneName)) {
|
||||
throw new Error(`unknown Docker E2E lane: ${laneName}\n\n${formatQaDockerE2eLaneUsage()}`);
|
||||
}
|
||||
const lane = QA_DOCKER_E2E_LANES[laneName];
|
||||
return {
|
||||
env: { ...env, ...lane.env?.(env) },
|
||||
name: laneName,
|
||||
script: lane.script,
|
||||
};
|
||||
}
|
||||
|
||||
export function runQaDockerE2eLane(
|
||||
laneName: string,
|
||||
deps: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
spawn?: SpawnQaDockerE2eLane;
|
||||
} = {},
|
||||
): QaDockerE2eLaneRunResult {
|
||||
const lane = resolveQaDockerE2eLane(laneName, deps.env);
|
||||
const spawn = deps.spawn ?? spawnSync;
|
||||
return spawn("bash", [lane.script], {
|
||||
env: lane.env,
|
||||
stdio: "inherit",
|
||||
});
|
||||
}
|
||||
|
||||
function isQaDockerE2eLaneName(laneName: string): laneName is QaDockerE2eLaneName {
|
||||
return Object.hasOwn(QA_DOCKER_E2E_LANES, laneName);
|
||||
}
|
||||
28
test/e2e/qa-lab/runtime/docker-e2e-lane.ts
Normal file
28
test/e2e/qa-lab/runtime/docker-e2e-lane.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
// Runs an existing Docker E2E lane through the QA Lab script scenario contract.
|
||||
import {
|
||||
formatQaDockerE2eLaneUsage,
|
||||
listQaDockerE2eLaneNames,
|
||||
parseQaDockerE2eLaneArgs,
|
||||
runQaDockerE2eLane,
|
||||
} from "./docker-e2e-lane.fixture.ts";
|
||||
|
||||
const args = parseQaDockerE2eLaneArgs(process.argv.slice(2));
|
||||
if (args.kind === "help") {
|
||||
console.log(formatQaDockerE2eLaneUsage());
|
||||
process.exit(0);
|
||||
}
|
||||
if (args.kind === "list") {
|
||||
console.log(listQaDockerE2eLaneNames().join("\n"));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const result = runQaDockerE2eLane(args.laneName);
|
||||
|
||||
if (result.error) {
|
||||
console.error(result.error);
|
||||
process.exit(1);
|
||||
}
|
||||
if (result.signal) {
|
||||
process.kill(process.pid, result.signal);
|
||||
}
|
||||
process.exit(result.status ?? 1);
|
||||
99
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts
Normal file
99
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../../helpers/temp-dir.js";
|
||||
import { testing } from "./gateway-mcp-real-transports.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
function createRepoRoot() {
|
||||
return tempDirs.make("openclaw-qalab-cli-entry-");
|
||||
}
|
||||
|
||||
async function writeEntry(root: string, relativePath: string) {
|
||||
const entryPath = path.join(root, relativePath);
|
||||
await mkdir(path.dirname(entryPath), { recursive: true });
|
||||
await writeFile(entryPath, "", "utf8");
|
||||
return entryPath;
|
||||
}
|
||||
|
||||
describe("gateway MCP real transport producer", () => {
|
||||
it("uses the source CLI entry when build output is absent", async () => {
|
||||
const root = createRepoRoot();
|
||||
const entryPath = await writeEntry(root, "src/entry.ts");
|
||||
|
||||
const cli = testing.resolveOpenClawCliInvocation(root);
|
||||
|
||||
expect(cli.command).toBe(process.execPath);
|
||||
expect(cli.argsPrefix).toStrictEqual(["--import", "tsx", entryPath]);
|
||||
expect(cli.cwd).toBe(root);
|
||||
expect(cli.gatewayCommand).toMatchObject({
|
||||
executablePath: process.execPath,
|
||||
argsPrefix: ["--import", "tsx", entryPath],
|
||||
cwd: root,
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers built package output when it exists", async () => {
|
||||
const root = createRepoRoot();
|
||||
const distEntry = await writeEntry(root, "dist/index.mjs");
|
||||
await writeEntry(root, "src/entry.ts");
|
||||
|
||||
const cli = testing.resolveOpenClawCliInvocation(root);
|
||||
|
||||
expect(cli.argsPrefix).toStrictEqual([distEntry]);
|
||||
expect(cli.gatewayCommand).toMatchObject({
|
||||
argsPrefix: [distEntry],
|
||||
usePackagedPlugins: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the source channel MCP module when build output is absent", async () => {
|
||||
const root = createRepoRoot();
|
||||
const channelServerPath = await writeEntry(root, "src/mcp/channel-server.ts");
|
||||
|
||||
const mcp = testing.resolveChannelMcpInvocation({
|
||||
gatewayToken: "secret-token",
|
||||
gatewayUrl: "ws://127.0.0.1:12345",
|
||||
repoRoot: root,
|
||||
tokenFile: "/tmp/token-file",
|
||||
});
|
||||
|
||||
expect(mcp.command).toBe(process.execPath);
|
||||
expect(mcp.args.slice(0, 3)).toStrictEqual(["--import", "tsx", "--eval"]);
|
||||
expect(mcp.args[3]).toContain(channelServerPath);
|
||||
expect(mcp.args[3]).toContain("serveOpenClawChannelMcp");
|
||||
expect(mcp.cwd).toBe(root);
|
||||
expect(mcp.envPatch).toStrictEqual({
|
||||
OPENCLAW_QA_GATEWAY_TOKEN: "secret-token",
|
||||
OPENCLAW_QA_GATEWAY_URL: "ws://127.0.0.1:12345",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the packaged CLI for channel MCP when build output exists", async () => {
|
||||
const root = createRepoRoot();
|
||||
const distEntry = await writeEntry(root, "dist/index.js");
|
||||
await writeEntry(root, "src/mcp/channel-server.ts");
|
||||
|
||||
const mcp = testing.resolveChannelMcpInvocation({
|
||||
gatewayToken: "secret-token",
|
||||
gatewayUrl: "ws://127.0.0.1:12345",
|
||||
repoRoot: root,
|
||||
tokenFile: "/tmp/token-file",
|
||||
});
|
||||
|
||||
expect(mcp.args).toStrictEqual([
|
||||
distEntry,
|
||||
"mcp",
|
||||
"serve",
|
||||
"--url",
|
||||
"ws://127.0.0.1:12345",
|
||||
"--token-file",
|
||||
"/tmp/token-file",
|
||||
"--claude-channel-mode",
|
||||
"off",
|
||||
"--verbose",
|
||||
]);
|
||||
expect(mcp.envPatch).toStrictEqual({});
|
||||
});
|
||||
});
|
||||
817
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
Normal file
817
test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
Normal file
@@ -0,0 +1,817 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
// QA Lab producer proves Gateway and MCP scenarios across real process and protocol boundaries.
|
||||
import { createServer, type Server } from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
||||
import {
|
||||
QA_EVIDENCE_FILENAME,
|
||||
startQaGatewayChild,
|
||||
type QaGatewayChildCommand,
|
||||
type QaEvidenceSummaryJson,
|
||||
type QaGatewayChildListeningContext,
|
||||
} from "../../../../extensions/qa-lab/api.js";
|
||||
import {
|
||||
PROTOCOL_VERSION,
|
||||
MIN_CLIENT_PROTOCOL_VERSION,
|
||||
} from "../../../../packages/gateway-protocol/src/version.js";
|
||||
import { runGatewaySmoke } from "../../../../scripts/dev/gateway-smoke.js";
|
||||
import type { OpenClawConfig } from "../../../../src/config/types.openclaw.js";
|
||||
import { formatErrorMessage } from "../../../../src/infra/errors.js";
|
||||
import { createMcpClientTempState } from "./mcp-client-temp-state.fixture.ts";
|
||||
import { createQaScriptEvidenceWriter, type QaScriptEvidenceStatus } from "./script-evidence.ts";
|
||||
|
||||
const FIXTURE_PLUGIN_ID = "qa-real-transports-fixture";
|
||||
const FIXTURE_TOOL_NAME = "memory_search";
|
||||
const FIXTURE_FACT = "MCP fact: the codename is ORBIT-9.";
|
||||
const STARTUP_GATE_TIMEOUT_MS = 30_000;
|
||||
const MCP_CONNECT_TIMEOUT_MS = 30_000;
|
||||
const SOURCE_PATH = "test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts";
|
||||
|
||||
type ScenarioId = "gateway-smoke" | "mcp-gateway-connect-startup-retry" | "mcp-plugin-tools-call";
|
||||
|
||||
type ProducerOptions = {
|
||||
artifactBase: string;
|
||||
repoRoot: string;
|
||||
scenarioId: ScenarioId;
|
||||
};
|
||||
|
||||
type ProofResult = {
|
||||
details?: string;
|
||||
durationMs: number;
|
||||
status: QaScriptEvidenceStatus;
|
||||
};
|
||||
|
||||
type GatewayFrameCapture = {
|
||||
connectFrames: Array<{ minProtocol: number; maxProtocol: number }>;
|
||||
helloProtocols: number[];
|
||||
startupUnavailableResponses: number;
|
||||
};
|
||||
|
||||
type GatewayProxy = {
|
||||
capture: GatewayFrameCapture;
|
||||
stop: () => Promise<void>;
|
||||
url: string;
|
||||
};
|
||||
|
||||
type OpenClawCliInvocation = {
|
||||
argsPrefix: string[];
|
||||
command: string;
|
||||
cwd: string;
|
||||
gatewayCommand: QaGatewayChildCommand;
|
||||
};
|
||||
|
||||
type ChannelMcpInvocation = {
|
||||
args: string[];
|
||||
command: string;
|
||||
cwd: string;
|
||||
envPatch: NodeJS.ProcessEnv;
|
||||
};
|
||||
|
||||
type McpClientHandle = {
|
||||
client: Client;
|
||||
cleanup: () => void;
|
||||
stderr: () => string;
|
||||
transport: StdioClientTransport;
|
||||
};
|
||||
|
||||
const SCENARIOS = {
|
||||
"gateway-smoke": {
|
||||
title: "Gateway smoke evidence",
|
||||
sourcePath: "qa/scenarios/runtime/gateway-smoke.yaml",
|
||||
primaryCoverageIds: [
|
||||
"gateway.websocket-transport",
|
||||
"gateway.health-apis",
|
||||
"gateway.hello-ok-snapshot",
|
||||
],
|
||||
docsRefs: ["docs/gateway/index.md", "docs/concepts/qa-e2e-automation.md"],
|
||||
codeRefs: [
|
||||
SOURCE_PATH,
|
||||
"extensions/qa-lab/src/gateway-child.ts",
|
||||
"scripts/dev/gateway-smoke.ts",
|
||||
],
|
||||
},
|
||||
"mcp-gateway-connect-startup-retry": {
|
||||
title: "MCP Gateway connect startup retry",
|
||||
sourcePath: "qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml",
|
||||
primaryCoverageIds: [
|
||||
"gateway.connect-request",
|
||||
"gateway.protocol-version-negotiation",
|
||||
"gateway.startup-retry",
|
||||
],
|
||||
docsRefs: ["docs/gateway/protocol.md", "docs/cli/mcp.md"],
|
||||
codeRefs: [SOURCE_PATH, "extensions/qa-lab/src/gateway-child.ts", "src/mcp/channel-bridge.ts"],
|
||||
},
|
||||
"mcp-plugin-tools-call": {
|
||||
title: "MCP plugin-tools call",
|
||||
sourcePath: "qa/scenarios/plugins/mcp-plugin-tools-call.yaml",
|
||||
primaryCoverageIds: ["plugins.mcp-tools", "tools.invocation"],
|
||||
docsRefs: ["docs/cli/mcp.md", "docs/gateway/protocol.md"],
|
||||
codeRefs: [SOURCE_PATH, "src/mcp/plugin-tools-serve.ts", "src/mcp/plugin-tools-handlers.ts"],
|
||||
},
|
||||
} as const;
|
||||
|
||||
function parseOptions(argv: readonly string[]): ProducerOptions {
|
||||
const readValue = (name: string) => {
|
||||
const index = argv.indexOf(name);
|
||||
return index >= 0 ? argv[index + 1] : undefined;
|
||||
};
|
||||
const scenarioId = readValue("--scenario");
|
||||
if (!scenarioId || !(scenarioId in SCENARIOS)) {
|
||||
throw new Error(`--scenario must be one of: ${Object.keys(SCENARIOS).join(", ")}`);
|
||||
}
|
||||
const artifactBase = readValue("--artifact-base");
|
||||
if (!artifactBase) {
|
||||
throw new Error("--artifact-base is required");
|
||||
}
|
||||
return {
|
||||
artifactBase: path.resolve(artifactBase),
|
||||
repoRoot: path.resolve(readValue("--repo-root") ?? process.cwd()),
|
||||
scenarioId: scenarioId as ScenarioId,
|
||||
};
|
||||
}
|
||||
|
||||
async function createFixturePlugin() {
|
||||
// openclaw-temp-dir: allow standalone producer cleans this root in each scenario finally block
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gateway-mcp-fixture-"));
|
||||
const pluginDir = path.join(root, FIXTURE_PLUGIN_ID);
|
||||
const startupGatePath = path.join(root, "startup-connect-observed");
|
||||
await fs.mkdir(pluginDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(pluginDir, "openclaw.plugin.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
id: FIXTURE_PLUGIN_ID,
|
||||
activation: { onStartup: true },
|
||||
configSchema: { type: "object", additionalProperties: false, properties: {} },
|
||||
contracts: { tools: [FIXTURE_TOOL_NAME] },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(pluginDir, "index.js"),
|
||||
`const fs = require("node:fs");
|
||||
|
||||
module.exports = {
|
||||
id: ${JSON.stringify(FIXTURE_PLUGIN_ID)},
|
||||
register(api) {
|
||||
api.registerTool({
|
||||
name: ${JSON.stringify(FIXTURE_TOOL_NAME)},
|
||||
description: "Search fixture memory",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { query: { type: "string" }, maxResults: { type: "number" } },
|
||||
required: ["query"],
|
||||
},
|
||||
async execute(_toolCallId, params) {
|
||||
return { content: [{ type: "text", text: ${JSON.stringify(FIXTURE_FACT)} + " query=" + String(params.query) }] };
|
||||
},
|
||||
});
|
||||
api.registerService({
|
||||
id: "qa-startup-delay",
|
||||
async start() {
|
||||
const deadline = Date.now() + ${STARTUP_GATE_TIMEOUT_MS};
|
||||
while (!fs.existsSync(${JSON.stringify(startupGatePath)})) {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error("timed out waiting for the QA MCP startup connect frame");
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
},
|
||||
stop() {},
|
||||
});
|
||||
},
|
||||
};\n`,
|
||||
"utf8",
|
||||
);
|
||||
return {
|
||||
pluginDir,
|
||||
startupGatePath,
|
||||
cleanup: () => fs.rm(root, { force: true, recursive: true }),
|
||||
};
|
||||
}
|
||||
|
||||
function withFixturePlugin(config: OpenClawConfig, pluginDir: string): OpenClawConfig {
|
||||
const existingPaths = config.plugins?.load?.paths ?? [];
|
||||
const existingAllow = config.plugins?.allow ?? [];
|
||||
return {
|
||||
...config,
|
||||
plugins: {
|
||||
...config.plugins,
|
||||
enabled: true,
|
||||
allow: [...new Set([...existingAllow, FIXTURE_PLUGIN_ID])],
|
||||
load: {
|
||||
...config.plugins?.load,
|
||||
paths: [...new Set([...existingPaths, pluginDir])],
|
||||
},
|
||||
entries: {
|
||||
...config.plugins?.entries,
|
||||
[FIXTURE_PLUGIN_ID]: { enabled: true },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function emptyTransport() {
|
||||
return {
|
||||
requiredPluginIds: [] as string[],
|
||||
createGatewayConfig: () => ({}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveOpenClawCliInvocation(repoRoot: string): OpenClawCliInvocation {
|
||||
for (const relativePath of ["dist/index.mjs", "dist/index.js"]) {
|
||||
const entryPath = path.join(repoRoot, relativePath);
|
||||
if (existsSync(entryPath)) {
|
||||
const argsPrefix = [entryPath];
|
||||
return {
|
||||
argsPrefix,
|
||||
command: process.execPath,
|
||||
cwd: repoRoot,
|
||||
gatewayCommand: {
|
||||
executablePath: process.execPath,
|
||||
argsPrefix,
|
||||
cwd: repoRoot,
|
||||
usePackagedPlugins: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const sourceEntryPath = path.join(repoRoot, "src/entry.ts");
|
||||
if (existsSync(sourceEntryPath)) {
|
||||
const argsPrefix = ["--import", "tsx", sourceEntryPath];
|
||||
return {
|
||||
argsPrefix,
|
||||
command: process.execPath,
|
||||
cwd: repoRoot,
|
||||
gatewayCommand: {
|
||||
executablePath: process.execPath,
|
||||
argsPrefix,
|
||||
cwd: repoRoot,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error("OpenClaw CLI entry not found: expected dist/index.(m)js or src/entry.ts");
|
||||
}
|
||||
|
||||
function resolveChannelMcpInvocation(params: {
|
||||
gatewayToken: string;
|
||||
gatewayUrl: string;
|
||||
repoRoot: string;
|
||||
tokenFile: string;
|
||||
}): ChannelMcpInvocation {
|
||||
for (const relativePath of ["dist/index.mjs", "dist/index.js"]) {
|
||||
const entryPath = path.join(params.repoRoot, relativePath);
|
||||
if (existsSync(entryPath)) {
|
||||
return {
|
||||
args: [
|
||||
entryPath,
|
||||
"mcp",
|
||||
"serve",
|
||||
"--url",
|
||||
params.gatewayUrl,
|
||||
"--token-file",
|
||||
params.tokenFile,
|
||||
"--claude-channel-mode",
|
||||
"off",
|
||||
"--verbose",
|
||||
],
|
||||
command: process.execPath,
|
||||
cwd: params.repoRoot,
|
||||
envPatch: {},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const channelServerPath = path.join(params.repoRoot, "src/mcp/channel-server.ts");
|
||||
if (existsSync(channelServerPath)) {
|
||||
const channelServerUrl = pathToFileURL(channelServerPath).href;
|
||||
return {
|
||||
args: [
|
||||
"--import",
|
||||
"tsx",
|
||||
"--eval",
|
||||
[
|
||||
`import(${JSON.stringify(channelServerUrl)})`,
|
||||
`.then((module) => module.serveOpenClawChannelMcp({`,
|
||||
`gatewayUrl: process.env.OPENCLAW_QA_GATEWAY_URL,`,
|
||||
`gatewayToken: process.env.OPENCLAW_QA_GATEWAY_TOKEN,`,
|
||||
`claudeChannelMode: "off",`,
|
||||
`verbose: true`,
|
||||
`}))`,
|
||||
].join(""),
|
||||
],
|
||||
command: process.execPath,
|
||||
cwd: params.repoRoot,
|
||||
envPatch: {
|
||||
OPENCLAW_QA_GATEWAY_TOKEN: params.gatewayToken,
|
||||
OPENCLAW_QA_GATEWAY_URL: params.gatewayUrl,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"OpenClaw channel MCP entry not found: expected dist/index.(m)js or src/mcp/channel-server.ts",
|
||||
);
|
||||
}
|
||||
|
||||
function parseJsonFrame(data: RawData): Record<string, unknown> | null {
|
||||
try {
|
||||
const text = Array.isArray(data)
|
||||
? Buffer.concat(data).toString("utf8")
|
||||
: Buffer.from(data).toString("utf8");
|
||||
const value = JSON.parse(text);
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function startGatewayProxy(
|
||||
upstreamUrl: string,
|
||||
onConnectFrame?: () => void,
|
||||
): Promise<GatewayProxy> {
|
||||
const capture: GatewayFrameCapture = {
|
||||
connectFrames: [],
|
||||
helloProtocols: [],
|
||||
startupUnavailableResponses: 0,
|
||||
};
|
||||
const connectRequestIds = new Set<string>();
|
||||
const server: Server = createServer();
|
||||
const wss = new WebSocketServer({ server });
|
||||
wss.on("connection", (downstream) => {
|
||||
const upstream = new WebSocket(upstreamUrl);
|
||||
const pending: RawData[] = [];
|
||||
downstream.on("message", (data) => {
|
||||
const frame = parseJsonFrame(data);
|
||||
if (frame?.method === "connect" && typeof frame.id === "string") {
|
||||
const params = frame.params as Record<string, unknown> | undefined;
|
||||
if (typeof params?.minProtocol === "number" && typeof params.maxProtocol === "number") {
|
||||
capture.connectFrames.push({
|
||||
minProtocol: params.minProtocol,
|
||||
maxProtocol: params.maxProtocol,
|
||||
});
|
||||
connectRequestIds.add(frame.id);
|
||||
onConnectFrame?.();
|
||||
}
|
||||
}
|
||||
if (upstream.readyState === WebSocket.OPEN) {
|
||||
upstream.send(data);
|
||||
} else {
|
||||
pending.push(data);
|
||||
}
|
||||
});
|
||||
upstream.on("open", () => {
|
||||
for (const data of pending.splice(0)) {
|
||||
upstream.send(data);
|
||||
}
|
||||
});
|
||||
upstream.on("message", (data) => {
|
||||
const frame = parseJsonFrame(data);
|
||||
if (typeof frame?.id === "string" && connectRequestIds.has(frame.id)) {
|
||||
const error = frame.error as Record<string, unknown> | undefined;
|
||||
const details = error?.details as Record<string, unknown> | undefined;
|
||||
if (error?.retryable === true && details?.reason === "startup-sidecars") {
|
||||
capture.startupUnavailableResponses += 1;
|
||||
}
|
||||
const payload = frame.payload as Record<string, unknown> | undefined;
|
||||
if (payload?.type === "hello-ok" && typeof payload.protocol === "number") {
|
||||
capture.helloProtocols.push(payload.protocol);
|
||||
}
|
||||
}
|
||||
if (downstream.readyState === WebSocket.OPEN) {
|
||||
downstream.send(data);
|
||||
}
|
||||
});
|
||||
const closeDownstream = () => {
|
||||
if (downstream.readyState === WebSocket.OPEN) {
|
||||
downstream.close(1013, "gateway unavailable");
|
||||
}
|
||||
};
|
||||
upstream.on("error", closeDownstream);
|
||||
upstream.on("close", (code, reason) => {
|
||||
if (downstream.readyState === WebSocket.OPEN) {
|
||||
downstream.close(code, reason.toString());
|
||||
}
|
||||
});
|
||||
downstream.on("close", () => upstream.close());
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => resolve());
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("gateway frame proxy did not bind a TCP port");
|
||||
}
|
||||
return {
|
||||
capture,
|
||||
url: `ws://127.0.0.1:${address.port}`,
|
||||
async stop() {
|
||||
for (const client of wss.clients) {
|
||||
client.terminate();
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
wss.close(() => resolve());
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function connectChannelMcpClient(params: {
|
||||
gatewayUrl: string;
|
||||
gatewayToken: string;
|
||||
repoRoot: string;
|
||||
}): Promise<McpClientHandle> {
|
||||
const tempState = createMcpClientTempState({ gatewayToken: params.gatewayToken });
|
||||
const mcpInvocation = resolveChannelMcpInvocation({
|
||||
gatewayToken: params.gatewayToken,
|
||||
gatewayUrl: params.gatewayUrl,
|
||||
repoRoot: params.repoRoot,
|
||||
tokenFile: tempState.tokenFile,
|
||||
});
|
||||
const stderrChunks: Buffer[] = [];
|
||||
const transport = new StdioClientTransport({
|
||||
command: mcpInvocation.command,
|
||||
args: mcpInvocation.args,
|
||||
cwd: mcpInvocation.cwd,
|
||||
env: {
|
||||
...process.env,
|
||||
...mcpInvocation.envPatch,
|
||||
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1",
|
||||
OPENCLAW_LOG_LEVEL: "debug",
|
||||
OPENCLAW_STATE_DIR: tempState.stateDir,
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
|
||||
const client = new Client({ name: "qa-gateway-mcp-client", version: "1.0.0" });
|
||||
let connectTimeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
client.connect(transport),
|
||||
new Promise<never>((_, reject) => {
|
||||
connectTimeout = setTimeout(
|
||||
() => reject(new Error("MCP channel client connect timed out")),
|
||||
MCP_CONNECT_TIMEOUT_MS,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
return {
|
||||
client,
|
||||
cleanup: tempState.cleanup,
|
||||
transport,
|
||||
stderr: () => Buffer.concat(stderrChunks).toString("utf8"),
|
||||
};
|
||||
} catch (error) {
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
tempState.cleanup();
|
||||
throw error;
|
||||
} finally {
|
||||
if (connectTimeout) {
|
||||
clearTimeout(connectTimeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function closeMcpClient(handle: McpClientHandle | undefined) {
|
||||
if (!handle) {
|
||||
return;
|
||||
}
|
||||
await Promise.allSettled([handle.client.close(), handle.transport.close()]);
|
||||
handle.cleanup();
|
||||
}
|
||||
|
||||
async function approvePendingMcpPairing(gateway: Awaited<ReturnType<typeof startQaGatewayChild>>) {
|
||||
const pairing = (await gateway.call("device.pair.list", {})) as {
|
||||
pending?: Array<{ requestId?: string; role?: string }>;
|
||||
};
|
||||
const pending = pairing.pending?.find((entry) => entry.role === "operator");
|
||||
if (!pending?.requestId) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await gateway.call("device.pair.approve", { requestId: pending.requestId });
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (formatErrorMessage(error).includes("unknown requestId")) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function runGatewaySmokeProof(options: ProducerOptions): Promise<string> {
|
||||
const gateway = await startQaGatewayChild({
|
||||
repoRoot: options.repoRoot,
|
||||
command: resolveOpenClawCliInvocation(options.repoRoot).gatewayCommand,
|
||||
transport: emptyTransport(),
|
||||
transportBaseUrl: "http://127.0.0.1",
|
||||
controlUiEnabled: false,
|
||||
});
|
||||
const tempRoot = gateway.tempRoot;
|
||||
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1";
|
||||
let details = "";
|
||||
try {
|
||||
const stdout: string[] = [];
|
||||
const stderr: string[] = [];
|
||||
const exitCode = await runGatewaySmoke(
|
||||
{ token: gateway.token, urlRaw: gateway.wsUrl },
|
||||
{
|
||||
stdout: (message) => stdout.push(message),
|
||||
stderr: (message) => stderr.push(message),
|
||||
},
|
||||
);
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(`gateway smoke exited ${exitCode}: ${stderr.join("\n")}`);
|
||||
}
|
||||
const health = (await gateway.call("health", {})) as { ok?: boolean };
|
||||
if (health.ok !== true) {
|
||||
throw new Error(`gateway health RPC returned ${JSON.stringify(health)}`);
|
||||
}
|
||||
details = `real Gateway pid=${gateway.pid ?? "unknown"}; ${stdout.join("; ")}; health.ok=true`;
|
||||
} finally {
|
||||
await gateway.stop();
|
||||
}
|
||||
if (!keepTemp && existsSync(tempRoot)) {
|
||||
throw new Error(`Gateway temp root was not cleaned up: ${tempRoot}`);
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
async function runMcpGatewayStartupRetryProof(options: ProducerOptions): Promise<string> {
|
||||
const fixture = await createFixturePlugin();
|
||||
let proxy: GatewayProxy | undefined;
|
||||
let mcp: McpClientHandle | undefined;
|
||||
let gateway: Awaited<ReturnType<typeof startQaGatewayChild>> | undefined;
|
||||
let beforeSpawnAt = 0;
|
||||
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1";
|
||||
let details = "";
|
||||
let proofError: Error | undefined;
|
||||
try {
|
||||
const onListening = async (context: QaGatewayChildListeningContext) => {
|
||||
await closeMcpClient(mcp);
|
||||
await proxy?.stop();
|
||||
proxy = await startGatewayProxy(context.wsUrl, () => {
|
||||
void fs.writeFile(fixture.startupGatePath, "observed\n", "utf8");
|
||||
});
|
||||
beforeSpawnAt = Date.now();
|
||||
mcp = await connectChannelMcpClient({
|
||||
gatewayUrl: proxy.url,
|
||||
gatewayToken: context.token,
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
};
|
||||
gateway = await startQaGatewayChild({
|
||||
repoRoot: options.repoRoot,
|
||||
command: resolveOpenClawCliInvocation(options.repoRoot).gatewayCommand,
|
||||
transport: emptyTransport(),
|
||||
transportBaseUrl: "http://127.0.0.1",
|
||||
controlUiEnabled: false,
|
||||
onListening,
|
||||
mutateConfig: (config) => withFixturePlugin(config, fixture.pluginDir),
|
||||
});
|
||||
if (!proxy || !mcp) {
|
||||
throw new Error("MCP client was not started by the Gateway before-spawn hook");
|
||||
}
|
||||
const gatewayReadyAt = Date.now();
|
||||
if (beforeSpawnAt >= gatewayReadyAt) {
|
||||
throw new Error("MCP client did not start before Gateway readiness");
|
||||
}
|
||||
if (await approvePendingMcpPairing(gateway)) {
|
||||
await closeMcpClient(mcp);
|
||||
mcp = await connectChannelMcpClient({
|
||||
gatewayUrl: proxy.url,
|
||||
gatewayToken: gateway.token,
|
||||
repoRoot: options.repoRoot,
|
||||
});
|
||||
}
|
||||
const tools = await mcp.client.listTools();
|
||||
if (!tools.tools.some((tool) => tool.name === "conversations_list")) {
|
||||
throw new Error("real MCP channel server did not expose conversations_list");
|
||||
}
|
||||
const conversations = await mcp.client.callTool({
|
||||
name: "conversations_list",
|
||||
arguments: { limit: 1 },
|
||||
});
|
||||
if (conversations.isError) {
|
||||
throw new Error(`conversations_list failed: ${JSON.stringify(conversations.content)}`);
|
||||
}
|
||||
const capture = proxy.capture;
|
||||
if (capture.startupUnavailableResponses < 1) {
|
||||
throw new Error(
|
||||
`expected a retryable startup-unavailable response; captured=${JSON.stringify(capture)}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
!capture.connectFrames.some(
|
||||
(frame) =>
|
||||
frame.minProtocol === MIN_CLIENT_PROTOCOL_VERSION &&
|
||||
frame.maxProtocol === PROTOCOL_VERSION,
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`MCP Gateway connect frame used unexpected protocol range: ${JSON.stringify(capture)}`,
|
||||
);
|
||||
}
|
||||
if (!capture.helloProtocols.includes(PROTOCOL_VERSION)) {
|
||||
throw new Error(`MCP Gateway negotiation did not select protocol ${PROTOCOL_VERSION}`);
|
||||
}
|
||||
details = [
|
||||
`MCP started ${gatewayReadyAt - beforeSpawnAt}ms before Gateway readiness`,
|
||||
`startup retries=${capture.startupUnavailableResponses}`,
|
||||
`connect frames=${capture.connectFrames.length}`,
|
||||
`negotiated protocol=${PROTOCOL_VERSION}`,
|
||||
].join("; ");
|
||||
} catch (error) {
|
||||
const diagnostics = [
|
||||
mcp?.stderr(),
|
||||
proxy ? `captured Gateway frames: ${JSON.stringify(proxy.capture)}` : undefined,
|
||||
gateway?.logs(),
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join("\n");
|
||||
proofError = new Error(`${formatErrorMessage(error)}${diagnostics ? `\n${diagnostics}` : ""}`, {
|
||||
cause: error,
|
||||
});
|
||||
} finally {
|
||||
await closeMcpClient(mcp);
|
||||
await proxy?.stop().catch(() => undefined);
|
||||
const tempRoot = gateway?.tempRoot;
|
||||
await gateway?.stop().catch(() => undefined);
|
||||
await fixture.cleanup();
|
||||
if (!keepTemp && tempRoot && existsSync(tempRoot) && !proofError) {
|
||||
proofError = new Error(`Gateway temp root was not cleaned up: ${tempRoot}`);
|
||||
}
|
||||
}
|
||||
if (proofError) {
|
||||
throw proofError;
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
async function writePluginToolsConfig(root: string, pluginDir: string) {
|
||||
const configPath = path.join(root, "openclaw.json");
|
||||
const config = withFixturePlugin({} as OpenClawConfig, pluginDir);
|
||||
await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
|
||||
return configPath;
|
||||
}
|
||||
|
||||
async function runMcpPluginToolsProof(options: ProducerOptions): Promise<string> {
|
||||
const fixture = await createFixturePlugin();
|
||||
// openclaw-temp-dir: allow standalone producer cleans and verifies this root in its finally block
|
||||
const runtimeRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-plugin-tools-mcp-"));
|
||||
const stateDir = path.join(runtimeRoot, "state");
|
||||
const homeDir = path.join(runtimeRoot, "home");
|
||||
await Promise.all([
|
||||
fs.mkdir(stateDir, { recursive: true }),
|
||||
fs.mkdir(homeDir, { recursive: true }),
|
||||
]);
|
||||
const configPath = await writePluginToolsConfig(runtimeRoot, fixture.pluginDir);
|
||||
const stderrChunks: Buffer[] = [];
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [
|
||||
"--import",
|
||||
"tsx",
|
||||
"--eval",
|
||||
`import(${JSON.stringify(pathToFileURL(path.join(options.repoRoot, "src/mcp/plugin-tools-serve.ts")).href)}).then((module) => module.servePluginToolsMcp())`,
|
||||
],
|
||||
cwd: options.repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: homeDir,
|
||||
OPENCLAW_CONFIG_PATH: configPath,
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk)));
|
||||
const client = new Client({ name: "qa-plugin-tools-client", version: "1.0.0" });
|
||||
let details = "";
|
||||
let proofError: Error | undefined;
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const listed = await client.listTools();
|
||||
if (!listed.tools.some((tool) => tool.name === FIXTURE_TOOL_NAME)) {
|
||||
throw new Error(
|
||||
`fixture plugin tool was not listed: ${listed.tools.map((tool) => tool.name).join(", ")}`,
|
||||
);
|
||||
}
|
||||
const result = await client.callTool({
|
||||
name: FIXTURE_TOOL_NAME,
|
||||
arguments: { query: "ORBIT-9 codename", maxResults: 3 },
|
||||
});
|
||||
if (result.isError || !JSON.stringify(result.content).includes(FIXTURE_FACT)) {
|
||||
throw new Error(`fixture plugin tool returned unexpected payload: ${JSON.stringify(result)}`);
|
||||
}
|
||||
details = `real plugin-tools pid=${transport.pid ?? "unknown"}; listed and called ${FIXTURE_TOOL_NAME}; received ORBIT-9`;
|
||||
} catch (error) {
|
||||
const stderr = Buffer.concat(stderrChunks).toString("utf8");
|
||||
proofError = new Error(
|
||||
`${formatErrorMessage(error)}${stderr ? `\nplugin-tools stderr:\n${stderr}` : ""}`,
|
||||
{
|
||||
cause: error,
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
await Promise.all([fixture.cleanup(), fs.rm(runtimeRoot, { force: true, recursive: true })]);
|
||||
if (existsSync(runtimeRoot) && !proofError) {
|
||||
proofError = new Error(`plugin-tools runtime root was not cleaned up: ${runtimeRoot}`);
|
||||
}
|
||||
}
|
||||
if (proofError) {
|
||||
throw proofError;
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
async function produceProof(options: ProducerOptions): Promise<ProofResult> {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const details =
|
||||
options.scenarioId === "gateway-smoke"
|
||||
? await runGatewaySmokeProof(options)
|
||||
: options.scenarioId === "mcp-gateway-connect-startup-retry"
|
||||
? await runMcpGatewayStartupRetryProof(options)
|
||||
: await runMcpPluginToolsProof(options);
|
||||
return { details, durationMs: Math.max(1, Date.now() - startedAt), status: "pass" };
|
||||
} catch (error) {
|
||||
return {
|
||||
details: formatErrorMessage(error),
|
||||
durationMs: Math.max(1, Date.now() - startedAt),
|
||||
status: "fail",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function runGatewayMcpRealTransportProducer(
|
||||
options: ProducerOptions,
|
||||
): Promise<QaEvidenceSummaryJson> {
|
||||
const scenario = SCENARIOS[options.scenarioId];
|
||||
const writer = createQaScriptEvidenceWriter({
|
||||
artifactBase: options.artifactBase,
|
||||
logFileName: `${options.scenarioId}.log`,
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: options.repoRoot,
|
||||
target: {
|
||||
id: options.scenarioId,
|
||||
title: scenario.title,
|
||||
sourcePath: scenario.sourcePath,
|
||||
primaryCoverageIds: scenario.primaryCoverageIds,
|
||||
docsRefs: scenario.docsRefs,
|
||||
codeRefs: scenario.codeRefs,
|
||||
},
|
||||
});
|
||||
const result = await produceProof(options);
|
||||
writer.appendLog(`${result.status}: ${result.details ?? "no details"}\n`);
|
||||
return await writer.write(result);
|
||||
}
|
||||
|
||||
async function main(argv: readonly string[]) {
|
||||
const options = parseOptions(argv);
|
||||
const evidence = await runGatewayMcpRealTransportProducer(options);
|
||||
const status = evidence.entries[0]?.result.status;
|
||||
console.log(`Gateway/MCP real transport evidence: ${QA_EVIDENCE_FILENAME}`);
|
||||
console.log(`Gateway/MCP real transport status: ${status}`);
|
||||
return status === "pass" ? 0 : 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
main(process.argv.slice(2))
|
||||
.then((exitCode) => {
|
||||
process.exit(exitCode);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
console.error(formatErrorMessage(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
resolveChannelMcpInvocation,
|
||||
resolveOpenClawCliInvocation,
|
||||
};
|
||||
424
test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
Normal file
424
test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
Normal file
@@ -0,0 +1,424 @@
|
||||
// Gateway Smoke tests cover QA Lab gateway smoke evidence.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { WebSocket, WebSocketServer } from "ws";
|
||||
import { runGatewaySmoke } from "../../../../scripts/dev/gateway-smoke.js";
|
||||
|
||||
let server: Server | undefined;
|
||||
let wss: WebSocketServer | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
await new Promise<void>((resolve) => {
|
||||
wss?.close(() => resolve());
|
||||
if (!wss) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
wss = undefined;
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server?.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
if (!server) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
server = undefined;
|
||||
});
|
||||
|
||||
describe("gateway-smoke", () => {
|
||||
function healthResponse() {
|
||||
return {
|
||||
ok: true,
|
||||
payload: {
|
||||
agents: [],
|
||||
channelOrder: [],
|
||||
channels: {},
|
||||
defaultAgentId: "codex",
|
||||
durationMs: 3,
|
||||
ok: true,
|
||||
sessions: { count: 0, path: "/state/sessions", recent: [] },
|
||||
ts: Date.now(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function connectHelloResponse(scopes: string[] = []) {
|
||||
return {
|
||||
ok: true,
|
||||
payload: {
|
||||
auth: { role: "operator", scopes },
|
||||
features: { events: [], methods: ["health"] },
|
||||
policy: {
|
||||
maxBufferedBytes: 1024 * 1024,
|
||||
maxPayload: 256 * 1024,
|
||||
tickIntervalMs: 1000,
|
||||
},
|
||||
protocol: 1,
|
||||
server: { connId: "test-conn", version: "dev" },
|
||||
snapshot: {},
|
||||
type: "hello-ok",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function listenGatewaySmokeServer() {
|
||||
const requests: Array<{ method: string; params?: unknown; timeout?: number }> = [];
|
||||
server = createServer();
|
||||
wss = new WebSocketServer({ server });
|
||||
wss.on("connection", (ws: WebSocket) => {
|
||||
ws.on("message", (data) => {
|
||||
const frame = JSON.parse(data.toString()) as {
|
||||
id: string;
|
||||
method: string;
|
||||
params?: unknown;
|
||||
type: string;
|
||||
};
|
||||
requests.push({ method: frame.method, params: frame.params });
|
||||
if (frame.method === "connect") {
|
||||
ws.send(JSON.stringify({ id: frame.id, type: "res", ...connectHelloResponse() }));
|
||||
return;
|
||||
}
|
||||
if (frame.method === "health") {
|
||||
ws.send(JSON.stringify({ id: frame.id, type: "res", ...healthResponse() }));
|
||||
return;
|
||||
}
|
||||
if (frame.method === "chat.history") {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
error: "missing scope: operator.read",
|
||||
id: frame.id,
|
||||
ok: false,
|
||||
type: "res",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
error: `unexpected method ${frame.method}`,
|
||||
id: frame.id,
|
||||
ok: false,
|
||||
type: "res",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server?.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test gateway smoke server did not get a TCP address");
|
||||
}
|
||||
return { requests, url: `ws://127.0.0.1:${address.port}` };
|
||||
}
|
||||
|
||||
function createSmokeDeps(
|
||||
responses: Record<string, { error?: string; ok: boolean } & Record<string, unknown>>,
|
||||
calls: Array<{ method: string; timeout?: number }> = [],
|
||||
) {
|
||||
const stdout: string[] = [];
|
||||
const stderr: string[] = [];
|
||||
let closed = 0;
|
||||
|
||||
return {
|
||||
calls,
|
||||
get closed() {
|
||||
return closed;
|
||||
},
|
||||
stderr,
|
||||
stdout,
|
||||
deps: {
|
||||
createClient: () =>
|
||||
({
|
||||
close: () => {
|
||||
closed += 1;
|
||||
},
|
||||
request: async (method: string, _params?: unknown, timeout?: number) => {
|
||||
calls.push({ method, timeout });
|
||||
const response = responses[method];
|
||||
return {
|
||||
id: method,
|
||||
...response,
|
||||
ok: response?.ok ?? false,
|
||||
error: response?.error,
|
||||
type: "res",
|
||||
};
|
||||
},
|
||||
waitOpen: async () => {},
|
||||
}) as never,
|
||||
stderr: (message: string) => {
|
||||
stderr.push(message);
|
||||
},
|
||||
stdout: (message: string) => {
|
||||
stdout.push(message);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("prints CLI help without connecting", () => {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/gateway-smoke.ts", "--help"],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain("Usage: bun scripts/dev/gateway-smoke.ts");
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("rejects unknown CLI args before connecting", () => {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"scripts/dev/gateway-smoke.ts",
|
||||
"--url",
|
||||
"ws://127.0.0.1:9",
|
||||
"--token",
|
||||
"token",
|
||||
"--wat",
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe("Unknown argument: --wat");
|
||||
});
|
||||
|
||||
it("rejects option-looking CLI values before connecting", () => {
|
||||
for (const [flag, args] of [
|
||||
["--url", ["--url", "-h", "--token", "token"]],
|
||||
["--token", ["--url", "ws://127.0.0.1:9", "--token", "-h"]],
|
||||
] as const) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/gateway-smoke.ts", ...args],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe(`${flag} requires a value`);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects duplicate CLI args before connecting", () => {
|
||||
for (const [flag, args] of [
|
||||
[
|
||||
"--url",
|
||||
["--url", "ws://127.0.0.1:9", "--url", "ws://127.0.0.1:10", "--token", "token"],
|
||||
],
|
||||
[
|
||||
"--token",
|
||||
["--url", "ws://127.0.0.1:9", "--token", "one", "--token", "two"],
|
||||
],
|
||||
["--help", ["--help", "--help"]],
|
||||
] as const) {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/gateway-smoke.ts", ...args],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe("");
|
||||
expect(result.stderr.trim()).toBe(`${flag} was provided more than once`);
|
||||
}
|
||||
});
|
||||
|
||||
it("passes against a loopback gateway websocket using the real client", async () => {
|
||||
const stdout: string[] = [];
|
||||
const stderr: string[] = [];
|
||||
const loopback = await listenGatewaySmokeServer();
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: loopback.url },
|
||||
{
|
||||
stderr: (message) => {
|
||||
stderr.push(message);
|
||||
},
|
||||
stdout: (message) => {
|
||||
stdout.push(message);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(code).toBe(0);
|
||||
expect(loopback.requests.map((request) => request.method)).toEqual(["connect", "health"]);
|
||||
expect(loopback.requests[0]?.params).toMatchObject({
|
||||
auth: { token: "secret-token" },
|
||||
client: { id: "openclaw-ios" },
|
||||
role: "operator",
|
||||
scopes: ["operator.read", "operator.write", "operator.admin"],
|
||||
});
|
||||
expect(stdout).toEqual(["ok: connected + health"]);
|
||||
expect(stderr).toEqual([]);
|
||||
});
|
||||
|
||||
it("closes the websocket client when connect fails", async () => {
|
||||
const stderr: string[] = [];
|
||||
const methods: string[] = [];
|
||||
let closed = 0;
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
{
|
||||
createClient: () =>
|
||||
({
|
||||
close: () => {
|
||||
closed += 1;
|
||||
},
|
||||
request: async (method: string) => {
|
||||
methods.push(method);
|
||||
return { error: "bad token", id: "connect", ok: false, type: "res" };
|
||||
},
|
||||
waitOpen: async () => {},
|
||||
}) as never,
|
||||
stderr: (message) => {
|
||||
stderr.push(message);
|
||||
},
|
||||
stdout: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
expect(code).toBe(2);
|
||||
expect(closed).toBe(1);
|
||||
expect(methods).toEqual(["connect"]);
|
||||
expect(stderr).toEqual(["connect failed: bad token"]);
|
||||
});
|
||||
|
||||
it("requires connect and health in order", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: healthResponse(),
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(0);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([
|
||||
{ method: "connect", timeout: undefined },
|
||||
{ method: "health", timeout: undefined },
|
||||
]);
|
||||
expect(fake.stdout).toEqual(["ok: connected + health"]);
|
||||
expect(fake.stderr).toEqual([]);
|
||||
});
|
||||
|
||||
it("fails when connect success is missing hello evidence", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: { ok: true },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(2);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([{ method: "connect", timeout: undefined }]);
|
||||
expect(fake.stdout).toEqual([]);
|
||||
expect(fake.stderr).toEqual(["connect failed: missing hello-ok payload"]);
|
||||
});
|
||||
|
||||
it("fails when the unpaired iOS-shaped connect keeps operator scopes", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(["operator.read"]),
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(2);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([{ method: "connect", timeout: undefined }]);
|
||||
expect(fake.stderr).toEqual([
|
||||
"connect failed: unpaired iOS smoke unexpectedly received operator scopes",
|
||||
]);
|
||||
});
|
||||
|
||||
it("fails after connect when health is unavailable", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: { ok: false, error: "not healthy" },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(3);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls.map((call) => call.method)).toEqual(["connect", "health"]);
|
||||
expect(fake.stderr).toEqual(["health failed: not healthy"]);
|
||||
});
|
||||
|
||||
it("fails when health success is missing summary evidence", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: { ok: true },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(3);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls.map((call) => call.method)).toEqual(["connect", "health"]);
|
||||
expect(fake.stderr).toEqual(["health failed: missing health summary payload"]);
|
||||
});
|
||||
|
||||
it("does not call scoped chat history for an unpaired iOS-shaped client", async () => {
|
||||
const fake = createSmokeDeps({
|
||||
connect: connectHelloResponse(),
|
||||
health: healthResponse(),
|
||||
"chat.history": { ok: false, error: "session store unavailable" },
|
||||
});
|
||||
|
||||
const code = await runGatewaySmoke(
|
||||
{ token: "secret-token", urlRaw: "ws://127.0.0.1:12345" },
|
||||
fake.deps,
|
||||
);
|
||||
|
||||
expect(code).toBe(0);
|
||||
expect(fake.closed).toBe(1);
|
||||
expect(fake.calls).toEqual([
|
||||
{ method: "connect", timeout: undefined },
|
||||
{ method: "health", timeout: undefined },
|
||||
]);
|
||||
expect(fake.stderr).toEqual([]);
|
||||
});
|
||||
});
|
||||
464
test/e2e/qa-lab/runtime/mcp-channels-docker-client.ts
Normal file
464
test/e2e/qa-lab/runtime/mcp-channels-docker-client.ts
Normal file
@@ -0,0 +1,464 @@
|
||||
// MCP channels Docker client drives the QA-owned channel bridge smoke.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import {
|
||||
assert,
|
||||
ClaudeChannelNotificationSchema,
|
||||
ClaudePermissionNotificationSchema,
|
||||
connectGateway,
|
||||
connectMcpClient,
|
||||
extractTextFromGatewayPayload,
|
||||
type ClaudeChannelNotification,
|
||||
type GatewayRpcClient,
|
||||
maybeApprovePendingBridgePairing,
|
||||
waitFor,
|
||||
} from "./mcp-channels.fixture.ts";
|
||||
import {
|
||||
connectMcpClientWithPairingReconnect,
|
||||
createMcpClientTempState,
|
||||
} from "./mcp-client-temp-state.fixture.ts";
|
||||
|
||||
function summarizeSessionRows(rows: Array<Record<string, unknown>> | undefined) {
|
||||
return (rows ?? []).map((entry) => ({
|
||||
key: entry.key,
|
||||
channel: entry.channel,
|
||||
deliveryContext: entry.deliveryContext,
|
||||
lastChannel: entry.lastChannel,
|
||||
lastTo: entry.lastTo,
|
||||
lastAccountId: entry.lastAccountId,
|
||||
lastThreadId: entry.lastThreadId,
|
||||
}));
|
||||
}
|
||||
|
||||
function findEventByText(events: Array<Record<string, unknown>> | undefined, text: string) {
|
||||
return (events ?? []).find((entry) => entry.text === text);
|
||||
}
|
||||
|
||||
const NON_OWNER_PERMISSION_QUIET_WINDOW_MS = 1_000;
|
||||
|
||||
function formatUnknownError(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
if (error === undefined || error === null) {
|
||||
return "";
|
||||
}
|
||||
if (typeof error === "string") {
|
||||
return error;
|
||||
}
|
||||
if (typeof error === "number" || typeof error === "boolean" || typeof error === "bigint") {
|
||||
return `${error}`;
|
||||
}
|
||||
if (typeof error === "symbol") {
|
||||
return error.description ?? "symbol";
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(error) ?? "";
|
||||
} catch {
|
||||
return Object.prototype.toString.call(error);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForGatewaySeededConversation(gateway: GatewayRpcClient) {
|
||||
let lastList: { sessions?: Array<Record<string, unknown>> } | undefined;
|
||||
let lastError: unknown;
|
||||
try {
|
||||
return await waitFor(
|
||||
"seeded conversation in gateway sessions.list",
|
||||
async () => {
|
||||
try {
|
||||
lastList = await gateway.request<{ sessions?: Array<Record<string, unknown>> }>(
|
||||
"sessions.list",
|
||||
{ limit: 50, includeDerivedTitles: false, includeLastMessage: false },
|
||||
);
|
||||
lastError = undefined;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
return undefined;
|
||||
}
|
||||
return lastList.sessions?.find((entry) => entry.key === "agent:main:main");
|
||||
},
|
||||
180_000,
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`gateway sessions.list did not include seeded conversation: ${JSON.stringify(
|
||||
{
|
||||
count: lastList?.sessions?.length ?? 0,
|
||||
sessions: summarizeSessionRows(lastList?.sessions),
|
||||
lastError: formatUnknownError(lastError),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const gatewayUrl = process.env.GW_URL?.trim();
|
||||
const gatewayToken = process.env.GW_TOKEN?.trim();
|
||||
assert(gatewayUrl, "missing GW_URL");
|
||||
assert(gatewayToken, "missing GW_TOKEN");
|
||||
|
||||
const gateway = await connectGateway({ url: gatewayUrl, token: gatewayToken });
|
||||
let nonOwnerGateway: GatewayRpcClient | undefined;
|
||||
let mcpHandle: Awaited<ReturnType<typeof connectMcpClient>> | undefined;
|
||||
const mcpTempState = createMcpClientTempState({ gatewayToken });
|
||||
|
||||
try {
|
||||
const gatewayConversation = await waitForGatewaySeededConversation(gateway);
|
||||
assert(
|
||||
(gatewayConversation.deliveryContext as { channel?: unknown } | undefined)?.channel ===
|
||||
"imessage",
|
||||
"expected seeded gateway deliveryContext channel",
|
||||
);
|
||||
assert(
|
||||
(gatewayConversation.deliveryContext as { to?: unknown } | undefined)?.to === "+15551234567",
|
||||
"expected seeded gateway deliveryContext target",
|
||||
);
|
||||
|
||||
mcpHandle = await connectMcpClientWithPairingReconnect({
|
||||
tempState: mcpTempState,
|
||||
connect: (tempState) =>
|
||||
connectMcpClient({
|
||||
gatewayUrl,
|
||||
gatewayToken,
|
||||
tempState,
|
||||
}),
|
||||
maybeApprovePairing: () => maybeApprovePendingBridgePairing(gateway),
|
||||
});
|
||||
const mcp = mcpHandle.client;
|
||||
const callTool = <T>(params: Parameters<typeof mcp.callTool>[0]) =>
|
||||
mcp.callTool(params, undefined, { timeout: 240_000 }) as Promise<T>;
|
||||
|
||||
let lastMcpConversationList: unknown;
|
||||
const conversation = await waitFor(
|
||||
"seeded conversation in conversations_list",
|
||||
async () => {
|
||||
const listed = await callTool<{
|
||||
structuredContent?: { conversations?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "conversations_list",
|
||||
arguments: {
|
||||
includeDerivedTitles: false,
|
||||
includeLastMessage: false,
|
||||
},
|
||||
});
|
||||
lastMcpConversationList = listed;
|
||||
return listed.structuredContent?.conversations?.find(
|
||||
(entry) => entry.sessionKey === "agent:main:main",
|
||||
);
|
||||
},
|
||||
240_000,
|
||||
).catch((error: unknown) => {
|
||||
throw new Error(
|
||||
`timeout waiting for seeded MCP conversation: ${JSON.stringify(
|
||||
lastMcpConversationList,
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
});
|
||||
assert(conversation.channel === "imessage", "expected seeded channel");
|
||||
assert(conversation.to === "+15551234567", "expected seeded target");
|
||||
|
||||
const fetched = await callTool<{
|
||||
structuredContent?: { conversation?: Record<string, unknown> };
|
||||
isError?: boolean;
|
||||
}>({
|
||||
name: "conversation_get",
|
||||
arguments: { session_key: "agent:main:main" },
|
||||
});
|
||||
assert(!fetched.isError, "conversation_get should succeed");
|
||||
assert(
|
||||
fetched.structuredContent?.conversation?.sessionKey === "agent:main:main",
|
||||
"conversation_get returned wrong session",
|
||||
);
|
||||
|
||||
let lastHistory: unknown;
|
||||
const messages = await waitFor(
|
||||
"seeded transcript messages",
|
||||
async () => {
|
||||
const history = await callTool<{
|
||||
structuredContent?: { messages?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "messages_read",
|
||||
arguments: { session_key: "agent:main:main", limit: 10 },
|
||||
});
|
||||
lastHistory = history;
|
||||
const currentMessages = history.structuredContent?.messages ?? [];
|
||||
return currentMessages.length >= 2 ? currentMessages : undefined;
|
||||
},
|
||||
240_000,
|
||||
).catch((error: unknown) => {
|
||||
throw new Error(
|
||||
`timeout waiting for seeded transcript messages: ${JSON.stringify(lastHistory, null, 2)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
});
|
||||
await waitFor(
|
||||
"seeded attachment message",
|
||||
() =>
|
||||
messages.find((entry) => {
|
||||
const raw = entry["__openclaw"];
|
||||
return (
|
||||
raw && typeof raw === "object" && (raw as { id?: unknown }).id === "msg-attachment"
|
||||
);
|
||||
}),
|
||||
240_000,
|
||||
);
|
||||
|
||||
const attachments = await callTool<{
|
||||
structuredContent?: { attachments?: Array<Record<string, unknown>> };
|
||||
isError?: boolean;
|
||||
}>({
|
||||
name: "attachments_fetch",
|
||||
arguments: { session_key: "agent:main:main", message_id: "msg-attachment" },
|
||||
});
|
||||
assert(!attachments.isError, "attachments_fetch should succeed");
|
||||
assert(
|
||||
(attachments.structuredContent?.attachments?.length ?? 0) === 1,
|
||||
"expected one seeded attachment",
|
||||
);
|
||||
|
||||
const waited = (await Promise.all([
|
||||
callTool<{
|
||||
structuredContent?: { event?: Record<string, unknown> };
|
||||
}>({
|
||||
name: "events_wait",
|
||||
arguments: {
|
||||
session_key: "agent:main:main",
|
||||
after_cursor: 0,
|
||||
timeout_ms: 10_000,
|
||||
},
|
||||
}),
|
||||
gateway.request("chat.inject", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: "assistant live event",
|
||||
}),
|
||||
]).then(([result]) => result)) as {
|
||||
structuredContent?: { event?: Record<string, unknown> };
|
||||
};
|
||||
const assistantEvent = waited.structuredContent?.event;
|
||||
assert(assistantEvent, "expected events_wait result");
|
||||
assert(assistantEvent.type === "message", "expected message event");
|
||||
assert(assistantEvent.role === "assistant", "expected assistant event role");
|
||||
assert(assistantEvent.text === "assistant live event", "expected assistant event text");
|
||||
const assistantCursor = typeof assistantEvent.cursor === "number" ? assistantEvent.cursor : 0;
|
||||
|
||||
const polled = await callTool<{
|
||||
structuredContent?: { events?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "events_poll",
|
||||
arguments: { session_key: "agent:main:main", after_cursor: 0, limit: 10 },
|
||||
});
|
||||
assert(
|
||||
(polled.structuredContent?.events ?? []).some(
|
||||
(entry) => entry.text === "assistant live event",
|
||||
),
|
||||
"expected assistant event in events_poll",
|
||||
);
|
||||
|
||||
const channelMessage = `hello from docker ${randomUUID()}`;
|
||||
await gateway.request("chat.send", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: channelMessage,
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
const rawGatewayUserMessage = await waitFor(
|
||||
"raw gateway user session.message",
|
||||
() =>
|
||||
gateway.events.find(
|
||||
(entry) =>
|
||||
entry.event === "session.message" &&
|
||||
entry.payload.sessionKey === "agent:main:main" &&
|
||||
extractTextFromGatewayPayload(entry.payload) === channelMessage,
|
||||
),
|
||||
10_000,
|
||||
).catch(() => undefined);
|
||||
let userEvent = await waitFor(
|
||||
"MCP user session.message event",
|
||||
async () => {
|
||||
const polledValue = await callTool<{
|
||||
structuredContent?: { events?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "events_poll",
|
||||
arguments: { session_key: "agent:main:main", after_cursor: assistantCursor, limit: 50 },
|
||||
});
|
||||
return findEventByText(polledValue.structuredContent?.events, channelMessage);
|
||||
},
|
||||
60_000,
|
||||
).catch(() => undefined);
|
||||
let finalPolledEvents: Array<Record<string, unknown>> | undefined;
|
||||
if (userEvent?.text !== channelMessage) {
|
||||
const polledLocal = await callTool<{
|
||||
structuredContent?: { events?: Array<Record<string, unknown>> };
|
||||
}>({
|
||||
name: "events_poll",
|
||||
arguments: { session_key: "agent:main:main", after_cursor: assistantCursor, limit: 50 },
|
||||
});
|
||||
finalPolledEvents = polledLocal.structuredContent?.events ?? [];
|
||||
const finalUserEvent = findEventByText(finalPolledEvents, channelMessage);
|
||||
if (finalUserEvent?.text === channelMessage) {
|
||||
userEvent = finalUserEvent;
|
||||
}
|
||||
}
|
||||
if (userEvent?.text !== channelMessage) {
|
||||
throw new Error(
|
||||
`expected user event after chat.send: ${JSON.stringify(
|
||||
{
|
||||
userEvent: userEvent ?? null,
|
||||
rawGatewayUserMessage: rawGatewayUserMessage ?? null,
|
||||
mcpEventsAfterAssistant: finalPolledEvents ?? [],
|
||||
recentGatewayEvents: gateway.events.slice(-10).map((entry) => ({
|
||||
event: entry.event,
|
||||
sessionKey: entry.payload.sessionKey,
|
||||
text: extractTextFromGatewayPayload(entry.payload),
|
||||
})),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
let helpNotification: ClaudeChannelNotification;
|
||||
try {
|
||||
helpNotification = await waitFor(
|
||||
"Claude channel notification",
|
||||
() =>
|
||||
mcpHandle.rawMessages
|
||||
.map((entry) => ClaudeChannelNotificationSchema.safeParse(entry))
|
||||
.find(
|
||||
(entry) =>
|
||||
entry.success &&
|
||||
entry.data.params.meta.session_key === "agent:main:main" &&
|
||||
entry.data.params.content === channelMessage,
|
||||
)?.data.params,
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`timeout waiting for Claude channel notification: ${JSON.stringify(
|
||||
{
|
||||
rawMessages: mcpHandle.rawMessages.slice(-10),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
assert(helpNotification.content === channelMessage, "expected Claude channel content");
|
||||
|
||||
await mcp.notification({
|
||||
method: "notifications/claude/channel/permission_request",
|
||||
params: {
|
||||
request_id: "abcde",
|
||||
tool_name: "Bash",
|
||||
description: "run npm test",
|
||||
input_preview: '{"cmd":"npm test"}',
|
||||
},
|
||||
});
|
||||
|
||||
nonOwnerGateway = await connectGateway({
|
||||
url: gatewayUrl,
|
||||
token: gatewayToken,
|
||||
scopes: ["operator.read", "operator.write"],
|
||||
});
|
||||
await nonOwnerGateway.request("chat.send", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: "yes abcde",
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
await waitFor(
|
||||
"non-owner reply forwarded as an ordinary Claude channel message",
|
||||
() =>
|
||||
mcpHandle.rawMessages
|
||||
.map((entry) => ClaudeChannelNotificationSchema.safeParse(entry))
|
||||
.find(
|
||||
(entry) =>
|
||||
entry.success &&
|
||||
entry.data.params.meta.session_key === "agent:main:main" &&
|
||||
entry.data.params.content === "yes abcde",
|
||||
)?.data.params,
|
||||
60_000,
|
||||
);
|
||||
await delay(NON_OWNER_PERMISSION_QUIET_WINDOW_MS);
|
||||
const nonOwnerPermission = mcpHandle.rawMessages
|
||||
.map((entry) => ClaudePermissionNotificationSchema.safeParse(entry))
|
||||
.find((entry) => entry.success && entry.data.params.request_id === "abcde");
|
||||
assert(!nonOwnerPermission, "non-owner reply must not resolve the Claude permission");
|
||||
|
||||
const ownerNotificationStart = mcpHandle.rawMessages.length;
|
||||
await gateway.request("chat.send", {
|
||||
sessionKey: "agent:main:main",
|
||||
message: "yes abcde",
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
let permission: { request_id: string; behavior: "allow" | "deny" };
|
||||
try {
|
||||
permission = await waitFor(
|
||||
"Claude permission notification",
|
||||
() =>
|
||||
mcpHandle.rawMessages
|
||||
.slice(ownerNotificationStart)
|
||||
.map((entry) => ClaudePermissionNotificationSchema.safeParse(entry))
|
||||
.find((entry) => entry.success && entry.data.params.request_id === "abcde")?.data
|
||||
.params,
|
||||
60_000,
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`timeout waiting for Claude permission notification: ${JSON.stringify(
|
||||
{
|
||||
rawMessages: mcpHandle.rawMessages.slice(-10),
|
||||
recentGatewayEvents: gateway.events.slice(-10).map((entry) => ({
|
||||
event: entry.event,
|
||||
sessionKey: entry.payload.sessionKey,
|
||||
text: extractTextFromGatewayPayload(entry.payload),
|
||||
})),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
assert(permission.behavior === "allow", "expected allow permission reply");
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify(
|
||||
{
|
||||
ok: true,
|
||||
sessionKey: "agent:main:main",
|
||||
nonOwnerReplyForwarded: true,
|
||||
nonOwnerPermissionBlocked: true,
|
||||
ownerPermissionAllowed: permission.behavior === "allow",
|
||||
rawNotifications: mcpHandle.rawMessages.filter(
|
||||
(entry) =>
|
||||
ClaudeChannelNotificationSchema.safeParse(entry).success ||
|
||||
ClaudePermissionNotificationSchema.safeParse(entry).success,
|
||||
).length,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
);
|
||||
} finally {
|
||||
const closeTasks: Array<Promise<unknown>> = [gateway.close()];
|
||||
if (nonOwnerGateway) {
|
||||
closeTasks.push(nonOwnerGateway.close());
|
||||
}
|
||||
if (mcpHandle) {
|
||||
closeTasks.push(mcpHandle.client.close(), mcpHandle.transport.close());
|
||||
}
|
||||
await Promise.allSettled(closeTasks);
|
||||
mcpHandle?.cleanup();
|
||||
mcpTempState.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
325
test/e2e/qa-lab/runtime/mcp-channels.fixture.ts
Normal file
325
test/e2e/qa-lab/runtime/mcp-channels.fixture.ts
Normal file
@@ -0,0 +1,325 @@
|
||||
// Shared MCP-channel QA/Docker E2E fixture helpers.
|
||||
// The mounted test harness imports packaged dist modules so bridge assertions run
|
||||
// against the OpenClaw npm tarball installed in the functional image.
|
||||
import process from "node:process";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
import { z } from "zod";
|
||||
import { PROTOCOL_VERSION } from "../../../../dist/gateway/protocol/index.js";
|
||||
import { formatErrorMessage } from "../../../../dist/infra/errors.js";
|
||||
import { readStringValue } from "../../../../dist/normalization-core/string-coerce.js";
|
||||
import { resolveGatewaySuccessPayload } from "../../../../scripts/e2e/lib/gateway-frame-payload.mjs";
|
||||
import { readMcpChannelLimits } from "../../../../scripts/e2e/mcp-channel-limits.ts";
|
||||
import {
|
||||
createGatewayWsClient,
|
||||
type GatewayEventFrame,
|
||||
} from "../../../../scripts/lib/gateway-ws-client.ts";
|
||||
import {
|
||||
connectMcpWithTimeout,
|
||||
createMcpClientTempState,
|
||||
type McpClientTempState,
|
||||
} from "./mcp-client-temp-state.fixture.ts";
|
||||
|
||||
export const ClaudeChannelNotificationSchema = z.object({
|
||||
method: z.literal("notifications/claude/channel"),
|
||||
params: z.object({
|
||||
content: z.string(),
|
||||
meta: z.record(z.string(), z.string()),
|
||||
}),
|
||||
});
|
||||
|
||||
export const ClaudePermissionNotificationSchema = z.object({
|
||||
method: z.literal("notifications/claude/channel/permission"),
|
||||
params: z.object({
|
||||
request_id: z.string(),
|
||||
behavior: z.enum(["allow", "deny"]),
|
||||
}),
|
||||
});
|
||||
|
||||
export type ClaudeChannelNotification = z.infer<typeof ClaudeChannelNotificationSchema>["params"];
|
||||
|
||||
export type GatewayRpcClient = {
|
||||
request<T>(method: string, params?: unknown, opts?: { timeoutMs?: number }): Promise<T>;
|
||||
events: Array<{ event: string; payload: Record<string, unknown> }>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export type McpClientHandle = {
|
||||
client: Client;
|
||||
cleanup(): void;
|
||||
transport: StdioClientTransport;
|
||||
rawMessages: unknown[];
|
||||
};
|
||||
|
||||
const GATEWAY_WS_OPEN_TIMEOUT_MS = 45_000;
|
||||
const GATEWAY_RPC_TIMEOUT_MS = 60_000;
|
||||
const GATEWAY_REQUEST_TIMEOUT_MS = 45_000;
|
||||
const GATEWAY_CONNECT_RETRY_WINDOW_MS = 420_000;
|
||||
const MCP_CHANNEL_LIMITS = readMcpChannelLimits();
|
||||
const MCP_CONNECT_TIMEOUT_MS = MCP_CHANNEL_LIMITS.connectTimeoutMs;
|
||||
const GATEWAY_EVENT_RETAIN_LIMIT = MCP_CHANNEL_LIMITS.gatewayEventRetainLimit;
|
||||
const MCP_RAW_MESSAGE_RETAIN_LIMIT = MCP_CHANNEL_LIMITS.rawMessageRetainLimit;
|
||||
|
||||
export function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function pushBounded<T>(items: T[], item: T, limit: number): void {
|
||||
items.push(item);
|
||||
if (items.length > limit) {
|
||||
items.splice(0, items.length - limit);
|
||||
}
|
||||
}
|
||||
|
||||
export function extractTextFromGatewayPayload(
|
||||
payload: Record<string, unknown> | undefined,
|
||||
): string | undefined {
|
||||
const message = payload?.message;
|
||||
if (!message || typeof message !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (typeof content === "string" && content.trim().length > 0) {
|
||||
return content;
|
||||
}
|
||||
if (!Array.isArray(content)) {
|
||||
return undefined;
|
||||
}
|
||||
const first = content[0];
|
||||
if (!first || typeof first !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return readStringValue((first as { text?: unknown }).text);
|
||||
}
|
||||
|
||||
export async function waitFor<T>(
|
||||
label: string,
|
||||
predicate: () => Promise<T | undefined> | T | undefined,
|
||||
timeoutMs = 10_000,
|
||||
): Promise<T> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
const value = await predicate();
|
||||
if (value !== undefined) {
|
||||
return value;
|
||||
}
|
||||
await delay(50);
|
||||
}
|
||||
throw new Error(`timeout waiting for ${label}`);
|
||||
}
|
||||
|
||||
export async function connectGateway(params: {
|
||||
url: string;
|
||||
token: string;
|
||||
scopes?: readonly string[];
|
||||
}): Promise<GatewayRpcClient> {
|
||||
const startedAt = Date.now();
|
||||
let attempt = 0;
|
||||
let lastError: Error | null = null;
|
||||
|
||||
while (Date.now() - startedAt < GATEWAY_CONNECT_RETRY_WINDOW_MS) {
|
||||
attempt += 1;
|
||||
try {
|
||||
return await connectGatewayOnce(params);
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
if (!isRetryableGatewayConnectError(lastError)) {
|
||||
throw lastError;
|
||||
}
|
||||
await delay(Math.min(500 * attempt, 2_000));
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError ?? new Error("gateway ws open timeout");
|
||||
}
|
||||
|
||||
async function connectGatewayOnce(params: {
|
||||
url: string;
|
||||
token: string;
|
||||
scopes?: readonly string[];
|
||||
}): Promise<GatewayRpcClient> {
|
||||
const requestedScopes = params.scopes ?? [
|
||||
"operator.read",
|
||||
"operator.write",
|
||||
"operator.pairing",
|
||||
"operator.admin",
|
||||
];
|
||||
const events: Array<{ event: string; payload: Record<string, unknown> }> = [];
|
||||
const gatewayClient = createGatewayWsClient({
|
||||
handshakeTimeoutMs: GATEWAY_WS_OPEN_TIMEOUT_MS,
|
||||
onEvent(event: GatewayEventFrame) {
|
||||
pushBounded(
|
||||
events,
|
||||
{
|
||||
event: event.event,
|
||||
payload:
|
||||
event.payload && typeof event.payload === "object"
|
||||
? (event.payload as Record<string, unknown>)
|
||||
: {},
|
||||
},
|
||||
GATEWAY_EVENT_RETAIN_LIMIT,
|
||||
);
|
||||
},
|
||||
openTimeoutMs: GATEWAY_WS_OPEN_TIMEOUT_MS,
|
||||
openTimeoutMessage: "gateway ws open timeout",
|
||||
url: params.url,
|
||||
});
|
||||
await gatewayClient.waitOpen();
|
||||
|
||||
const sendGatewayRequest = <T = unknown>(
|
||||
method: string,
|
||||
requestParams: unknown,
|
||||
timeoutMs: number,
|
||||
): Promise<T> => {
|
||||
return gatewayClient.request(method, requestParams ?? {}, timeoutMs).then((response) => {
|
||||
if (response.ok) {
|
||||
return resolveGatewaySuccessPayload(response) as T;
|
||||
}
|
||||
throw new Error(
|
||||
response.error && typeof response.error === "object" && "message" in response.error
|
||||
? String(response.error.message)
|
||||
: "gateway request failed",
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
await sendGatewayRequest(
|
||||
"connect",
|
||||
{
|
||||
minProtocol: PROTOCOL_VERSION,
|
||||
maxProtocol: PROTOCOL_VERSION,
|
||||
client: {
|
||||
id: "openclaw-tui",
|
||||
displayName: "docker-mcp-channels",
|
||||
version: "1.0.0",
|
||||
platform: process.platform,
|
||||
mode: "ui",
|
||||
},
|
||||
role: "operator",
|
||||
scopes: requestedScopes,
|
||||
caps: [],
|
||||
auth: { token: params.token },
|
||||
},
|
||||
GATEWAY_RPC_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
await sendGatewayRequest("sessions.subscribe", {}, GATEWAY_RPC_TIMEOUT_MS);
|
||||
|
||||
return {
|
||||
request(method, requestParams, opts) {
|
||||
return sendGatewayRequest(
|
||||
method,
|
||||
requestParams,
|
||||
opts?.timeoutMs ?? GATEWAY_REQUEST_TIMEOUT_MS,
|
||||
);
|
||||
},
|
||||
events,
|
||||
async close() {
|
||||
gatewayClient.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function isRetryableGatewayConnectError(error: Error): boolean {
|
||||
const message = error.message.toLowerCase();
|
||||
return (
|
||||
message.includes("gateway ws open timeout") ||
|
||||
message.includes("gateway connect timeout") ||
|
||||
message.includes("closed before open") ||
|
||||
message.includes("gateway closed") ||
|
||||
message.includes("gateway websocket closed") ||
|
||||
message.includes("econnrefused") ||
|
||||
message.includes("socket hang up")
|
||||
);
|
||||
}
|
||||
|
||||
export async function connectMcpClient(params: {
|
||||
gatewayUrl: string;
|
||||
gatewayToken: string;
|
||||
tempState?: McpClientTempState;
|
||||
}): Promise<McpClientHandle> {
|
||||
const ownsTempState = !params.tempState;
|
||||
const tempState =
|
||||
params.tempState ?? createMcpClientTempState({ gatewayToken: params.gatewayToken });
|
||||
const transport = new StdioClientTransport({
|
||||
command: "node",
|
||||
args: [
|
||||
"/app/openclaw.mjs",
|
||||
"mcp",
|
||||
"serve",
|
||||
"--url",
|
||||
params.gatewayUrl,
|
||||
"--token-file",
|
||||
tempState.tokenFile,
|
||||
"--claude-channel-mode",
|
||||
"on",
|
||||
],
|
||||
cwd: "/app",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1",
|
||||
OPENCLAW_STATE_DIR: tempState.stateDir,
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", (chunk) => {
|
||||
process.stderr.write(`[openclaw mcp] ${String(chunk)}`);
|
||||
});
|
||||
const rawMessages: unknown[] = [];
|
||||
Reflect.set(transport, "onmessage", (message: unknown) => {
|
||||
pushBounded(rawMessages, message, MCP_RAW_MESSAGE_RETAIN_LIMIT);
|
||||
});
|
||||
|
||||
const client = new Client({ name: "docker-mcp-channels", version: "1.0.0" });
|
||||
try {
|
||||
await connectMcpWithTimeout(client, transport, MCP_CONNECT_TIMEOUT_MS);
|
||||
return {
|
||||
client,
|
||||
cleanup: ownsTempState ? tempState.cleanup : () => {},
|
||||
transport,
|
||||
rawMessages,
|
||||
};
|
||||
} catch (error) {
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
if (ownsTempState) {
|
||||
tempState.cleanup();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function maybeApprovePendingBridgePairing(
|
||||
gateway: GatewayRpcClient,
|
||||
): Promise<boolean> {
|
||||
let pairingState:
|
||||
| {
|
||||
pending?: Array<{ requestId?: string; role?: string }>;
|
||||
}
|
||||
| undefined;
|
||||
try {
|
||||
pairingState = await gateway.request<{
|
||||
pending?: Array<{ requestId?: string; role?: string }>;
|
||||
}>("device.pair.list", {});
|
||||
} catch (error) {
|
||||
const message = formatErrorMessage(error);
|
||||
if (
|
||||
message.includes("missing scope: operator.pairing") ||
|
||||
message.includes("device.pair.list")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!pairingState) {
|
||||
return false;
|
||||
}
|
||||
const pendingRequest = pairingState.pending?.find((entry) => entry.role === "operator");
|
||||
if (!pendingRequest?.requestId) {
|
||||
return false;
|
||||
}
|
||||
await gateway.request("device.pair.approve", { requestId: pendingRequest.requestId });
|
||||
return true;
|
||||
}
|
||||
116
test/e2e/qa-lab/runtime/mcp-client-temp-state.fixture.ts
Normal file
116
test/e2e/qa-lab/runtime/mcp-client-temp-state.fixture.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
// MCP client temp-state helpers used by QA-owned MCP E2E fixtures.
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
export type McpClientTempState = {
|
||||
cleanup: () => void;
|
||||
root: string;
|
||||
stateDir: string;
|
||||
tokenFile: string;
|
||||
};
|
||||
|
||||
export type ReconnectableMcpClientHandle = {
|
||||
cleanup: () => void;
|
||||
client: { close: () => Promise<unknown> };
|
||||
transport: { close: () => Promise<unknown> };
|
||||
};
|
||||
|
||||
type McpConnectTransport = {
|
||||
close?(): Promise<void> | void;
|
||||
};
|
||||
|
||||
const MCP_TIMEOUT_CLOSE_GRACE_MS = 5_000;
|
||||
|
||||
export function createMcpClientTempState(params: {
|
||||
gatewayToken: string;
|
||||
tempRoot?: string;
|
||||
}): McpClientTempState {
|
||||
const root = mkdtempSync(path.join(params.tempRoot ?? tmpdir(), "openclaw-mcp-client-"));
|
||||
const stateDir = path.join(root, "state");
|
||||
const tokenFile = path.join(root, "gateway.token");
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
writeFileSync(tokenFile, `${params.gatewayToken}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
return {
|
||||
cleanup: () => {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
},
|
||||
root,
|
||||
stateDir,
|
||||
tokenFile,
|
||||
};
|
||||
}
|
||||
|
||||
export async function connectMcpWithTimeout<TTransport extends McpConnectTransport>(
|
||||
client: { connect(transport: TTransport): Promise<void> },
|
||||
transport: TTransport,
|
||||
timeoutMs: number,
|
||||
): Promise<void> {
|
||||
let timedOut = false;
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
reject(new Error(`MCP stdio connect timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
timeout.unref?.();
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.race([client.connect(transport), timeoutPromise]);
|
||||
} catch (error) {
|
||||
if (timedOut) {
|
||||
await closeTimedOutTransport(transport);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function closeTimedOutTransport(transport: McpConnectTransport): Promise<void> {
|
||||
if (!transport.close) {
|
||||
return;
|
||||
}
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
Promise.resolve(transport.close()).catch(() => undefined),
|
||||
new Promise<void>((resolve) => {
|
||||
timer = setTimeout(resolve, MCP_TIMEOUT_CLOSE_GRACE_MS);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function connectMcpClientWithPairingReconnect<
|
||||
T extends ReconnectableMcpClientHandle,
|
||||
>(params: {
|
||||
connect: (tempState: McpClientTempState) => Promise<T>;
|
||||
maybeApprovePairing: () => Promise<boolean>;
|
||||
tempState: McpClientTempState;
|
||||
}): Promise<T> {
|
||||
let handle = await params.connect(params.tempState);
|
||||
let shouldReconnect: boolean;
|
||||
try {
|
||||
shouldReconnect = await params.maybeApprovePairing();
|
||||
} catch (error) {
|
||||
await Promise.allSettled([handle.client.close(), handle.transport.close()]);
|
||||
handle.cleanup();
|
||||
throw error;
|
||||
}
|
||||
if (!shouldReconnect) {
|
||||
return handle;
|
||||
}
|
||||
await Promise.allSettled([handle.client.close(), handle.transport.close()]);
|
||||
handle.cleanup();
|
||||
handle = await params.connect(params.tempState);
|
||||
return handle;
|
||||
}
|
||||
185
test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts
Normal file
185
test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
// QA Lab MCP gateway transport tests cover script-backed MCP client state.
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
connectMcpClientWithPairingReconnect,
|
||||
connectMcpWithTimeout,
|
||||
createMcpClientTempState,
|
||||
type McpClientTempState,
|
||||
} from "./mcp-client-temp-state.fixture.ts";
|
||||
|
||||
describe("MCP gateway transport fixture", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("creates unique client temp state and removes token files on cleanup", () => {
|
||||
const tempRoot = mkdtempSync(path.join(tmpdir(), "openclaw-mcp-harness-test-"));
|
||||
try {
|
||||
const first = createMcpClientTempState({ gatewayToken: "first-token", tempRoot });
|
||||
const second = createMcpClientTempState({ gatewayToken: "second-token", tempRoot });
|
||||
|
||||
expect(first.root).not.toBe(second.root);
|
||||
expect(first.stateDir).toBe(path.join(first.root, "state"));
|
||||
expect(readFileSync(first.tokenFile, "utf8")).toBe("first-token\n");
|
||||
expect(statSync(first.tokenFile).mode & 0o777).toBe(0o600);
|
||||
expect(readFileSync(second.tokenFile, "utf8")).toBe("second-token\n");
|
||||
|
||||
first.cleanup();
|
||||
second.cleanup();
|
||||
|
||||
expect(existsSync(first.root)).toBe(false);
|
||||
expect(existsSync(second.root)).toBe(false);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses one MCP temp state across the pairing reconnect path", async () => {
|
||||
const tempState = createMcpClientTempState({ gatewayToken: "pairing-token" });
|
||||
const firstHandle = {
|
||||
cleanup: vi.fn(),
|
||||
client: { close: vi.fn(async () => undefined) },
|
||||
transport: { close: vi.fn(async () => undefined) },
|
||||
};
|
||||
const secondHandle = {
|
||||
cleanup: vi.fn(),
|
||||
client: { close: vi.fn(async () => undefined) },
|
||||
transport: { close: vi.fn(async () => undefined) },
|
||||
};
|
||||
const connectCalls: McpClientTempState[] = [];
|
||||
const connect = vi.fn(async (state: McpClientTempState) => {
|
||||
connectCalls.push(state);
|
||||
return connectCalls.length === 1 ? firstHandle : secondHandle;
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
connectMcpClientWithPairingReconnect({
|
||||
connect,
|
||||
maybeApprovePairing: async () => true,
|
||||
tempState,
|
||||
}),
|
||||
).resolves.toBe(secondHandle);
|
||||
|
||||
expect(connect).toHaveBeenCalledTimes(2);
|
||||
expect(connectCalls).toEqual([tempState, tempState]);
|
||||
expect(firstHandle.client.close).toHaveBeenCalledOnce();
|
||||
expect(firstHandle.transport.close).toHaveBeenCalledOnce();
|
||||
expect(firstHandle.cleanup).toHaveBeenCalledOnce();
|
||||
expect(secondHandle.cleanup).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
tempState.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans up the first MCP client when pairing approval fails", async () => {
|
||||
const tempState = createMcpClientTempState({ gatewayToken: "pairing-token" });
|
||||
const handle = {
|
||||
cleanup: vi.fn(),
|
||||
client: { close: vi.fn(async () => undefined) },
|
||||
transport: { close: vi.fn(async () => undefined) },
|
||||
};
|
||||
const failure = new Error("pairing approval failed");
|
||||
|
||||
try {
|
||||
await expect(
|
||||
connectMcpClientWithPairingReconnect({
|
||||
connect: async () => handle,
|
||||
maybeApprovePairing: async () => {
|
||||
throw failure;
|
||||
},
|
||||
tempState,
|
||||
}),
|
||||
).rejects.toBe(failure);
|
||||
|
||||
expect(handle.client.close).toHaveBeenCalledOnce();
|
||||
expect(handle.transport.close).toHaveBeenCalledOnce();
|
||||
expect(handle.cleanup).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
tempState.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves when the MCP client connects before the timeout", async () => {
|
||||
const client = {
|
||||
connect: vi.fn(async () => undefined),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
await expect(connectMcpWithTimeout(client, transport, 1000)).resolves.toBeUndefined();
|
||||
|
||||
expect(client.connect).toHaveBeenCalledWith(transport);
|
||||
expect(transport.close).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("closes the transport when MCP initialize hangs", async () => {
|
||||
vi.useFakeTimers();
|
||||
const client = {
|
||||
connect: vi.fn(() => new Promise<void>(() => {})),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
const result = connectMcpWithTimeout(client, transport, 100);
|
||||
const rejection = expect(result).rejects.toThrow("MCP stdio connect timed out after 100ms");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await rejection;
|
||||
expect(transport.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("waits for timed-out transport cleanup before rejecting", async () => {
|
||||
vi.useFakeTimers();
|
||||
let closeSettled = false;
|
||||
const client = {
|
||||
connect: vi.fn(() => new Promise<void>(() => {})),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
setTimeout(() => {
|
||||
closeSettled = true;
|
||||
resolve();
|
||||
}, 25);
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
const result = connectMcpWithTimeout(client, transport, 100);
|
||||
const rejection = expect(result).rejects.toThrow("MCP stdio connect timed out after 100ms");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(transport.close).toHaveBeenCalledOnce();
|
||||
expect(closeSettled).toBe(false);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
await rejection;
|
||||
expect(closeSettled).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the original timeout error when cleanup rejects", async () => {
|
||||
vi.useFakeTimers();
|
||||
const client = {
|
||||
connect: vi.fn(() => new Promise<void>(() => {})),
|
||||
};
|
||||
const transport = {
|
||||
close: vi.fn(async () => {
|
||||
throw new Error("close failed");
|
||||
}),
|
||||
};
|
||||
|
||||
const result = connectMcpWithTimeout(client, transport, 100);
|
||||
const rejection = expect(result).rejects.toThrow("MCP stdio connect timed out after 100ms");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await rejection;
|
||||
expect(transport.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
428
test/e2e/qa-lab/runtime/openai-compatible-chat-tools.e2e.test.ts
Normal file
428
test/e2e/qa-lab/runtime/openai-compatible-chat-tools.e2e.test.ts
Normal file
@@ -0,0 +1,428 @@
|
||||
// OpenAI-compatible chat tools tests cover QA Lab HTTP tool-call evidence.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import { createBoundedChildOutput } from "../../../helpers/bounded-child-output.js";
|
||||
import { cleanupTempDirs, makeTempDir } from "../../../helpers/temp-dir.js";
|
||||
|
||||
const clientPath = path.resolve("scripts/e2e/lib/openai-chat-tools/client.mjs");
|
||||
const dockerRunnerPath = path.resolve("scripts/e2e/openai-chat-tools-docker.sh");
|
||||
const writeConfigPath = path.resolve("scripts/e2e/lib/openai-chat-tools/write-config.mjs");
|
||||
|
||||
interface ClientResult {
|
||||
error?: Error;
|
||||
status: number | null;
|
||||
stderr: string;
|
||||
stdout: string;
|
||||
}
|
||||
|
||||
async function listen(server: Server): Promise<number> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not expose a TCP port");
|
||||
}
|
||||
return address.port;
|
||||
}
|
||||
|
||||
function runClient(
|
||||
port: number | string,
|
||||
env: Record<string, string> = {},
|
||||
timeout = 5_000,
|
||||
): Promise<ClientResult> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(process.execPath, [clientPath], {
|
||||
env: {
|
||||
...process.env,
|
||||
MODEL_REF: "openai/gpt-5.4-mini",
|
||||
OPENCLAW_GATEWAY_TOKEN: "test-token",
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "1",
|
||||
PORT: String(port),
|
||||
...env,
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = createBoundedChildOutput();
|
||||
const stderr = createBoundedChildOutput();
|
||||
let timedOut = false;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr.append(chunk);
|
||||
});
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGKILL");
|
||||
}, timeout);
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ error, status: null, stderr: stderr.text(), stdout: stdout.text() });
|
||||
});
|
||||
child.on("exit", (status) => {
|
||||
clearTimeout(timer);
|
||||
resolve({
|
||||
error: timedOut ? new Error(`client timed out after ${timeout}ms`) : undefined,
|
||||
status,
|
||||
stderr: stderr.text(),
|
||||
stdout: stdout.text(),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function runWriteConfig(root: string, env: Record<string, string> = {}) {
|
||||
return spawnSync(process.execPath, [writeConfigPath], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_CONFIG_PATH: path.join(root, "openclaw.json"),
|
||||
OPENCLAW_GATEWAY_TOKEN: "test-token",
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_MODEL: "openai/gpt-5.5",
|
||||
OPENCLAW_STATE_DIR: path.join(root, "state"),
|
||||
OPENCLAW_TEST_WORKSPACE_DIR: path.join(root, "workspace"),
|
||||
PORT: "18789",
|
||||
...env,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function runDockerRunnerAuthPreflight(root: string, env: Record<string, string> = {}) {
|
||||
return spawnSync("bash", [dockerRunnerPath], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: root,
|
||||
OPENAI_API_KEY: "",
|
||||
OPENAI_BASE_URL: "",
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_PROFILE_FILE: path.join(root, "missing.profile"),
|
||||
...env,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function toolCallResponse(messageOverrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
choices: [
|
||||
{
|
||||
finish_reason: "tool_calls",
|
||||
message: {
|
||||
...messageOverrides,
|
||||
tool_calls: [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "get_weather",
|
||||
arguments: JSON.stringify({ city: "Paris, France" }),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("scripts/e2e/lib/openai-chat-tools/client.mjs", () => {
|
||||
let bodyReadTimeoutProbe: {
|
||||
elapsedMs: number;
|
||||
responseClosed: boolean;
|
||||
result: ClientResult;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
let responseClosed = false;
|
||||
const server = createServer((_request, response) => {
|
||||
response.on("close", () => {
|
||||
responseClosed = true;
|
||||
});
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.write('{"choices":');
|
||||
});
|
||||
const port = await listen(server);
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
bodyReadTimeoutProbe = {
|
||||
result: await runClient(port, {}, 4_000),
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
responseClosed,
|
||||
};
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps full profile exports out of the Docker build phase", () => {
|
||||
const runner = readFileSync(dockerRunnerPath, "utf8");
|
||||
const preflightSourceIndex = runner.indexOf('source "$profile_file"');
|
||||
const buildIndex = runner.indexOf("docker_e2e_build_or_reuse");
|
||||
const fullProfileSourceIndex = runner.indexOf('source "$PROFILE_FILE"', buildIndex);
|
||||
|
||||
expect(preflightSourceIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(buildIndex).toBeGreaterThan(preflightSourceIndex);
|
||||
expect(fullProfileSourceIndex).toBeGreaterThan(buildIndex);
|
||||
});
|
||||
|
||||
it("fails auth preflight before Docker build work starts", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runDockerRunnerAuthPreflight(root);
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(output).toContain("OPENAI_API_KEY was not available");
|
||||
expect(output).not.toContain("Building Docker image:");
|
||||
expect(output).not.toContain("Reusing Docker image:");
|
||||
expect(output).not.toContain("Running OpenAI Chat Completions tools Docker E2E");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("treats placeholder profile auth as missing before Docker build work starts", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const profile = path.join(root, "profile");
|
||||
writeFileSync(profile, "OPENAI_API_KEY=undefined\n");
|
||||
const result = runDockerRunnerAuthPreflight(root, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_PROFILE_FILE: profile,
|
||||
});
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(output).toContain("OPENAI_API_KEY was not available");
|
||||
expect(output).not.toContain("Building Docker image:");
|
||||
expect(output).not.toContain("Reusing Docker image:");
|
||||
expect(output).not.toContain("Running OpenAI Chat Completions tools Docker E2E");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["timeout", "OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS", "1e3"],
|
||||
["body limit", "OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES", "64bytes"],
|
||||
])(
|
||||
"rejects invalid Docker runner %s before auth or Docker build work starts",
|
||||
(_label, envName, value) => {
|
||||
const tempDirs: string[] = [];
|
||||
const root = makeTempDir(tempDirs, "openclaw-openai-chat-tools-");
|
||||
try {
|
||||
const result = runDockerRunnerAuthPreflight(root, { [envName]: value });
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
|
||||
expect(result.status).toBe(2);
|
||||
expect(output).toContain(`invalid ${envName}: ${value}`);
|
||||
expect(output).not.toContain("OPENAI_API_KEY was not available");
|
||||
expect(output).not.toContain("Building Docker image:");
|
||||
expect(output).not.toContain("Reusing Docker image:");
|
||||
expect(output).not.toContain("Running OpenAI Chat Completions tools Docker E2E");
|
||||
} finally {
|
||||
cleanupTempDirs(tempDirs);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("passes normalized timeout and body limits into the Docker runner", () => {
|
||||
const runner = readFileSync(dockerRunnerPath, "utf8");
|
||||
|
||||
expect(runner).toContain(
|
||||
"docker_e2e_read_positive_int_env OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS 180",
|
||||
);
|
||||
expect(runner).toContain(
|
||||
"docker_e2e_read_positive_int_env OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES 1048576",
|
||||
);
|
||||
expect(runner).toContain('-e "OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS=$TIMEOUT_SECONDS"');
|
||||
expect(runner).toContain('-e "OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES=$MAX_BODY_BYTES"');
|
||||
});
|
||||
|
||||
it("rejects loose timeout env values instead of parsing numeric prefixes", async () => {
|
||||
const result = await runClient(1, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "1e3",
|
||||
});
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: 1e3");
|
||||
});
|
||||
|
||||
it("rejects loose body limit env values instead of parsing numeric prefixes", async () => {
|
||||
const result = await runClient(1, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64bytes",
|
||||
});
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: 64bytes");
|
||||
});
|
||||
|
||||
it("rejects out-of-range client gateway ports", async () => {
|
||||
const result = await runClient("65536");
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid PORT: 65536");
|
||||
});
|
||||
|
||||
it("rejects loose write-config timeout env values", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runWriteConfig(root, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "1e3",
|
||||
});
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: 1e3");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects out-of-range write-config gateway ports", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runWriteConfig(root, { PORT: "65536" });
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("invalid PORT: 65536");
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("writes strict positive timeout and port values into generated config", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-openai-chat-tools-"));
|
||||
try {
|
||||
const result = runWriteConfig(root, {
|
||||
OPENCLAW_OPENAI_CHAT_TOOLS_TIMEOUT_SECONDS: "240",
|
||||
PORT: "19001",
|
||||
});
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
const config = JSON.parse(readFileSync(path.join(root, "openclaw.json"), "utf8"));
|
||||
expect(config.gateway.port).toBe(19001);
|
||||
expect(config.models.providers.openai.timeoutSeconds).toBe(240);
|
||||
expect(config.agents.defaults.timeoutSeconds).toBe(240);
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts a matching chat completions tool call response", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
expect(request.method).toBe("POST");
|
||||
expect(request.url).toBe("/v1/chat/completions");
|
||||
expect(request.headers.authorization).toBe("Bearer test-token");
|
||||
expect(request.headers["x-openclaw-model"]).toBe("openai/gpt-5.4-mini");
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify(toolCallResponse()));
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const result = await runClient(port);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
args: { city: "Paris, France" },
|
||||
finishReason: "tool_calls",
|
||||
ok: true,
|
||||
toolName: "get_weather",
|
||||
});
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects chat completions responses that include content beside the tool call", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify(toolCallResponse({ content: "I will call the tool now." })));
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const result = await runClient(port);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("expected tool call only response");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the request timeout active while reading the response body", async () => {
|
||||
expect(bodyReadTimeoutProbe.result.error).toBeUndefined();
|
||||
expect(bodyReadTimeoutProbe.result.status).not.toBe(0);
|
||||
expect(bodyReadTimeoutProbe.result.stderr).toMatch(/timed out|aborted|AbortError/iu);
|
||||
expect(bodyReadTimeoutProbe.elapsedMs).toBeLessThan(3_500);
|
||||
expect(bodyReadTimeoutProbe.responseClosed).toBe(true);
|
||||
});
|
||||
|
||||
it("caps chat completion response bodies before JSON parsing", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end("x".repeat(256));
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const result = await runClient(port, { OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64" });
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat completions response body exceeded 64 bytes");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects declared oversized chat completion bodies before waiting on the stream", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, {
|
||||
"content-length": "65",
|
||||
"content-type": "application/json",
|
||||
});
|
||||
response.flushHeaders();
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
const result = await runClient(port, { OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64" });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat completions response body exceeded 64 bytes");
|
||||
expect(Date.now() - startedAt).toBeLessThan(3_500);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects unsafe declared chat completion body lengths before waiting on the stream", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, {
|
||||
"content-length": "9007199254740993",
|
||||
"content-type": "application/json",
|
||||
});
|
||||
response.flushHeaders();
|
||||
});
|
||||
const port = await listen(server);
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
const result = await runClient(port, { OPENCLAW_OPENAI_CHAT_TOOLS_MAX_BODY_BYTES: "64" });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat completions response body exceeded 64 bytes");
|
||||
expect(result.stderr).not.toContain("timed out");
|
||||
expect(Date.now() - startedAt).toBeLessThan(3_500);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
255
test/e2e/qa-lab/runtime/openai-image-auth-docker-client.ts
Normal file
255
test/e2e/qa-lab/runtime/openai-image-auth-docker-client.ts
Normal file
@@ -0,0 +1,255 @@
|
||||
// Openai Image Auth Docker Client script supports OpenClaw repository automation.
|
||||
import http from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
isRequestBodyTooLargeError,
|
||||
readBody,
|
||||
} from "../../../../scripts/e2e/lib/mock-openai-http.mjs";
|
||||
|
||||
const DIRECT_IMAGE_BYTES = Buffer.from("docker-direct-image");
|
||||
const CODEX_IMAGE_BYTES = Buffer.from("docker-codex-image");
|
||||
const DIRECT_TOKEN = "sk-openclaw-image-auth-e2e";
|
||||
const CODEX_TOKEN = "docker-codex-oauth-token";
|
||||
|
||||
export type RequestRecord = {
|
||||
method?: string;
|
||||
url?: string;
|
||||
authorization?: string;
|
||||
accept?: string;
|
||||
contentType?: string;
|
||||
body: string;
|
||||
};
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function writeJson(res: http.ServerResponse, status: number, body: unknown): void {
|
||||
res.writeHead(status, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
function writeCodexSse(res: http.ServerResponse): void {
|
||||
const events = [
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "image_generation_call",
|
||||
result: CODEX_IMAGE_BYTES.toString("base64"),
|
||||
revised_prompt: "docker codex revised prompt",
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
usage: { input_tokens: 1, output_tokens: 2, total_tokens: 3 },
|
||||
tool_usage: { image_gen: { total_tokens: 3 } },
|
||||
},
|
||||
},
|
||||
];
|
||||
res.writeHead(200, { "content-type": "text/event-stream" });
|
||||
for (const event of events) {
|
||||
res.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
res.end("data: [DONE]\n\n");
|
||||
}
|
||||
|
||||
export async function startMockServer(records: RequestRecord[]): Promise<{
|
||||
baseUrl: string;
|
||||
close: () => Promise<void>;
|
||||
}> {
|
||||
const server = http.createServer((req, res) => {
|
||||
void (async () => {
|
||||
try {
|
||||
let body: string;
|
||||
try {
|
||||
body = await readBody(req);
|
||||
} catch (error) {
|
||||
if (isRequestBodyTooLargeError(error)) {
|
||||
writeJson(res, 413, { error: { message: error.message } });
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
records.push({
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
authorization: req.headers.authorization,
|
||||
accept: req.headers.accept,
|
||||
contentType: req.headers["content-type"],
|
||||
body,
|
||||
});
|
||||
|
||||
if (req.method === "POST" && req.url === "/v1/images/generations") {
|
||||
assert(
|
||||
req.headers.authorization === `Bearer ${DIRECT_TOKEN}`,
|
||||
`direct image route used wrong auth: ${req.headers.authorization}`,
|
||||
);
|
||||
const parsed = JSON.parse(body) as { model?: string; prompt?: string; size?: string };
|
||||
assert(parsed.model === "gpt-image-2", `direct route model mismatch: ${body}`);
|
||||
assert(
|
||||
parsed.prompt === "docker direct image auth",
|
||||
`direct route prompt mismatch: ${body}`,
|
||||
);
|
||||
assert(parsed.size === "1024x1024", `direct route size mismatch: ${body}`);
|
||||
writeJson(res, 200, {
|
||||
data: [
|
||||
{
|
||||
b64_json: DIRECT_IMAGE_BYTES.toString("base64"),
|
||||
revised_prompt: "docker direct revised prompt",
|
||||
},
|
||||
],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === "POST" && req.url === "/backend-api/codex/responses") {
|
||||
assert(
|
||||
req.headers.authorization === `Bearer ${CODEX_TOKEN}`,
|
||||
`codex image route used wrong auth: ${req.headers.authorization}`,
|
||||
);
|
||||
const parsed = JSON.parse(body) as {
|
||||
tools?: Array<{ type?: string; model?: string; size?: string }>;
|
||||
input?: Array<{ content?: Array<{ type?: string; text?: string }> }>;
|
||||
};
|
||||
assert(
|
||||
parsed.tools?.[0]?.type === "image_generation" &&
|
||||
parsed.tools[0].model === "gpt-image-2" &&
|
||||
parsed.tools[0].size === "1024x1024",
|
||||
`codex image tool mismatch: ${body}`,
|
||||
);
|
||||
assert(
|
||||
parsed.input?.[0]?.content?.some(
|
||||
(entry) =>
|
||||
entry.type === "input_text" && entry.text === "docker codex oauth image auth",
|
||||
),
|
||||
`codex prompt missing: ${body}`,
|
||||
);
|
||||
writeCodexSse(res);
|
||||
return;
|
||||
}
|
||||
|
||||
writeJson(res, 404, { error: `unexpected ${req.method} ${req.url}` });
|
||||
} catch (error) {
|
||||
writeJson(res, 500, { error: String(error instanceof Error ? error.message : error) });
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address() as AddressInfo;
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${address.port}`,
|
||||
close: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function createCodexOAuthStore() {
|
||||
return {
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:chatgpt": {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: CODEX_TOKEN,
|
||||
refresh: "docker-codex-refresh-token",
|
||||
expires: Date.now() + 60 * 60 * 1000,
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
}
|
||||
|
||||
export async function main() {
|
||||
assert(
|
||||
process.env.OPENAI_API_KEY === DIRECT_TOKEN,
|
||||
"Docker lane must expose the direct OpenAI API key",
|
||||
);
|
||||
const records: RequestRecord[] = [];
|
||||
const mock = await startMockServer(records);
|
||||
try {
|
||||
const { buildOpenAIImageGenerationProvider } =
|
||||
await import("../../../../dist/extensions/openai/image-generation-provider.js");
|
||||
const provider = buildOpenAIImageGenerationProvider();
|
||||
|
||||
const directResult = await provider.generateImage({
|
||||
provider: "openai",
|
||||
model: "gpt-image-2",
|
||||
prompt: "docker direct image auth",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: `${mock.baseUrl}/v1`,
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
assert(
|
||||
directResult.images?.[0]?.buffer?.equals(DIRECT_IMAGE_BYTES),
|
||||
"direct image route did not return expected bytes",
|
||||
);
|
||||
assert(
|
||||
records.some((entry) => entry.url === "/v1/images/generations"),
|
||||
"direct image route was not called",
|
||||
);
|
||||
|
||||
records.length = 0;
|
||||
const codexResult = await provider.generateImage({
|
||||
provider: "openai",
|
||||
model: "gpt-image-2",
|
||||
prompt: "docker codex oauth image auth",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: `${mock.baseUrl}/backend-api/codex`,
|
||||
api: "openai-chatgpt-responses",
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
authStore: createCodexOAuthStore(),
|
||||
});
|
||||
assert(
|
||||
codexResult.images?.[0]?.buffer?.equals(CODEX_IMAGE_BYTES),
|
||||
"Codex OAuth image route did not return expected bytes",
|
||||
);
|
||||
assert(
|
||||
records.some((entry) => entry.url === "/backend-api/codex/responses"),
|
||||
"Codex OAuth image route was not called",
|
||||
);
|
||||
assert(
|
||||
!records.some((entry) => entry.url === "/v1/images/generations"),
|
||||
"Codex OAuth image route fell back to the direct OpenAI API key",
|
||||
);
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify({
|
||||
ok: true,
|
||||
routes: records.map((entry) => entry.url),
|
||||
directBytes: directResult.images[0]?.buffer.length,
|
||||
codexBytes: codexResult.images[0]?.buffer.length,
|
||||
}) + "\n",
|
||||
);
|
||||
} finally {
|
||||
await mock.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// OpenAI web-search minimal assertion tests cover QA Lab native web_search evidence.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const ASSERTIONS_SCRIPT = "scripts/e2e/lib/openai-web-search-minimal/assertions.mjs";
|
||||
|
||||
function runAssertSuccessRequest(logPath: string) {
|
||||
return spawnSync(process.execPath, [ASSERTIONS_SCRIPT, "assert-success-request", logPath], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
}
|
||||
|
||||
describe("openai web-search minimal assertions", () => {
|
||||
it("accepts a success request with web_search and non-minimal reasoning", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
input: "OPENCLAW_SCHEMA_E2E_OK",
|
||||
reasoning: { effort: "low" },
|
||||
tools: [{ type: "web_search" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
expect(runAssertSuccessRequest(logPath).status).toBe(0);
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("finds success requests split across large scan chunks", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({ path: "/health", body: { pad: "x".repeat(70 * 1024) } })}\n${JSON.stringify(
|
||||
{
|
||||
body: {
|
||||
input: "OPENCLAW_SCHEMA_E2E_OK",
|
||||
reasoning: { effort: "low" },
|
||||
tools: [{ type: "web_search" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
},
|
||||
)}\n`,
|
||||
);
|
||||
|
||||
expect(runAssertSuccessRequest(logPath).status).toBe(0);
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds diagnostics when the OpenAI responses endpoint was not used", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
old: `DO_NOT_DUMP_OLD_REQUESTS${"x".repeat(70 * 1024)}`,
|
||||
},
|
||||
path: "/health",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
const result = runAssertSuccessRequest(logPath);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Request log tail:");
|
||||
expect(result.stderr).not.toContain("DO_NOT_DUMP_OLD_REQUESTS");
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds diagnostics when no success response is present", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
input: `DO_NOT_DUMP_OLD_RESPONSE${"x".repeat(70 * 1024)}recent response tail`,
|
||||
tools: [{ type: "web_search" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
const result = runAssertSuccessRequest(logPath);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Recent /v1/responses:");
|
||||
expect(result.stderr).toContain("recent response tail");
|
||||
expect(result.stderr).not.toContain("DO_NOT_DUMP_OLD_RESPONSE");
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects function-shaped web_search as native Responses proof", () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-web-search-minimal-"));
|
||||
try {
|
||||
const logPath = path.join(dir, "requests.jsonl");
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
body: {
|
||||
input: "OPENCLAW_SCHEMA_E2E_OK",
|
||||
reasoning: { effort: "low" },
|
||||
tools: [{ name: "web_search", type: "function" }],
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/responses",
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
const result = runAssertSuccessRequest(logPath);
|
||||
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("success request did not include native web_search");
|
||||
} finally {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
119
test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts
Normal file
119
test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
// OpenAI web-search minimal tests cover QA Lab hosted provider schema evidence.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { testing } from "../../../../scripts/e2e/lib/openai-web-search-minimal/client.mjs";
|
||||
|
||||
describe("scripts/e2e/lib/openai-web-search-minimal/client.mjs", () => {
|
||||
it("accepts only the expected raw schema rejection in reject mode", () => {
|
||||
expect(
|
||||
testing.validateRejectResult({
|
||||
ok: false,
|
||||
error: new Error(`gateway failed: ${testing.DEFAULT_RAW_SCHEMA_ERROR}`),
|
||||
}),
|
||||
).toContain(testing.DEFAULT_RAW_SCHEMA_ERROR);
|
||||
});
|
||||
|
||||
it("accepts the gateway schema rejection wrapper in reject mode", () => {
|
||||
expect(
|
||||
testing.validateRejectResult({
|
||||
ok: false,
|
||||
error: new Error(
|
||||
`GatewayClientRequestError: FailoverError: ${testing.DEFAULT_GATEWAY_SCHEMA_ERROR}.`,
|
||||
),
|
||||
}),
|
||||
).toContain(testing.DEFAULT_GATEWAY_SCHEMA_ERROR);
|
||||
});
|
||||
|
||||
it("fails reject mode when the agent run unexpectedly succeeds", () => {
|
||||
expect(() =>
|
||||
testing.validateRejectResult({
|
||||
ok: true,
|
||||
value: { status: "ok" },
|
||||
}),
|
||||
).toThrow(/reject mode unexpectedly completed/u);
|
||||
});
|
||||
|
||||
it("fails reject mode on unrelated transport errors", () => {
|
||||
expect(() =>
|
||||
testing.validateRejectResult({
|
||||
ok: false,
|
||||
error: new Error("connect ECONNREFUSED 127.0.0.1:9"),
|
||||
}),
|
||||
).toThrow(/reject mode failed for an unexpected reason/u);
|
||||
});
|
||||
|
||||
it("rejects out-of-range gateway ports before connecting", () => {
|
||||
expect(() => testing.resolveGatewayPort({ PORT: "65536" })).toThrow("invalid PORT: 65536");
|
||||
});
|
||||
|
||||
it("accepts success mode only when the final assistant reply contains the marker", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
meta: { finalAssistantVisibleText: `done: ${testing.SUCCESS_MARKER}` },
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts success markers from non-error reply payload text", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
payloads: [{ text: testing.SUCCESS_MARKER }],
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts success markers from the gateway agent result envelope", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
result: {
|
||||
meta: { finalAssistantVisibleText: testing.SUCCESS_MARKER },
|
||||
payloads: [{ text: "secondary reply" }],
|
||||
},
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("fails success mode when the agent run completes without the marker", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: { status: "ok" },
|
||||
}),
|
||||
).toThrow(/completed without success marker/u);
|
||||
});
|
||||
|
||||
it("does not accept success markers from error payload text", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
payloads: [{ isError: true, text: testing.SUCCESS_MARKER }],
|
||||
status: "ok",
|
||||
},
|
||||
}),
|
||||
).toThrow(/completed without success marker/u);
|
||||
});
|
||||
|
||||
it("keeps non-ok success mode failures distinct from marker failures", () => {
|
||||
expect(() =>
|
||||
testing.validateSuccessResult({
|
||||
ok: true,
|
||||
value: {
|
||||
meta: { finalAssistantVisibleText: testing.SUCCESS_MARKER },
|
||||
status: "blocked",
|
||||
},
|
||||
}),
|
||||
).toThrow(/agent run did not complete successfully/u);
|
||||
});
|
||||
});
|
||||
497
test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts
Normal file
497
test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts
Normal file
@@ -0,0 +1,497 @@
|
||||
// OpenWebUI probe tests cover QA Lab OpenAI-compatible API evidence.
|
||||
import { spawn } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createServer, type IncomingMessage, type Server as HttpServer } from "node:http";
|
||||
import { createServer as createTcpServer, type Server as TcpServer, type Socket } from "node:net";
|
||||
import path from "node:path";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createBoundedChildOutput } from "../../../helpers/bounded-child-output.js";
|
||||
|
||||
const probePath = path.resolve("scripts/e2e/openwebui-probe.mjs");
|
||||
|
||||
interface ProbeResult {
|
||||
error?: Error;
|
||||
status: number | null;
|
||||
stderr: string;
|
||||
stdout: string;
|
||||
}
|
||||
|
||||
async function listen(server: HttpServer | TcpServer): Promise<string> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("test server did not expose a TCP port");
|
||||
}
|
||||
return `http://127.0.0.1:${address.port}`;
|
||||
}
|
||||
|
||||
function runProbe(baseUrl: string, env: Record<string, string> = {}, timeout = 3_000) {
|
||||
return new Promise<ProbeResult>((resolve) => {
|
||||
const child = spawn(process.execPath, [probePath], {
|
||||
env: {
|
||||
...process.env,
|
||||
OPENWEBUI_ADMIN_EMAIL: "openwebui-e2e@example.com",
|
||||
OPENWEBUI_ADMIN_PASSWORD: "test-password",
|
||||
OPENWEBUI_BASE_URL: baseUrl,
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "250",
|
||||
OPENWEBUI_EXPECTED_NONCE: "nonce-123",
|
||||
OPENWEBUI_MODEL_ATTEMPTS: "1",
|
||||
OPENWEBUI_MODEL_RETRY_MS: "0",
|
||||
OPENWEBUI_PROMPT: "reply with nonce-123",
|
||||
OPENWEBUI_SMOKE_MODE: "models",
|
||||
...env,
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const stdout = createBoundedChildOutput();
|
||||
const stderr = createBoundedChildOutput();
|
||||
let timedOut = false;
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr.append(chunk);
|
||||
});
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGKILL");
|
||||
}, timeout);
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ error, status: null, stderr: stderr.text(), stdout: stdout.text() });
|
||||
});
|
||||
child.on("exit", (status) => {
|
||||
clearTimeout(timer);
|
||||
resolve({
|
||||
error: timedOut ? new Error(`probe timed out after ${timeout}ms`) : undefined,
|
||||
status,
|
||||
stderr: stderr.text(),
|
||||
stdout: stdout.text(),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function readRequestBody(request: IncomingMessage): Promise<string> {
|
||||
let body = "";
|
||||
request.setEncoding("utf8");
|
||||
for await (const chunk of request) {
|
||||
body += chunk;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
describe("scripts/e2e/openwebui-probe.mjs", () => {
|
||||
it("rejects loose numeric timeout env values instead of parsing prefixes", async () => {
|
||||
const result = await runProbe("http://127.0.0.1:9", {
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "25ms",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain(
|
||||
"OPENWEBUI_CONTROL_TIMEOUT_MS must be a positive integer; got: 25ms",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects zero where positive retry counts are required", async () => {
|
||||
const result = await runProbe("http://127.0.0.1:9", {
|
||||
OPENWEBUI_MODEL_ATTEMPTS: "0",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("OPENWEBUI_MODEL_ATTEMPTS must be a positive integer; got: 0");
|
||||
});
|
||||
|
||||
it("rejects loose response body cap env values instead of parsing prefixes", async () => {
|
||||
const result = await runProbe("http://127.0.0.1:9", {
|
||||
OPENWEBUI_RESPONSE_BODY_MAX_BYTES: "1mb",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain(
|
||||
"OPENWEBUI_RESPONSE_BODY_MAX_BYTES must be a positive integer; got: 1mb",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses a short control-plane timeout for stalled sign-in requests", async () => {
|
||||
const sockets = new Set<Socket>();
|
||||
const server = createTcpServer((socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on("close", () => sockets.delete(socket));
|
||||
socket.on("data", () => {});
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const result = await runProbe(
|
||||
baseUrl,
|
||||
{
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "25",
|
||||
OPENWEBUI_FETCH_TIMEOUT_MS: "5000",
|
||||
},
|
||||
2_000,
|
||||
);
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Open WebUI signin timed out after 25ms");
|
||||
expect(elapsedMs).toBeLessThan(1500);
|
||||
} finally {
|
||||
for (const socket of sockets) {
|
||||
socket.destroy();
|
||||
}
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the control-plane timeout active while reading sign-in bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.flushHeaders();
|
||||
response.write("{");
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(
|
||||
baseUrl,
|
||||
{
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: "25",
|
||||
OPENWEBUI_FETCH_TIMEOUT_MS: "5000",
|
||||
},
|
||||
2_000,
|
||||
);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Open WebUI signin timed out after 25ms");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("passes Open WebUI request timeouts into bounded body reads", () => {
|
||||
const script = readFileSync(probePath, "utf8");
|
||||
|
||||
expect(script).toContain("run(controller.signal, timeoutPromise)");
|
||||
expect(script).toMatch(
|
||||
/readBoundedResponseTextWithLimit\(\s*response,\s*label,\s*responseBodyMaxBytes,\s*timeoutPromise,/u,
|
||||
);
|
||||
expect(script.match(/async \(signal, timeoutPromise\)/gu)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("bounds sign-in error response bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(500, { "content-type": "text/plain" });
|
||||
response.end("x".repeat(64));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, {
|
||||
OPENWEBUI_RESPONSE_BODY_MAX_BYTES: "16",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Open WebUI signin response body exceeded 16 bytes");
|
||||
expect(result.stderr).not.toContain("x".repeat(64));
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("redacts admin credentials from sign-in error bodies", async () => {
|
||||
const adminEmail = "openwebui-e2e" + "@example.com";
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(401, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
error: "invalid credentials",
|
||||
email: adminEmail,
|
||||
password: 'pa"ss',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, { OPENWEBUI_ADMIN_PASSWORD: 'pa"ss' });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("signin failed: HTTP 401");
|
||||
expect(result.stderr).toContain("<redacted>");
|
||||
expect(result.stderr).not.toContain(adminEmail);
|
||||
expect(result.stderr).not.toContain('pa"ss');
|
||||
expect(result.stderr).not.toContain('pa\\"ss');
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds model-list error response bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=chat=secret=cookie; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "chat-secret-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(502, { "content-type": "text/plain" });
|
||||
response.end("y".repeat(96));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, {
|
||||
OPENWEBUI_RESPONSE_BODY_MAX_BYTES: "32",
|
||||
});
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain(
|
||||
"Open WebUI models attempt 1 response body exceeded 32 bytes",
|
||||
);
|
||||
expect(result.stderr).not.toContain("y".repeat(96));
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("redacts auth material from model-list error bodies", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=model=secret=cookie; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "model-secret-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(502, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
error: "upstream rejected Authorization Bearer model-secret-token",
|
||||
cookieValue: "model=secret=cookie",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("HTTP 502");
|
||||
expect(result.stderr).toContain("<redacted>");
|
||||
expect(result.stderr).not.toContain("model-secret-token");
|
||||
expect(result.stderr).not.toContain("model=secret=cookie");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not sleep after the final model-list attempt", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=chat=secret=cookie; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "chat-secret-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "other-model" }] }));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(
|
||||
baseUrl,
|
||||
{
|
||||
OPENWEBUI_MODEL_ATTEMPTS: "1",
|
||||
OPENWEBUI_MODEL_RETRY_MS: "1500",
|
||||
},
|
||||
1_000,
|
||||
);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("openclaw model missing from Open WebUI model list");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("passes in models mode when Open WebUI exposes the OpenClaw model", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
setTimeout(() => {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=test; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "test-token" }));
|
||||
}, 25);
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
expect(request.headers.authorization).toBe("Bearer test-token");
|
||||
expect(request.headers.cookie).toContain("openwebui-session=test");
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] }));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, {
|
||||
OPENWEBUI_CONTROL_TIMEOUT_MS: String(MAX_TIMER_TIMEOUT_MS + 1),
|
||||
OPENWEBUI_FETCH_TIMEOUT_MS: String(MAX_TIMER_TIMEOUT_MS + 1),
|
||||
});
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
mode: "models",
|
||||
model: "openclaw/default",
|
||||
ok: true,
|
||||
});
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("runs chat mode through Open WebUI chat completions and validates the nonce", async () => {
|
||||
const chatRequests: unknown[] = [];
|
||||
const server = createServer(async (request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=test; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "test-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/chat/completions") {
|
||||
expect(request.headers.authorization).toBe("Bearer test-token");
|
||||
expect(request.headers.cookie).toContain("openwebui-session=test");
|
||||
chatRequests.push(JSON.parse(await readRequestBody(request)));
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
choices: [{ message: { content: "OpenClaw replied with nonce-123" } }],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, { OPENWEBUI_SMOKE_MODE: "chat" });
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
model: "openclaw/default",
|
||||
ok: true,
|
||||
reply: "OpenClaw replied with nonce-123",
|
||||
});
|
||||
expect(chatRequests).toEqual([
|
||||
{
|
||||
messages: [{ content: "reply with nonce-123", role: "user" }],
|
||||
model: "openclaw/default",
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails chat mode when the Open WebUI reply omits the expected nonce", async () => {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/api/v1/auths/signin") {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/json",
|
||||
"set-cookie": "openwebui-session=test; Path=/",
|
||||
});
|
||||
response.end(JSON.stringify({ token: "test-token" }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/models") {
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] }));
|
||||
return;
|
||||
}
|
||||
if (request.url === "/api/chat/completions") {
|
||||
const authorization = request.headers.authorization ?? "";
|
||||
const cookie = request.headers.cookie ?? "";
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
message: {
|
||||
content: `missing the marker with ${authorization} and ${cookie}`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.writeHead(404).end();
|
||||
});
|
||||
const baseUrl = await listen(server);
|
||||
try {
|
||||
const result = await runProbe(baseUrl, { OPENWEBUI_SMOKE_MODE: "chat" });
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("chat reply missing nonce");
|
||||
expect(result.stderr).toContain("<redacted>");
|
||||
expect(result.stderr).not.toContain("chat-secret-token");
|
||||
expect(result.stderr).not.toContain("chat=secret=cookie");
|
||||
expect(result.stderr).not.toContain("openwebui-session=chat");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
646
test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts
Normal file
646
test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts
Normal file
@@ -0,0 +1,646 @@
|
||||
// Package OpenClaw For Docker tests cover QA Lab package artifact evidence.
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildPackageArtifacts,
|
||||
packOpenClawPackageForDocker,
|
||||
parseArgs,
|
||||
prepareBundledAiRuntimePackage,
|
||||
runCommandForTest,
|
||||
} from "../../../../scripts/package-openclaw-for-docker.mjs";
|
||||
|
||||
const skipBundledAiRuntime = async (): Promise<() => Promise<void>> => async () => {};
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
if (!Number.isSafeInteger(pid) || pid <= 0) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
async function readPid(filePath: string, timeoutMs: number): Promise<number> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (fs.existsSync(filePath)) {
|
||||
const pid = Number(fs.readFileSync(filePath, "utf8").trim());
|
||||
if (Number.isSafeInteger(pid) && pid > 0) {
|
||||
return pid;
|
||||
}
|
||||
}
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error(`timeout waiting for a positive pid in ${filePath}`);
|
||||
}
|
||||
|
||||
async function waitForDead(pid: number, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (!isProcessAlive(pid)) {
|
||||
return;
|
||||
}
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error(`process still alive: ${pid}`);
|
||||
}
|
||||
|
||||
async function waitForExit(
|
||||
child: ReturnType<typeof spawn>,
|
||||
timeoutMs: number,
|
||||
): Promise<{ signal: NodeJS.Signals | null; status: number | null }> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(
|
||||
() => reject(new Error("timeout waiting for child exit")),
|
||||
timeoutMs,
|
||||
);
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
resolve({ signal, status });
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe("package-openclaw-for-docker", () => {
|
||||
it("parses package artifact output options", () => {
|
||||
expect(
|
||||
parseArgs([
|
||||
"--output-dir",
|
||||
".artifacts/docker",
|
||||
"--output-name=openclaw-current.tgz",
|
||||
"--source-dir",
|
||||
"/repo",
|
||||
"--skip-build",
|
||||
]),
|
||||
).toEqual({
|
||||
outputDir: ".artifacts/docker",
|
||||
outputName: "openclaw-current.tgz",
|
||||
skipBuild: true,
|
||||
sourceDir: "/repo",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects missing package artifact option values", () => {
|
||||
for (const flag of ["--output-dir", "--output-name", "--source-dir"]) {
|
||||
expect(() => parseArgs([flag])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([flag, "--skip-build"])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([flag, "-h"])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([`${flag}=`])).toThrow(`${flag} requires a value`);
|
||||
expect(() => parseArgs([`${flag}=-h`])).toThrow(`${flag} requires a value`);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects duplicate package artifact CLI options", () => {
|
||||
const duplicateCases = [
|
||||
["--output-dir", ["--output-dir", "one", "--output-dir=two"]],
|
||||
["--output-name", ["--output-name", "one.tgz", "--output-name=two.tgz"]],
|
||||
["--source-dir", ["--source-dir", "/repo-a", "--source-dir=/repo-b"]],
|
||||
["--skip-build", ["--skip-build", "--skip-build"]],
|
||||
] satisfies Array<[string, string[]]>;
|
||||
|
||||
for (const [flag, args] of duplicateCases) {
|
||||
expect(() => parseArgs(args), flag).toThrow(`${flag} was provided more than once`);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects package artifact output names that escape the output directory", () => {
|
||||
for (const outputName of [
|
||||
"../openclaw-current.tgz",
|
||||
"nested/openclaw-current.tgz",
|
||||
"openclaw-current.zip",
|
||||
".openclaw-current.tgz",
|
||||
]) {
|
||||
expect(() => parseArgs(["--output-name", outputName])).toThrow(
|
||||
`--output-name must be a tarball filename, not a path: ${outputName}`,
|
||||
);
|
||||
}
|
||||
|
||||
expect(parseArgs(["--output-name", "openclaw-current.tar.gz"]).outputName).toBe(
|
||||
"openclaw-current.tar.gz",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses build-all with declaration generation for package artifacts", async () => {
|
||||
const calls: Array<{
|
||||
command: string;
|
||||
args: string[];
|
||||
cwd: string;
|
||||
noPnpm: string | undefined;
|
||||
skipDts: string | undefined;
|
||||
timeoutMs: number | undefined;
|
||||
}> = [];
|
||||
const previousTimeout = process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
const previousSkipDts = process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD;
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = "1234";
|
||||
process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD = "1";
|
||||
|
||||
try {
|
||||
await buildPackageArtifacts("/repo", {
|
||||
runImpl: async (
|
||||
command: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
options: { env?: NodeJS.ProcessEnv; timeoutMs?: number },
|
||||
) => {
|
||||
calls.push({
|
||||
command,
|
||||
args,
|
||||
cwd,
|
||||
noPnpm: options.env?.OPENCLAW_BUILD_ALL_NO_PNPM,
|
||||
skipDts: options.env?.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD,
|
||||
timeoutMs: options.timeoutMs,
|
||||
});
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
if (previousTimeout === undefined) {
|
||||
delete process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
} else {
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = previousTimeout;
|
||||
}
|
||||
if (previousSkipDts === undefined) {
|
||||
delete process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD;
|
||||
} else {
|
||||
process.env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD = previousSkipDts;
|
||||
}
|
||||
}
|
||||
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
command: "node",
|
||||
args: ["scripts/build-all.mjs", "ciArtifacts"],
|
||||
cwd: "/repo",
|
||||
noPnpm: "1",
|
||||
skipDts: "0",
|
||||
timeoutMs: 1234,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects loose package artifact timeout env values", async () => {
|
||||
const previousTimeout = process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
try {
|
||||
for (const value of ["1e3", "123.9", "9007199254740993", "0"]) {
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = value;
|
||||
|
||||
await expect(
|
||||
buildPackageArtifacts("/repo", {
|
||||
runImpl: async () => undefined,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS must be a positive timeout in milliseconds",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if (previousTimeout === undefined) {
|
||||
delete process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS;
|
||||
} else {
|
||||
process.env.OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS = previousTimeout;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("bundles and restores the separately packed AI runtime", async () => {
|
||||
const sourceDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-ai-source-"));
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-ai-output-"));
|
||||
const packageJsonPath = path.join(sourceDir, "package.json");
|
||||
const originalPackageJson = `${JSON.stringify(
|
||||
{
|
||||
dependencies: { "@openclaw/ai": "workspace:*", "dep-a": "1.2.3" },
|
||||
files: ["dist"],
|
||||
name: "openclaw",
|
||||
version: "2026.6.17",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`;
|
||||
const installedAiPath = path.join(sourceDir, "node_modules", "@openclaw", "ai");
|
||||
fs.mkdirSync(path.join(sourceDir, "packages", "ai"), { recursive: true });
|
||||
fs.mkdirSync(installedAiPath, { recursive: true });
|
||||
fs.writeFileSync(path.join(installedAiPath, "original-marker"), "workspace package");
|
||||
fs.writeFileSync(packageJsonPath, originalPackageJson);
|
||||
|
||||
try {
|
||||
const cleanup = await prepareBundledAiRuntimePackage(
|
||||
sourceDir,
|
||||
outputDir,
|
||||
async (command: string, args: string[], cwd: string) => {
|
||||
expect({ args, command, cwd }).toEqual({
|
||||
args: ["--dir", "packages/ai", "pack", "--silent", "--pack-destination", outputDir],
|
||||
command: "pnpm",
|
||||
cwd: sourceDir,
|
||||
});
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-ai-2026.6.17.tgz"), "ai package");
|
||||
return "";
|
||||
},
|
||||
{
|
||||
extractAiRuntime: async (_tarballPath: string, destination: string) => {
|
||||
fs.writeFileSync(
|
||||
path.join(destination, "package.json"),
|
||||
`${JSON.stringify({
|
||||
dependencies: { "dep-a": "1.2.3" },
|
||||
name: "@openclaw/ai",
|
||||
version: "2026.6.17",
|
||||
})}\n`,
|
||||
);
|
||||
fs.writeFileSync(path.join(destination, "runtime.js"), "export {};\n");
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as {
|
||||
bundleDependencies: string[];
|
||||
dependencies: Record<string, string>;
|
||||
};
|
||||
expect(packageJson.dependencies["@openclaw/ai"]).toBe("2026.6.17");
|
||||
expect(packageJson.bundleDependencies).toContain("@openclaw/ai");
|
||||
expect(fs.existsSync(path.join(installedAiPath, "original-marker"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(installedAiPath, "runtime.js"))).toBe(true);
|
||||
const stagedAiPackageJson = JSON.parse(
|
||||
fs.readFileSync(path.join(installedAiPath, "package.json"), "utf8"),
|
||||
) as { dependencies?: Record<string, string> };
|
||||
expect(stagedAiPackageJson.dependencies).toBeUndefined();
|
||||
|
||||
await cleanup();
|
||||
expect(fs.readFileSync(packageJsonPath, "utf8")).toBe(originalPackageJson);
|
||||
expect(fs.readFileSync(path.join(installedAiPath, "original-marker"), "utf8")).toBe(
|
||||
"workspace package",
|
||||
);
|
||||
expect(fs.existsSync(path.join(outputDir, "openclaw-ai-2026.6.17.tgz"))).toBe(false);
|
||||
} finally {
|
||||
fs.rmSync(sourceDir, { recursive: true, force: true });
|
||||
fs.rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("trims and restores the changelog around ignore-scripts package artifacts", async () => {
|
||||
const calls: string[] = [];
|
||||
const tarball = await packOpenClawPackageForDocker("/repo", "/out", {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async (cwd: string) => {
|
||||
calls.push(`prepare:${cwd}`);
|
||||
},
|
||||
restoreChangelog: async (cwd: string) => {
|
||||
calls.push(`restore:${cwd}`);
|
||||
},
|
||||
runCaptureImpl: async (
|
||||
command: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
options: { deferForwardedSignalExit?: boolean },
|
||||
) => {
|
||||
calls.push(`${command}:${args.join(" ")}:${cwd}`);
|
||||
expect(options.deferForwardedSignalExit).toBe(true);
|
||||
return "openclaw-2026.5.28.tgz\n";
|
||||
},
|
||||
});
|
||||
|
||||
expect(tarball).toBe(path.join("/out", "openclaw-2026.5.28.tgz"));
|
||||
expect(calls).toEqual([
|
||||
"prepare:/repo",
|
||||
"npm:pack --silent --ignore-scripts --pack-destination /out:/repo",
|
||||
"restore:/repo",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects path-like npm pack stdout before resolving Docker package tarballs", async () => {
|
||||
for (const filename of [
|
||||
"../openclaw-2026.6.17.tgz",
|
||||
"/tmp/openclaw-2026.6.17.tgz",
|
||||
String.raw`C:\temp\openclaw-2026.6.17.tgz`,
|
||||
"openclaw-nested/evil.tgz",
|
||||
String.raw`openclaw-nested\evil.tgz`,
|
||||
"openclaw-C:evil.tgz",
|
||||
]) {
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", "/out", {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => `${filename}\n`,
|
||||
}),
|
||||
).rejects.toThrow("npm pack reported unsafe OpenClaw tarball filename");
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores unsafe output directory tarball names when npm stdout is not usable", async () => {
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-"));
|
||||
try {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-C:evil.tgz"), "");
|
||||
fs.writeFileSync(path.join(outputDir, String.raw`openclaw-nested\evil.tgz`), "");
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => "npm notice\n",
|
||||
}),
|
||||
).rejects.toThrow("missing packed OpenClaw tarball");
|
||||
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-2026.6.17.tgz"), "");
|
||||
return "npm notice\n";
|
||||
},
|
||||
}),
|
||||
).resolves.toBe(path.join(outputDir, "openclaw-2026.6.17.tgz"));
|
||||
} finally {
|
||||
fs.rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores stale package tarballs before fallback scanning npm output", async () => {
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-stale-"));
|
||||
try {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-9999.1.1.tgz"), "stale");
|
||||
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", outputDir, {
|
||||
prepareBundledAiRuntime: skipBundledAiRuntime,
|
||||
prepareChangelog: async () => {},
|
||||
restoreChangelog: async () => {},
|
||||
runCaptureImpl: async () => {
|
||||
fs.writeFileSync(path.join(outputDir, "openclaw-2026.6.17.tgz"), "current");
|
||||
return "npm notice\n";
|
||||
},
|
||||
}),
|
||||
).resolves.toBe(path.join(outputDir, "openclaw-2026.6.17.tgz"));
|
||||
|
||||
expect(fs.existsSync(path.join(outputDir, "openclaw-9999.1.1.tgz"))).toBe(false);
|
||||
expect(fs.readFileSync(path.join(outputDir, "openclaw-2026.6.17.tgz"), "utf8")).toBe(
|
||||
"current",
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("restores the changelog when ignore-scripts packaging fails", async () => {
|
||||
const calls: string[] = [];
|
||||
|
||||
await expect(
|
||||
packOpenClawPackageForDocker("/repo", "/out", {
|
||||
prepareBundledAiRuntime: async () => {
|
||||
calls.push("embed");
|
||||
return async () => {
|
||||
calls.push("cleanup");
|
||||
};
|
||||
},
|
||||
prepareChangelog: async (cwd: string) => {
|
||||
calls.push(`prepare:${cwd}`);
|
||||
},
|
||||
restoreChangelog: async (cwd: string) => {
|
||||
calls.push(`restore:${cwd}`);
|
||||
},
|
||||
runCaptureImpl: async () => {
|
||||
calls.push("pack");
|
||||
throw new Error("pack failed");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("pack failed");
|
||||
|
||||
expect(calls).toEqual(["prepare:/repo", "embed", "pack", "cleanup", "restore:/repo"]);
|
||||
});
|
||||
|
||||
it("clamps oversized command timers before scheduling", async () => {
|
||||
await expect(
|
||||
runCommandForTest(
|
||||
process.execPath,
|
||||
["-e", "setTimeout(() => process.exit(0), 25);"],
|
||||
process.cwd(),
|
||||
{
|
||||
killAfterMs: MAX_TIMER_TIMEOUT_MS + 1,
|
||||
timeoutMs: MAX_TIMER_TIMEOUT_MS + 1,
|
||||
},
|
||||
),
|
||||
).resolves.toBe("");
|
||||
});
|
||||
|
||||
it("kills timed-out child process groups", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-timeout-"));
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
let childPid;
|
||||
try {
|
||||
const childScript = ["process.on('SIGTERM', () => {});", "setInterval(() => {}, 1000);"].join(
|
||||
"",
|
||||
);
|
||||
const parentScript = [
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"const fs = require('node:fs');",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"fs.writeFileSync(process.env.OPENCLAW_TEST_CHILD_PID, String(child.pid));",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
const runPromise = runCommandForTest(process.execPath, ["-e", parentScript], process.cwd(), {
|
||||
env: { ...process.env, OPENCLAW_TEST_CHILD_PID: childPidPath },
|
||||
killAfterMs: 25,
|
||||
timeoutMs: 500,
|
||||
});
|
||||
const timeoutAssertion = expect(runPromise).rejects.toThrow(/timed out after 500ms/u);
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
await timeoutAssertion;
|
||||
await waitForDead(childPid, 2000);
|
||||
} finally {
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("clamps oversized kill grace before scheduling", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-grace-"));
|
||||
const donePath = path.join(tempDir, "done");
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
let childPid;
|
||||
try {
|
||||
const script = [
|
||||
"const fs = require('node:fs');",
|
||||
`fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`,
|
||||
"process.on('SIGTERM', () => {",
|
||||
` setTimeout(() => { fs.writeFileSync(${JSON.stringify(donePath)}, 'done'); process.exit(0); }, 75);`,
|
||||
"});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n");
|
||||
|
||||
const runPromise = runCommandForTest(process.execPath, ["-e", script], process.cwd(), {
|
||||
killAfterMs: MAX_TIMER_TIMEOUT_MS + 1,
|
||||
timeoutMs: 500,
|
||||
});
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
|
||||
await expect(runPromise).rejects.toThrow(/timed out after 500ms/u);
|
||||
expect(fs.readFileSync(donePath, "utf8")).toBe("done");
|
||||
} finally {
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps fallback SIGKILL armed for descendants after the direct child exits", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-descendant-"));
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
let childPid;
|
||||
try {
|
||||
const childScript = ["process.on('SIGTERM', () => {});", "setInterval(() => {}, 1000);"].join(
|
||||
"",
|
||||
);
|
||||
const parentScript = [
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"const fs = require('node:fs');",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"fs.writeFileSync(process.env.OPENCLAW_TEST_CHILD_PID, String(child.pid));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
await expect(
|
||||
runCommandForTest(process.execPath, ["-e", parentScript], process.cwd(), {
|
||||
env: { ...process.env, OPENCLAW_TEST_CHILD_PID: childPidPath },
|
||||
killAfterMs: 25,
|
||||
timeoutMs: 500,
|
||||
}),
|
||||
).rejects.toThrow(/timed out after 500ms/u);
|
||||
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
await waitForDead(childPid, 2000);
|
||||
} finally {
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not fire delayed SIGKILL after a timed-out child exits during grace", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const killSpy = vi.spyOn(process, "kill");
|
||||
try {
|
||||
const script = [
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
await expect(
|
||||
runCommandForTest(process.execPath, ["-e", script], process.cwd(), {
|
||||
killAfterMs: 100,
|
||||
timeoutMs: 25,
|
||||
}),
|
||||
).rejects.toThrow(/timed out after 25ms/u);
|
||||
|
||||
const sigkillCallsAfterExit = killSpy.mock.calls.filter(
|
||||
([, signal]) => signal === "SIGKILL",
|
||||
).length;
|
||||
await sleep(150);
|
||||
expect(killSpy.mock.calls.filter(([, signal]) => signal === "SIGKILL")).toHaveLength(
|
||||
sigkillCallsAfterExit,
|
||||
);
|
||||
} finally {
|
||||
killSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails captured commands that exceed the stdout limit", async () => {
|
||||
const script = [
|
||||
"process.stdout.write('x'.repeat(2048));",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
|
||||
await expect(
|
||||
runCommandForTest(process.execPath, ["-e", script], process.cwd(), {
|
||||
captureStdout: true,
|
||||
killAfterMs: 50,
|
||||
maxCapturedStdoutBytes: 1024,
|
||||
timeoutMs: 5000,
|
||||
}),
|
||||
).rejects.toThrow(/exceeded captured stdout limit \(1024 bytes\)/u);
|
||||
});
|
||||
|
||||
it("forwards external termination to active child process groups", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-signal-"));
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
const scriptUrl = pathToFileURL(path.resolve("scripts/package-openclaw-for-docker.mjs")).href;
|
||||
let childPid = 0;
|
||||
let runnerPid;
|
||||
try {
|
||||
const childScript = "setInterval(() => {}, 1000);";
|
||||
const parentScript = [
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"const fs = require('node:fs');",
|
||||
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
|
||||
"fs.writeFileSync(process.env.OPENCLAW_TEST_CHILD_PID, String(child.pid));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("");
|
||||
const runnerScript = [
|
||||
`import { runCommandForTest } from ${JSON.stringify(scriptUrl)};`,
|
||||
`await runCommandForTest(process.execPath, ['-e', ${JSON.stringify(parentScript)}], process.cwd(), { timeoutMs: 60000 });`,
|
||||
].join("\n");
|
||||
const runner = spawn(process.execPath, ["--input-type=module", "-e", runnerScript], {
|
||||
cwd: process.cwd(),
|
||||
env: { ...process.env, OPENCLAW_TEST_CHILD_PID: childPidPath },
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
});
|
||||
runnerPid = runner.pid ?? 0;
|
||||
|
||||
childPid = await readPid(childPidPath, 2000);
|
||||
runner.kill("SIGTERM");
|
||||
const result = await waitForExit(runner, 5000);
|
||||
|
||||
expect(result).toEqual({ signal: null, status: 143 });
|
||||
await waitForDead(childPid, 2000);
|
||||
} finally {
|
||||
if (runnerPid && isProcessAlive(runnerPid)) {
|
||||
process.kill(runnerPid, "SIGKILL");
|
||||
}
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
2063
test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts
Normal file
2063
test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts
Normal file
File diff suppressed because it is too large
Load Diff
867
test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts
Normal file
867
test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts
Normal file
@@ -0,0 +1,867 @@
|
||||
// QA OTEL Smoke tests cover QA Lab telemetry evidence.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { createConnection as createNetConnection } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { resolveWindowsTaskkillPath } from "../../../../scripts/lib/windows-taskkill.mjs";
|
||||
import { testing } from "./qa-otel-smoke-runtime.js";
|
||||
|
||||
function expectedTaskkillPath(): string {
|
||||
return resolveWindowsTaskkillPath();
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("qa-otel-smoke receiver bounds", () => {
|
||||
let configuredBodyLimitLoad: ReturnType<typeof spawnSync>;
|
||||
|
||||
beforeAll(() => {
|
||||
configuredBodyLimitLoad = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
'await import("./test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts");',
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_QA_OTEL_MAX_CAPTURED_BODY_TEXT_BYTES: "1024",
|
||||
OPENCLAW_QA_OTEL_MAX_COMPRESSED_BODY_BYTES: "2048",
|
||||
OPENCLAW_QA_OTEL_MAX_DECODED_BODY_BYTES: "4096",
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
function makePassingSmokeAssertionInput(): Parameters<typeof testing.assertSmoke>[0] {
|
||||
return {
|
||||
bodyText: {
|
||||
logs: ["diagnostics-otel: logs exporter enabled"],
|
||||
},
|
||||
childExitCode: 0,
|
||||
disallowedBodyNeedles: ["OTEL-QA-SECRET"],
|
||||
logsExporter: "otlp",
|
||||
logRecords: [
|
||||
{
|
||||
body: "diagnostics-otel: logs exporter enabled",
|
||||
traceId: "trace",
|
||||
spanId: "span",
|
||||
},
|
||||
],
|
||||
metrics: [{ name: "openclaw.harness.duration_ms" }],
|
||||
requests: [
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 16,
|
||||
contentEncoding: undefined,
|
||||
status: 200,
|
||||
spanCount: 5,
|
||||
metricCount: 0,
|
||||
logCount: 0,
|
||||
},
|
||||
{
|
||||
path: "/v1/metrics",
|
||||
signal: "metrics",
|
||||
bytes: 16,
|
||||
contentEncoding: undefined,
|
||||
status: 200,
|
||||
spanCount: 0,
|
||||
metricCount: 1,
|
||||
logCount: 0,
|
||||
},
|
||||
{
|
||||
path: "/v1/logs",
|
||||
signal: "logs",
|
||||
bytes: 16,
|
||||
contentEncoding: undefined,
|
||||
status: 200,
|
||||
spanCount: 0,
|
||||
metricCount: 0,
|
||||
logCount: 1,
|
||||
},
|
||||
],
|
||||
stdoutLogLines: [],
|
||||
stdoutLogRecords: [],
|
||||
spans: [
|
||||
{ name: "openclaw.run", parent: false, attributes: {} },
|
||||
{ name: "openclaw.harness.run", parent: true, attributes: {} },
|
||||
{ name: "openclaw.context.assembled", parent: true, attributes: {} },
|
||||
{ name: "openclaw.message.delivery", parent: true, attributes: {} },
|
||||
{
|
||||
name: "chat gpt-5.5",
|
||||
parent: true,
|
||||
attributes: {
|
||||
"gen_ai.operation.name": "chat",
|
||||
"gen_ai.request.model": "gpt-5.5",
|
||||
"openclaw.model": "gpt-5.5",
|
||||
"openclaw.provider": "openai",
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
it("accepts package-manager forwarded arguments", () => {
|
||||
expect(
|
||||
testing.parseArgs([
|
||||
"--",
|
||||
"--collector",
|
||||
"docker",
|
||||
"--provider-mode",
|
||||
"mock-openai",
|
||||
"--scenario",
|
||||
"otel-stdout-log-smoke",
|
||||
"--logs-exporter",
|
||||
"stdout",
|
||||
]),
|
||||
).toMatchObject({
|
||||
collectorMode: "docker",
|
||||
logsExporter: "stdout",
|
||||
providerMode: "mock-openai",
|
||||
scenarioId: "otel-stdout-log-smoke",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["--collector", ["--collector", "--logs-exporter"]],
|
||||
["--logs-exporter", ["--logs-exporter", "--collector"]],
|
||||
["--output-dir", ["--output-dir", "--collector"]],
|
||||
["--provider-mode", ["--provider-mode", "--collector"]],
|
||||
["--scenario", ["--scenario", "--collector"]],
|
||||
["--model", ["--model", "--collector"]],
|
||||
["--alt-model", ["--alt-model", "--collector"]],
|
||||
])("rejects missing values for %s before shifting parser state", (flag, args) => {
|
||||
expect(() => testing.parseArgs(args)).toThrow(`${flag} requires a value`);
|
||||
});
|
||||
|
||||
it("rejects duplicate OTEL smoke CLI options", () => {
|
||||
const duplicateCases = [
|
||||
["--collector", ["--collector", "local", "--collector", "docker"]],
|
||||
["--logs-exporter", ["--logs-exporter", "otlp", "--logs-exporter", "stdout"]],
|
||||
["--output-dir", ["--output-dir", ".artifacts/one", "--output-dir", ".artifacts/two"]],
|
||||
["--provider-mode", ["--provider-mode", "mock-openai", "--provider-mode", "live-frontier"]],
|
||||
["--scenario", ["--scenario", "custom-one", "--scenario", "custom-two"]],
|
||||
["--model", ["--model", "openai/gpt-5.5", "--model", "openai/gpt-5.4"]],
|
||||
["--alt-model", ["--alt-model", "openai/gpt-5.5", "--alt-model", "openai/gpt-5.4"]],
|
||||
] satisfies Array<[string, string[]]>;
|
||||
|
||||
for (const [flag, args] of duplicateCases) {
|
||||
expect(() => testing.parseArgs(args), flag).toThrow(`${flag} was provided more than once`);
|
||||
}
|
||||
});
|
||||
|
||||
it("selects the matching scenario for the requested log exporter", () => {
|
||||
expect(testing.parseArgs(["--logs-exporter", "otlp"]).scenarioId).toBe("otel-trace-smoke");
|
||||
expect(testing.parseArgs(["--logs-exporter", "stdout"]).scenarioId).toBe(
|
||||
"otel-stdout-log-smoke",
|
||||
);
|
||||
expect(testing.parseArgs(["--logs-exporter", "both"]).scenarioId).toBe("otel-both-log-smoke");
|
||||
});
|
||||
|
||||
it("rejects explicit scenarios that do not match the log exporter", () => {
|
||||
expect(() =>
|
||||
testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "otel-trace-smoke"]),
|
||||
).toThrow("--logs-exporter stdout requires --scenario otel-stdout-log-smoke");
|
||||
});
|
||||
|
||||
it("allows explicit custom scenarios to own their exporter config", () => {
|
||||
expect(testing.parseArgs(["--scenario", "custom-otel-smoke"]).scenarioId).toBe(
|
||||
"custom-otel-smoke",
|
||||
);
|
||||
expect(
|
||||
testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "custom-stdout-smoke"])
|
||||
.scenarioId,
|
||||
).toBe("custom-stdout-smoke");
|
||||
});
|
||||
|
||||
it("uses unique default output dirs", () => {
|
||||
const firstOutputDir = testing.parseArgs([]).outputDir;
|
||||
const secondOutputDir = testing.parseArgs([]).outputDir;
|
||||
|
||||
expect(path.dirname(firstOutputDir)).toBe(path.join(".artifacts", "qa-e2e"));
|
||||
expect(path.basename(firstOutputDir)).toMatch(/^otel-smoke-[a-z0-9]+-[a-f0-9]{8}$/u);
|
||||
expect(secondOutputDir).not.toBe(firstOutputDir);
|
||||
expect(testing.parseArgs(["--output-dir", ".artifacts/custom"]).outputDir).toBe(
|
||||
".artifacts/custom",
|
||||
);
|
||||
});
|
||||
|
||||
it("passes a repo-relative output dir to the child QA suite", () => {
|
||||
const repoRoot = path.join(path.sep, "repo");
|
||||
const options = testing.parseArgs([
|
||||
"--output-dir",
|
||||
path.join(repoRoot, ".artifacts", "qa-e2e", "otel-smoke"),
|
||||
]);
|
||||
|
||||
expect(testing.buildQaArgs(options, repoRoot)).toContain(".artifacts/qa-e2e/otel-smoke");
|
||||
const repoRootArgs = testing.buildQaArgs(
|
||||
testing.parseArgs(["--output-dir", repoRoot]),
|
||||
repoRoot,
|
||||
);
|
||||
expect(repoRootArgs[repoRootArgs.indexOf("--output-dir") + 1]).toBe(".");
|
||||
expect(() =>
|
||||
testing.buildQaArgs(
|
||||
testing.parseArgs(["--output-dir", path.join(path.sep, "outside", "otel-smoke")]),
|
||||
repoRoot,
|
||||
),
|
||||
).toThrow("--output-dir must stay within the repo root");
|
||||
});
|
||||
|
||||
it("parses body-size limit env values as strict positive integers", () => {
|
||||
expect(testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, {})).toBe(64);
|
||||
expect(
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: " 128 " }),
|
||||
).toBe(128);
|
||||
|
||||
expect(() =>
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: "1e3" }),
|
||||
).toThrow("OTEL_TEST_LIMIT must be a positive integer");
|
||||
expect(() =>
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: "1024bytes" }),
|
||||
).toThrow("OTEL_TEST_LIMIT must be a positive integer");
|
||||
expect(() =>
|
||||
testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, { OTEL_TEST_LIMIT: "0" }),
|
||||
).toThrow("OTEL_TEST_LIMIT must be a positive integer");
|
||||
});
|
||||
|
||||
it("loads with configured body-size limit env values", () => {
|
||||
expect(configuredBodyLimitLoad.status).toBe(0);
|
||||
expect(configuredBodyLimitLoad.stderr).not.toContain("ReferenceError");
|
||||
});
|
||||
|
||||
it("scrubs inherited OpenTelemetry exporter env before running the QA suite", () => {
|
||||
vi.stubEnv("OTEL_SDK_DISABLED", "true");
|
||||
vi.stubEnv("OTEL_TRACES_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_METRICS_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_LOGS_EXPORTER", "none");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.example.test:4318");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_PROTOCOL", "grpc");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=secret");
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "authorization=logs-secret");
|
||||
vi.stubEnv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=developer-laptop");
|
||||
|
||||
const env = testing.buildQaEnv(4318);
|
||||
|
||||
expect(env.OTEL_SDK_DISABLED).toBeUndefined();
|
||||
expect(env.OTEL_TRACES_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_METRICS_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_LOGS_EXPORTER).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_ENDPOINT).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_METRICS_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_HEADERS).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toBeUndefined();
|
||||
expect(env.OTEL_RESOURCE_ATTRIBUTES).toBeUndefined();
|
||||
expect(env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).toBe("http://127.0.0.1:4318/v1/traces");
|
||||
expect(env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT).toBe("http://127.0.0.1:4318/v1/metrics");
|
||||
expect(env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT).toBe("http://127.0.0.1:4318/v1/logs");
|
||||
expect(env.OTEL_SERVICE_NAME).toBe("openclaw-qa-lab-otel-smoke");
|
||||
});
|
||||
|
||||
it("rejects identity OTLP bodies above the decoded byte ceiling", () => {
|
||||
expect(() => testing.decodeRequestBody(Buffer.alloc(65), undefined, 64)).toThrow(
|
||||
"OTLP request body exceeded 64 bytes: 65 bytes",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects gzip OTLP bodies above the decoded byte ceiling", () => {
|
||||
const compressed = gzipSync(Buffer.alloc(256, "a"));
|
||||
|
||||
expect(() => testing.decodeRequestBody(compressed, "gzip", 64)).toThrow(
|
||||
"decoded OTLP request body exceeded 64 bytes",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps captured OTLP body text bounded per signal", () => {
|
||||
const captured: { traces?: string[] } = {};
|
||||
|
||||
testing.appendCapturedBodyText(captured, "traces", Buffer.from("a".repeat(20)), 16, [
|
||||
"OTEL-QA-SECRET",
|
||||
]);
|
||||
testing.appendCapturedBodyText(captured, "traces", Buffer.from("b".repeat(20)), 16);
|
||||
|
||||
expect(captured.traces).toHaveLength(1);
|
||||
expect(captured.traces?.[0]).toContain("[captured body text truncated to last 16 bytes]");
|
||||
expect(captured.traces?.[0]).toContain("b".repeat(16));
|
||||
expect(captured.traces?.[0]).not.toContain("a".repeat(20));
|
||||
});
|
||||
|
||||
it("returns a bounded failure for malformed local OTLP protobuf", async () => {
|
||||
const receiver = testing.startLocalOtlpReceiver(["OTEL-QA-SECRET"]);
|
||||
const port = await receiver.listen();
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/v1/traces`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-protobuf" },
|
||||
body: Buffer.concat([Buffer.from([0x0a]), Buffer.from("OTEL-QA-SECRET")]),
|
||||
});
|
||||
const text = await response.text();
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(text).toContain("truncated protobuf");
|
||||
expect(receiver.capturedRequests).toEqual([
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 15,
|
||||
contentEncoding: undefined,
|
||||
status: 400,
|
||||
spanCount: 0,
|
||||
metricCount: 0,
|
||||
logCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(receiver.capturedBodyText.traces).toEqual([
|
||||
"[detected leak needle] OTEL-QA-SECRET",
|
||||
"\nOTEL-QA-SECRET",
|
||||
]);
|
||||
} finally {
|
||||
await receiver.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects truncated unknown fixed-width protobuf fields", async () => {
|
||||
const receiver = testing.startLocalOtlpReceiver();
|
||||
const port = await receiver.listen();
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/v1/traces`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-protobuf" },
|
||||
body: Buffer.from([0x09]),
|
||||
});
|
||||
const text = await response.text();
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(text).toContain("truncated protobuf fixed64");
|
||||
expect(receiver.capturedRequests).toMatchObject([
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 1,
|
||||
status: 400,
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
await receiver.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("closes active local OTLP receiver sockets during cleanup", async () => {
|
||||
const receiver = testing.startLocalOtlpReceiver();
|
||||
const port = await receiver.listen();
|
||||
const socket = createNetConnection(port, "127.0.0.1");
|
||||
const socketClosed = new Promise<void>((resolve) => {
|
||||
socket.once("close", resolve);
|
||||
});
|
||||
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.once("connect", resolve);
|
||||
socket.once("error", reject);
|
||||
});
|
||||
socket.write(
|
||||
[
|
||||
"POST /v1/traces HTTP/1.1",
|
||||
"Host: 127.0.0.1",
|
||||
"Content-Type: application/x-protobuf",
|
||||
"Content-Length: 1048576",
|
||||
"",
|
||||
"x",
|
||||
].join("\r\n"),
|
||||
);
|
||||
|
||||
await Promise.race([
|
||||
receiver.close(),
|
||||
delay(1_000).then(() => {
|
||||
throw new Error("receiver close timed out");
|
||||
}),
|
||||
]);
|
||||
await Promise.race([
|
||||
socketClosed,
|
||||
delay(1_000).then(() => {
|
||||
throw new Error("socket close timed out");
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
socket.destroy();
|
||||
await receiver.close().catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
it("fails smoke assertions for captured non-2xx OTLP requests", () => {
|
||||
const assertion = testing.assertSmoke({
|
||||
bodyText: {},
|
||||
childExitCode: 0,
|
||||
disallowedBodyNeedles: [],
|
||||
logsExporter: "otlp",
|
||||
logRecords: [],
|
||||
metrics: [],
|
||||
requests: [
|
||||
{
|
||||
path: "/v1/traces",
|
||||
signal: "traces",
|
||||
bytes: 15,
|
||||
contentEncoding: undefined,
|
||||
status: 400,
|
||||
spanCount: 0,
|
||||
metricCount: 0,
|
||||
logCount: 0,
|
||||
},
|
||||
],
|
||||
stdoutLogLines: [],
|
||||
stdoutLogRecords: [],
|
||||
spans: [],
|
||||
});
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("OTLP traces request /v1/traces returned status 400");
|
||||
});
|
||||
|
||||
it("allows safe operational OTLP log bodies while leak checks inspect raw payloads", () => {
|
||||
const assertion = testing.assertSmoke(makePassingSmokeAssertionInput());
|
||||
|
||||
expect(assertion.passed).toBe(true);
|
||||
expect(assertion.failures).toEqual([]);
|
||||
});
|
||||
|
||||
it("allows stdout diagnostic logs without OTLP log requests", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.bodyText = {};
|
||||
input.logRecords = [];
|
||||
input.requests = input.requests.filter((request) => request.signal !== "logs");
|
||||
input.stdoutLogRecords = [
|
||||
{
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
signal: "openclaw.diagnostic.log",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "log",
|
||||
attributes: {
|
||||
"openclaw.log.level": "INFO",
|
||||
},
|
||||
},
|
||||
];
|
||||
input.stdoutLogLines = [JSON.stringify(input.stdoutLogRecords[0])];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(true);
|
||||
expect(assertion.failures).toEqual([]);
|
||||
expect(assertion.signalRequestCounts.logs).toBe(0);
|
||||
expect(assertion.stdoutLogRecordCount).toBe(1);
|
||||
});
|
||||
|
||||
it("fails stdout diagnostic mode when no stdout log records are captured", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.bodyText = {};
|
||||
input.logRecords = [];
|
||||
input.requests = input.requests.filter((request) => request.signal !== "logs");
|
||||
input.stdoutLogRecords = [];
|
||||
input.stdoutLogLines = [];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("no stdout diagnostic log records were captured");
|
||||
expect(assertion.signalRequestCounts.logs).toBe(0);
|
||||
expect(assertion.stdoutLogRecordCount).toBe(0);
|
||||
});
|
||||
|
||||
it("fails stdout diagnostic mode when OTLP log requests are still emitted", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.logRecords = [];
|
||||
input.stdoutLogRecords = [
|
||||
{
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
signal: "openclaw.diagnostic.log",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "log",
|
||||
attributes: {},
|
||||
},
|
||||
];
|
||||
input.stdoutLogLines = [JSON.stringify(input.stdoutLogRecords[0])];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain(
|
||||
"OTLP logs requests were received for stdout logs exporter",
|
||||
);
|
||||
});
|
||||
|
||||
it("still fails when OTLP log payload text leaks scenario content", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.bodyText = {
|
||||
logs: ["diagnostics-otel: log payload contains OTEL-QA-SECRET"],
|
||||
};
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("OTLP logs payload leaked content: OTEL-QA-SECRET");
|
||||
expect(assertion.leakContexts.logs?.[0]).toContain("[needle]");
|
||||
});
|
||||
|
||||
it("still fails when stdout diagnostic log payload text leaks scenario content", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logsExporter = "stdout";
|
||||
input.bodyText = {};
|
||||
input.logRecords = [];
|
||||
input.requests = input.requests.filter((request) => request.signal !== "logs");
|
||||
input.stdoutLogRecords = [
|
||||
{
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
signal: "openclaw.diagnostic.log",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "log",
|
||||
attributes: {},
|
||||
},
|
||||
];
|
||||
input.stdoutLogLines = [
|
||||
JSON.stringify({
|
||||
...input.stdoutLogRecords[0],
|
||||
body: "diagnostics-otel: log payload contains OTEL-QA-SECRET",
|
||||
}),
|
||||
];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain(
|
||||
"stdout diagnostic log payload leaked content: OTEL-QA-SECRET",
|
||||
);
|
||||
expect(assertion.leakContexts.logs?.[0]).toContain("[needle]");
|
||||
});
|
||||
|
||||
it("still requires OTLP log records to carry trace correlation", () => {
|
||||
const input = makePassingSmokeAssertionInput();
|
||||
input.logRecords = [
|
||||
{
|
||||
body: "diagnostics-otel: logs exporter enabled",
|
||||
traceId: "",
|
||||
spanId: "",
|
||||
},
|
||||
];
|
||||
|
||||
const assertion = testing.assertSmoke(input);
|
||||
|
||||
expect(assertion.passed).toBe(false);
|
||||
expect(assertion.failures).toContain("no OTLP log records included trace/span correlation ids");
|
||||
});
|
||||
|
||||
it("preserves leak markers even when later body text is truncated", () => {
|
||||
const captured: { traces?: string[] } = {};
|
||||
|
||||
testing.appendCapturedBodyText(
|
||||
captured,
|
||||
"traces",
|
||||
Buffer.from(`prefix OTEL-QA-SECRET ${"a".repeat(20)}`),
|
||||
16,
|
||||
["OTEL-QA-SECRET"],
|
||||
);
|
||||
testing.appendCapturedBodyText(captured, "traces", Buffer.from("b".repeat(128)), 16, [
|
||||
"OTEL-QA-SECRET",
|
||||
]);
|
||||
|
||||
expect(captured.traces?.join("\n")).toContain("OTEL-QA-SECRET");
|
||||
expect(captured.traces?.join("\n")).toContain("[captured body text truncated");
|
||||
});
|
||||
|
||||
it("keeps collector output tails bounded without retaining earlier chunks", () => {
|
||||
const output = testing.createBoundedTextAccumulator(64);
|
||||
|
||||
output.append("DO_NOT_RETAIN_COLLECTOR_PREFIX\n");
|
||||
output.append(Buffer.alloc(2048, "x"));
|
||||
output.append("\nCOLLECTOR_TAIL_MARKER\n");
|
||||
|
||||
expect(output.byteLength()).toBeLessThanOrEqual(64);
|
||||
expect(output.text()).toContain("COLLECTOR_TAIL_MARKER");
|
||||
expect(output.text()).toContain("...");
|
||||
expect(output.text()).not.toContain("DO_NOT_RETAIN_COLLECTOR_PREFIX");
|
||||
});
|
||||
|
||||
it("streams gateway stdout artifact records without requiring them in the tail", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-stdout-stream-"));
|
||||
const logPath = path.join(tempRoot, "gateway.stdout.log");
|
||||
const capture = testing.createStdoutDiagnosticLogCapture();
|
||||
const record = {
|
||||
signal: "openclaw.diagnostic.log",
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "early log",
|
||||
attributes: {},
|
||||
};
|
||||
try {
|
||||
writeFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify(record)}\n${"x".repeat(256 * 1024)}\nGATEWAY_STDOUT_TAIL\n`,
|
||||
);
|
||||
|
||||
await testing.appendUtf8FileToStdoutDiagnosticCapture(logPath, capture);
|
||||
capture.flush();
|
||||
|
||||
expect(capture.records).toEqual([record]);
|
||||
expect(capture.lines).toHaveLength(1);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps gateway stdout artifact fallback parsing bounded", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-stdout-artifact-"));
|
||||
const outputDir = path.join(tempRoot, "output");
|
||||
const artifactDir = path.join(outputDir, "artifacts", "gateway-runtime");
|
||||
const record = {
|
||||
signal: "openclaw.diagnostic.log",
|
||||
ts: "2026-06-18T00:00:00.000Z",
|
||||
"service.name": "openclaw-qa-lab-otel-smoke",
|
||||
severityText: "INFO",
|
||||
severityNumber: 9,
|
||||
body: "tail log",
|
||||
attributes: {},
|
||||
};
|
||||
try {
|
||||
mkdirSync(artifactDir, { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(artifactDir, "gateway.stdout.log"),
|
||||
`${JSON.stringify(record)}\n${"x".repeat(256 * 1024)}\n`,
|
||||
);
|
||||
const capture = testing.createStdoutDiagnosticLogCapture();
|
||||
|
||||
await testing.appendGatewayStdoutArtifactLogs({ capture, outputDir });
|
||||
|
||||
expect(capture.records).toEqual([record]);
|
||||
expect(capture.lines).toHaveLength(1);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("times out and kills a wedged QA suite child with a detached gateway", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
|
||||
const tempDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-child-"));
|
||||
const markerPath = path.join(tempDir, "marker.txt");
|
||||
try {
|
||||
const gatewayScript = [
|
||||
"import fs from 'node:fs';",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
`setInterval(() => fs.appendFileSync(${JSON.stringify(markerPath)}, "x"), 20);`,
|
||||
].join("\n");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
[
|
||||
"import childProcess from 'node:child_process';",
|
||||
`childProcess.spawn(process.execPath, ["--input-type=module", "--eval", ${JSON.stringify(
|
||||
gatewayScript,
|
||||
)}], { detached: true, stdio: "ignore" });`,
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join("\n"),
|
||||
],
|
||||
{
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
},
|
||||
);
|
||||
|
||||
await expect(testing.waitForChild(child, 100, 100)).rejects.toThrow(
|
||||
"openclaw qa suite timed out after 100ms",
|
||||
);
|
||||
const sizeAfterReturn = existsSync(markerPath) ? statSync(markerPath).size : 0;
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 150);
|
||||
});
|
||||
const sizeAfterWait = existsSync(markerPath) ? statSync(markerPath).size : 0;
|
||||
expect(sizeAfterWait).toBe(sizeAfterReturn);
|
||||
} finally {
|
||||
rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("clamps oversized QA suite child timers before scheduling", async () => {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
["--input-type=module", "--eval", "setTimeout(() => process.exit(0), 25);"],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
|
||||
await expect(testing.waitForChild(child, MAX_TIMER_TIMEOUT_MS + 1, 100)).resolves.toBe(0);
|
||||
});
|
||||
|
||||
it("uses taskkill for Windows QA suite timeout cleanup", () => {
|
||||
const kill = vi.fn();
|
||||
const runTaskkill = vi.fn(() => ({ status: 0 }));
|
||||
|
||||
testing.terminateChildTree(
|
||||
{ kill, pid: 1234 } as never,
|
||||
"SIGTERM",
|
||||
[],
|
||||
"win32",
|
||||
runTaskkill as never,
|
||||
);
|
||||
|
||||
expect(runTaskkill).toHaveBeenCalledWith(expectedTaskkillPath(), ["/PID", "1234", "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
expect(kill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves Docker collector telemetry off the default host port", async () => {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
stdout: EventEmitter;
|
||||
};
|
||||
child.stderr = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
let writtenConfig = "";
|
||||
const stopDockerContainer = vi.fn(async () => {});
|
||||
const removePath = vi.fn(async () => {});
|
||||
const ports = [4318, 4318, 45679];
|
||||
|
||||
const collector = await testing.startDockerOtelCollector(4317, {
|
||||
mkdtemp: async () => "/tmp/openclaw-otel-collector-test",
|
||||
platform: "linux",
|
||||
randomUUID: () => "00000000-0000-4000-8000-000000000000",
|
||||
reserveLocalPort: async () => ports.shift() ?? 49999,
|
||||
rm: removePath as never,
|
||||
spawn: vi.fn(() => child) as never,
|
||||
stopDockerContainer,
|
||||
waitForLocalPort: async () => {},
|
||||
writeFile: async (_path, config) => {
|
||||
writtenConfig = String(config);
|
||||
},
|
||||
});
|
||||
|
||||
expect(writtenConfig).toContain("endpoint: 127.0.0.1:4318");
|
||||
expect(writtenConfig).toContain("telemetry:");
|
||||
expect(writtenConfig).toContain("address: 127.0.0.1:45679");
|
||||
expect(writtenConfig).not.toContain("address: :8888");
|
||||
|
||||
await collector.close();
|
||||
expect(stopDockerContainer).toHaveBeenCalledWith(
|
||||
"openclaw-otel-smoke-00000000-0000-4000-8000-000000000000",
|
||||
);
|
||||
expect(removePath).toHaveBeenCalledWith("/tmp/openclaw-otel-collector-test", {
|
||||
force: true,
|
||||
recursive: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("cleans Docker collector containers and temp config after readiness failures", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-collector-"));
|
||||
const collectorDir = path.join(tempRoot, "collector");
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
stdout: EventEmitter;
|
||||
};
|
||||
child.stderr = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
const stopDockerContainer = vi.fn(async () => {});
|
||||
const ports = [4318, 45679];
|
||||
|
||||
try {
|
||||
await expect(
|
||||
testing.startDockerOtelCollector(4317, {
|
||||
mkdtemp: async () => {
|
||||
mkdirSync(collectorDir);
|
||||
return collectorDir;
|
||||
},
|
||||
randomUUID: () => "00000000-0000-4000-8000-000000000000",
|
||||
reserveLocalPort: async () => ports.shift() ?? 49999,
|
||||
spawn: vi.fn(() => child) as never,
|
||||
stopDockerContainer,
|
||||
waitForLocalPort: async () => {
|
||||
throw new Error("collector never became ready");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("collector never became ready");
|
||||
|
||||
expect(stopDockerContainer).toHaveBeenCalledWith(
|
||||
"openclaw-otel-smoke-00000000-0000-4000-8000-000000000000",
|
||||
);
|
||||
expect(existsSync(collectorDir)).toBe(false);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reports bounded Docker collector output when readiness exits", async () => {
|
||||
const tempRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-qa-otel-collector-output-"));
|
||||
const collectorDir = path.join(tempRoot, "collector");
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stderr: EventEmitter;
|
||||
stdout: EventEmitter;
|
||||
};
|
||||
child.stderr = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
const ports = [4318, 45679];
|
||||
|
||||
try {
|
||||
let thrown: unknown;
|
||||
await testing
|
||||
.startDockerOtelCollector(4317, {
|
||||
mkdtemp: async () => {
|
||||
mkdirSync(collectorDir);
|
||||
return collectorDir;
|
||||
},
|
||||
randomUUID: () => "00000000-0000-4000-8000-000000000000",
|
||||
reserveLocalPort: async () => ports.shift() ?? 49999,
|
||||
spawn: vi.fn(() => child) as never,
|
||||
stopDockerContainer: vi.fn(async () => {}),
|
||||
waitForLocalPort: async (_port, _timeout, readFailure) => {
|
||||
child.stdout.emit("data", "DO_NOT_DUMP_COLLECTOR_PREFIX\n");
|
||||
child.stderr.emit("data", Buffer.alloc(64 * 1024, "x"));
|
||||
child.stderr.emit("data", "\nCOLLECTOR_TAIL_MARKER\n");
|
||||
child.emit("close", 1);
|
||||
await delay(0);
|
||||
throw new Error(readFailure());
|
||||
},
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
thrown = error;
|
||||
});
|
||||
|
||||
expect(thrown).toBeInstanceOf(Error);
|
||||
const message = thrown instanceof Error ? thrown.message : String(thrown);
|
||||
expect(message).toContain("COLLECTOR_TAIL_MARKER");
|
||||
expect(message).not.toContain("DO_NOT_DUMP_COLLECTOR_PREFIX");
|
||||
expect(message.length).toBeLessThan(24 * 1024);
|
||||
} finally {
|
||||
rmSync(tempRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
199
test/e2e/qa-lab/runtime/script-evidence.test.ts
Normal file
199
test/e2e/qa-lab/runtime/script-evidence.test.ts
Normal file
@@ -0,0 +1,199 @@
|
||||
// QA script evidence writer tests cover status, bounded logs, and artifact paths.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { validateQaEvidenceSummaryJson } from "../../../../extensions/qa-lab/api.js";
|
||||
import {
|
||||
createQaScriptBlockedStatusTracker,
|
||||
createQaScriptEvidenceWriter,
|
||||
} from "./script-evidence.js";
|
||||
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
async function makeWriter(
|
||||
params: { maxDetailsBytes?: number; maxLogBytes?: number } = {},
|
||||
) {
|
||||
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-script-evidence-"));
|
||||
tempRoots.push(repoRoot);
|
||||
return {
|
||||
artifactBase: path.join(repoRoot, ".artifacts", "qa-e2e", "script"),
|
||||
repoRoot,
|
||||
writer: createQaScriptEvidenceWriter({
|
||||
artifactBase: path.join(repoRoot, ".artifacts", "qa-e2e", "script"),
|
||||
logFileName: "producer.log",
|
||||
maxDetailsBytes: params.maxDetailsBytes,
|
||||
maxLogBytes: params.maxLogBytes ?? 64,
|
||||
primaryModel: "mock-openai/gpt-5.5",
|
||||
providerMode: "mock-openai",
|
||||
repoRoot,
|
||||
target: {
|
||||
id: "script-evidence-test",
|
||||
title: "Script evidence test",
|
||||
sourcePath: "test/e2e/qa-lab/runtime/script-evidence.test.ts",
|
||||
primaryCoverageIds: ["qa.script-evidence"],
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await Promise.all(
|
||||
tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe("QA script evidence writer", () => {
|
||||
for (const status of ["pass", "fail", "blocked"] as const) {
|
||||
it(`writes ${status} evidence with normalized artifact paths`, async () => {
|
||||
const { artifactBase, writer } = await makeWriter();
|
||||
const summaryPath = path.join(artifactBase, "nested", "summary.json");
|
||||
await fs.mkdir(path.dirname(summaryPath), { recursive: true });
|
||||
await fs.writeFile(summaryPath, "{}\n", "utf8");
|
||||
writer.appendLog("producer output\n");
|
||||
|
||||
const evidence = await writer.write({
|
||||
artifacts: [{ kind: "summary", filePath: summaryPath }],
|
||||
details: `${status} details`,
|
||||
durationMs: 25,
|
||||
status,
|
||||
});
|
||||
|
||||
expect(evidence.entries[0]).toMatchObject({
|
||||
execution: {
|
||||
artifacts: [
|
||||
{ kind: "log", path: "producer.log", source: "script" },
|
||||
{ kind: "summary", path: path.join("nested", "summary.json"), source: "script" },
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status,
|
||||
timing: { wallMs: 25 },
|
||||
},
|
||||
});
|
||||
const diskEvidence = validateQaEvidenceSummaryJson(
|
||||
JSON.parse(await fs.readFile(path.join(artifactBase, "qa-evidence.json"), "utf8")),
|
||||
);
|
||||
expect(diskEvidence).toEqual(evidence);
|
||||
expect(
|
||||
JSON.parse(await fs.readFile(path.join(artifactBase, "latest-run.json"), "utf8")),
|
||||
).toEqual({ qaEvidence: "qa-evidence.json" });
|
||||
});
|
||||
}
|
||||
|
||||
it("keeps only the bounded log tail", async () => {
|
||||
const { artifactBase, writer } = await makeWriter({ maxLogBytes: 24 });
|
||||
writer.appendLog(`discard-me-${"x".repeat(64)}`);
|
||||
writer.appendLog("recent-tail");
|
||||
|
||||
await writer.write({ durationMs: 1, status: "pass" });
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
expect(log).toContain("recent-tail");
|
||||
expect(log).not.toContain("discard-me");
|
||||
expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(24);
|
||||
});
|
||||
|
||||
it("keeps only the bounded failure detail tail", async () => {
|
||||
const { writer } = await makeWriter({ maxDetailsBytes: 24 });
|
||||
|
||||
const evidence = writer.build({
|
||||
details: `discard-me-${"x".repeat(64)}recent-reason`,
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
});
|
||||
|
||||
const reason = evidence.entries[0]?.result.failure?.reason ?? "";
|
||||
expect(reason).toContain("recent-reason");
|
||||
expect(reason).not.toContain("discard-me");
|
||||
expect(Buffer.byteLength(reason, "utf8")).toBeLessThanOrEqual(24);
|
||||
});
|
||||
|
||||
it("keeps UTF-8 logs and failure details within byte limits", async () => {
|
||||
const { artifactBase, writer } = await makeWriter({
|
||||
maxDetailsBytes: 9,
|
||||
maxLogBytes: 9,
|
||||
});
|
||||
const diagnostic = `${"🙂".repeat(32)}done`;
|
||||
writer.appendLog(diagnostic);
|
||||
|
||||
const evidence = await writer.write({
|
||||
details: diagnostic,
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
});
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
const reason = evidence.entries[0]?.result.failure?.reason ?? "";
|
||||
expect(log).toContain("done");
|
||||
expect(reason).toContain("done");
|
||||
expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(9);
|
||||
expect(Buffer.byteLength(reason, "utf8")).toBeLessThanOrEqual(9);
|
||||
});
|
||||
|
||||
it("applies built-in and configured redaction before bounding logs and details", async () => {
|
||||
const { artifactBase, repoRoot, writer } = await makeWriter({
|
||||
maxDetailsBytes: 4096,
|
||||
maxLogBytes: 4096,
|
||||
});
|
||||
const configPath = path.join(repoRoot, "openclaw.json");
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
`${JSON.stringify({ logging: { redactPatterns: ["/internal-\\d+/g"] } })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
vi.stubEnv("OPENCLAW_CONFIG_PATH", configPath);
|
||||
writer.appendLog(`${"x".repeat(16_380)}inter`);
|
||||
writer.appendLog("nal-12345 should hide password=s");
|
||||
writer.appendLog("k-split-secret-1234567890");
|
||||
|
||||
const evidence = await writer.write({
|
||||
details: "reason internal-67890 should hide",
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
});
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
expect(log).toContain("*** should hide password=sk-spl…7890");
|
||||
expect(log).not.toContain("internal-12345");
|
||||
expect(evidence.entries[0]?.result.failure?.reason).toBe("reason *** should hide");
|
||||
});
|
||||
|
||||
it("omits oversized raw logs instead of truncating before redaction", async () => {
|
||||
const secretTail = "private-secret-tail-1234567890";
|
||||
const { artifactBase, writer } = await makeWriter({ maxLogBytes: 128 });
|
||||
writer.appendLog(`password=${"x".repeat(40 * 1024)}${secretTail}`);
|
||||
|
||||
await writer.write({ durationMs: 1, status: "fail" });
|
||||
|
||||
const log = await fs.readFile(path.join(artifactBase, "producer.log"), "utf8");
|
||||
expect(log).toBe("QA evidence log omitted: safe redaction buffer exceeded.\n");
|
||||
expect(log).not.toContain(secretTail);
|
||||
expect(Buffer.byteLength(log, "utf8")).toBeLessThanOrEqual(128);
|
||||
});
|
||||
|
||||
it.each(["..", "../outside.log"])(
|
||||
"rejects artifact path %s outside the producer output directory",
|
||||
async (filePath) => {
|
||||
const { writer } = await makeWriter();
|
||||
|
||||
expect(() =>
|
||||
writer.build({
|
||||
artifacts: [{ kind: "log", filePath }],
|
||||
durationMs: 1,
|
||||
status: "fail",
|
||||
}),
|
||||
).toThrow("QA evidence artifact must be inside artifact base");
|
||||
},
|
||||
);
|
||||
|
||||
it("tracks blocked output before discarded diagnostic tails", () => {
|
||||
const tracker = createQaScriptBlockedStatusTracker([/missing provider auth/i]);
|
||||
|
||||
tracker.append("missing provider ");
|
||||
tracker.append(`auth\n${"x".repeat(4096)}`);
|
||||
|
||||
expect(tracker.status()).toBe("blocked");
|
||||
});
|
||||
});
|
||||
236
test/e2e/qa-lab/runtime/script-evidence.ts
Normal file
236
test/e2e/qa-lab/runtime/script-evidence.ts
Normal file
@@ -0,0 +1,236 @@
|
||||
// Shared QA script evidence writer keeps producer logs and artifact paths bounded.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
buildScriptEvidenceSummary,
|
||||
QA_EVIDENCE_FILENAME,
|
||||
type QaEvidencePackageSource,
|
||||
type QaEvidenceStatus,
|
||||
type QaEvidenceSummaryJson,
|
||||
type QaProviderMode,
|
||||
} from "../../../../extensions/qa-lab/api.js";
|
||||
import { readLoggingConfig } from "../../../../src/logging/config.js";
|
||||
import { withFullContextToolPayloadRedaction } from "../../../../src/logging/redact-internal.js";
|
||||
import { redactToolPayloadTextWithConfig } from "../../../../src/logging/redact.js";
|
||||
import { DEFAULT_CHILD_OUTPUT_TAIL_BYTES } from "../../../helpers/bounded-child-output.js";
|
||||
|
||||
export const DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES = 32 * 1024;
|
||||
const QA_SCRIPT_STATUS_MATCH_CARRY_CHARS = 1024;
|
||||
const QA_SCRIPT_LOG_OVERFLOW_MESSAGE = "QA evidence log omitted: safe redaction buffer exceeded.\n";
|
||||
const QA_SCRIPT_DETAILS_OVERFLOW_MESSAGE =
|
||||
"QA evidence details omitted: safe redaction buffer exceeded.";
|
||||
|
||||
type QaScriptEvidenceArtifactInput = {
|
||||
filePath: string;
|
||||
kind: string;
|
||||
};
|
||||
|
||||
type QaScriptEvidenceTarget = {
|
||||
codeRefs?: readonly string[];
|
||||
docsRefs?: readonly string[];
|
||||
id: string;
|
||||
primaryCoverageIds?: readonly string[];
|
||||
secondaryCoverageIds?: readonly string[];
|
||||
sourcePath: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
export type QaScriptEvidenceStatus = Exclude<QaEvidenceStatus, "skipped">;
|
||||
|
||||
type QaScriptEvidenceResult = {
|
||||
artifacts?: readonly QaScriptEvidenceArtifactInput[];
|
||||
details?: string;
|
||||
durationMs: number;
|
||||
status: QaScriptEvidenceStatus;
|
||||
};
|
||||
|
||||
type QaScriptEvidenceWriterOptions = {
|
||||
artifactBase: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
evidenceMode?: "full" | "slim";
|
||||
logFileName: string;
|
||||
maxDetailsBytes?: number;
|
||||
maxLogBytes?: number;
|
||||
packageSource?: QaEvidencePackageSource;
|
||||
primaryModel: string;
|
||||
providerMode: QaProviderMode;
|
||||
repoRoot: string;
|
||||
target: QaScriptEvidenceTarget;
|
||||
};
|
||||
|
||||
function resolveArtifactPath(artifactBase: string, filePath: string) {
|
||||
const absoluteArtifactBase = path.resolve(artifactBase);
|
||||
const absoluteFilePath = path.resolve(absoluteArtifactBase, filePath);
|
||||
const relativePath = path.relative(absoluteArtifactBase, absoluteFilePath);
|
||||
const escapesArtifactBase =
|
||||
relativePath === ".." ||
|
||||
relativePath.startsWith(`..${path.sep}`) ||
|
||||
path.isAbsolute(relativePath);
|
||||
if (!relativePath || escapesArtifactBase) {
|
||||
throw new Error(`QA evidence artifact must be inside artifact base: ${filePath}`);
|
||||
}
|
||||
return { absoluteFilePath, relativePath };
|
||||
}
|
||||
|
||||
async function writeJson(filePath: string, value: unknown) {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function resolveByteLimit(value: number | undefined, fallback: number) {
|
||||
return Number.isInteger(value) && Number(value) > 0 ? Number(value) : fallback;
|
||||
}
|
||||
|
||||
function utf8Tail(text: string, maxBytes: number) {
|
||||
const buffer = Buffer.from(text, "utf8");
|
||||
if (buffer.byteLength <= maxBytes) {
|
||||
return text;
|
||||
}
|
||||
let start = buffer.byteLength - maxBytes;
|
||||
while (start < buffer.byteLength && (buffer[start]! & 0xc0) === 0x80) {
|
||||
start += 1;
|
||||
}
|
||||
return buffer.subarray(start).toString("utf8");
|
||||
}
|
||||
|
||||
function createSafeRedactionBuffer(maxBytes: number) {
|
||||
const chunks: Buffer[] = [];
|
||||
let totalBytes = 0;
|
||||
let overflowed = false;
|
||||
|
||||
return {
|
||||
append(chunk: string) {
|
||||
if (overflowed) {
|
||||
return;
|
||||
}
|
||||
const buffer = Buffer.from(chunk);
|
||||
if (totalBytes + buffer.byteLength > maxBytes) {
|
||||
// Arbitrary configured patterns need the complete raw context. Omit an
|
||||
// oversized log instead of truncating before redaction and leaking a tail.
|
||||
chunks.length = 0;
|
||||
totalBytes = 0;
|
||||
overflowed = true;
|
||||
return;
|
||||
}
|
||||
chunks.push(buffer);
|
||||
totalBytes += buffer.byteLength;
|
||||
},
|
||||
overflowed() {
|
||||
return overflowed;
|
||||
},
|
||||
text() {
|
||||
return Buffer.concat(chunks, totalBytes).toString("utf8");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createQaScriptBlockedStatusTracker(blockedPatterns: readonly RegExp[]) {
|
||||
let blocked = false;
|
||||
let carry = "";
|
||||
|
||||
return {
|
||||
append(chunk: unknown) {
|
||||
if (blocked) {
|
||||
return;
|
||||
}
|
||||
const text = `${carry}${String(chunk)}`;
|
||||
blocked = blockedPatterns.some((pattern) => {
|
||||
pattern.lastIndex = 0;
|
||||
return pattern.test(text);
|
||||
});
|
||||
// Keep enough overlap for a prerequisite phrase split across stream chunks.
|
||||
carry = text.slice(-QA_SCRIPT_STATUS_MATCH_CARRY_CHARS);
|
||||
},
|
||||
status(): QaScriptEvidenceStatus {
|
||||
return blocked ? "blocked" : "fail";
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOptions) {
|
||||
const maxLogBytes = resolveByteLimit(options.maxLogBytes, DEFAULT_CHILD_OUTPUT_TAIL_BYTES);
|
||||
const logFile = resolveArtifactPath(options.artifactBase, options.logFileName);
|
||||
const maxDetailsBytes = resolveByteLimit(
|
||||
options.maxDetailsBytes,
|
||||
DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES,
|
||||
);
|
||||
const log = createSafeRedactionBuffer(maxLogBytes + DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES);
|
||||
const redact = (text: string) =>
|
||||
redactToolPayloadTextWithConfig(text, withFullContextToolPayloadRedaction(readLoggingConfig()));
|
||||
const boundedLogText = () => {
|
||||
if (log.overflowed()) {
|
||||
return utf8Tail(QA_SCRIPT_LOG_OVERFLOW_MESSAGE, maxLogBytes);
|
||||
}
|
||||
return utf8Tail(redact(log.text()), maxLogBytes);
|
||||
};
|
||||
|
||||
const boundedDetails = (details: string | undefined) => {
|
||||
if (!details) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
Buffer.byteLength(details, "utf8") >
|
||||
maxDetailsBytes + DEFAULT_QA_SCRIPT_EVIDENCE_DETAILS_BYTES
|
||||
) {
|
||||
return utf8Tail(QA_SCRIPT_DETAILS_OVERFLOW_MESSAGE, maxDetailsBytes);
|
||||
}
|
||||
return utf8Tail(redact(details), maxDetailsBytes);
|
||||
};
|
||||
|
||||
const normalizeArtifacts = (artifacts: readonly QaScriptEvidenceArtifactInput[] = []) => {
|
||||
const normalized = [
|
||||
{ kind: "log", path: logFile.relativePath },
|
||||
...artifacts.map((artifact) => ({
|
||||
kind: artifact.kind,
|
||||
path: resolveArtifactPath(options.artifactBase, artifact.filePath).relativePath,
|
||||
})),
|
||||
];
|
||||
return [
|
||||
...new Map(
|
||||
normalized.map((artifact) => [`${artifact.kind}:${artifact.path}`, artifact]),
|
||||
).values(),
|
||||
];
|
||||
};
|
||||
|
||||
const build = (result: QaScriptEvidenceResult): QaEvidenceSummaryJson =>
|
||||
buildScriptEvidenceSummary({
|
||||
artifactPaths: normalizeArtifacts(result.artifacts),
|
||||
evidenceMode: options.evidenceMode ?? "full",
|
||||
env: options.env ?? process.env,
|
||||
generatedAt: new Date().toISOString(),
|
||||
packageSource: options.packageSource,
|
||||
primaryModel: options.primaryModel,
|
||||
providerMode: options.providerMode,
|
||||
repoRoot: options.repoRoot,
|
||||
runner: "script",
|
||||
targets: [options.target],
|
||||
results: [
|
||||
{
|
||||
id: options.target.id,
|
||||
status: result.status,
|
||||
durationMs: result.durationMs,
|
||||
failureMessage: boundedDetails(result.details),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
appendLog(chunk: unknown) {
|
||||
log.append(String(chunk));
|
||||
},
|
||||
build,
|
||||
logText() {
|
||||
return boundedLogText();
|
||||
},
|
||||
async write(result: QaScriptEvidenceResult) {
|
||||
const evidence = build(result);
|
||||
await fs.mkdir(options.artifactBase, { recursive: true });
|
||||
await fs.writeFile(logFile.absoluteFilePath, boundedLogText(), "utf8");
|
||||
await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence);
|
||||
await writeJson(path.join(options.artifactBase, "latest-run.json"), {
|
||||
qaEvidence: QA_EVIDENCE_FILENAME,
|
||||
});
|
||||
return evidence;
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user