title: "Kitchen Sink live OpenAI plugin gauntlet" scenario: id: kitchen-sink-live-openai surface: plugins category: pre-release coverage: primary: - plugins.kitchen-sink secondary: - plugins.lifecycle - plugins.plugin-tools - models.live-openai - gateway.performance risk: high objective: Verify the external Kitchen Sink plugin can be installed into a qa-lab gateway, expose its major runtime surfaces, and coexist with a live OpenAI provider turn. successCriteria: - The npm Kitchen Sink package installs, enables, and inspects as loaded. - Kitchen Sink command inventory, MCP tool, and channel status work after gateway restart. - A live OpenAI turn either completes or surfaces the Kitchen Sink plugin's before-agent-run block without hanging. - Gateway logs and process metrics are captured and stay under broad anomaly thresholds. docsRefs: - docs/concepts/qa-e2e-automation.md - docs/channels/qa-channel.md - docs/plugins/manifest.md codeRefs: - extensions/qa-lab/src/suite.ts - extensions/qa-lab/src/gateway-child.ts - scripts/e2e/kitchen-sink-plugin-docker.sh execution: kind: flow suiteIsolation: isolated isolationReason: Mutates gateway plugin/channel/tool config across gateway restarts. summary: Install @openclaw/kitchen-sink, restart the gateway, exercise command inventory/tool/channel/OpenAI-or-block paths, and record CPU/RSS/log evidence. config: requiredProviderMode: live-frontier requiredProvider: openai pluginSpec: npm:@openclaw/kitchen-sink@latest pluginId: openclaw-kitchen-sink-fixture pluginPersonality: conformance adversarialPersonality: adversarial channelId: kitchen-sink-channel channelAccountId: local textProviderId: kitchen-sink-llm textModel: kitchen-sink-text-v1 expectedProviderAny: - kitchen-sink-provider - kitchen-sink-llm expectedToolAny: - kitchen_sink_text - kitchen_sink_search - kitchen_sink_image_job expectedSurfaceIds: speechProviderIds: - kitchen-sink-speech - kitchen-sink-speech-provider realtimeTranscriptionProviderIds: - kitchen-sink-realtime-transcription - kitchen-sink-realtime-transcription-provider realtimeVoiceProviderIds: - kitchen-sink-realtime-voice - kitchen-sink-realtime-voice-provider mediaUnderstandingProviderIds: - kitchen-sink-media - kitchen-sink-media-understanding-provider imageGenerationProviderIds: - kitchen-sink-image - kitchen-sink-image-generation-provider videoGenerationProviderIds: - kitchen-sink-video - kitchen-sink-video-generation-provider musicGenerationProviderIds: - kitchen-sink-music - kitchen-sink-music-generation-provider webFetchProviderIds: - kitchen-sink-fetch - kitchen-sink-web-fetch-provider webSearchProviderIds: - kitchen-sink-search - kitchen-sink-web-search-provider migrationProviderIds: - kitchen-sink-migration-providers - kitchen-sink-migration-provider maxGatewayCpuCoreRatio: 1.5 maxGatewayRssMiB: 2048 agentTurnTimeoutMs: 120000 outboundTimeoutMs: 60000 livePrompt: "Kitchen Sink OpenAI marker. Reply exactly: KITCHEN-SINK-OPENAI-OK" blockedReplyNeedle: "blocked by openclaw-kitchen-sink-fixture" expectedAdversarialDiagnostics: - agent event subscription registration requires id and handle - agent tool result middleware must be a function - agent harness "kitchen-sink-agent-harness" registration missing required runtime methods - channel "kitchen-sink-channel-probe" registration missing required config helpers - cli registration missing explicit commands metadata - only bundled plugins can register Codex app-server extension factories - compaction provider "kitchen-sink-compaction-provider" registration missing summarize - context engine registration missing id - hosted media resolver registration missing resolver - "http route registration missing or invalid auth: /kitchen-sink/http-route" - "plugin must declare contracts.embeddingProviders for adapter: kitchen-sink-embedding-provider" - "plugin must own memory slot or declare contracts.memoryEmbeddingProviders for adapter: kitchen-sink-memory-embedding-provider" - "trusted tool policy registration requires id, description, and evaluate()" - memory prompt supplement registration missing builder - model catalog provider registration missing provider - node invoke policy registration missing commands - session extension registration requires namespace and description - session scheduler job registration requires unique id, sessionKey, and kind - "plugin must declare contracts.tools for: kitchen-sink-tool" - tool metadata registration missing toolName flow: steps: - name: installs and inspects the Kitchen Sink plugin actions: - call: runQaCli args: - ref: env - - plugins - install - expr: config.pluginSpec - timeoutMs: 180000 - call: runQaCli args: - ref: env - - plugins - enable - expr: config.pluginId - timeoutMs: 60000 - set: configuredPluginPath value: expr: |- (async () => { const raw = await fs.readFile(env.gateway.configPath, "utf8").catch(() => "{}"); const cfg = JSON.parse(raw || "{}"); cfg.plugins = cfg.plugins || {}; cfg.plugins.allow = [...new Set([...(cfg.plugins.allow || []), config.pluginId])]; cfg.plugins.entries = cfg.plugins.entries || {}; cfg.plugins.entries[config.pluginId] = { ...(cfg.plugins.entries[config.pluginId] || {}), enabled: true, config: { ...(cfg.plugins.entries[config.pluginId]?.config || {}), personality: config.pluginPersonality, }, hooks: { ...(cfg.plugins.entries[config.pluginId]?.hooks || {}), allowConversationAccess: true, }, }; cfg.channels = { ...(cfg.channels || {}), [config.channelId]: { enabled: true, token: "kitchen-sink-qa" }, }; cfg.tools = { ...(cfg.tools || {}), alsoAllow: [...new Set([...(cfg.tools?.alsoAllow || []), ...config.expectedToolAny])], }; await fs.writeFile(env.gateway.configPath, `${JSON.stringify(cfg, null, 2)}\n`, "utf8"); return env.gateway.configPath; })() - call: runQaCli saveAs: pluginList args: - ref: env - - plugins - list - --json - json: true timeoutMs: 60000 - call: runQaCli saveAs: inspect args: - ref: env - - plugins - inspect - expr: config.pluginId - --runtime - --json - json: true timeoutMs: 60000 - set: inspectFacts value: expr: |- (() => { const plugin = inspect.plugin ?? {}; const namesFromTools = Array.isArray(inspect.tools) ? inspect.tools.flatMap((entry) => Array.isArray(entry?.names) ? entry.names : [entry?.name]).filter(Boolean) : []; const contracts = plugin.contracts && typeof plugin.contracts === "object" ? plugin.contracts : {}; return { id: plugin.id, enabled: plugin.enabled, status: plugin.status, channels: [...new Set([...(plugin.channelIds ?? []), ...(plugin.channels ?? [])])], providers: [...new Set([...(plugin.providerIds ?? []), ...(plugin.providers ?? [])])], tools: [...new Set([...namesFromTools, ...(contracts.tools ?? [])])], commands: inspect.commands ?? [], services: inspect.services ?? [], typedHookCount: Array.isArray(inspect.typedHooks) ? inspect.typedHooks.length : 0, hookCount: plugin.hookCount ?? 0, surfaceIds: Object.fromEntries( Object.keys(config.expectedSurfaceIds ?? {}) .map((field) => [field, Array.isArray(plugin[field]) ? plugin[field] : []]) ), agentHarnessIds: plugin.agentHarnessIds ?? [], diagnostics: [...(pluginList.diagnostics ?? []), ...(inspect.diagnostics ?? [])] .filter((entry) => entry?.level === "error") .map((entry) => String(entry.message ?? "")), unexpectedDiagnostics: [...new Set([...(pluginList.diagnostics ?? []), ...(inspect.diagnostics ?? [])] .filter((entry) => entry?.level === "error") .map((entry) => String(entry.message ?? "")) .filter((message) => !config.expectedAdversarialDiagnostics.includes(message)))], }; })() - assert: expr: "inspectFacts.id === config.pluginId && inspectFacts.enabled === true && inspectFacts.status === 'loaded'" message: expr: "`Kitchen Sink plugin did not inspect as enabled+loaded: ${JSON.stringify(inspectFacts)}`" - assert: expr: "inspectFacts.channels.includes(config.channelId)" message: expr: "`Kitchen Sink channel missing from inspect output: ${JSON.stringify(inspectFacts.channels)}`" - assert: expr: "config.expectedProviderAny.some((provider) => inspectFacts.providers.includes(provider))" message: expr: "`Kitchen Sink providers missing from inspect output: ${JSON.stringify(inspectFacts.providers)}`" - assert: expr: "config.expectedToolAny.some((tool) => inspectFacts.tools.includes(tool))" message: expr: "`Kitchen Sink tools missing from inspect output: ${JSON.stringify(inspectFacts.tools)}`" - assert: expr: "Object.entries(config.expectedSurfaceIds).every(([field, expected]) => expected.some((id) => (inspectFacts.surfaceIds[field] ?? []).includes(id)))" message: expr: "`Kitchen Sink SDK provider surface missing from inspect output: ${JSON.stringify(inspectFacts.surfaceIds)}`" - assert: expr: "inspectFacts.commands.includes('kitchen') && inspectFacts.services.includes('kitchen-sink-service')" message: expr: "`Kitchen Sink command/service surfaces missing: ${JSON.stringify({ commands: inspectFacts.commands, services: inspectFacts.services })}`" - assert: expr: "inspectFacts.hookCount >= 30 && inspectFacts.typedHookCount >= 30" message: expr: "`Kitchen Sink hook surfaces missing: ${JSON.stringify({ hookCount: inspectFacts.hookCount, typedHookCount: inspectFacts.typedHookCount })}`" - assert: expr: "!inspectFacts.agentHarnessIds.includes('kitchen-sink-agent-harness')" message: expr: "`External Kitchen Sink plugin unexpectedly registered bundled-only agent harness: ${JSON.stringify(inspectFacts.agentHarnessIds)}`" - assert: expr: "inspectFacts.unexpectedDiagnostics.length === 0" message: expr: "`Kitchen Sink conformance personality emitted unexpected diagnostics: ${JSON.stringify(inspectFacts.unexpectedDiagnostics)}`" detailsExpr: inspectFacts - name: restarts gateway with Kitchen Sink configured actions: - assert: expr: "typeof env.gateway.restartAfterStateMutation === 'function'" message: "qa gateway child does not expose restartAfterStateMutation" - call: env.gateway.restartAfterStateMutation args: - lambda: async: true params: [ctx] expr: |- (async () => { const raw = await fs.readFile(ctx.configPath, "utf8").catch(() => "{}"); const cfg = JSON.parse(raw || "{}"); cfg.plugins = cfg.plugins || {}; cfg.plugins.allow = [...new Set([...(cfg.plugins.allow || []), config.pluginId])]; cfg.plugins.entries = cfg.plugins.entries || {}; cfg.plugins.entries[config.pluginId] = { ...(cfg.plugins.entries[config.pluginId] || {}), enabled: true, config: { ...(cfg.plugins.entries[config.pluginId]?.config || {}), personality: config.pluginPersonality, }, hooks: { ...(cfg.plugins.entries[config.pluginId]?.hooks || {}), allowConversationAccess: true, }, }; cfg.channels = { ...(cfg.channels || {}), [config.channelId]: { enabled: true, token: "kitchen-sink-qa" }, }; cfg.tools = { ...(cfg.tools || {}), alsoAllow: [...new Set([...(cfg.tools?.alsoAllow || []), ...config.expectedToolAny])], }; await fs.writeFile(ctx.configPath, `${JSON.stringify(cfg, null, 2)}\n`, "utf8"); })() - call: waitForGatewayHealthy args: - ref: env - 120000 - call: fetchJson saveAs: healthz args: - expr: "`${env.gateway.baseUrl}/healthz`" - call: fetchJson saveAs: readyz args: - expr: "`${env.gateway.baseUrl}/readyz`" - assert: expr: "healthz?.ok === true && healthz?.status === 'live'" message: expr: "`/healthz did not report live: ${JSON.stringify(healthz)}`" - assert: expr: "readyz?.ready === true" message: expr: "`/readyz did not report ready: ${JSON.stringify(readyz)}`" - call: waitForQaChannelReady args: - ref: env - 120000 - set: perfStartedAtMs value: expr: "Date.now()" - set: cpuStartMs value: expr: "env.gateway.getProcessCpuMs?.() ?? null" - set: rssStartBytes value: expr: "env.gateway.getProcessRssBytes?.() ?? null" - call: env.gateway.call saveAs: channelStatus args: - channels.status - probe: true timeoutMs: 10000 - timeoutMs: 15000 - set: kitchenChannelAccount value: expr: "(channelStatus.channelAccounts?.[config.channelId] ?? []).find((entry) => entry.accountId === config.channelAccountId) ?? null" - assert: expr: "kitchenChannelAccount?.running === true && kitchenChannelAccount?.configured === true" message: expr: "`Kitchen Sink channel did not report running+configured: ${JSON.stringify(kitchenChannelAccount)}`" detailsExpr: "{ healthz, readyz, kitchenChannelAccount }" - name: exercises command inventory and MCP tool surfaces actions: - call: env.gateway.call saveAs: commandList args: - commands.list - agentId: qa scope: text - timeoutMs: 15000 - set: pluginCommandNames value: expr: "(commandList.commands ?? []).filter((entry) => entry.source === 'plugin').map((entry) => entry.name).sort()" - assert: expr: "pluginCommandNames.includes('kitchen') && pluginCommandNames.includes('kitchen-sink')" message: expr: "`Kitchen Sink plugin commands missing from commands.list: ${JSON.stringify(pluginCommandNames)}`" - call: callPluginToolsMcp saveAs: mcpTool args: - env: ref: env toolName: kitchen_sink_search args: query: "kitchen sink qa live openai" - set: mcpToolText value: expr: "JSON.stringify(mcpTool.content ?? mcpTool)" - assert: expr: "mcpToolText.includes('Kitchen Sink image fixture')" message: expr: "`Kitchen Sink MCP tool output missed expected fixture: ${mcpToolText.slice(0, 500)}`" detailsExpr: "{ pluginCommandNames, mcpToolText: mcpToolText.slice(0, 500) }" - name: runs live OpenAI turn with Kitchen Sink loaded actions: - call: reset - call: runAgentPrompt args: - ref: env - sessionKey: expr: "`agent:qa:kitchen-sink-openai:${randomUUID().slice(0, 8)}`" message: expr: config.livePrompt timeoutMs: expr: liveTurnTimeoutMs(env, config.agentTurnTimeoutMs) - call: waitForOutboundMessage saveAs: openaiReply args: - ref: state - lambda: params: [candidate] expr: "candidate.conversation.id === 'qa-operator' && (candidate.text.includes('KITCHEN-SINK-OPENAI-OK') || normalizeLowercaseStringOrEmpty(candidate.text).includes(config.blockedReplyNeedle))" - expr: liveTurnTimeoutMs(env, config.outboundTimeoutMs) - set: normalizedOpenAiReply value: expr: "normalizeLowercaseStringOrEmpty(openaiReply.text)" - assert: expr: "openaiReply.text.includes('KITCHEN-SINK-OPENAI-OK') || normalizedOpenAiReply.includes(config.blockedReplyNeedle)" message: expr: "`Kitchen Sink live turn neither completed nor surfaced the plugin block: ${openaiReply.text}`" detailsExpr: "{ openaiReply: openaiReply.text }" - name: records gateway CPU RSS and log anomaly evidence actions: - set: perfEvidence value: expr: |- (() => { const cpuStart = typeof vars.cpuStartMs === "number" ? vars.cpuStartMs : null; const cpuEnd = env.gateway.getProcessCpuMs?.() ?? null; const rssStart = typeof vars.rssStartBytes === "number" ? vars.rssStartBytes : null; const rssEnd = env.gateway.getProcessRssBytes?.() ?? null; const logs = env.gateway.logs?.() ?? ""; const deny = [ /\buncaught exception\b/iu, /\bunhandled rejection\b/iu, /\bfatal\b/iu, /\bpanic\b/iu, ]; const findings = logs .split(/\r?\n/u) .filter((line) => deny.some((pattern) => pattern.test(line))) .slice(0, 10) .map((line) => line.replaceAll(env.repoRoot, "").slice(0, 500)); const wallMs = Date.now() - Number(vars.perfStartedAtMs ?? Date.now()); const cpuDeltaMs = cpuStart === null || cpuEnd === null ? null : Math.max(0, cpuEnd - cpuStart); const cpuCoreRatio = cpuDeltaMs === null || wallMs <= 0 ? null : Math.round((cpuDeltaMs / wallMs) * 1000) / 1000; const rssMiB = rssEnd === null ? null : Math.round((rssEnd / 1024 / 1024) * 10) / 10; return { wallMs, cpuStart, cpuEnd, cpuDeltaMs, cpuCoreRatio, rssStartBytes: rssStart, rssEndBytes: rssEnd, rssMiB, logBytes: logs.length, findings, }; })() - assert: expr: "perfEvidence.findings.length === 0" message: expr: "`Gateway logs contain fatal runtime lines: ${JSON.stringify(perfEvidence.findings)}`" - assert: expr: "perfEvidence.cpuCoreRatio === null || perfEvidence.cpuCoreRatio <= config.maxGatewayCpuCoreRatio" message: expr: "`Gateway CPU ratio exceeded Kitchen Sink anomaly threshold: ${JSON.stringify(perfEvidence)}`" - assert: expr: "perfEvidence.rssMiB === null || perfEvidence.rssMiB <= config.maxGatewayRssMiB" message: expr: "`Gateway RSS exceeded Kitchen Sink anomaly threshold: ${JSON.stringify(perfEvidence)}`" detailsExpr: perfEvidence - name: verifies adversarial diagnostics personality actions: - call: env.gateway.restartAfterStateMutation args: - lambda: async: true params: [ctx] expr: |- (async () => { const raw = await fs.readFile(ctx.configPath, "utf8").catch(() => "{}"); const cfg = JSON.parse(raw || "{}"); cfg.plugins = cfg.plugins || {}; cfg.plugins.allow = [...new Set([...(cfg.plugins.allow || []), config.pluginId])]; cfg.plugins.entries = cfg.plugins.entries || {}; cfg.plugins.entries[config.pluginId] = { ...(cfg.plugins.entries[config.pluginId] || {}), enabled: true, config: { ...(cfg.plugins.entries[config.pluginId]?.config || {}), personality: config.adversarialPersonality, }, hooks: { ...(cfg.plugins.entries[config.pluginId]?.hooks || {}), allowConversationAccess: true, }, }; await fs.writeFile(ctx.configPath, `${JSON.stringify(cfg, null, 2)}\n`, "utf8"); })() - call: waitForGatewayHealthy args: - ref: env - 120000 - call: runQaCli saveAs: adversarialInspect args: - ref: env - - plugins - inspect - expr: config.pluginId - --runtime - --json - json: true timeoutMs: 60000 - set: adversarialDiagnostics value: expr: |- (adversarialInspect.diagnostics ?? []) .filter((entry) => entry?.level === "error") .map((entry) => String(entry.message ?? "")) - assert: expr: "config.expectedAdversarialDiagnostics.every((message) => adversarialDiagnostics.includes(message))" message: expr: "`Kitchen Sink adversarial diagnostics missing expected messages: ${JSON.stringify({ expected: config.expectedAdversarialDiagnostics, actual: adversarialDiagnostics })}`" - assert: expr: "adversarialDiagnostics.every((message) => config.expectedAdversarialDiagnostics.includes(message))" message: expr: "`Kitchen Sink adversarial diagnostics contained unexpected messages: ${JSON.stringify(adversarialDiagnostics)}`" detailsExpr: "{ diagnostics: adversarialDiagnostics }"