Vendor OpenClaw source as Adolf fork baseline

Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11),
free to diverge. Tree copied sans upstream .git; upstream remote added for
future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19.
Preserves docs/ARCHITECTURE.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2

qa/scenarios/models/anthropic-opus-api-key-smoke.yaml

@@ -0,0 +1,87 @@
+title: Anthropic Opus API key smoke
+
+scenario:
+  id: anthropic-opus-api-key-smoke
+  surface: model-provider
+  coverage:
+    primary:
+      - models.provider-auth
+    secondary:
+      - models.anthropic
+  objective: Verify the regular Anthropic Opus lane can complete a quick chat turn using API-key auth.
+  successCriteria:
+    - A live-frontier run fails fast unless the selected primary provider is anthropic.
+    - The selected primary model is Anthropic Opus 4.8.
+    - The QA gateway worker has an Anthropic API key available through environment auth.
+    - The agent replies through the regular Anthropic provider.
+  docsRefs:
+    - docs/concepts/model-providers.md
+    - docs/help/testing.md
+  codeRefs:
+    - extensions/anthropic/register.runtime.ts
+    - extensions/qa-lab/src/gateway-child.ts
+    - extensions/qa-lab/src/suite.ts
+  execution:
+    kind: flow
+    summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-4-8 --alt-model anthropic/claude-opus-4-8 --scenario anthropic-opus-api-key-smoke`.
+    config:
+      requiredProvider: anthropic
+      requiredModel: claude-opus-4-8
+      chatPrompt: "Anthropic Opus API key smoke. Reply exactly: ANTHROPIC-OPUS-API-KEY-OK"
+      chatExpected: ANTHROPIC-OPUS-API-KEY-OK
+
+flow:
+  steps:
+    - name: confirms regular Anthropic API-key lane
+      actions:
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            message:
+              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            message:
+              expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())"
+            message: expected ANTHROPIC_API_KEY to be available for API-key QA mode
+      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=env-api-key` : `mock-compatible provider=${selected?.provider}`"
+    - name: talks through regular Anthropic Opus
+      actions:
+        - if:
+            expr: "env.providerMode !== 'live-frontier'"
+            then:
+              - assert: "true"
+            else:
+              - call: reset
+              - set: selected
+                value:
+                  expr: splitModelRef(env.primaryModel)
+              - call: runAgentPrompt
+                args:
+                  - ref: env
+                  - sessionKey: agent:qa:anthropic-opus-api-key
+                    message:
+                      expr: config.chatPrompt
+                    provider:
+                      expr: selected?.provider
+                    model:
+                      expr: selected?.model
+                    timeoutMs:
+                      expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+              - call: waitForOutboundMessage
+                saveAs: chatOutbound
+                args:
+                  - ref: state
+                  - lambda:
+                      params: [candidate]
+                      expr: "candidate.conversation.id === 'qa-operator'"
+                  - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+              - assert:
+                  expr: "chatOutbound.text.includes(config.chatExpected)"
+                  message:
+                    expr: "`chat marker missing: ${chatOutbound.text}`"
+      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"

qa/scenarios/models/anthropic-opus-setup-token-smoke.yaml

@@ -0,0 +1,92 @@
+title: Anthropic Opus setup-token smoke
+
+scenario:
+  id: anthropic-opus-setup-token-smoke
+  surface: model-provider
+  coverage:
+    primary:
+      - models.provider-auth
+    secondary:
+      - models.anthropic
+  objective: Verify the regular Anthropic Opus lane can complete a quick chat turn using setup-token auth.
+  successCriteria:
+    - A live-frontier run fails fast unless the selected primary provider is anthropic.
+    - The selected primary model is Anthropic Opus 4.8.
+    - The QA gateway worker stages a token auth profile in the isolated agent store.
+    - The agent replies through the regular Anthropic provider.
+  docsRefs:
+    - docs/concepts/model-providers.md
+    - docs/help/testing.md
+  codeRefs:
+    - extensions/anthropic/register.runtime.ts
+    - extensions/qa-lab/src/gateway-child.ts
+    - extensions/qa-lab/src/suite.ts
+  execution:
+    kind: flow
+    summary: Run with `OPENCLAW_LIVE_SETUP_TOKEN_VALUE=<setup-token> pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-4-8 --alt-model anthropic/claude-opus-4-8 --scenario anthropic-opus-setup-token-smoke`.
+    config:
+      requiredProvider: anthropic
+      requiredModel: claude-opus-4-8
+      profileId: "anthropic:qa-setup-token"
+      chatPrompt: "Anthropic Opus setup-token smoke. Reply exactly: ANTHROPIC-OPUS-SETUP-TOKEN-OK"
+      chatExpected: ANTHROPIC-OPUS-SETUP-TOKEN-OK
+
+flow:
+  steps:
+    - name: confirms regular Anthropic setup-token lane
+      actions:
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            message:
+              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            message:
+              expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'"
+            message:
+              expr: "`expected token profile ${config.profileId} in QA config`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || !env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE"
+            message: setup-token value should not be passed to the gateway child env
+      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}` : `mock-compatible provider=${selected?.provider}`"
+    - name: talks through regular Anthropic Opus
+      actions:
+        - if:
+            expr: "env.providerMode !== 'live-frontier'"
+            then:
+              - assert: "true"
+            else:
+              - call: reset
+              - set: selected
+                value:
+                  expr: splitModelRef(env.primaryModel)
+              - call: runAgentPrompt
+                args:
+                  - ref: env
+                  - sessionKey: agent:qa:anthropic-opus-setup-token
+                    message:
+                      expr: config.chatPrompt
+                    provider:
+                      expr: selected?.provider
+                    model:
+                      expr: selected?.model
+                    timeoutMs:
+                      expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+              - call: waitForOutboundMessage
+                saveAs: chatOutbound
+                args:
+                  - ref: state
+                  - lambda:
+                      params: [candidate]
+                      expr: "candidate.conversation.id === 'qa-operator'"
+                  - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+              - assert:
+                  expr: "chatOutbound.text.includes(config.chatExpected)"
+                  message:
+                    expr: "`chat marker missing: ${chatOutbound.text}`"
+      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"

qa/scenarios/models/claude-cli-provider-capabilities-subscription.yaml

@@ -0,0 +1,259 @@
+title: Claude CLI provider capabilities subscription
+
+scenario:
+  id: claude-cli-provider-capabilities-subscription
+  surface: model-provider
+  coverage:
+    primary:
+      - models.provider-capabilities
+    secondary:
+      - models.claude-cli
+  objective: Verify the Claude CLI model-provider lane can use native Claude subscription auth to talk, read an attached image, use bundled MCP tools, and apply workspace skills.
+  successCriteria:
+    - A live-frontier run fails fast unless the selected primary provider is claude-cli.
+    - The Claude CLI backend does not preserve ANTHROPIC_API_KEY for this run, forcing native Claude subscription auth.
+    - The agent replies through the Claude CLI provider in a direct chat turn.
+    - The agent describes an attached image through the Claude CLI image path.
+    - The agent can reach memory via the bundled MCP/tool bridge.
+    - The agent sees and follows a workspace skill.
+  docsRefs:
+    - docs/gateway/cli-backends.md
+    - docs/tools/skills.md
+    - docs/cli/mcp.md
+    - docs/tools/index.md
+  codeRefs:
+    - extensions/anthropic/cli-backend.ts
+    - src/agents/cli-backends.ts
+    - src/mcp/plugin-tools-serve.ts
+    - extensions/qa-lab/src/suite.ts
+  execution:
+    kind: flow
+    summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode subscription --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities-subscription`.
+    config:
+      authMode: subscription
+      requiredProviderMode: live-frontier
+      requiredProvider: claude-cli
+      chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK"
+      chatExpected: CLAUDE-CLI-CHAT-OK
+      imagePrompt: "Image understanding check: describe the top and bottom colors in the attached image in one short sentence."
+      imageColorGroups:
+        - [red, scarlet, crimson]
+        - [blue, azure, teal, cyan, aqua]
+      memoryFact: "Hidden Claude CLI MCP fact: the provider bridge codename is ORBIT-9."
+      memoryQuery: "provider bridge codename ORBIT-9"
+      memoryExpected: ORBIT-9
+      memoryPrompt: "Memory tools check: use the available memory search MCP/tool bridge to find the hidden provider bridge codename stored only in memory. Reply with the codename."
+      memoryPromptSnippet: "Memory tools check"
+      skillName: qa-claude-cli-skill
+      skillExpected: VISIBLE-SKILL-OK
+      skillBody: |-
+        ---
+        name: qa-claude-cli-skill
+        description: Claude CLI QA skill marker
+        ---
+        When the user asks for the Claude CLI skill marker exactly, or explicitly asks you to use qa-claude-cli-skill, reply with exactly: VISIBLE-SKILL-OK
+      skillPrompt: "Use qa-claude-cli-skill now. Reply exactly with the visible skill marker and nothing else."
+
+flow:
+  steps:
+    - name: confirms the selected live provider and Claude CLI auth mode
+      actions:
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - set: preserveEnv
+          value:
+            expr: "String(env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV ?? '')"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            message:
+              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE === config.authMode"
+            message:
+              expr: "`expected Claude CLI auth mode ${config.authMode}, got ${env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE ?? 'unset'}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || !preserveEnv.includes('ANTHROPIC_API_KEY')"
+            message:
+              expr: "`expected ANTHROPIC_API_KEY not to be preserved for Claude CLI subscription QA mode, got ${preserveEnv}`"
+      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=${env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE} preserve=${preserveEnv}` : `mock-compatible provider=${selected?.provider}`"
+    - name: talks through the selected provider
+      actions:
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                expr: "`agent:qa:claude-cli-${config.authMode}-chat`"
+              message:
+                expr: config.chatPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 45000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: chatOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 20000, env.primaryModel)
+        - assert:
+            expr: "chatOutbound.text.includes(config.chatExpected)"
+            message:
+              expr: "`chat marker missing: ${chatOutbound.text}`"
+      detailsExpr: chatOutbound.text
+    - name: describes an attached image through the selected provider
+      actions:
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                expr: "`agent:qa:claude-cli-${config.authMode}-image`"
+              message:
+                expr: config.imagePrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              attachments:
+                - mimeType: image/png
+                  fileName: claude-cli-red-top-blue-bottom.png
+                  content:
+                    expr: imageUnderstandingValidPngBase64
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: imageOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+        - assert:
+            expr: "config.imageColorGroups.every((group) => group.some((color) => normalizeLowercaseStringOrEmpty(imageOutbound.text).includes(color)))"
+            message:
+              expr: "`missing expected image colors: ${imageOutbound.text}`"
+        - assert:
+            expr: "!env.mock || (((await fetchJson(`${env.mock.baseUrl}/debug/requests`)).find((request) => String(request.prompt ?? '').includes('Image understanding check'))?.imageInputCount ?? 0) >= 1)"
+            message: expected image input to reach mock provider
+      detailsExpr: imageOutbound.text
+    - name: reaches memory through the MCP/tool bridge
+      actions:
+        - call: reset
+        - call: fs.writeFile
+          args:
+            - expr: "path.join(env.gateway.workspaceDir, 'MEMORY.md')"
+            - expr: "`${config.memoryFact}\\n`"
+            - utf8
+        - call: forceMemoryIndex
+          args:
+            - env:
+                ref: env
+              query:
+                expr: config.memoryQuery
+              expectedNeedle:
+                expr: config.memoryExpected
+        - call: createSession
+          saveAs: mcpSessionKey
+          args:
+            - ref: env
+            - expr: "`Claude CLI MCP bridge ${config.authMode}`"
+        - call: readEffectiveTools
+          saveAs: mcpTools
+          args:
+            - ref: env
+            - ref: mcpSessionKey
+        - assert:
+            expr: "mcpTools.has('memory_search')"
+            message: memory_search missing from effective tools before MCP bridge check
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                ref: mcpSessionKey
+              message:
+                expr: config.memoryPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 90000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: mcpOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 45000, env.primaryModel)
+        - assert:
+            expr: "mcpOutbound.text.includes(config.memoryExpected)"
+            message:
+              expr: "`MCP memory result missing ${config.memoryExpected}: ${mcpOutbound.text}`"
+        - assert:
+            expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).filter((request) => String(request.allInputText ?? '').includes(config.memoryPromptSnippet)).some((request) => request.plannedToolName === 'memory_search')"
+            message: expected mock model to plan memory_search for MCP bridge prompt
+      detailsExpr: mcpOutbound.text
+    - name: applies a workspace skill through the selected provider
+      actions:
+        - call: reset
+        - call: writeWorkspaceSkill
+          args:
+            - env:
+                ref: env
+              name:
+                expr: config.skillName
+              body:
+                expr: config.skillBody
+        - call: waitForCondition
+          args:
+            - lambda:
+                async: true
+                expr: "((await readSkillStatus(env)).find((skill) => skill.name === config.skillName)?.eligible ? true : undefined)"
+            - 15000
+            - 200
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                expr: "`agent:qa:claude-cli-${config.authMode}-skill`"
+              message:
+                expr: config.skillPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: skillOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+        - assert:
+            expr: "skillOutbound.text.includes(config.skillExpected)"
+            message:
+              expr: "`skill marker missing: ${skillOutbound.text}`"
+      detailsExpr: skillOutbound.text

qa/scenarios/models/claude-cli-provider-capabilities.yaml

@@ -0,0 +1,259 @@
+title: Claude CLI provider capabilities API key
+
+scenario:
+  id: claude-cli-provider-capabilities
+  surface: model-provider
+  coverage:
+    primary:
+      - models.provider-capabilities
+    secondary:
+      - models.claude-cli
+  objective: Verify the Claude CLI model-provider lane can use the Anthropic API key path to talk, read an attached image, use bundled MCP tools, and apply workspace skills.
+  successCriteria:
+    - A live-frontier run fails fast unless the selected primary provider is claude-cli.
+    - The Claude CLI backend preserves ANTHROPIC_API_KEY for this run instead of using native subscription auth.
+    - The agent replies through the Claude CLI provider in a direct chat turn.
+    - The agent describes an attached image through the Claude CLI image path.
+    - The agent can reach memory via the bundled MCP/tool bridge.
+    - The agent sees and follows a workspace skill.
+  docsRefs:
+    - docs/gateway/cli-backends.md
+    - docs/tools/skills.md
+    - docs/cli/mcp.md
+    - docs/tools/index.md
+  codeRefs:
+    - extensions/anthropic/cli-backend.ts
+    - src/agents/cli-backends.ts
+    - src/mcp/plugin-tools-serve.ts
+    - extensions/qa-lab/src/suite.ts
+  execution:
+    kind: flow
+    summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode api-key --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities`.
+    config:
+      authMode: api-key
+      requiredProviderMode: live-frontier
+      requiredProvider: claude-cli
+      chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK"
+      chatExpected: CLAUDE-CLI-CHAT-OK
+      imagePrompt: "Image understanding check: describe the top and bottom colors in the attached image in one short sentence."
+      imageColorGroups:
+        - [red, scarlet, crimson]
+        - [blue, azure, teal, cyan, aqua]
+      memoryFact: "Hidden Claude CLI MCP fact: the provider bridge codename is ORBIT-9."
+      memoryQuery: "provider bridge codename ORBIT-9"
+      memoryExpected: ORBIT-9
+      memoryPrompt: "Memory tools check: use the available memory search MCP/tool bridge to find the hidden provider bridge codename stored only in memory. Reply with the codename."
+      memoryPromptSnippet: "Memory tools check"
+      skillName: qa-claude-cli-skill
+      skillExpected: VISIBLE-SKILL-OK
+      skillBody: |-
+        ---
+        name: qa-claude-cli-skill
+        description: Claude CLI QA skill marker
+        ---
+        When the user asks for the Claude CLI skill marker exactly, or explicitly asks you to use qa-claude-cli-skill, reply with exactly: VISIBLE-SKILL-OK
+      skillPrompt: "Use qa-claude-cli-skill now. Reply exactly with the visible skill marker and nothing else."
+
+flow:
+  steps:
+    - name: confirms the selected live provider and Claude CLI auth mode
+      actions:
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - set: preserveEnv
+          value:
+            expr: "String(env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV ?? '')"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            message:
+              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE === config.authMode"
+            message:
+              expr: "`expected Claude CLI auth mode ${config.authMode}, got ${env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE ?? 'unset'}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || preserveEnv.includes('ANTHROPIC_API_KEY')"
+            message:
+              expr: "`expected ANTHROPIC_API_KEY to be preserved for Claude CLI API-key QA mode, got ${preserveEnv}`"
+      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=${env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE} preserve=${preserveEnv}` : `mock-compatible provider=${selected?.provider}`"
+    - name: talks through the selected provider
+      actions:
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                expr: "`agent:qa:claude-cli-${config.authMode}-chat`"
+              message:
+                expr: config.chatPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 45000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: chatOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 20000, env.primaryModel)
+        - assert:
+            expr: "chatOutbound.text.includes(config.chatExpected)"
+            message:
+              expr: "`chat marker missing: ${chatOutbound.text}`"
+      detailsExpr: chatOutbound.text
+    - name: describes an attached image through the selected provider
+      actions:
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                expr: "`agent:qa:claude-cli-${config.authMode}-image`"
+              message:
+                expr: config.imagePrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              attachments:
+                - mimeType: image/png
+                  fileName: claude-cli-red-top-blue-bottom.png
+                  content:
+                    expr: imageUnderstandingValidPngBase64
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: imageOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+        - assert:
+            expr: "config.imageColorGroups.every((group) => group.some((color) => normalizeLowercaseStringOrEmpty(imageOutbound.text).includes(color)))"
+            message:
+              expr: "`missing expected image colors: ${imageOutbound.text}`"
+        - assert:
+            expr: "!env.mock || (((await fetchJson(`${env.mock.baseUrl}/debug/requests`)).find((request) => String(request.prompt ?? '').includes('Image understanding check'))?.imageInputCount ?? 0) >= 1)"
+            message: expected image input to reach mock provider
+      detailsExpr: imageOutbound.text
+    - name: reaches memory through the MCP/tool bridge
+      actions:
+        - call: reset
+        - call: fs.writeFile
+          args:
+            - expr: "path.join(env.gateway.workspaceDir, 'MEMORY.md')"
+            - expr: "`${config.memoryFact}\\n`"
+            - utf8
+        - call: forceMemoryIndex
+          args:
+            - env:
+                ref: env
+              query:
+                expr: config.memoryQuery
+              expectedNeedle:
+                expr: config.memoryExpected
+        - call: createSession
+          saveAs: mcpSessionKey
+          args:
+            - ref: env
+            - expr: "`Claude CLI MCP bridge ${config.authMode}`"
+        - call: readEffectiveTools
+          saveAs: mcpTools
+          args:
+            - ref: env
+            - ref: mcpSessionKey
+        - assert:
+            expr: "mcpTools.has('memory_search')"
+            message: memory_search missing from effective tools before MCP bridge check
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                ref: mcpSessionKey
+              message:
+                expr: config.memoryPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 90000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: mcpOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 45000, env.primaryModel)
+        - assert:
+            expr: "mcpOutbound.text.includes(config.memoryExpected)"
+            message:
+              expr: "`MCP memory result missing ${config.memoryExpected}: ${mcpOutbound.text}`"
+        - assert:
+            expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).filter((request) => String(request.allInputText ?? '').includes(config.memoryPromptSnippet)).some((request) => request.plannedToolName === 'memory_search')"
+            message: expected mock model to plan memory_search for MCP bridge prompt
+      detailsExpr: mcpOutbound.text
+    - name: applies a workspace skill through the selected provider
+      actions:
+        - call: reset
+        - call: writeWorkspaceSkill
+          args:
+            - env:
+                ref: env
+              name:
+                expr: config.skillName
+              body:
+                expr: config.skillBody
+        - call: waitForCondition
+          args:
+            - lambda:
+                async: true
+                expr: "((await readSkillStatus(env)).find((skill) => skill.name === config.skillName)?.eligible ? true : undefined)"
+            - 15000
+            - 200
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey:
+                expr: "`agent:qa:claude-cli-${config.authMode}-skill`"
+              message:
+                expr: config.skillPrompt
+              provider:
+                expr: selected?.provider
+              model:
+                expr: selected?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+        - call: waitForOutboundMessage
+          saveAs: skillOutbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+        - assert:
+            expr: "skillOutbound.text.includes(config.skillExpected)"
+            message:
+              expr: "`skill marker missing: ${skillOutbound.text}`"
+      detailsExpr: skillOutbound.text

qa/scenarios/models/codex-harness-no-meta-leak.yaml

@@ -0,0 +1,136 @@
+title: Codex harness no meta leak
+
+scenario:
+  id: codex-harness-no-meta-leak
+  surface: dm
+  coverage:
+    primary:
+      - models.codex-cli
+    secondary:
+      - runtime.no-meta-leak
+  objective: Verify the Codex app-server harness keeps coordination/meta chatter out of the visible reply.
+  successCriteria:
+    - The scenario forces the Codex embedded harness.
+    - The final visible reply includes the requested confirmation token.
+    - The visible reply does not include internal coordination or progress chatter.
+  docsRefs:
+    - docs/plugins/sdk-agent-harness.md
+    - docs/concepts/qa-e2e-automation.md
+    - docs/help/testing.md
+  codeRefs:
+    - extensions/codex/src/app-server/event-projector.ts
+    - extensions/qa-lab/src/reply-failure.ts
+    - extensions/qa-lab/src/suite.ts
+  execution:
+    kind: flow
+    summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.5 --alt-model openai/gpt-5.5 --fast --thinking medium --scenario codex-harness-no-meta-leak`.
+    config:
+      requiredProvider: codex
+      requiredModel: gpt-5.5
+      harnessRuntime: codex
+      expectedReply: QA_LEAK_OK
+      prompt: |-
+        Think through your answer privately, but do not expose any internal planning, thread-context checks, or progress narration.
+
+        Reply with exactly one line:
+        QA_LEAK_OK
+      forbiddenReplySubstrings:
+        - checking thread context
+        - thread context thin
+        - post a tight progress reply here
+        - posting a coordination nudge
+        - posted a short coordination reply
+        - not inventing status
+
+flow:
+  steps:
+    - name: confirms GPT-5.5 Codex harness target
+      actions:
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            message:
+              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            message:
+              expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
+        - if:
+            expr: "env.providerMode !== 'live-frontier'"
+            then:
+              - assert: "true"
+            else:
+              - call: patchConfig
+                saveAs: patchResult
+                args:
+                  - env:
+                      ref: env
+                    patch:
+                      agents:
+                        defaults:
+                          models:
+                            expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime } } })"
+              - call: waitForGatewayHealthy
+                args:
+                  - ref: env
+                  - 60000
+              - call: waitForQaChannelReady
+                args:
+                  - ref: env
+                  - 60000
+              - call: readConfigSnapshot
+                saveAs: snapshot
+                args:
+                  - ref: env
+              - assert:
+                  expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id === config.harnessRuntime"
+                  message:
+                    expr: "`expected ${env.primaryModel} agentRuntime.id=${config.harnessRuntime}, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime)}`"
+      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} runtime=${snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id}` : `mock mode: parsed ${scenario.id}`"
+    - name: keeps codex coordination chatter out of the visible reply
+      actions:
+        - if:
+            expr: "env.providerMode !== 'live-frontier'"
+            then:
+              - assert: "true"
+            else:
+              - call: reset
+              - call: runAgentPrompt
+                args:
+                  - ref: env
+                  - sessionKey: agent:qa:codex-meta-leak
+                    message:
+                      expr: config.prompt
+                    provider:
+                      expr: selected?.provider
+                    model:
+                      expr: selected?.model
+                    timeoutMs:
+                      expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
+              - call: waitForOutboundMessage
+                saveAs: outbound
+                args:
+                  - ref: state
+                  - lambda:
+                      params: [candidate]
+                      expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.expectedReply)"
+                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+              - set: outboundLower
+                value:
+                  expr: normalizeLowercaseStringOrEmpty(outbound.text)
+              - assert:
+                  expr: "outbound.text.trim() === config.expectedReply"
+                  message:
+                    expr: "`expected exact visible reply ${config.expectedReply}, got ${outbound.text}`"
+              - forEach:
+                  items:
+                    expr: "config.forbiddenReplySubstrings ?? []"
+                  item: forbidden
+                  actions:
+                    - assert:
+                        expr: "!outboundLower.includes(normalizeLowercaseStringOrEmpty(forbidden))"
+                        message:
+                          expr: "`visible reply leaked internal meta text (${forbidden}): ${outbound.text}`"
+      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live codex leak check' : outbound.text"

qa/scenarios/models/gpt55-thinking-visibility-switch.yaml

@@ -0,0 +1,218 @@
+title: GPT-5.5 thinking visibility switch
+
+scenario:
+  id: gpt55-thinking-visibility-switch
+  surface: models
+  coverage:
+    primary:
+      - models.thinking
+    secondary:
+      - runtime.reasoning-visibility
+  objective: Verify GPT-5.5 can switch from disabled thinking to medium thinking while reasoning display stays enabled.
+  successCriteria:
+    - Live runs target openai/gpt-5.5, not a mini or pro variant.
+    - The session enables reasoning display before the comparison turns.
+    - The disabled-thinking turn returns its visible marker without sending a reasoning payload to OpenAI-compatible providers.
+    - The medium-thinking turn sends a medium reasoning request and returns its visible marker.
+    - Transports with a visible reasoning lane expose a separate Reasoning-prefixed message; qa-channel validates provider behavior because generic delivery suppresses reasoning payloads by design.
+  docsRefs:
+    - docs/tools/thinking.md
+    - docs/help/testing.md
+    - docs/concepts/qa-e2e-automation.md
+  codeRefs:
+    - src/auto-reply/reply/directives.ts
+    - src/auto-reply/thinking.shared.ts
+    - src/agents/embedded-agent-runner/run/payloads.ts
+    - extensions/openai/openai-provider.ts
+    - extensions/qa-lab/src/providers/mock-openai/server.ts
+  execution:
+    kind: flow
+    summary: Toggle reasoning display and GPT-5.5 thinking between off/none and medium, then verify visible reasoning only on the medium turn.
+    config:
+      requiredProvider: openai
+      requiredModel: gpt-5.5
+      offDirective: /think off
+      maxDirective: /think medium
+      reasoningDirective: /reasoning on
+      conversationId: qa-thinking-visibility
+      offPrompt: "QA thinking visibility check off: answer exactly THINKING-OFF-OK."
+      maxPrompt: "QA thinking visibility check max: verify 17+24=41 internally, then answer exactly THINKING-MAX-OK."
+      offMarker: THINKING-OFF-OK
+      maxMarker: THINKING-MAX-OK
+
+flow:
+  steps:
+    - name: enables reasoning display and disables thinking
+      actions:
+        - call: waitForGatewayHealthy
+          args:
+            - ref: env
+            - 60000
+        - call: waitForQaChannelReady
+          args:
+            - ref: env
+            - 60000
+        - call: reset
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || (selected?.provider === config.requiredProvider && selected?.model === config.requiredModel)"
+            message:
+              expr: "`expected live GPT-5.5, got ${env.primaryModel}`"
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text:
+              expr: config.reasoningDirective
+        - call: waitForCondition
+          saveAs: reasoningAck
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Reasoning visibility enabled/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+        - set: thinkOffCursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text:
+              expr: config.offDirective
+        - call: waitForCondition
+          saveAs: thinkOffAck
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(thinkOffCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking disabled/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+        - set: offCursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text:
+              expr: config.offPrompt
+        - call: waitForCondition
+          saveAs: offAnswer
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(offCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.includes(config.offMarker)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 90000)
+        - set: offMessages
+          value:
+            expr: "state.getSnapshot().messages.slice(offCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId)"
+        - assert:
+            expr: "offMessages.some((candidate) => candidate.text.includes(config.offMarker))"
+            message:
+              expr: "`missing off marker; saw ${offMessages.map((message) => message.text).join(' | ')}`"
+        - assert:
+            expr: "!offMessages.some((candidate) => candidate.text.trimStart().startsWith('Reasoning:'))"
+            message:
+              expr: "`disabled thinking unexpectedly emitted reasoning: ${offMessages.map((message) => message.text).join(' | ')}`"
+        - if:
+            expr: "Boolean(env.mock)"
+            then:
+              - set: requests
+                value:
+                  expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests`)"
+              - set: offRequest
+                value:
+                  expr: "requests.find((request) => String(request.allInputText ?? '').includes(config.offPrompt))"
+              - assert:
+                  expr: "String(offRequest?.model ?? '').includes('gpt-5.5')"
+                  message:
+                    expr: "`expected GPT-5.5 off mock request, got ${String(offRequest?.model ?? '')}`"
+              - assert:
+                  expr: "offRequest?.body && !Object.prototype.hasOwnProperty.call(offRequest.body, 'reasoning')"
+                  message:
+                    expr: "`disabled thinking should omit OpenAI reasoning payload, got ${JSON.stringify(offRequest?.body?.reasoning ?? null)}`"
+      detailsExpr: "`reasoning ack=${reasoningAck.text}; thinking off=${thinkOffAck.text}; off answer=${offAnswer.text}`"
+    - name: switches to medium thinking
+      actions:
+        - set: thinkMediumCursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text:
+              expr: config.maxDirective
+        - call: waitForCondition
+          saveAs: thinkMediumAck
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(thinkMediumCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking level set to medium/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+      detailsExpr: "`thinking medium=${thinkMediumAck.text}`"
+    - name: verifies medium thinking reaches the provider
+      actions:
+        - set: maxCursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text:
+              expr: config.maxPrompt
+        - call: waitForCondition
+          saveAs: maxAnswer
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(maxCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.includes(config.maxMarker)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 120000)
+        - assert:
+            expr: "maxAnswer.text.includes(config.maxMarker)"
+            message:
+              expr: "`missing max marker near answer: ${recentOutboundSummary(state, 6)}`"
+        - if:
+            expr: "Boolean(env.mock)"
+            then:
+              - set: requests
+                value:
+                  expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests`)"
+              - set: maxRequest
+                value:
+                  expr: "requests.find((request) => String(request.allInputText ?? '').includes(config.maxPrompt))"
+              - assert:
+                  expr: "String(maxRequest?.model ?? '').includes('gpt-5.5')"
+                  message:
+                    expr: "`expected GPT-5.5 mock request, got ${String(maxRequest?.model ?? '')}`"
+              - assert:
+                  expr: "maxRequest?.body?.reasoning?.effort === 'medium'"
+                  message:
+                    expr: "`expected medium OpenAI reasoning payload, got ${JSON.stringify(maxRequest?.body?.reasoning ?? null)}`"
+        - if:
+            expr: "env.transport.id !== 'qa-channel' && env.transport.id !== 'crabline'"
+            then:
+              - call: waitForCondition
+                saveAs: maxReasoning
+                args:
+                  - lambda:
+                      expr: "state.getSnapshot().messages.slice(maxCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.trimStart().startsWith('Reasoning:')).at(-1)"
+                  - expr: liveTurnTimeoutMs(env, 120000)
+              - assert:
+                  expr: "maxReasoning.text.trimStart().startsWith('Reasoning:')"
+                  message:
+                    expr: "`missing max reasoning message near answer: ${recentOutboundSummary(state, 6)}`"
+      detailsExpr: "env.transport.id === 'qa-channel' || env.transport.id === 'crabline' ? `answer=${maxAnswer.text}; medium reasoning=${env.mock ? String(maxRequest?.body?.reasoning?.effort ?? '') : 'live'}; ${env.transport.id} suppresses reasoning delivery` : `answer=${maxAnswer.text}; reasoning=${maxReasoning.text}`"

qa/scenarios/models/model-switch-follow-up.yaml

@@ -0,0 +1,76 @@
+title: Model switch follow-up
+
+scenario:
+  id: model-switch-follow-up
+  surface: models
+  coverage:
+    primary:
+      - models.switching
+    secondary:
+      - runtime.session-continuity
+  objective: Verify the agent can switch to a different configured model and continue coherently.
+  successCriteria:
+    - Agent reflects the model switch request.
+    - Follow-up answer remains coherent with prior context.
+    - Final report notes whether the switch actually happened.
+  docsRefs:
+    - docs/help/testing.md
+    - docs/web/dashboard.md
+  codeRefs:
+    - extensions/qa-lab/src/report.ts
+  execution:
+    kind: flow
+    summary: Verify the agent can switch to a different configured model and continue coherently.
+    config:
+      initialPrompt: "Say hello from the default configured model."
+      followupPrompt: "Continue the exchange after switching models and note the handoff."
+
+flow:
+  steps:
+    - name: runs on the default configured model
+      actions:
+        - call: reset
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey: agent:qa:model-switch
+              message:
+                expr: config.initialPrompt
+              timeoutMs:
+                expr: liveTurnTimeoutMs(env, 30000)
+        - call: waitForOutboundMessage
+          saveAs: outbound
+          args:
+            - ref: state
+            - lambda:
+                params: [candidate]
+                expr: "candidate.conversation.id === 'qa-operator'"
+      detailsExpr: "env.mock ? String((await fetchJson(`${env.mock.baseUrl}/debug/last-request`))?.body?.model ?? '') : outbound.text"
+    - name: switches to the alternate model and continues
+      actions:
+        - set: alternate
+          value:
+            expr: splitModelRef(env.alternateModel)
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey: agent:qa:model-switch
+              message:
+                expr: config.followupPrompt
+              provider:
+                expr: alternate?.provider
+              model:
+                expr: alternate?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.alternateModel)
+        - call: waitForCondition
+          saveAs: outbound
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && (() => { const lower = normalizeLowercaseStringOrEmpty(candidate.text); return lower.includes('switch') || lower.includes('handoff'); })()).at(-1)"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 20000, env.alternateModel)
+        - assert:
+            expr: "!env.mock || ((await fetchJson(`${env.mock.baseUrl}/debug/last-request`))?.body?.model === 'gpt-5.5-alt')"
+            message:
+              expr: "`expected gpt-5.5-alt, got ${String((await fetchJson(`${env.mock.baseUrl}/debug/last-request`))?.body?.model ?? '')}`"
+      detailsExpr: outbound.text

qa/scenarios/models/model-switch-tool-continuity.yaml

@@ -0,0 +1,92 @@
+title: Model switch with tool continuity
+
+scenario:
+  id: model-switch-tool-continuity
+  surface: models
+  coverage:
+    primary:
+      - models.switching
+    secondary:
+      - runtime.tool-continuity
+  objective: Verify switching models preserves session context and tool use instead of dropping into plain-text only behavior.
+  successCriteria:
+    - Alternate model is actually requested.
+    - A tool call still happens after the model switch.
+    - Final answer acknowledges the handoff and reread QA mission.
+  docsRefs:
+    - docs/help/testing.md
+    - docs/concepts/model-failover.md
+  codeRefs:
+    - extensions/qa-lab/src/suite.ts
+    - extensions/qa-lab/src/mock-openai-server.ts
+  execution:
+    kind: flow
+    summary: Verify switching models preserves session context and tool use instead of dropping into plain-text only behavior.
+    config:
+      initialPrompt: "Read repo/qa/scenarios/index.yaml and summarize the QA scenario pack mission in one clause before any model switch."
+      followupPrompt: "The harness has already requested the alternate model for this turn. Do not call session_status or change models yourself. Tool continuity check: use the read tool to reread repo/qa/scenarios/index.yaml, then mention the model handoff and QA mission in one short sentence."
+      promptSnippet: "Tool continuity check"
+
+flow:
+  steps:
+    - name: keeps using tools after switching models
+      actions:
+        - call: waitForGatewayHealthy
+          args:
+            - ref: env
+            - 60000
+        - call: reset
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey: agent:qa:model-switch-tools
+              message:
+                expr: config.initialPrompt
+              timeoutMs:
+                expr: liveTurnTimeoutMs(env, 30000)
+        - set: alternate
+          value:
+            expr: splitModelRef(env.alternateModel)
+        - set: beforeSwitchCursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - call: runAgentPrompt
+          args:
+            - ref: env
+            - sessionKey: agent:qa:model-switch-tools
+              message:
+                expr: config.followupPrompt
+              provider:
+                expr: alternate?.provider
+              model:
+                expr: alternate?.model
+              timeoutMs:
+                expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.alternateModel)
+        - call: waitForCondition
+          saveAs: outbound
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(beforeSwitchCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && hasModelSwitchContinuitySignal(candidate.text)).at(-1)"
+            - expr: resolveQaLiveTurnTimeoutMs(env, 20000, env.alternateModel)
+        - assert:
+            expr: hasModelSwitchContinuitySignal(outbound.text)
+            message:
+              expr: "`switch reply missed kickoff continuity: ${outbound.text}`"
+        - if:
+            expr: "Boolean(env.mock)"
+            then:
+              - set: switchDebugRequests
+                value:
+                  expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests`)"
+              - set: switchRequest
+                value:
+                  expr: "switchDebugRequests.find((request) => String(request.allInputText ?? '').includes(config.promptSnippet))"
+              - assert:
+                  expr: "switchRequest?.plannedToolName === 'read'"
+                  message:
+                    expr: "`expected read after switch, got ${String(switchRequest?.plannedToolName ?? '')}`"
+              - assert:
+                  expr: "String(switchRequest?.model ?? '') === String(alternate?.model ?? '')"
+                  message:
+                    expr: "`expected alternate model, got ${String(switchRequest?.model ?? '')}`"
+      detailsExpr: outbound.text

qa/scenarios/models/openai-native-web-search-live.yaml

@@ -0,0 +1,143 @@
+title: OpenAI native web search live
+
+scenario:
+  id: openai-native-web-search-live
+  surface: model-provider
+  coverage:
+    primary:
+      - tools.web-search
+    secondary:
+      - models.openai
+  objective: Verify a live OpenAI GPT model can use OpenAI native web_search when OpenClaw web search is enabled in auto mode.
+  successCriteria:
+    - A live-frontier run fails fast unless the selected primary provider is openai.
+    - The selected primary model is GPT-5.5, not a mini or pro variant.
+    - Web search is enabled without pinning a managed web_search provider.
+    - The live reply includes the required marker plus an official OpenAI News URL and headline found through web search.
+  gatewayConfigPatch:
+    tools:
+      web:
+        search:
+          enabled: true
+          provider: null
+  docsRefs:
+    - docs/tools/web.md
+    - docs/help/testing.md
+    - docs/concepts/qa-e2e-automation.md
+  codeRefs:
+    - extensions/openai/native-web-search.ts
+    - extensions/openai/shared.ts
+    - extensions/openai/openai-provider.ts
+    - extensions/qa-lab/src/suite.ts
+  execution:
+    kind: flow
+    summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.5 --alt-model openai/gpt-5.5 --fast --thinking medium --scenario openai-native-web-search-live`.
+    config:
+      requiredProvider: openai
+      requiredModel: gpt-5.5
+      expectedMarker: WEB-SEARCH-OK
+      failureMarker: WEB-SEARCH-FAILED
+      searchPrompt: |-
+        Web search QA: use web search now for `site:openai.com/news OpenAI latest news`.
+        Reply in exactly three lines:
+        WEB-SEARCH-OK
+        URL: <official openai.com/news URL from the search results>
+        HEADLINE: <article or page headline from the search results>
+        Do not answer from memory. If web search is unavailable, reply exactly WEB-SEARCH-FAILED.
+
+flow:
+  steps:
+    - name: confirms live OpenAI GPT-5.5 web search auto mode
+      actions:
+        - call: waitForGatewayHealthy
+          args:
+            - ref: env
+            - 60000
+        - call: waitForQaChannelReady
+          args:
+            - ref: env
+            - 60000
+        - set: selected
+          value:
+            expr: splitModelRef(env.primaryModel)
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+            message:
+              expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+            message:
+              expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
+        - call: readConfigSnapshot
+          saveAs: snapshot
+          args:
+            - ref: env
+        - set: searchConfig
+          value:
+            expr: "snapshot.config.tools?.web?.search ?? {}"
+        - assert:
+            expr: "searchConfig.enabled !== false"
+            message:
+              expr: "`expected web search enabled, got ${JSON.stringify(searchConfig)}`"
+        - assert:
+            expr: "typeof searchConfig.provider !== 'string' || ['auto', 'openai', ''].includes(searchConfig.provider.trim().toLowerCase())"
+            message:
+              expr: "`expected web search provider auto/openai/unset for native OpenAI search, got ${JSON.stringify(searchConfig)}`"
+        - assert:
+            expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.OPENAI_API_KEY?.trim() || env.gateway.runtimeEnv.OPENCLAW_LIVE_OPENAI_KEY?.trim())"
+            message: expected OPENAI_API_KEY or OPENCLAW_LIVE_OPENAI_KEY for live OpenAI QA
+      detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} webSearch=${JSON.stringify(searchConfig)}` : `mock-compatible provider=${selected?.provider}`"
+    - name: searches official OpenAI News through the live model
+      actions:
+        - if:
+            expr: "env.providerMode !== 'live-frontier'"
+            then:
+              - assert: "true"
+            else:
+              - call: reset
+              - set: selected
+                value:
+                  expr: splitModelRef(env.primaryModel)
+              - call: runAgentPrompt
+                args:
+                  - ref: env
+                  - sessionKey: agent:qa:openai-native-web-search
+                    message:
+                      expr: config.searchPrompt
+                    provider:
+                      expr: selected?.provider
+                    model:
+                      expr: selected?.model
+                    timeoutMs:
+                      expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
+              - call: waitForOutboundMessage
+                saveAs: searchOutbound
+                args:
+                  - ref: state
+                  - lambda:
+                      params: [candidate]
+                      expr: "candidate.conversation.id === 'qa-operator'"
+                  - expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+              - set: searchText
+                value:
+                  expr: searchOutbound.text
+              - set: searchTextLower
+                value:
+                  expr: normalizeLowercaseStringOrEmpty(searchText)
+              - assert:
+                  expr: "searchText.includes(config.expectedMarker)"
+                  message:
+                    expr: "`missing ${config.expectedMarker}: ${searchText}`"
+              - assert:
+                  expr: "!searchText.includes(config.failureMarker) && !/(web search is unavailable|unable to search|cannot search|can't search)/i.test(searchText)"
+                  message:
+                    expr: "`search looked unavailable: ${searchText}`"
+              - assert:
+                  expr: "/URL:\\s*https?:\\/\\/[^\\s]*openai\\.com\\/news/i.test(searchText)"
+                  message:
+                    expr: "`missing official OpenAI News URL: ${searchText}`"
+              - assert:
+                  expr: "/HEADLINE:\\s*\\S.{8,}/i.test(searchText)"
+                  message:
+                    expr: "`missing searched headline: ${searchText}`"
+      detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live OpenAI web search probe' : searchText"

qa/scenarios/models/thinking-slash-model-remap.yaml

@@ -0,0 +1,191 @@
+title: Thinking slash model remap
+
+scenario:
+  id: thinking-slash-model-remap
+  surface: models
+  coverage:
+    primary:
+      - models.thinking
+    secondary:
+      - models.switching
+      - runtime.session-continuity
+  objective: Verify /think lists provider-owned levels and remaps stored thinking levels when the session model changes provider capabilities.
+  plugins:
+    - anthropic
+  gatewayConfigPatch:
+    agents:
+      defaults:
+        models:
+          anthropic/claude-sonnet-4-6:
+            params: {}
+  successCriteria:
+    - Anthropic Claude Sonnet 4.6 advertises adaptive and native max, but not OpenAI-only xhigh.
+    - A stored adaptive level remaps to medium when switching to OpenAI GPT-5.5.
+    - OpenAI GPT-5.5 advertises xhigh but not adaptive or max.
+    - A stored xhigh level remaps to high when switching to an Anthropic model without xhigh support.
+  docsRefs:
+    - docs/tools/thinking.md
+    - docs/help/testing.md
+    - docs/concepts/qa-e2e-automation.md
+  codeRefs:
+    - src/auto-reply/thinking.ts
+    - src/auto-reply/thinking.shared.ts
+    - src/auto-reply/reply/directive-handling.impl.ts
+    - src/gateway/sessions-patch.ts
+    - extensions/anthropic/register.runtime.ts
+    - extensions/openai/openai-provider.ts
+  execution:
+    kind: flow
+    summary: Select Anthropic, set adaptive, switch to OpenAI and verify medium fallback, then set xhigh and verify high fallback on a model without xhigh support.
+    config:
+      requiredProviderMode: live-frontier
+      requiredProvider: openai
+      requiredModel: gpt-5.5
+      anthropicModelRef: anthropic/claude-sonnet-4-6
+      openAiXhighModelRef: openai/gpt-5.5
+      noXhighModelRef: anthropic/claude-sonnet-4-6
+      conversationId: thinking-slash-remap
+      sessionKey: agent:qa:main
+
+flow:
+  steps:
+    - name: selects Anthropic and verifies adaptive options
+      actions:
+        - call: waitForGatewayHealthy
+          args:
+            - ref: env
+            - 60000
+        - call: waitForQaChannelReady
+          args:
+            - ref: env
+            - 60000
+        - call: reset
+        - assert:
+            expr: "env.providerMode === config.requiredProviderMode"
+            message:
+              expr: "`thinking remap scenario requires ${config.requiredProviderMode}; got ${env.providerMode}`"
+        - set: anthropicModelAck
+          value:
+            expr: "await env.gateway.call('sessions.patch', { key: config.sessionKey, model: config.anthropicModelRef }, { timeoutMs: liveTurnTimeoutMs(env, 45000) })"
+        - set: cursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text: /think
+        - call: waitForCondition
+          saveAs: anthropicThinkStatus
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level:/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+        - assert:
+            expr: "/Options: .*adaptive/i.test(anthropicThinkStatus.text)"
+            message:
+              expr: "`expected Anthropic /think options to include adaptive, got ${anthropicThinkStatus.text}`"
+        - assert:
+            expr: "!/Options: .*\\bxhigh\\b/i.test(anthropicThinkStatus.text) && /Options: .*\\bmax\\b/i.test(anthropicThinkStatus.text)"
+            message:
+              expr: "`expected Sonnet /think options to include max and omit xhigh, got ${anthropicThinkStatus.text}`"
+      detailsExpr: "`model=${JSON.stringify(anthropicModelAck.resolved)}; think=${anthropicThinkStatus.text}`"
+    - name: maps adaptive to medium when switching to OpenAI
+      actions:
+        - set: cursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text: /think adaptive
+        - call: waitForCondition
+          saveAs: adaptiveAck
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking level set to adaptive/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+        - set: openAiModelAck
+          value:
+            expr: "await env.gateway.call('sessions.patch', { key: config.sessionKey, model: config.openAiXhighModelRef }, { timeoutMs: liveTurnTimeoutMs(env, 45000) })"
+        - assert:
+            expr: "openAiModelAck.entry?.thinkingLevel === 'medium'"
+            message:
+              expr: "`expected adaptive->medium remap, got ${JSON.stringify(openAiModelAck.entry)}`"
+        - set: cursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text: /think
+        - call: waitForCondition
+          saveAs: openAiThinkStatus
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level: medium/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+        - assert:
+            expr: "/Options: .*\\bxhigh\\b/i.test(openAiThinkStatus.text) && !/Options: .*\\badaptive\\b/i.test(openAiThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(openAiThinkStatus.text)"
+            message:
+              expr: "`expected OpenAI GPT-5.5 /think options to include xhigh only, got ${openAiThinkStatus.text}`"
+      detailsExpr: "`adaptive=${adaptiveAck.text}; switch=${JSON.stringify(openAiModelAck.resolved)}; think=${openAiThinkStatus.text}`"
+    - name: maps xhigh to high on a model without xhigh
+      actions:
+        - set: cursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text: /think xhigh
+        - call: waitForCondition
+          saveAs: xhighAck
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking level set to xhigh/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+        - set: noXhighModelAck
+          value:
+            expr: "await env.gateway.call('sessions.patch', { key: config.sessionKey, model: config.noXhighModelRef }, { timeoutMs: liveTurnTimeoutMs(env, 45000) })"
+        - assert:
+            expr: "noXhighModelAck.entry?.thinkingLevel === 'high'"
+            message:
+              expr: "`expected xhigh->high remap, got ${JSON.stringify(noXhighModelAck.entry)}`"
+        - set: cursor
+          value:
+            expr: state.getSnapshot().messages.length
+        - sendInbound:
+            conversation:
+              id:
+                expr: config.conversationId
+              kind: direct
+            senderId: qa-operator
+            senderName: QA Operator
+            text: /think
+        - call: waitForCondition
+          saveAs: noXhighThinkStatus
+          args:
+            - lambda:
+                expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level: high/i.test(candidate.text)).at(-1)"
+            - expr: liveTurnTimeoutMs(env, 20000)
+        - assert:
+            expr: "/Options: .*\\badaptive\\b/i.test(noXhighThinkStatus.text) && /Options: .*\\bmax\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bxhigh\\b/i.test(noXhighThinkStatus.text)"
+            message:
+              expr: "`expected non-xhigh model /think options to include adaptive/max and omit xhigh, got ${noXhighThinkStatus.text}`"
+      detailsExpr: "`xhigh=${xhighAck.text}; switch=${JSON.stringify(noXhighModelAck.resolved)}; think=${noXhighThinkStatus.text}`"