adolf/qa/scenarios/config/config-restart-capability-flip.yaml

title: "Config restart capability flip"

scenario:
  id: config-restart-capability-flip
  surface: config
  coverage:
    primary:
      - config.restart-apply
    secondary:
      - plugins.capabilities
  objective: Verify a restart-triggering config change flips capability inventory and the same session successfully uses the newly restored tool after wake-up.
  successCriteria:
    - Capability is absent before the restart-triggering patch.
    - Restart sentinel wakes the same session back up after config patch.
    - The restored capability appears in tools.effective and works in the follow-up turn.
  docsRefs:
    - docs/gateway/configuration.md
    - docs/gateway/protocol.md
    - docs/tools/image-generation.md
  codeRefs:
    - src/gateway/server-methods/config.ts
    - src/gateway/server-restart-sentinel.ts
    - src/gateway/server-methods/tools-effective.ts
    - extensions/qa-lab/src/suite.ts
  execution:
    kind: flow
    summary: Verify a restart-triggering config change flips capability inventory and the same session successfully uses the newly restored tool after wake-up.
    config:
      imagePrompt: "Capability flip image check: generate a QA lighthouse image in this turn right now. Do not acknowledge first, do not promise future work, and do not stop before using image_generate. Final reply must include the MEDIA path."
      imagePromptSnippet: "Capability flip image check"
      deniedTool: image_generate
      imageTurnTimeoutMs: 120000
      mediaPathTimeoutMs: 30000

flow:
  steps:
    - name: restores image_generate after restart and uses it in the same session
      actions:
        - call: ensureImageGenerationConfigured
          args:
            - ref: env
        - call: readConfigSnapshot
          saveAs: original
          args:
            - ref: env
        - set: originalTools
          value:
            expr: "original.config.tools && typeof original.config.tools === 'object' ? original.config.tools : null"
        - set: originalToolsDeny
          value:
            expr: "originalTools ? (Object.prototype.hasOwnProperty.call(originalTools, 'deny') ? structuredClone(originalTools.deny) : undefined) : undefined"
        - set: originalImageGenerationModelPrimary
          value:
            expr: "original.config.agents?.defaults?.imageGenerationModel?.primary ?? null"
        - set: denied
          value:
            expr: "Array.isArray(originalToolsDeny) ? originalToolsDeny.map((entry) => String(entry)) : []"
        - set: deniedWithImage
          value:
            expr: "denied.includes(config.deniedTool) ? denied : [...denied, config.deniedTool]"
        - set: sessionKey
          value: agent:qa:capability-flip
        - call: createSession
          args:
            - ref: env
            - Capability flip
            - ref: sessionKey
        - try:
            actions:
              - call: patchConfig
                args:
                  - env:
                      ref: env
                    patch:
                      tools:
                        deny:
                          ref: deniedWithImage
              - call: waitForGatewayHealthy
                args:
                  - ref: env
              - call: waitForQaChannelReady
                args:
                  - ref: env
                  - 60000
              - call: readEffectiveTools
                saveAs: beforeTools
                args:
                  - ref: env
                  - ref: sessionKey
              - assert:
                  expr: "!beforeTools.has(config.deniedTool)"
                  message:
                    expr: "`${config.deniedTool} still present before capability flip`"
              - set: wakeMarker
                value:
                  expr: "`QA-CAPABILITY-${randomUUID().slice(0, 8)}`"
              - call: patchConfig
                args:
                  - env:
                      ref: env
                    patch:
                      tools:
                        deny:
                          expr: "originalToolsDeny === undefined ? null : originalToolsDeny"
                      agents:
                        defaults:
                          imageGenerationModel:
                            primary:
                              ref: originalImageGenerationModelPrimary
                    sessionKey:
                      ref: sessionKey
                    note:
                      ref: wakeMarker
                    replacePaths:
                      - tools.deny
              - call: waitForGatewayHealthy
                args:
                  - ref: env
                  - 60000
              - call: waitForQaChannelReady
                args:
                  - ref: env
                  - 60000
              - call: waitForCondition
                saveAs: afterTools
                args:
                  - lambda:
                      async: true
                      expr: "(() => readEffectiveTools(env, sessionKey).then((tools) => (tools.has('image_generate') ? tools : undefined)))()"
                  - expr: liveTurnTimeoutMs(env, config.imageTurnTimeoutMs)
                  - 500
              - set: imageStartedAtMs
                value:
                  expr: "Date.now()"
              - set: mediaPath
                value: ""
              - set: imageReplyText
                value: ""
              - set: imageReplyStartIndex
                value:
                  expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
              - try:
                  actions:
                    - call: runAgentPrompt
                      args:
                        - ref: env
                        - sessionKey:
                            ref: sessionKey
                          message:
                            expr: config.imagePrompt
                          timeoutMs:
                            expr: liveTurnTimeoutMs(env, config.imageTurnTimeoutMs)
                  catchAs: imageRunError
                  catch:
                    - if:
                        expr: "!env.mock || !/agent run aborted/i.test(formatErrorMessage(imageRunError))"
                        then:
                          - throw:
                              message:
                                expr: "formatErrorMessage(imageRunError)"
              - try:
                  actions:
                    - call: resolveGeneratedImagePath
                      saveAs: mediaPath
                      args:
                        - env:
                            ref: env
                          promptSnippet:
                            expr: config.imagePromptSnippet
                          startedAtMs:
                            ref: imageStartedAtMs
                          timeoutMs:
                            expr: liveTurnTimeoutMs(env, config.mediaPathTimeoutMs)
                  catch:
                    - set: mediaPath
                      value: ""
              - if:
                  expr: "!mediaPath"
                  then:
                    - call: waitForOutboundMessage
                      saveAs: imageReply
                      args:
                        - ref: state
                        - lambda:
                            params: [candidate]
                            expr: "candidate.conversation.id === 'qa-operator' && (String(candidate.text ?? '').includes('MEDIA:') || /media failed|image generation failed/i.test(String(candidate.text ?? '')))"
                        - expr: liveTurnTimeoutMs(env, config.imageTurnTimeoutMs)
                        - sinceIndex:
                            ref: imageReplyStartIndex
                    - set: imageReplyText
                      value:
                        expr: "String(imageReply.text ?? '')"
                  else:
                    - set: imageReplyText
                      value:
                        expr: "`MEDIA:${mediaPath}`"
              - set: imageReplyLower
                value:
                  expr: "imageReplyText.toLowerCase()"
              - assert:
                  expr: "Boolean(mediaPath) || (!env.mock && /media failed|image generation failed/.test(imageReplyLower))"
                  message:
                    expr: "`expected restored ${config.deniedTool} to either produce media or, in live mode only, surface a provider-side image failure; got ${imageReplyText}`"
              # Tool-call assertion (criterion 2 of the parity completion
              # gate in #64227): the restored `image_generate` capability
              # must have actually fired as a real tool call. Without this
              # assertion, a prose reply that just mentions a MEDIA path
              # could satisfy the scenario, so strengthen it by requiring
              # the mock to have recorded `plannedToolName: "image_generate"`
              # against a post-restart request. The `!env.mock || ...`
              # guard means this check only runs in mock mode (where
              # `/debug/requests` is available); live-frontier runs skip
              # it and still pass the rest of the scenario.
              - assert:
                  expr: "!env.mock || [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].some((request) => String(request.allInputText ?? '').toLowerCase().includes('capability flip image check') && request.plannedToolName === 'image_generate')"
                  message:
                    expr: "`expected image_generate tool call during capability flip scenario, saw plannedToolNames=${JSON.stringify([...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].filter((request) => String(request.allInputText ?? '').toLowerCase().includes('capability flip image check')).map((request) => request.plannedToolName ?? null))}`"
            finally:
              - call: patchConfig
                args:
                  - env:
                      ref: env
                    patch:
                      tools:
                        deny:
                          expr: "originalToolsDeny === undefined ? null : originalToolsDeny"
                    replacePaths:
                      - tools.deny
              - call: waitForGatewayHealthy
                args:
                  - ref: env
              - call: waitForQaChannelReady
                args:
                  - ref: env
                  - 60000
      detailsExpr: "`${wakeMarker}\\n${config.deniedTool}=${String(afterTools.has(config.deniedTool))}\\n${mediaPath ? `MEDIA:${mediaPath}` : imageReplyText}`"