adolf/qa/scenarios/models/gpt55-thinking-visibility-switch.yaml

title: GPT-5.5 thinking visibility switch

scenario:
  id: gpt55-thinking-visibility-switch
  surface: models
  coverage:
    primary:
      - models.thinking
    secondary:
      - runtime.reasoning-visibility
  objective: Verify GPT-5.5 can switch from disabled thinking to medium thinking while reasoning display stays enabled.
  successCriteria:
    - Live runs target openai/gpt-5.5, not a mini or pro variant.
    - The session enables reasoning display before the comparison turns.
    - The disabled-thinking turn returns its visible marker without sending a reasoning payload to OpenAI-compatible providers.
    - The medium-thinking turn sends a medium reasoning request and returns its visible marker.
    - Transports with a visible reasoning lane expose a separate Reasoning-prefixed message; qa-channel validates provider behavior because generic delivery suppresses reasoning payloads by design.
  docsRefs:
    - docs/tools/thinking.md
    - docs/help/testing.md
    - docs/concepts/qa-e2e-automation.md
  codeRefs:
    - src/auto-reply/reply/directives.ts
    - src/auto-reply/thinking.shared.ts
    - src/agents/embedded-agent-runner/run/payloads.ts
    - extensions/openai/openai-provider.ts
    - extensions/qa-lab/src/providers/mock-openai/server.ts
  execution:
    kind: flow
    summary: Toggle reasoning display and GPT-5.5 thinking between off/none and medium, then verify visible reasoning only on the medium turn.
    config:
      requiredProvider: openai
      requiredModel: gpt-5.5
      offDirective: /think off
      maxDirective: /think medium
      reasoningDirective: /reasoning on
      conversationId: qa-thinking-visibility
      offPrompt: "QA thinking visibility check off: answer exactly THINKING-OFF-OK."
      maxPrompt: "QA thinking visibility check max: verify 17+24=41 internally, then answer exactly THINKING-MAX-OK."
      offMarker: THINKING-OFF-OK
      maxMarker: THINKING-MAX-OK

flow:
  steps:
    - name: enables reasoning display and disables thinking
      actions:
        - call: waitForGatewayHealthy
          args:
            - ref: env
            - 60000
        - call: waitForQaChannelReady
          args:
            - ref: env
            - 60000
        - call: reset
        - set: selected
          value:
            expr: splitModelRef(env.primaryModel)
        - assert:
            expr: "env.providerMode !== 'live-frontier' || (selected?.provider === config.requiredProvider && selected?.model === config.requiredModel)"
            message:
              expr: "`expected live GPT-5.5, got ${env.primaryModel}`"
        - sendInbound:
            conversation:
              id:
                expr: config.conversationId
              kind: direct
            senderId: qa-operator
            senderName: QA Operator
            text:
              expr: config.reasoningDirective
        - call: waitForCondition
          saveAs: reasoningAck
          args:
            - lambda:
                expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Reasoning visibility enabled/i.test(candidate.text)).at(-1)"
            - expr: liveTurnTimeoutMs(env, 20000)
        - set: thinkOffCursor
          value:
            expr: state.getSnapshot().messages.length
        - sendInbound:
            conversation:
              id:
                expr: config.conversationId
              kind: direct
            senderId: qa-operator
            senderName: QA Operator
            text:
              expr: config.offDirective
        - call: waitForCondition
          saveAs: thinkOffAck
          args:
            - lambda:
                expr: "state.getSnapshot().messages.slice(thinkOffCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking disabled/i.test(candidate.text)).at(-1)"
            - expr: liveTurnTimeoutMs(env, 20000)
        - set: offCursor
          value:
            expr: state.getSnapshot().messages.length
        - sendInbound:
            conversation:
              id:
                expr: config.conversationId
              kind: direct
            senderId: qa-operator
            senderName: QA Operator
            text:
              expr: config.offPrompt
        - call: waitForCondition
          saveAs: offAnswer
          args:
            - lambda:
                expr: "state.getSnapshot().messages.slice(offCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.includes(config.offMarker)).at(-1)"
            - expr: liveTurnTimeoutMs(env, 90000)
        - set: offMessages
          value:
            expr: "state.getSnapshot().messages.slice(offCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId)"
        - assert:
            expr: "offMessages.some((candidate) => candidate.text.includes(config.offMarker))"
            message:
              expr: "`missing off marker; saw ${offMessages.map((message) => message.text).join(' | ')}`"
        - assert:
            expr: "!offMessages.some((candidate) => candidate.text.trimStart().startsWith('Reasoning:'))"
            message:
              expr: "`disabled thinking unexpectedly emitted reasoning: ${offMessages.map((message) => message.text).join(' | ')}`"
        - if:
            expr: "Boolean(env.mock)"
            then:
              - set: requests
                value:
                  expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests`)"
              - set: offRequest
                value:
                  expr: "requests.find((request) => String(request.allInputText ?? '').includes(config.offPrompt))"
              - assert:
                  expr: "String(offRequest?.model ?? '').includes('gpt-5.5')"
                  message:
                    expr: "`expected GPT-5.5 off mock request, got ${String(offRequest?.model ?? '')}`"
              - assert:
                  expr: "offRequest?.body && !Object.prototype.hasOwnProperty.call(offRequest.body, 'reasoning')"
                  message:
                    expr: "`disabled thinking should omit OpenAI reasoning payload, got ${JSON.stringify(offRequest?.body?.reasoning ?? null)}`"
      detailsExpr: "`reasoning ack=${reasoningAck.text}; thinking off=${thinkOffAck.text}; off answer=${offAnswer.text}`"
    - name: switches to medium thinking
      actions:
        - set: thinkMediumCursor
          value:
            expr: state.getSnapshot().messages.length
        - sendInbound:
            conversation:
              id:
                expr: config.conversationId
              kind: direct
            senderId: qa-operator
            senderName: QA Operator
            text:
              expr: config.maxDirective
        - call: waitForCondition
          saveAs: thinkMediumAck
          args:
            - lambda:
                expr: "state.getSnapshot().messages.slice(thinkMediumCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking level set to medium/i.test(candidate.text)).at(-1)"
            - expr: liveTurnTimeoutMs(env, 20000)
      detailsExpr: "`thinking medium=${thinkMediumAck.text}`"
    - name: verifies medium thinking reaches the provider
      actions:
        - set: maxCursor
          value:
            expr: state.getSnapshot().messages.length
        - sendInbound:
            conversation:
              id:
                expr: config.conversationId
              kind: direct
            senderId: qa-operator
            senderName: QA Operator
            text:
              expr: config.maxPrompt
        - call: waitForCondition
          saveAs: maxAnswer
          args:
            - lambda:
                expr: "state.getSnapshot().messages.slice(maxCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.includes(config.maxMarker)).at(-1)"
            - expr: liveTurnTimeoutMs(env, 120000)
        - assert:
            expr: "maxAnswer.text.includes(config.maxMarker)"
            message:
              expr: "`missing max marker near answer: ${recentOutboundSummary(state, 6)}`"
        - if:
            expr: "Boolean(env.mock)"
            then:
              - set: requests
                value:
                  expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests`)"
              - set: maxRequest
                value:
                  expr: "requests.find((request) => String(request.allInputText ?? '').includes(config.maxPrompt))"
              - assert:
                  expr: "String(maxRequest?.model ?? '').includes('gpt-5.5')"
                  message:
                    expr: "`expected GPT-5.5 mock request, got ${String(maxRequest?.model ?? '')}`"
              - assert:
                  expr: "maxRequest?.body?.reasoning?.effort === 'medium'"
                  message:
                    expr: "`expected medium OpenAI reasoning payload, got ${JSON.stringify(maxRequest?.body?.reasoning ?? null)}`"
        - if:
            expr: "env.transport.id !== 'qa-channel' && env.transport.id !== 'crabline'"
            then:
              - call: waitForCondition
                saveAs: maxReasoning
                args:
                  - lambda:
                      expr: "state.getSnapshot().messages.slice(maxCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.trimStart().startsWith('Reasoning:')).at(-1)"
                  - expr: liveTurnTimeoutMs(env, 120000)
              - assert:
                  expr: "maxReasoning.text.trimStart().startsWith('Reasoning:')"
                  message:
                    expr: "`missing max reasoning message near answer: ${recentOutboundSummary(state, 6)}`"
      detailsExpr: "env.transport.id === 'qa-channel' || env.transport.id === 'crabline' ? `answer=${maxAnswer.text}; medium reasoning=${env.mock ? String(maxRequest?.body?.reasoning?.effort ?? '') : 'live'}; ${env.transport.id} suppresses reasoning delivery` : `answer=${maxAnswer.text}; reasoning=${maxReasoning.text}`"