adolf/qa/scenarios/channels/webchat-direct-reply-routing.yaml

title: WebChat direct reply routing

scenario:
  id: webchat-direct-reply-routing
  surface: qa-channel
  runtimeParityTier: live-only
  coverage:
    primary:
      - runtime.direct-reply-routing
    secondary:
      - tools.message
      - channels.webchat
  objective: Verify a current-chat reply is delivered as assistant text, not by calling `message(action=send)` and ending with `Sent.`.
  gatewayConfigPatch:
    session:
      dmScope: per-channel-peer
  successCriteria:
    - The visible outbound reply contains the requested marker exactly once.
    - The session transcript does not include a `message(action=send)` call followed by final assistant text `Sent.`.
  docsRefs:
    - docs/concepts/qa-e2e-automation.md
    - qa/scenarios/index.yaml
  codeRefs:
    - extensions/qa-lab/src/suite-runtime-agent-session.ts
    - extensions/qa-lab/src/gateway-log-sentinel.ts
  execution:
    kind: flow
    summary: Run a direct current-chat reply and inspect the actual transcript for self-message routing.
    config:
      conversationId: qa-operator
      expectedMarker: WEBCHAT-DIRECT-REPLY-OK
      promptSnippet: Reply exactly

flow:
  steps:
    - name: replies directly instead of sending a self-message
      actions:
        - call: waitForGatewayHealthy
          args:
            - ref: env
            - 60000
        - call: waitForQaChannelReady
          args:
            - ref: env
            - 60000
        - call: reset
        - set: conversationId
          value:
            expr: config.conversationId
        - set: delivery
          value:
            expr: "transport.buildAgentDelivery({ target: `dm:${conversationId}` })"
        - set: sessionKey
          value:
            expr: "buildAgentSessionKey({ agentId: 'qa', channel: delivery.channel, accountId: transport.accountId, peer: { kind: 'direct', id: delivery.replyTo }, dmScope: env.cfg.session?.dmScope, identityLinks: env.cfg.session?.identityLinks })"
        - set: startIndex
          value:
            expr: state.getSnapshot().messages.length
        - set: requestCountBefore
          value:
            expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
        - sendInbound:
            conversation:
              id:
                ref: conversationId
              kind: direct
            senderId:
              ref: conversationId
            senderName: WebChat QA
            text:
              expr: "`Reply exactly \\`${config.expectedMarker}\\` in this current chat. Do not call the message tool.`"
        - try:
            actions:
              - call: waitForCondition
                saveAs: scenarioRequest
                args:
                  - lambda:
                      async: true
                      expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true"
                  - expr: liveTurnTimeoutMs(env, 60000)
                  - 500
              - call: waitForOutboundMessage
                saveAs: outbound
                args:
                  - ref: state
                  - lambda:
                      params: [candidate]
                      expr: "candidate.conversation.id === conversationId && normalizeLowercaseStringOrEmpty(candidate.text).includes(normalizeLowercaseStringOrEmpty(config.expectedMarker))"
                  - expr: liveTurnTimeoutMs(env, 60000)
                  - sinceIndex:
                      ref: startIndex
            catchAs: directReplyError
            catch:
              - set: directReplyDebugRequests
                value:
                  expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, allInputText: String(request.allInputText ?? '').slice(0, 400), finalText: String(request.finalText ?? '').slice(0, 200), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 200) : null })) : []"
              - throw:
                  expr: "`direct reply marker missing: ${directReplyError?.message ?? directReplyError}; transcript=${formatTransportTranscript(state, { conversationId })}; requests=${JSON.stringify(directReplyDebugRequests)}`"
        - set: transcriptSummary
          value:
            expr: "await readSessionTranscriptSummary(env, sessionKey)"
        - assert:
            expr: "!transcriptSummary.hasDirectReplySelfMessage"
            message:
              expr: "`assistant self-sent direct reply through message(action=send); finalText=${transcriptSummary.finalText}`"
      detailsExpr: outbound.text