adolf/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.yaml

title: Control UI plus qa-channel image roundtrip

scenario:
  id: control-ui-qa-channel-image-roundtrip
  surface: control-ui
  coverage:
    primary:
      - ui.control
      - ui.dashboard-auth-bootstrap
    secondary:
      - media.image-understanding
      - channels.qa-channel
  objective: Verify the embedded Control UI can observe a qa-channel-backed session while the fake channel injects text and image turns that the agent answers correctly.
  successCriteria:
    - Control UI opens directly on the target qa-channel session.
    - A text prompt delivered through qa-channel produces a correct outbound reply.
    - A later qa-channel image message produces a correct image-aware reply.
    - The Control UI transcript shows both transport-side prompts and both final answers.
  docsRefs:
    - docs/concepts/qa-e2e-automation.md
    - docs/channels/qa-channel.md
  codeRefs:
    - extensions/qa-lab/src/scenario-runtime-api.ts
    - extensions/qa-lab/src/suite.ts
    - extensions/qa-lab/src/web-runtime.ts
    - ui/src/ui/views/chat.ts
  gatewayRuntime:
    forwardHostHome: true
  execution:
    kind: flow
    summary: Open the Control UI on a qa-channel session with the generic QA web driver, inject text and image turns through qa-channel, and verify the replies in both the transport log and the UI transcript.
    config:
      requiredChannelDriver: qa-channel
      conversationId: control-ui-e2e
      textPrompt: "Control UI bridge check. Marker exact marker: `ui bridge armed`"
      uiExpectedNeedle: ui bridge armed
      imagePrompt: "Image understanding check: describe the top and bottom colors in the attached image in one short sentence."
      imagePromptNeedle: image understanding check
      requiredColorGroups:
        - [red, scarlet, crimson]
        - [blue, azure, teal, cyan, aqua]

flow:
  steps:
    - name: opens control ui on the qa-channel-backed session
      actions:
        - call: reset
        - call: waitForGatewayHealthy
          args:
            - ref: env
            - expr: liveTurnTimeoutMs(env, 60000)
        - call: waitForQaChannelReady
          args:
            - ref: env
            - expr: liveTurnTimeoutMs(env, 60000)
        - call: fetchJson
          saveAs: bootstrap
          args:
            - expr: "`${lab.baseUrl}/api/bootstrap`"
        - assert:
            expr: "Boolean(bootstrap.controlUiEmbeddedUrl)"
            message: qa-lab bootstrap did not expose controlUiEmbeddedUrl
        - set: uiSessionKey
          value:
            expr: "buildAgentSessionKey({ agentId: env.cfg.agents?.list?.find((agent) => agent.default)?.id ?? env.cfg.agents?.list?.[0]?.id ?? 'main', channel: 'qa-channel', accountId: 'default', peer: { kind: 'direct', id: config.conversationId }, dmScope: env.cfg.session?.dmScope, identityLinks: env.cfg.session?.identityLinks })"
        - set: controlUiChatUrl
          value:
            expr: "(() => { const url = new URL(`${env.gateway.baseUrl}/`); url.searchParams.set('session', uiSessionKey); url.hash = `token=${encodeURIComponent(env.gateway.token ?? '')}`; return url.toString(); })()"
        - call: webOpenPage
          saveAs: uiTab
          args:
            - url:
                ref: controlUiChatUrl
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 60000)
        - set: uiPageId
          value:
            expr: "uiTab.pageId"
        - call: webWait
          args:
            - pageId:
                ref: uiPageId
              selector: openclaw-app
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 45000)
        - try:
            actions:
              - call: waitForCondition
                saveAs: uiReadySnapshot
                args:
                  - lambda:
                      async: true
                      expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiPageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); return text.includes('ready to chat') ? snapshot : undefined; })()"
                  - expr: liveTurnTimeoutMs(env, 45000)
                  - 500
            catch:
              - call: webSnapshot
                saveAs: uiReadyFailureSnapshot
                args:
                  - pageId:
                      ref: uiPageId
                    maxChars: 12000
                    timeoutMs:
                      expr: liveTurnTimeoutMs(env, 15000)
              - call: webEvaluate
                saveAs: uiReadyFailureState
                args:
                  - pageId:
                      ref: uiPageId
                    expression: "(() => { const app = document.querySelector('openclaw-app'); const resources = performance.getEntriesByType('resource').map((entry) => ({ name: entry.name, type: entry.initiatorType, duration: Math.round(entry.duration), transferSize: entry.transferSize, decodedBodySize: entry.decodedBodySize })); return { url: location.href, readyState: document.readyState, appDefined: Boolean(customElements.get('openclaw-app')), appState: app ? { sessionKey: app.sessionKey, settingsSessionKey: app.settings?.sessionKey, lastActiveSessionKey: app.settings?.lastActiveSessionKey, chatMessages: Array.isArray(app.chatMessages) ? app.chatMessages.length : null, chatLoading: app.chatLoading, lastError: app.lastError, connected: app.connected, tab: app.tab } : null, scripts: Array.from(document.scripts).map((script) => script.src || script.textContent?.slice(0, 80)), links: Array.from(document.querySelectorAll('link')).map((link) => link.href), resources, bodyHtml: document.body.innerHTML.slice(0, 400) }; })()"
                    timeoutMs:
                      expr: liveTurnTimeoutMs(env, 15000)
              - throw:
                  expr: "`control ui did not become ready. state=${JSON.stringify(uiReadyFailureState)} diagnostics=${JSON.stringify(uiReadyFailureSnapshot.diagnostics ?? [])} snapshot: ${uiReadyFailureSnapshot.text}`"
        - assert:
            expr: "Boolean(uiPageId)"
            message: control ui page was not available
      detailsExpr: "uiReadySnapshot.text"
    - name: text injected through qa-channel gets a correct transport reply
      actions:
        - set: firstInboundStartIndex
          value:
            expr: "state.getSnapshot().messages.filter((message) => message.direction === 'inbound').length"
        - set: firstOutboundStartIndex
          value:
            expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
        - sendInbound:
            accountId: default
            conversation:
              id:
                expr: config.conversationId
              kind: direct
            senderId:
              expr: config.conversationId
            senderName: Control UI QA
            text:
              expr: config.textPrompt
        - call: waitForOutboundMessage
          saveAs: uiOutbound
          args:
            - ref: state
            - lambda:
                params: [candidate]
                expr: "candidate.conversation.id === config.conversationId && normalizeLowercaseStringOrEmpty(candidate.text).includes(config.uiExpectedNeedle)"
            - expr: liveTurnTimeoutMs(env, 45000)
            - sinceIndex:
                ref: firstOutboundStartIndex
        - call: readRawQaSessionStore
          saveAs: rawSessionStore
          args:
            - ref: env
        - set: rawSessionStoreKeys
          value:
            expr: "Object.keys(rawSessionStore)"
      detailsExpr: "`${uiOutbound.text}\\nSTORE:${JSON.stringify(rawSessionStoreKeys)}`"
    - name: text injected through qa-channel renders in a fresh control ui load
      actions:
        - call: webOpenPage
          saveAs: uiAckTab
          args:
            - url:
                ref: controlUiChatUrl
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 60000)
        - set: uiAckPageId
          value:
            expr: "uiAckTab.pageId"
        - call: webWait
          args:
            - pageId:
                ref: uiAckPageId
              selector: openclaw-app
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 45000)
        - try:
            actions:
              - call: waitForCondition
                saveAs: uiAckSnapshot
                args:
                  - lambda:
                      async: true
                      expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiAckPageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); return text.includes(config.uiExpectedNeedle) && text.includes('control ui bridge check') ? snapshot : undefined; })()"
                  - expr: liveTurnTimeoutMs(env, 45000)
                  - 500
            catch:
              - call: webSnapshot
                saveAs: uiAckFailureSnapshot
                args:
                  - pageId:
                      ref: uiAckPageId
                    maxChars: 12000
                    timeoutMs:
                      expr: liveTurnTimeoutMs(env, 15000)
              - call: webEvaluate
                saveAs: uiAckFailureState
                args:
                  - pageId:
                      ref: uiAckPageId
                    expression: "(() => { const app = document.querySelector('openclaw-app'); return app ? { sessionKey: app.sessionKey, settingsSessionKey: app.settings?.sessionKey, lastActiveSessionKey: app.settings?.lastActiveSessionKey, chatMessages: Array.isArray(app.chatMessages) ? app.chatMessages.length : null, chatLoading: app.chatLoading, lastError: app.lastError, connected: app.connected } : null; })()"
                    timeoutMs:
                      expr: liveTurnTimeoutMs(env, 15000)
              - throw:
                  expr: "`control ui text transcript missing after fresh load. state=${JSON.stringify(uiAckFailureState)} snapshot: ${uiAckFailureSnapshot.text}`"
      detailsExpr: "uiAckSnapshot.text"
    - name: image injected through qa-channel gets a correct transport reply
      actions:
        - set: secondOutboundStartIndex
          value:
            expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
        - sendInbound:
            accountId: default
            conversation:
              id:
                expr: config.conversationId
              kind: direct
            senderId:
              expr: config.conversationId
            senderName: Control UI QA
            text:
              expr: config.imagePrompt
            attachments:
              - kind: image
                mimeType: image/png
                fileName: red-top-blue-bottom.png
                altText: red on top blue on bottom
                contentBase64:
                  expr: imageUnderstandingValidPngBase64
        - try:
            actions:
              - call: waitForOutboundMessage
                saveAs: imageOutbound
                args:
                  - ref: state
                  - lambda:
                      params: [candidate]
                      expr: "candidate.conversation.id === config.conversationId && config.requiredColorGroups.every((group) => group.some((color) => normalizeLowercaseStringOrEmpty(candidate.text).includes(color)))"
                  - expr: liveTurnTimeoutMs(env, 90000)
                  - sinceIndex:
                      ref: secondOutboundStartIndex
            catchAs: imageWaitError
            catch:
              - set: imageDebugRequests
                value:
                  expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(-16).map((request) => ({ plannedToolName: request.plannedToolName ?? null, prompt: String(request.prompt ?? '').slice(0, 260), allInputText: String(request.allInputText ?? '').slice(0, 260), imageInputCount: request.imageInputCount ?? null })) : []"
              - throw:
                  expr: "`qa-channel image reply missing: ${imageWaitError?.message ?? imageWaitError}; outbound=${recentOutboundSummary(state, 8)} requests=${JSON.stringify(imageDebugRequests)}`"
        - set: missingColorGroup
          value:
            expr: "config.requiredColorGroups.find((group) => !group.some((color) => normalizeLowercaseStringOrEmpty(imageOutbound.text).includes(color)))"
        - assert:
            expr: "!missingColorGroup"
            message:
              expr: "`missing expected colors in image reply: ${imageOutbound.text}`"
      detailsExpr: "imageOutbound.text"
    - name: image injected through qa-channel renders in a fresh control ui load
      actions:
        - call: webOpenPage
          saveAs: uiImageTab
          args:
            - url:
                ref: controlUiChatUrl
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 60000)
        - set: uiImagePageId
          value:
            expr: "uiImageTab.pageId"
        - call: webWait
          args:
            - pageId:
                ref: uiImagePageId
              selector: openclaw-app
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 45000)
        - try:
            actions:
              - call: waitForCondition
                saveAs: uiImageSnapshot
                args:
                  - lambda:
                      async: true
                      expr: "await (async () => { const snapshot = await webSnapshot({ pageId: uiImagePageId, maxChars: 12000, timeoutMs: liveTurnTimeoutMs(env, 30000) }); const text = normalizeLowercaseStringOrEmpty(snapshot.text); const hasPrompt = text.includes(config.imagePromptNeedle); const hasColors = config.requiredColorGroups.every((group) => group.some((color) => text.includes(color))); return hasPrompt && hasColors ? snapshot : undefined; })()"
                  - expr: liveTurnTimeoutMs(env, 90000)
                  - 500
            catch:
              - call: webSnapshot
                saveAs: uiImageFailureSnapshot
                args:
                  - pageId:
                      ref: uiImagePageId
                    maxChars: 12000
                    timeoutMs:
                      expr: liveTurnTimeoutMs(env, 15000)
              - call: webEvaluate
                saveAs: uiImageFailureState
                args:
                  - pageId:
                      ref: uiImagePageId
                    expression: "(() => { const app = document.querySelector('openclaw-app'); return app ? { sessionKey: app.sessionKey, settingsSessionKey: app.settings?.sessionKey, lastActiveSessionKey: app.settings?.lastActiveSessionKey, chatMessages: Array.isArray(app.chatMessages) ? app.chatMessages.length : null, chatLoading: app.chatLoading, lastError: app.lastError, connected: app.connected } : null; })()"
                    timeoutMs:
                      expr: liveTurnTimeoutMs(env, 15000)
              - throw:
                  expr: "`control ui image transcript missing after fresh load. state=${JSON.stringify(uiImageFailureState)} snapshot: ${uiImageFailureSnapshot.text}`"
      detailsExpr: "uiImageSnapshot.text"