adolf/qa/scenarios/runtime/tool-search-gateway-e2e.yaml

title: Tool Search gateway E2E

scenario:
  id: tool-search-gateway-e2e
  surface: runtime
  coverage:
    primary:
      - runtime.hosted-tool-use
    secondary:
      - plugins.plugin-tools
      - tools.invocation
  objective: Verify the Tool Search gateway QA Lab flow keeps a large plugin-owned tool catalog behind the compact bridge while still invoking the selected plugin tool.
  successCriteria:
    - Direct mode exposes the fake plugin tool schemas and calls the selected plugin tool.
    - Tool Search code mode exposes only the compact bridge to the provider.
    - The compact bridge calls the same selected plugin tool and records bridge plus target tool mentions in session logs.
    - The Tool Search request payload is smaller than direct tool exposure for the large fake catalog.
  docsRefs:
    - docs/tools/tool-search.md
    - docs/gateway/protocol.md
    - docs/concepts/qa-e2e-automation.md
  codeRefs:
    - extensions/qa-lab/src/tool-search-gateway.fixture.ts
    - extensions/qa-lab/src/tool-search-gateway.fixture.test.ts
  execution:
    kind: flow
    suiteIsolation: isolated
    isolationReason: Mutates gateway plugin/tool-search config and restarts the QA gateway between direct and compact lanes.
    summary: Flow-backed gateway E2E for compact Tool Search bridge and plugin-owned tool invocation.
    config:
      requiredProviderMode: mock-openai
      targetTool: fake_plugin_tool_17
      toolCount: 36

flow:
  steps:
    - name: stages fake plugin tool catalog
      actions:
        - assert:
            expr: "env.providerMode === config.requiredProviderMode"
            message: this seeded scenario is mock-openai only
        - call: qaImport
          saveAs: toolSearch
          args:
            - ./tool-search-gateway.fixture.js
        - call: toolSearch.stageToolSearchGatewayFixture
          saveAs: toolSearchFixture
          args:
            - env:
                ref: env
              targetTool:
                expr: config.targetTool
              toolCount:
                expr: config.toolCount
      detailsExpr: toolSearchFixture

    - name: compares direct and compact Tool Search gateway lanes
      actions:
        - call: toolSearch.runToolSearchGatewayLane
          saveAs: normalLane
          args:
            - env:
                ref: env
              fixture:
                ref: toolSearchFixture
              lane: normal
        - call: toolSearch.runToolSearchGatewayLane
          saveAs: codeLane
          args:
            - env:
                ref: env
              fixture:
                ref: toolSearchFixture
              lane: code
        - call: toolSearch.assertToolSearchLaneResults
          args:
            - normal:
                ref: normalLane
              code:
                ref: codeLane
              targetTool:
                ref: toolSearchFixture.targetTool
      detailsExpr: |-
        ({
          targetTool: toolSearchFixture.targetTool,
          directDeclaredTools: normalLane.providerDeclaredToolCount,
          compactDeclaredTools: codeLane.providerDeclaredToolCount,
          directRawBytes: normalLane.providerRawBytes,
          compactRawBytes: codeLane.providerRawBytes,
          directPlannedTools: normalLane.providerPlannedTools,
          compactPlannedTools: codeLane.providerPlannedTools,
          directMentions: normalLane.sessionLogToolMentions,
          compactMentions: codeLane.sessionLogToolMentions,
        })