adolf/qa/scenarios/runtime/otel-trace-smoke.yaml

title: OTEL trace smoke

scenario:
  id: otel-trace-smoke
  surface: telemetry
  coverage:
    primary:
      - telemetry.otel
    secondary:
      - harness.qa-lab
  objective: Verify a QA-lab gateway run emits bounded OpenTelemetry traces, metrics, and logs through the diagnostics-otel plugin.
  successCriteria:
    - The diagnostics-otel plugin starts with trace, metric, and log export enabled.
    - A minimal QA-channel agent turn completes.
    - The trace includes the selected agent harness lifecycle span.
    - The run emits low-cardinality OpenTelemetry signals without content or raw diagnostic identifiers.
  plugins:
    - diagnostics-otel
  gatewayConfigPatch:
    logging:
      file: .artifacts/qa-e2e/otel-smoke-gateway.jsonl
      level: info
    diagnostics:
      enabled: true
      otel:
        enabled: true
        protocol: http/protobuf
        traces: true
        metrics: true
        logs: true
        sampleRate: 1
        flushIntervalMs: 1000
        captureContent:
          enabled: false
  docsRefs:
    - docs/gateway/opentelemetry.md
    - docs/concepts/qa-e2e-automation.md
  codeRefs:
    - extensions/diagnostics-otel/src/service.ts
    - src/agents/harness/lifecycle.ts
    - extensions/qa-lab/src/suite.ts
  execution:
    kind: flow
    summary: Emit minimal QA-lab telemetry with diagnostics-otel enabled.
    config:
      prompt: "OTEL QA marker: reply exactly `OTEL-QA-OK`. Do not repeat OTEL-QA-SECRET."
      expectedReply: OTEL-QA-OK

flow:
  steps:
    - name: emits a traced qa-channel turn
      actions:
        - call: waitForGatewayHealthy
          args:
            - ref: env
            - 60000
        - call: waitForQaChannelReady
          args:
            - ref: env
            - 60000
        - call: reset
        - set: startCursor
          value:
            expr: state.getSnapshot().messages.length
        - call: runAgentPrompt
          args:
            - ref: env
            - sessionKey: agent:qa:otel-trace-smoke
              message:
                expr: config.prompt
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 30000)
        - call: waitForCondition
          saveAs: outbound
          args:
            - lambda:
                expr: "state.getSnapshot().messages.slice(startCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && String(candidate.text ?? '').trim().length > 0).at(-1)"
            - expr: liveTurnTimeoutMs(env, 30000)
            - expr: "env.providerMode === 'mock-openai' ? 100 : 250"
        - assert:
            expr: "String(outbound.text ?? '').trim().length > 0"
            message: "expected non-empty qa output"
        - assert:
            expr: "String(outbound.text ?? '').includes(config.expectedReply)"
            message: "expected qa output to include the response sentinel"