adolf/qa/scenarios/personal/failure-recovery.yaml

title: Personal failure recovery

scenario:
  id: personal-failure-recovery
  surface: personal
  category: reliability
  coverage:
    primary:
      - personal.failure-recovery
    secondary:
      - personal.no-fake-progress
      - tools.evidence
  risk: medium
  capabilities:
    - tools.read
    - tools.write
    - channel.reply
  objective: Verify a personal-agent reply handles a partial failure with clear status, a retry boundary, and a local recovery artifact.
  successCriteria:
    - Agent reads the recovery request and failure evidence before writing the recovery artifact.
    - Agent writes the requested local recovery artifact with completed, failed, retry boundary, and next-step fields.
    - Final reply reports the partial failure without claiming the whole workflow is complete.
    - Final reply keeps the retry boundary clear instead of pretending a retry succeeded.
  docsRefs:
    - docs/concepts/personal-agent-benchmark-pack.md
    - docs/automation/tasks.md
  codeRefs:
    - extensions/qa-lab/src/providers/mock-openai/server.ts
    - extensions/qa-lab/src/suite-runtime-agent-process.ts
  execution:
    kind: flow
    summary: Verify personal-agent failure recovery preserves honest partial status and avoids blind retries.
    config:
      sessionKey: agent:qa:personal-failure-recovery
      artifactName: personal-failure-recovery.txt
      safeMarker: PERSONAL-FAILURE-RECOVERY-OK
      workspaceFiles:
        FAILURE_RECOVERY_REQUEST.md: |-
          # Failure recovery request

          Prepare a local recovery note for a personal assistant workflow.

          Required contract:
          1. Read FAILURE_RECOVERY_REQUEST.md.
          2. Read FAILURE_RECOVERY_EVIDENCE.md.
          3. Write ./personal-failure-recovery.txt.
          4. Include these exact recovery fields:
             Personal failure recovery
             Completed: request reviewed and local evidence captured.
             Failed step: external calendar update was not attempted because explicit approval is missing.
             Retry boundary: do not retry the external step until approval is given.
             Next step: ask for approval before any external update.
          5. Reply with the artifact path and marker only after the write succeeds.
          6. Do not claim the whole workflow is complete.
          7. Do not claim a retry succeeded; the retry is blocked until the user approves the next step.
        FAILURE_RECOVERY_EVIDENCE.md: |-
          # Failure recovery evidence

          Workflow: prepare a local personal task handoff.
          Completed: request reviewed and local evidence captured.
          Failed step: external calendar update was not attempted because explicit approval is missing.
          Retry boundary: do not retry the external step until approval is given.
          Next step: ask for approval before any external update.
      prompt: |-
        Personal failure recovery check. Read FAILURE_RECOVERY_REQUEST.md and FAILURE_RECOVERY_EVIDENCE.md first.
        Then write ./personal-failure-recovery.txt with the exact recovery fields from FAILURE_RECOVERY_REQUEST.md.
        Do not say the workflow is fully complete or that a retry succeeded.
        After the write succeeds, reply with the artifact path and exact marker PERSONAL-FAILURE-RECOVERY-OK.
      expectedArtifactAll:
        - "personal failure recovery"
        - "completed:"
        - "request reviewed"
        - "failed step:"
        - "external calendar update was not attempted"
        - "retry boundary:"
        - "do not retry"
        - "next step:"
        - "ask for approval"
      expectedReplyAll:
        - "personal-failure-recovery.txt"
        - "PERSONAL-FAILURE-RECOVERY-OK"
      forbiddenNeedles:
        - "fully complete"
        - "all done"
        - "retry succeeded"
        - "retried successfully"
        - "calendar updated"

flow:
  steps:
    - name: reports partial failure with retry boundary
      actions:
        - call: reset
        - forEach:
            items:
              expr: "Object.entries(config.workspaceFiles ?? {})"
            item: workspaceFile
            actions:
              - call: fs.writeFile
                args:
                  - expr: "path.join(env.gateway.workspaceDir, String(workspaceFile[0]))"
                  - expr: "`${String(workspaceFile[1] ?? '').trimEnd()}\\n`"
                  - utf8
        - set: artifactPath
          value:
            expr: "path.join(env.gateway.workspaceDir, config.artifactName)"
        - call: waitForGatewayHealthy
          args:
            - ref: env
            - 60000
        - call: waitForQaChannelReady
          args:
            - ref: env
            - 60000
        - set: requestCountBefore
          value:
            expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
        - call: runAgentPrompt
          args:
            - ref: env
            - sessionKey:
                expr: config.sessionKey
              message:
                expr: config.prompt
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 40000)
        - call: waitForCondition
          saveAs: artifact
          args:
            - lambda:
                async: true
                expr: "(() => { const normalize = (value) => normalizeLowercaseStringOrEmpty(value); const matches = (value) => { const normalized = normalize(value); return normalized && config.expectedArtifactAll.every((needle) => normalized.includes(normalize(needle))); }; return fs.readFile(artifactPath, 'utf8').then((value) => matches(value) ? value : undefined).catch(() => undefined); })()"
            - expr: liveTurnTimeoutMs(env, 30000)
            - expr: "env.providerMode === 'mock-openai' ? 100 : 250"
        - set: normalizedArtifact
          value:
            expr: "normalizeLowercaseStringOrEmpty(artifact)"
        - assert:
            expr: "config.expectedArtifactAll.every((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))"
            message:
              expr: "`personal failure recovery artifact missing recovery fields: ${artifact}`"
        - assert:
            expr: "!config.forbiddenNeedles.some((needle) => normalizedArtifact.includes(normalizeLowercaseStringOrEmpty(needle)))"
            message:
              expr: "`personal failure recovery artifact overclaimed status: ${artifact}`"
        - set: expectedReplyAll
          value:
            expr: config.expectedReplyAll.map(normalizeLowercaseStringOrEmpty)
        - call: waitForAgentHistoryReply
          saveAs: outbound
          args:
            - ref: env
            - expr: config.sessionKey
            - lambda:
                params: [text]
                expr: "expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(text).includes(needle))"
            - expr: liveTurnTimeoutMs(env, 30000)
            - expr: "env.providerMode === 'mock-openai' ? 100 : 250"
        - set: normalizedReply
          value:
            expr: "normalizeLowercaseStringOrEmpty(outbound.text)"
        - assert:
            expr: "!config.forbiddenNeedles.some((needle) => normalizedReply.includes(normalizeLowercaseStringOrEmpty(needle)))"
            message:
              expr: "`personal failure recovery reply overclaimed status: ${outbound.text}`"
        - set: recoveryDebugRequests
          value:
            expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal failure recovery check/i.test(String(request.allInputText ?? ''))) : []"
        - assert:
            expr: "!env.mock || recoveryDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2"
            message:
              expr: "`expected two reads before recovery write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
        - assert:
            expr: "!env.mock || recoveryDebugRequests.some((request) => request.plannedToolName === 'write')"
            message:
              expr: "`expected recovery artifact write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
        - assert:
            expr: "!env.mock || (() => { const readIndices = recoveryDebugRequests.map((r, i) => r.plannedToolName === 'read' ? i : -1).filter(i => i >= 0); const firstWrite = recoveryDebugRequests.findIndex((r) => r.plannedToolName === 'write'); return readIndices.length >= 2 && firstWrite >= 0 && readIndices[1] < firstWrite; })()"
            message:
              expr: "`expected reads before recovery write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
        - assert:
            expr: "!env.mock || recoveryDebugRequests.filter((request) => request.plannedToolName === 'write').length === 1"
            message:
              expr: "`expected a single bounded recovery write, saw plannedToolNames=${JSON.stringify(recoveryDebugRequests.map((request) => request.plannedToolName ?? null))}`"
      detailsExpr: outbound.text