// Hosted media provider live producer tests cover QA evidence wiring. import { spawnSync } from "node:child_process"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { MEDIA_SUITES, buildRunPlan, buildHostedMediaCommand, buildHostedMediaEvidence, classifyHostedMediaFailureStatus, findSkippedExplicitProviderSelections, parseArgs, parseHostedMediaOptions, runCli, type SuiteRunPlan, } from "./hosted-media-provider-live.js"; const SOURCE_PATH = "test/e2e/qa-lab/media/hosted-media-provider-live.ts"; const loadShellEnvFallbackMock = vi.fn(); const collectProviderApiKeysMock = vi.fn((provider: string) => process.env[`TEST_AUTH_${provider.toUpperCase()}`] ? ["test-key"] : [], ); function requirePlanEntry(plan: SuiteRunPlan[], suiteId: string) { const entry = plan.find((candidate) => candidate.suite.id === suiteId); if (!entry) { throw new Error(`expected ${suiteId} run plan entry`); } return entry; } afterEach(() => { collectProviderApiKeysMock.mockClear(); loadShellEnvFallbackMock.mockReset(); vi.unstubAllEnvs(); }); describe("hosted media provider live QA producer", () => { it("builds the image live media command with provider filters from env", () => { const options = parseHostedMediaOptions([ "--suite", "image", "--artifact-base", ".artifacts/qa-e2e/image", "--providers-env", "OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS", ]); const command = buildHostedMediaCommand({ env: { OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS: "openai,google" }, options, }); expect(command.args).toContain(SOURCE_PATH); expect(command.args).toContain("image"); expect(command.args).toContain("--image-providers"); expect(command.args).toContain("openai,google"); expect(command.env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES).toBeUndefined(); }); it("forces full video modes so reference image and video inputs are covered", () => { const options = parseHostedMediaOptions([ "--suite", "video", "--artifact-base", ".artifacts/qa-e2e/video", ]); const command = buildHostedMediaCommand({ env: {}, options }); expect(command.args).toContain("video"); expect(command.env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES).toBe("1"); }); it("classifies missing live media auth as blocked evidence", () => { expect( classifyHostedMediaFailureStatus( "[live:media] no runnable providers matched available auth; pass --allow-empty", ), ).toBe("blocked"); expect(classifyHostedMediaFailureStatus("provider response was malformed")).toBe("fail"); }); it("maps video provider live coverage roles without making tool invocation primary", () => { const artifactBase = path.join(os.tmpdir(), "openclaw-hosted-media-live-test"); const options = parseHostedMediaOptions(["--suite", "video", "--artifact-base", artifactBase]); const evidence = buildHostedMediaEvidence({ options, result: { durationMs: 10, status: "pass", }, }); expect(evidence.entries[0]?.coverage).toEqual([ { id: "hosted-providers.video-generation-providers", role: "primary" }, { id: "media.reference-image-video-and-audio-inputs", role: "primary" }, { id: "media.video-generation-tool-invocation", role: "secondary" }, ]); }); }); describe("hosted media provider live CLI", () => { it("prints help through the real node --import tsx entrypoint", () => { const result = spawnSync(process.execPath, ["--import", "tsx", SOURCE_PATH, "--help"], { cwd: process.cwd(), encoding: "utf8", }); expect(result.status).toBe(0); expect(result.stdout).toContain("Media live harness"); expect(result.stdout).toContain("pnpm test:live:media"); expect(result.stderr).toBe(""); }); it("rejects unknown global providers for the selected suites", () => { expect(() => parseArgs(["image", "--providers", "definitely-not-a-provider", "--all-providers"]), ).toThrow("Unknown provider(s) for selected media suite(s): definitely-not-a-provider"); }); it("rejects unknown suite-specific providers", () => { expect(() => parseArgs(["image", "--image-providers", "runway", "--all-providers"])).toThrow( "Unknown image provider(s): runway", ); }); it("accepts providers supported by the wrapped live suites", () => { expect( parseArgs(["image", "--image-providers", "openrouter", "--all-providers"]).suiteProviders .image, ).toEqual(new Set(["openrouter"])); expect( parseArgs(["music", "--music-providers", "fal,openrouter", "--all-providers"]).suiteProviders .music, ).toEqual(new Set(["fal", "openrouter"])); expect( parseArgs(["video", "--video-providers", "openrouter", "--all-providers"]).suiteProviders .video, ).toEqual(new Set(["openrouter"])); }); it("rejects suite-specific provider filters for unselected suites", () => { expect(() => parseArgs(["image", "--music-providers", "fal", "--all-providers"])).toThrow( "Provider filter(s) target unselected media suite(s): music", ); }); it("passes single-dash Vitest args after the option separator", () => { expect( parseArgs(["image", "--all-providers", "--project", "tooling", "--", "-t", "media-smoke"]), ).toMatchObject({ suites: ["image"], requireAuth: false, passthroughArgs: ["--project", "tooling", "-t", "media-smoke"], }); }); it("parses the explicit empty-run escape hatch", () => { expect(parseArgs(["--allow-empty"])).toMatchObject({ allowEmpty: true, requireAuth: true, }); }); it("fails explicit suite selections that auth filtering would skip", () => { const options = parseArgs([ "image", "music", "--image-providers", "openai", "--music-providers", "minimax", ]); const skipped = findSkippedExplicitProviderSelections(options, [ { suite: MEDIA_SUITES.image, providers: ["openai"] }, { suite: MEDIA_SUITES.music, providers: [], skippedReason: "no providers with usable auth", }, ]); expect(skipped.map((entry) => entry.suite.id)).toEqual(["music"]); }); it("does not fail global provider filters for suites without provider overlap", () => { const options = parseArgs(["image", "music", "video", "--providers", "openai"]); const skipped = findSkippedExplicitProviderSelections(options, [ { suite: MEDIA_SUITES.image, providers: ["openai"] }, { suite: MEDIA_SUITES.music, providers: [], skippedReason: "no providers selected", }, { suite: MEDIA_SUITES.video, providers: ["openai"] }, ]); expect(skipped).toEqual([]); }); it("fails default live media runs when auth filtering leaves no providers", async () => { await expect( runCli(["image"], { buildRunPlanImpl: () => [ { providers: [], skippedReason: "no providers with usable auth", suite: MEDIA_SUITES.image, }, ], }), ).resolves.toBe(1); }); it("allows empty live media runs only with an explicit escape hatch", async () => { await expect( runCli(["image", "--allow-empty"], { buildRunPlanImpl: () => [ { providers: [], skippedReason: "no providers with usable auth", suite: MEDIA_SUITES.image, }, ], }), ).resolves.toBe(0); }); it("defaults to all suites with auth filtering", async () => { vi.stubEnv("TEST_AUTH_OPENAI", "1"); vi.stubEnv("TEST_AUTH_GOOGLE", "1"); vi.stubEnv("TEST_AUTH_MINIMAX", "1"); vi.stubEnv("TEST_AUTH_FAL", "1"); vi.stubEnv("TEST_AUTH_VYDRA", "1"); const plan = await buildRunPlan(parseArgs([]), { collectProviderApiKeysImpl: collectProviderApiKeysMock, getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], loadShellEnvFallbackImpl: loadShellEnvFallbackMock, }); expect(plan.map((entry) => entry.suite.id)).toEqual(["image", "music", "video"]); expect(requirePlanEntry(plan, "image").providers).toEqual([ "fal", "google", "minimax", "openai", "vydra", ]); expect(requirePlanEntry(plan, "music").providers).toEqual(["fal", "google", "minimax"]); expect(requirePlanEntry(plan, "video").providers).toEqual([ "google", "minimax", "openai", "vydra", ]); }); it("supports suite-specific provider filters without auth narrowing", async () => { const plan = await buildRunPlan( parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]), { collectProviderApiKeysImpl: collectProviderApiKeysMock, getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], loadShellEnvFallbackImpl: loadShellEnvFallbackMock, }, ); expect(plan).toHaveLength(1); const [entry] = plan; expect(entry?.suite.id).toBe("video"); expect(entry?.providers).toEqual(["fal", "openai", "runway"]); }); it("forwards quiet flags separately from passthrough args", () => { const options = parseArgs(["image", "--quiet", "--reporter", "dot"]); expect(options.suites).toEqual(["image"]); expect(options.quietArgs).toEqual(["--quiet"]); expect(options.passthroughArgs).toEqual(["--reporter", "dot"]); }); });