Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
714
extensions/fal/video-generation-provider.ts
Normal file
714
extensions/fal/video-generation-provider.ts
Normal file
@@ -0,0 +1,714 @@
|
||||
// Fal provider module implements model/runtime integration.
|
||||
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
|
||||
import { resolvePositiveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
createProviderOperationDeadline,
|
||||
readProviderJsonResponse,
|
||||
type ProviderOperationDeadline,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
|
||||
import {
|
||||
fetchWithSsrFGuard,
|
||||
type SsrFPolicy,
|
||||
ssrfPolicyFromDangerouslyAllowPrivateNetwork,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
isRecord,
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import { resolveFalHttpRequestConfig } from "./http-config.js";
|
||||
|
||||
const DEFAULT_FAL_QUEUE_BASE_URL = "https://queue.fal.run";
|
||||
const DEFAULT_FAL_VIDEO_MODEL = "fal-ai/minimax/video-01-live";
|
||||
const HEYGEN_VIDEO_AGENT_MODEL = "fal-ai/heygen/v2/video-agent";
|
||||
const SEEDANCE_2_TEXT_IMAGE_VIDEO_MODELS = [
|
||||
"bytedance/seedance-2.0/fast/text-to-video",
|
||||
"bytedance/seedance-2.0/fast/image-to-video",
|
||||
"bytedance/seedance-2.0/text-to-video",
|
||||
"bytedance/seedance-2.0/image-to-video",
|
||||
] as const;
|
||||
const SEEDANCE_2_REFERENCE_VIDEO_MODELS = [
|
||||
"bytedance/seedance-2.0/fast/reference-to-video",
|
||||
"bytedance/seedance-2.0/reference-to-video",
|
||||
] as const;
|
||||
const SEEDANCE_2_VIDEO_MODELS = [
|
||||
...SEEDANCE_2_TEXT_IMAGE_VIDEO_MODELS,
|
||||
...SEEDANCE_2_REFERENCE_VIDEO_MODELS,
|
||||
] as const;
|
||||
const SEEDANCE_2_DURATION_SECONDS = [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] as const;
|
||||
const SEEDANCE_REFERENCE_MAX_IMAGES = 9;
|
||||
const SEEDANCE_REFERENCE_MAX_VIDEOS = 3;
|
||||
const SEEDANCE_REFERENCE_MAX_AUDIOS = 3;
|
||||
const SEEDANCE_REFERENCE_MAX_FILES = 12;
|
||||
const SEEDANCE_REFERENCE_MAX_IMAGES_BY_MODEL = Object.fromEntries(
|
||||
SEEDANCE_2_REFERENCE_VIDEO_MODELS.map((model) => [model, SEEDANCE_REFERENCE_MAX_IMAGES]),
|
||||
);
|
||||
const SEEDANCE_REFERENCE_MAX_VIDEOS_BY_MODEL = Object.fromEntries(
|
||||
SEEDANCE_2_REFERENCE_VIDEO_MODELS.map((model) => [model, SEEDANCE_REFERENCE_MAX_VIDEOS]),
|
||||
);
|
||||
const SEEDANCE_REFERENCE_MAX_AUDIOS_BY_MODEL = Object.fromEntries(
|
||||
SEEDANCE_2_REFERENCE_VIDEO_MODELS.map((model) => [model, SEEDANCE_REFERENCE_MAX_AUDIOS]),
|
||||
);
|
||||
const DEFAULT_HTTP_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_OPERATION_TIMEOUT_MS = 1_200_000;
|
||||
const DEFAULT_GENERATED_VIDEO_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const POLL_INTERVAL_MS = 5_000;
|
||||
const FAL_VIDEO_MALFORMED_RESPONSE = "fal video generation response malformed";
|
||||
const FAL_VIDEO_PENDING_STATUSES = new Set([
|
||||
"IN_QUEUE",
|
||||
"IN_PROGRESS",
|
||||
"PROCESSING",
|
||||
"QUEUED",
|
||||
"STARTED",
|
||||
]);
|
||||
|
||||
type FalVideoResponse = {
|
||||
video?: {
|
||||
url?: string;
|
||||
content_type?: string;
|
||||
};
|
||||
videos?: Array<{
|
||||
url?: string;
|
||||
content_type?: string;
|
||||
}>;
|
||||
prompt?: string;
|
||||
seed?: number;
|
||||
};
|
||||
|
||||
type FalQueueResponse = {
|
||||
status?: string;
|
||||
request_id?: string;
|
||||
response_url?: string;
|
||||
status_url?: string;
|
||||
cancel_url?: string;
|
||||
detail?: string;
|
||||
response?: FalVideoResponse;
|
||||
prompt?: string;
|
||||
error?: {
|
||||
message?: string;
|
||||
};
|
||||
};
|
||||
|
||||
let falFetchGuard = fetchWithSsrFGuard;
|
||||
|
||||
export function setFalVideoFetchGuardForTesting(impl: typeof fetchWithSsrFGuard | null): void {
|
||||
falFetchGuard = impl ?? fetchWithSsrFGuard;
|
||||
}
|
||||
|
||||
function normalizeFalVideoUrl(value: unknown): string | undefined {
|
||||
const normalized = normalizeOptionalString(value);
|
||||
if (!normalized && value !== undefined && value !== null) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function readFalVideoPayload(payload: unknown): FalVideoResponse {
|
||||
if (!isRecord(payload)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
const video = payload.video;
|
||||
const videos = payload.videos;
|
||||
if (video !== undefined && video !== null && !isRecord(video)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
if (videos !== undefined && videos !== null && !Array.isArray(videos)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
return {
|
||||
video: isRecord(video)
|
||||
? {
|
||||
url: normalizeFalVideoUrl(video.url),
|
||||
content_type: normalizeOptionalString(video.content_type),
|
||||
}
|
||||
: undefined,
|
||||
videos: Array.isArray(videos)
|
||||
? videos.map((entry) => {
|
||||
if (!isRecord(entry)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
return {
|
||||
url: normalizeFalVideoUrl(entry.url),
|
||||
content_type: normalizeOptionalString(entry.content_type),
|
||||
};
|
||||
})
|
||||
: undefined,
|
||||
prompt: normalizeOptionalString(payload.prompt),
|
||||
seed: typeof payload.seed === "number" ? payload.seed : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function readFalQueueResponse(payload: unknown): FalQueueResponse {
|
||||
if (!isRecord(payload)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
const error = payload.error;
|
||||
if (error !== undefined && error !== null && !isRecord(error)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
return {
|
||||
status: normalizeOptionalString(payload.status),
|
||||
request_id: normalizeOptionalString(payload.request_id),
|
||||
response_url: normalizeOptionalString(payload.response_url),
|
||||
status_url: normalizeOptionalString(payload.status_url),
|
||||
cancel_url: normalizeOptionalString(payload.cancel_url),
|
||||
detail: normalizeOptionalString(payload.detail),
|
||||
response: payload.response === undefined ? undefined : readFalVideoPayload(payload.response),
|
||||
prompt: normalizeOptionalString(payload.prompt),
|
||||
error: isRecord(error) ? { message: normalizeOptionalString(error.message) } : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function readFalCompletedQueueResult(payload: unknown): FalQueueResponse {
|
||||
if (!isRecord(payload)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
if (
|
||||
payload.response !== undefined ||
|
||||
(payload.video === undefined && payload.videos === undefined)
|
||||
) {
|
||||
return readFalQueueResponse(payload);
|
||||
}
|
||||
return {
|
||||
response: readFalVideoPayload(payload),
|
||||
};
|
||||
}
|
||||
|
||||
function toDataUrl(buffer: Buffer, mimeType: string): string {
|
||||
return `data:${mimeType};base64,${buffer.toString("base64")}`;
|
||||
}
|
||||
|
||||
function buildPolicy(allowPrivateNetwork: boolean): SsrFPolicy | undefined {
|
||||
return allowPrivateNetwork ? ssrfPolicyFromDangerouslyAllowPrivateNetwork(true) : undefined;
|
||||
}
|
||||
|
||||
function extractFalVideoEntry(payload: FalVideoResponse) {
|
||||
if (normalizeOptionalString(payload.video?.url)) {
|
||||
return payload.video;
|
||||
}
|
||||
return payload.videos?.find((entry) => normalizeOptionalString(entry.url));
|
||||
}
|
||||
|
||||
function resolveGeneratedVideoMaxBytes(req: VideoGenerationRequest): number {
|
||||
const configured = req.cfg.agents?.defaults?.mediaMaxMb;
|
||||
if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) {
|
||||
return Math.floor(configured * 1024 * 1024);
|
||||
}
|
||||
return DEFAULT_GENERATED_VIDEO_MAX_BYTES;
|
||||
}
|
||||
|
||||
async function downloadFalVideo(
|
||||
url: string,
|
||||
policy: SsrFPolicy | undefined,
|
||||
maxBytes: number,
|
||||
): Promise<GeneratedVideoAsset> {
|
||||
const { response, release } = await falFetchGuard({
|
||||
url,
|
||||
timeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
|
||||
policy,
|
||||
auditContext: "fal-video-download",
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "fal generated video download failed");
|
||||
const mimeType = normalizeOptionalString(response.headers.get("content-type")) ?? "video/mp4";
|
||||
const fileName = `video-1.${extensionForMime(mimeType)?.slice(1) ?? "mp4"}`;
|
||||
let exceededMaxBytes = false;
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = await readResponseWithLimit(response, maxBytes, {
|
||||
onOverflow: ({ maxBytes: maxBytesLocal }) => {
|
||||
exceededMaxBytes = true;
|
||||
return new Error(`fal generated video download exceeds ${maxBytesLocal} bytes`);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (exceededMaxBytes) {
|
||||
return {
|
||||
url,
|
||||
mimeType,
|
||||
fileName,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
url,
|
||||
buffer,
|
||||
mimeType,
|
||||
fileName,
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
function resolveFalQueueBaseUrl(baseUrl: string): string {
|
||||
try {
|
||||
const url = new URL(baseUrl);
|
||||
if (url.hostname === "fal.run") {
|
||||
url.hostname = "queue.fal.run";
|
||||
return url.toString().replace(/\/$/, "");
|
||||
}
|
||||
return baseUrl.replace(/\/$/, "");
|
||||
} catch {
|
||||
return DEFAULT_FAL_QUEUE_BASE_URL;
|
||||
}
|
||||
}
|
||||
|
||||
function isFalMiniMaxLiveModel(model: string): boolean {
|
||||
return normalizeLowercaseStringOrEmpty(model) === DEFAULT_FAL_VIDEO_MODEL;
|
||||
}
|
||||
|
||||
function isFalSeedance2Model(model: string): boolean {
|
||||
return SEEDANCE_2_VIDEO_MODELS.includes(model as (typeof SEEDANCE_2_VIDEO_MODELS)[number]);
|
||||
}
|
||||
|
||||
function isFalSeedance2ReferenceModel(model: string): boolean {
|
||||
return SEEDANCE_2_REFERENCE_VIDEO_MODELS.includes(
|
||||
model as (typeof SEEDANCE_2_REFERENCE_VIDEO_MODELS)[number],
|
||||
);
|
||||
}
|
||||
|
||||
function isFalHeyGenVideoAgentModel(model: string): boolean {
|
||||
return normalizeLowercaseStringOrEmpty(model) === HEYGEN_VIDEO_AGENT_MODEL;
|
||||
}
|
||||
|
||||
function resolveFalResolution(resolution: VideoGenerationRequest["resolution"], model: string) {
|
||||
if (!resolution) {
|
||||
return undefined;
|
||||
}
|
||||
if (isFalSeedance2Model(model)) {
|
||||
return resolution.toLowerCase();
|
||||
}
|
||||
return resolution;
|
||||
}
|
||||
|
||||
function resolveFalDuration(
|
||||
durationSeconds: number | undefined,
|
||||
model: string,
|
||||
): number | string | undefined {
|
||||
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||
return undefined;
|
||||
}
|
||||
const duration = Math.max(1, Math.round(durationSeconds));
|
||||
if (isFalSeedance2Model(model)) {
|
||||
return SEEDANCE_2_DURATION_SECONDS.includes(
|
||||
duration as (typeof SEEDANCE_2_DURATION_SECONDS)[number],
|
||||
)
|
||||
? String(duration)
|
||||
: undefined;
|
||||
}
|
||||
return duration;
|
||||
}
|
||||
|
||||
function resolveFalReferenceUrl(
|
||||
asset: NonNullable<VideoGenerationRequest["inputImages"]>[number] | undefined,
|
||||
defaultMimeType: string,
|
||||
label: string,
|
||||
): string {
|
||||
const assetUrl = normalizeOptionalString(asset?.url);
|
||||
if (assetUrl) {
|
||||
return assetUrl;
|
||||
}
|
||||
if (!asset?.buffer) {
|
||||
throw new Error(`fal ${label} is missing media data.`);
|
||||
}
|
||||
return toDataUrl(asset.buffer, normalizeOptionalString(asset.mimeType) ?? defaultMimeType);
|
||||
}
|
||||
|
||||
function resolveFalReferenceUrls(
|
||||
assets: VideoGenerationRequest["inputImages"],
|
||||
defaultMimeType: string,
|
||||
label: string,
|
||||
): string[] {
|
||||
return (assets ?? []).map((asset) => resolveFalReferenceUrl(asset, defaultMimeType, label));
|
||||
}
|
||||
|
||||
function applyFalSeedanceControls(params: {
|
||||
req: VideoGenerationRequest;
|
||||
model: string;
|
||||
body: Record<string, unknown>;
|
||||
}): void {
|
||||
const aspectRatio = normalizeOptionalString(params.req.aspectRatio);
|
||||
if (aspectRatio) {
|
||||
params.body.aspect_ratio = aspectRatio;
|
||||
}
|
||||
const size = normalizeOptionalString(params.req.size);
|
||||
if (size) {
|
||||
params.body.size = size;
|
||||
}
|
||||
const resolution = resolveFalResolution(params.req.resolution, params.model);
|
||||
if (resolution) {
|
||||
params.body.resolution = resolution;
|
||||
}
|
||||
const duration = resolveFalDuration(params.req.durationSeconds, params.model);
|
||||
if (duration) {
|
||||
params.body.duration = duration;
|
||||
}
|
||||
if (isFalSeedance2Model(params.model) && typeof params.req.audio === "boolean") {
|
||||
params.body.generate_audio = params.req.audio;
|
||||
}
|
||||
}
|
||||
|
||||
function buildFalVideoRequestBody(params: {
|
||||
req: VideoGenerationRequest;
|
||||
model: string;
|
||||
}): Record<string, unknown> {
|
||||
const requestBody: Record<string, unknown> = {
|
||||
prompt: params.req.prompt,
|
||||
};
|
||||
|
||||
if (isFalSeedance2ReferenceModel(params.model)) {
|
||||
const imageUrls = resolveFalReferenceUrls(
|
||||
params.req.inputImages,
|
||||
"image/png",
|
||||
"reference image",
|
||||
);
|
||||
const videoUrls = resolveFalReferenceUrls(
|
||||
params.req.inputVideos,
|
||||
"video/mp4",
|
||||
"reference video",
|
||||
);
|
||||
const audioUrls = resolveFalReferenceUrls(
|
||||
params.req.inputAudios,
|
||||
"audio/mpeg",
|
||||
"reference audio",
|
||||
);
|
||||
if (imageUrls.length > 0) {
|
||||
requestBody.image_urls = imageUrls;
|
||||
}
|
||||
if (videoUrls.length > 0) {
|
||||
requestBody.video_urls = videoUrls;
|
||||
}
|
||||
if (audioUrls.length > 0) {
|
||||
requestBody.audio_urls = audioUrls;
|
||||
}
|
||||
applyFalSeedanceControls({ req: params.req, model: params.model, body: requestBody });
|
||||
return requestBody;
|
||||
}
|
||||
|
||||
const input = params.req.inputImages?.[0];
|
||||
if (input) {
|
||||
requestBody.image_url = normalizeOptionalString(input.url)
|
||||
? normalizeOptionalString(input.url)
|
||||
: input.buffer
|
||||
? toDataUrl(input.buffer, normalizeOptionalString(input.mimeType) ?? "image/png")
|
||||
: undefined;
|
||||
}
|
||||
// MiniMax Live on fal currently documents prompt + optional image_url only.
|
||||
// Keep the default model conservative so queue requests do not hang behind
|
||||
// unsupported knobs such as duration/resolution/aspect-ratio overrides.
|
||||
if (isFalMiniMaxLiveModel(params.model) || isFalHeyGenVideoAgentModel(params.model)) {
|
||||
return requestBody;
|
||||
}
|
||||
applyFalSeedanceControls({ req: params.req, model: params.model, body: requestBody });
|
||||
return requestBody;
|
||||
}
|
||||
|
||||
function validateFalVideoReferenceInputs(params: {
|
||||
req: VideoGenerationRequest;
|
||||
model: string;
|
||||
}): void {
|
||||
const imageCount = params.req.inputImages?.length ?? 0;
|
||||
const videoCount = params.req.inputVideos?.length ?? 0;
|
||||
const audioCount = params.req.inputAudios?.length ?? 0;
|
||||
if (isFalSeedance2ReferenceModel(params.model)) {
|
||||
if (imageCount > SEEDANCE_REFERENCE_MAX_IMAGES) {
|
||||
throw new Error(
|
||||
`fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_IMAGES} reference images.`,
|
||||
);
|
||||
}
|
||||
if (videoCount > SEEDANCE_REFERENCE_MAX_VIDEOS) {
|
||||
throw new Error(
|
||||
`fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_VIDEOS} reference videos.`,
|
||||
);
|
||||
}
|
||||
if (audioCount > SEEDANCE_REFERENCE_MAX_AUDIOS) {
|
||||
throw new Error(
|
||||
`fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_AUDIOS} reference audios.`,
|
||||
);
|
||||
}
|
||||
const totalFiles = imageCount + videoCount + audioCount;
|
||||
if (totalFiles > SEEDANCE_REFERENCE_MAX_FILES) {
|
||||
throw new Error(
|
||||
`fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_FILES} total reference files.`,
|
||||
);
|
||||
}
|
||||
if (audioCount > 0 && imageCount === 0 && videoCount === 0) {
|
||||
throw new Error(
|
||||
"fal Seedance reference-to-video requires at least one image or video reference when audio references are provided.",
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (videoCount > 0) {
|
||||
throw new Error("fal video generation does not support video reference inputs.");
|
||||
}
|
||||
if (audioCount > 0) {
|
||||
throw new Error("fal video generation does not support audio reference inputs.");
|
||||
}
|
||||
if (imageCount > 1) {
|
||||
throw new Error("fal video generation supports at most one image reference.");
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchFalJson(params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
timeoutMs: number;
|
||||
policy: SsrFPolicy | undefined;
|
||||
dispatcherPolicy: Parameters<typeof fetchWithSsrFGuard>[0]["dispatcherPolicy"];
|
||||
auditContext: string;
|
||||
errorContext: string;
|
||||
}): Promise<unknown> {
|
||||
const { response, release } = await falFetchGuard({
|
||||
url: params.url,
|
||||
init: params.init,
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: params.auditContext,
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, params.errorContext);
|
||||
try {
|
||||
return await readProviderJsonResponse<unknown>(response, params.errorContext);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.endsWith(": malformed JSON response")) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE, { cause: error });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForFalQueueResult(params: {
|
||||
statusUrl: string;
|
||||
responseUrl: string;
|
||||
headers: Headers;
|
||||
deadline: ProviderOperationDeadline;
|
||||
policy: SsrFPolicy | undefined;
|
||||
dispatcherPolicy: Parameters<typeof fetchWithSsrFGuard>[0]["dispatcherPolicy"];
|
||||
}): Promise<FalQueueResponse> {
|
||||
let lastStatus = "unknown";
|
||||
for (;;) {
|
||||
const requestTimeoutMs = resolveFalQueueRemainingMs(
|
||||
params.deadline,
|
||||
lastStatus,
|
||||
DEFAULT_HTTP_TIMEOUT_MS,
|
||||
);
|
||||
const payload = readFalQueueResponse(
|
||||
await fetchFalJson({
|
||||
url: params.statusUrl,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
},
|
||||
timeoutMs: requestTimeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: "fal-video-status",
|
||||
errorContext: "fal video status request failed",
|
||||
}),
|
||||
);
|
||||
const status = normalizeOptionalString(payload.status)?.toUpperCase();
|
||||
if (!status) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
lastStatus = status;
|
||||
if (status === "COMPLETED") {
|
||||
return readFalCompletedQueueResult(
|
||||
await fetchFalJson({
|
||||
url: params.responseUrl,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
},
|
||||
timeoutMs: resolveFalQueueRemainingMs(
|
||||
params.deadline,
|
||||
lastStatus,
|
||||
DEFAULT_HTTP_TIMEOUT_MS,
|
||||
),
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: "fal-video-result",
|
||||
errorContext: "fal video result request failed",
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (status === "FAILED" || status === "CANCELLED") {
|
||||
throw new Error(
|
||||
normalizeOptionalString(payload.detail) ||
|
||||
normalizeOptionalString(payload.error?.message) ||
|
||||
`fal video generation ${normalizeLowercaseStringOrEmpty(status)}`,
|
||||
);
|
||||
}
|
||||
if (!FAL_VIDEO_PENDING_STATUSES.has(status)) {
|
||||
throw new Error(FAL_VIDEO_MALFORMED_RESPONSE);
|
||||
}
|
||||
const pollDelayMs = resolveFalQueueRemainingMs(params.deadline, lastStatus, POLL_INTERVAL_MS);
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, pollDelayMs);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function resolveFalQueueRemainingMs(
|
||||
deadline: ProviderOperationDeadline,
|
||||
lastStatus: string,
|
||||
defaultTimeoutMs: number,
|
||||
): number {
|
||||
const defaultMs = resolvePositiveTimerTimeoutMs(defaultTimeoutMs, 1);
|
||||
if (typeof deadline.deadlineAtMs !== "number") {
|
||||
return defaultMs;
|
||||
}
|
||||
const remainingMs = deadline.deadlineAtMs - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
throw new Error(`fal video generation did not finish in time (last status: ${lastStatus})`);
|
||||
}
|
||||
return Math.max(1, Math.min(defaultMs, remainingMs));
|
||||
}
|
||||
|
||||
function extractFalVideoPayload(payload: FalQueueResponse): FalVideoResponse {
|
||||
if (payload.response) {
|
||||
return payload.response;
|
||||
}
|
||||
return readFalVideoPayload(payload);
|
||||
}
|
||||
|
||||
export function buildFalVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
id: "fal",
|
||||
label: "fal",
|
||||
defaultModel: DEFAULT_FAL_VIDEO_MODEL,
|
||||
models: [
|
||||
DEFAULT_FAL_VIDEO_MODEL,
|
||||
HEYGEN_VIDEO_AGENT_MODEL,
|
||||
...SEEDANCE_2_VIDEO_MODELS,
|
||||
"fal-ai/kling-video/v2.1/master/text-to-video",
|
||||
"fal-ai/wan/v2.2-a14b/text-to-video",
|
||||
"fal-ai/wan/v2.2-a14b/image-to-video",
|
||||
],
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "fal",
|
||||
agentDir,
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
supportedDurationSecondsByModel: Object.fromEntries(
|
||||
SEEDANCE_2_VIDEO_MODELS.map((model) => [model, SEEDANCE_2_DURATION_SECONDS]),
|
||||
),
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxInputImagesByModel: SEEDANCE_REFERENCE_MAX_IMAGES_BY_MODEL,
|
||||
maxInputAudiosByModel: SEEDANCE_REFERENCE_MAX_AUDIOS_BY_MODEL,
|
||||
supportedDurationSecondsByModel: Object.fromEntries(
|
||||
SEEDANCE_2_VIDEO_MODELS.map((model) => [model, SEEDANCE_2_DURATION_SECONDS]),
|
||||
),
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 0,
|
||||
maxInputImagesByModel: SEEDANCE_REFERENCE_MAX_IMAGES_BY_MODEL,
|
||||
maxInputVideos: 0,
|
||||
maxInputVideosByModel: SEEDANCE_REFERENCE_MAX_VIDEOS_BY_MODEL,
|
||||
maxInputAudiosByModel: SEEDANCE_REFERENCE_MAX_AUDIOS_BY_MODEL,
|
||||
supportedDurationSecondsByModel: Object.fromEntries(
|
||||
SEEDANCE_2_REFERENCE_VIDEO_MODELS.map((model) => [model, SEEDANCE_2_DURATION_SECONDS]),
|
||||
),
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
},
|
||||
async generateVideo(req) {
|
||||
const model = normalizeOptionalString(req.model) || DEFAULT_FAL_VIDEO_MODEL;
|
||||
validateFalVideoReferenceInputs({ req, model });
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
await resolveFalHttpRequestConfig({ req, capability: "video" });
|
||||
const requestBody = buildFalVideoRequestBody({ req, model });
|
||||
const policy = buildPolicy(allowPrivateNetwork);
|
||||
const queueBaseUrl = resolveFalQueueBaseUrl(baseUrl);
|
||||
const submitted = readFalQueueResponse(
|
||||
await fetchFalJson({
|
||||
url: `${queueBaseUrl}/${model}`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
},
|
||||
timeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
|
||||
policy,
|
||||
dispatcherPolicy,
|
||||
auditContext: "fal-video-submit",
|
||||
errorContext: "fal video generation failed",
|
||||
}),
|
||||
);
|
||||
const statusUrl = normalizeOptionalString(submitted.status_url);
|
||||
const responseUrl = normalizeOptionalString(submitted.response_url);
|
||||
if (!statusUrl || !responseUrl) {
|
||||
throw new Error("fal video generation response missing queue URLs");
|
||||
}
|
||||
const operationTimeoutMs = resolvePositiveTimerTimeoutMs(
|
||||
req.timeoutMs,
|
||||
DEFAULT_OPERATION_TIMEOUT_MS,
|
||||
);
|
||||
const operationDeadline = createProviderOperationDeadline({
|
||||
timeoutMs: operationTimeoutMs,
|
||||
label: "fal video generation",
|
||||
});
|
||||
const payload = await waitForFalQueueResult({
|
||||
statusUrl,
|
||||
responseUrl,
|
||||
headers,
|
||||
deadline: operationDeadline,
|
||||
policy,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
const videoPayload = extractFalVideoPayload(payload);
|
||||
const entry = extractFalVideoEntry(videoPayload);
|
||||
const url = normalizeOptionalString(entry?.url);
|
||||
if (!url) {
|
||||
throw new Error("fal video generation response missing output URL");
|
||||
}
|
||||
const video = await downloadFalVideo(url, policy, resolveGeneratedVideoMaxBytes(req));
|
||||
return {
|
||||
videos: [video],
|
||||
model,
|
||||
metadata: {
|
||||
...(normalizeOptionalString(submitted.request_id)
|
||||
? { requestId: normalizeOptionalString(submitted.request_id) }
|
||||
: {}),
|
||||
...(videoPayload.prompt ? { prompt: videoPayload.prompt } : {}),
|
||||
...(typeof videoPayload.seed === "number" ? { seed: videoPayload.seed } : {}),
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user