Vendor OpenClaw source as Adolf fork baseline

Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11),
free to diverge. Tree copied sans upstream .git; upstream remote added for
future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19.
Preserves docs/ARCHITECTURE.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2

packages/media-understanding-common/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@openclaw/media-understanding-common",
+  "version": "0.0.0-private",
+  "private": true,
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    },
+    "./active-model": {
+      "types": "./dist/active-model.d.mts",
+      "import": "./dist/active-model.mjs",
+      "default": "./dist/active-model.mjs"
+    },
+    "./defaults": {
+      "types": "./dist/defaults.d.mts",
+      "import": "./dist/defaults.mjs",
+      "default": "./dist/defaults.mjs"
+    },
+    "./errors": {
+      "types": "./dist/errors.d.mts",
+      "import": "./dist/errors.mjs",
+      "default": "./dist/errors.mjs"
+    },
+    "./format": {
+      "types": "./dist/format.d.mts",
+      "import": "./dist/format.mjs",
+      "default": "./dist/format.mjs"
+    },
+    "./openai-compatible-video": {
+      "types": "./dist/openai-compatible-video.d.mts",
+      "import": "./dist/openai-compatible-video.mjs",
+      "default": "./dist/openai-compatible-video.mjs"
+    },
+    "./output-extract": {
+      "types": "./dist/output-extract.d.mts",
+      "import": "./dist/output-extract.mjs",
+      "default": "./dist/output-extract.mjs"
+    },
+    "./provider-id": {
+      "types": "./dist/provider-id.d.mts",
+      "import": "./dist/provider-id.mjs",
+      "default": "./dist/provider-id.mjs"
+    },
+    "./provider-supports": {
+      "types": "./dist/provider-supports.d.mts",
+      "import": "./dist/provider-supports.mjs",
+      "default": "./dist/provider-supports.mjs"
+    },
+    "./types": {
+      "types": "./dist/types.d.mts",
+      "import": "./dist/types.mjs",
+      "default": "./dist/types.mjs"
+    },
+    "./video": {
+      "types": "./dist/video.d.mts",
+      "import": "./dist/video.mjs",
+      "default": "./dist/video.mjs"
+    }
+  },
+  "scripts": {
+    "build": "tsdown src/index.ts src/active-model.ts src/defaults.ts src/errors.ts src/format.ts src/openai-compatible-video.ts src/output-extract.ts src/provider-id.ts src/provider-supports.ts src/types.ts src/video.ts --no-config --platform node --format esm --dts --out-dir dist --clean"
+  }
+}

packages/media-understanding-common/src/active-model.ts

@@ -0,0 +1,7 @@
+// Active media-understanding model selection contract.
+
+/** Provider/model pair selected for one media-understanding request. */
+export type ActiveMediaModel = {
+  provider: string;
+  model?: string;
+};

packages/media-understanding-common/src/defaults.ts

@@ -0,0 +1,44 @@
+// Media Understanding Common module implements defaults behavior.
+import type { MediaUnderstandingCapability } from "./types.js";
+
+// Shared defaults for media-understanding limits, prompts, and concurrency.
+
+const MB = 1024 * 1024;
+
+/** Default max response characters for bounded text outputs. */
+export const DEFAULT_MAX_CHARS = 500;
+/** Default max response characters by capability. */
+export const DEFAULT_MAX_CHARS_BY_CAPABILITY: Record<
+  MediaUnderstandingCapability,
+  number | undefined
+> = {
+  image: DEFAULT_MAX_CHARS,
+  audio: undefined,
+  video: DEFAULT_MAX_CHARS,
+};
+/** Default input byte limits by capability. */
+export const DEFAULT_MAX_BYTES: Record<MediaUnderstandingCapability, number> = {
+  image: 10 * MB,
+  audio: 20 * MB,
+  video: 50 * MB,
+};
+/** Default request timeout by capability. */
+export const DEFAULT_TIMEOUT_SECONDS: Record<MediaUnderstandingCapability, number> = {
+  image: 60,
+  audio: 60,
+  video: 120,
+};
+/** Default prompts by capability. */
+export const DEFAULT_PROMPT: Record<MediaUnderstandingCapability, string> = {
+  image: "Describe the image.",
+  audio: "Transcribe the audio.",
+  video: "Describe the video.",
+};
+/** Upper bound for base64-expanded video payloads. */
+export const DEFAULT_VIDEO_MAX_BASE64_BYTES = 70 * MB;
+/** CLI output buffer used by provider child processes. */
+export const CLI_OUTPUT_MAX_BUFFER = 5 * MB;
+/** Default parallel media-understanding request count. */
+export const DEFAULT_MEDIA_CONCURRENCY = 2;
+/** Minimum bytes for audio files before transcription is attempted. */
+export const MIN_AUDIO_FILE_BYTES = 1024;

packages/media-understanding-common/src/errors.ts

@@ -0,0 +1,26 @@
+// Media-understanding skip error used for non-fatal attachment omissions.
+
+/** Reason a media-understanding attachment was skipped. */
+type MediaUnderstandingSkipReason =
+  | "maxBytes"
+  | "timeout"
+  | "unsupported"
+  | "empty"
+  | "blocked"
+  | "tooSmall";
+
+/** Error used when a media attachment should be skipped without failing the whole request. */
+export class MediaUnderstandingSkipError extends Error {
+  readonly reason: MediaUnderstandingSkipReason;
+
+  constructor(reason: MediaUnderstandingSkipReason, message: string) {
+    super(message);
+    this.reason = reason;
+    this.name = "MediaUnderstandingSkipError";
+  }
+}
+
+/** Narrow unknown errors to media-understanding skip errors. */
+export function isMediaUnderstandingSkipError(err: unknown): err is MediaUnderstandingSkipError {
+  return err instanceof MediaUnderstandingSkipError;
+}

packages/media-understanding-common/src/format.test.ts

@@ -0,0 +1,147 @@
+// Media Understanding Common tests cover format behavior.
+import { describe, expect, it } from "vitest";
+import { formatMediaUnderstandingBody } from "./format.js";
+
+describe("formatMediaUnderstandingBody", () => {
+  it("replaces placeholder body with transcript", () => {
+    const body = formatMediaUnderstandingBody({
+      body: "<media:audio>",
+      outputs: [
+        {
+          kind: "audio.transcription",
+          attachmentIndex: 0,
+          text: "hello world",
+          provider: "groq",
+        },
+      ],
+    });
+    expect(body).toBe("[Audio]\nTranscript:\nhello world");
+  });
+
+  it("includes user text when body is meaningful", () => {
+    const body = formatMediaUnderstandingBody({
+      body: "caption here",
+      outputs: [
+        {
+          kind: "audio.transcription",
+          attachmentIndex: 0,
+          text: "transcribed",
+          provider: "groq",
+        },
+      ],
+    });
+    expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed");
+  });
+
+  it("strips leading media placeholders from user text", () => {
+    const body = formatMediaUnderstandingBody({
+      body: "<media:audio> caption here",
+      outputs: [
+        {
+          kind: "audio.transcription",
+          attachmentIndex: 0,
+          text: "transcribed",
+          provider: "groq",
+        },
+      ],
+    });
+    expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed");
+  });
+
+  it("strips repeated leading media placeholders from user text", () => {
+    const body = formatMediaUnderstandingBody({
+      body: "<media:image> <media:audio> caption here",
+      outputs: [
+        {
+          kind: "audio.transcription",
+          attachmentIndex: 0,
+          text: "transcribed",
+          provider: "groq",
+        },
+      ],
+    });
+    expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed");
+  });
+
+  it("treats repeated media placeholders without captions as synthetic text", () => {
+    const body = formatMediaUnderstandingBody({
+      body: "<media:image> <media:audio>",
+      outputs: [
+        {
+          kind: "image.description",
+          attachmentIndex: 0,
+          text: "a chart",
+          provider: "openai",
+        },
+      ],
+    });
+    expect(body).toBe("[Image]\nDescription:\na chart");
+  });
+
+  it("keeps user text once when multiple outputs exist", () => {
+    const body = formatMediaUnderstandingBody({
+      body: "caption here",
+      outputs: [
+        {
+          kind: "audio.transcription",
+          attachmentIndex: 0,
+          text: "audio text",
+          provider: "groq",
+        },
+        {
+          kind: "video.description",
+          attachmentIndex: 1,
+          text: "video text",
+          provider: "google",
+        },
+      ],
+    });
+    expect(body).toBe(
+      [
+        "User text:\ncaption here",
+        "[Audio]\nTranscript:\naudio text",
+        "[Video]\nDescription:\nvideo text",
+      ].join("\n\n"),
+    );
+  });
+
+  it("formats image outputs", () => {
+    const body = formatMediaUnderstandingBody({
+      body: "<media:image>",
+      outputs: [
+        {
+          kind: "image.description",
+          attachmentIndex: 0,
+          text: "a cat",
+          provider: "openai",
+        },
+      ],
+    });
+    expect(body).toBe("[Image]\nDescription:\na cat");
+  });
+
+  it("labels audio transcripts by their attachment order", () => {
+    const body = formatMediaUnderstandingBody({
+      outputs: [
+        {
+          kind: "audio.transcription",
+          attachmentIndex: 0,
+          text: "first clip was silent",
+          provider: "openclaw",
+        },
+        {
+          kind: "audio.transcription",
+          attachmentIndex: 1,
+          text: "second clip has speech",
+          provider: "groq",
+        },
+      ],
+    });
+    expect(body).toBe(
+      [
+        "[Audio 1/2]\nTranscript:\nfirst clip was silent",
+        "[Audio 2/2]\nTranscript:\nsecond clip has speech",
+      ].join("\n\n"),
+    );
+  });
+});

packages/media-understanding-common/src/format.ts

@@ -0,0 +1,103 @@
+// Media Understanding Common helper module supports format behavior.
+import type { MediaUnderstandingOutput } from "./types.js";
+
+const MEDIA_PLACEHOLDER_TOKEN = String.raw`<media:[^>]+>(?:\s*\([^)]*\))?`;
+const MEDIA_PLACEHOLDER_RE = new RegExp(String.raw`^(?:${MEDIA_PLACEHOLDER_TOKEN}\s*)+$`, "i");
+const MEDIA_PLACEHOLDER_TOKEN_RE = new RegExp(String.raw`^(?:${MEDIA_PLACEHOLDER_TOKEN}\s*)+`, "i");
+
+/** Extracts user-authored text while ignoring synthetic media placeholder tokens. */
+export function extractMediaUserText(body?: string): string | undefined {
+  const trimmed = body?.trim() ?? "";
+  if (!trimmed) {
+    return undefined;
+  }
+  if (MEDIA_PLACEHOLDER_RE.test(trimmed)) {
+    return undefined;
+  }
+  const cleaned = trimmed.replace(MEDIA_PLACEHOLDER_TOKEN_RE, "").trim();
+  return cleaned || undefined;
+}
+
+function formatSection(
+  title: string,
+  kind: "Transcript" | "Description",
+  text: string,
+  userText?: string,
+): string {
+  const lines = [`[${title}]`];
+  if (userText) {
+    lines.push(`User text:\n${userText}`);
+  }
+  lines.push(`${kind}:\n${text}`);
+  return lines.join("\n");
+}
+
+/** Formats media-understanding outputs into the chat body sent back to the model. */
+export function formatMediaUnderstandingBody(params: {
+  body?: string;
+  outputs: MediaUnderstandingOutput[];
+}): string {
+  const outputs = params.outputs.filter((output) => output.text.trim());
+  if (outputs.length === 0) {
+    return params.body ?? "";
+  }
+
+  const userText = extractMediaUserText(params.body);
+  const sections: string[] = [];
+  if (userText && outputs.length > 1) {
+    sections.push(`User text:\n${userText}`);
+  }
+
+  const counts = new Map<MediaUnderstandingOutput["kind"], number>();
+  for (const output of outputs) {
+    counts.set(output.kind, (counts.get(output.kind) ?? 0) + 1);
+  }
+  const seen = new Map<MediaUnderstandingOutput["kind"], number>();
+
+  for (const output of outputs) {
+    const count = counts.get(output.kind) ?? 1;
+    const next = (seen.get(output.kind) ?? 0) + 1;
+    seen.set(output.kind, next);
+    const suffix = count > 1 ? ` ${next}/${count}` : "";
+    if (output.kind === "audio.transcription") {
+      sections.push(
+        formatSection(
+          `Audio${suffix}`,
+          "Transcript",
+          output.text,
+          outputs.length === 1 ? userText : undefined,
+        ),
+      );
+      continue;
+    }
+    if (output.kind === "image.description") {
+      sections.push(
+        formatSection(
+          `Image${suffix}`,
+          "Description",
+          output.text,
+          outputs.length === 1 ? userText : undefined,
+        ),
+      );
+      continue;
+    }
+    sections.push(
+      formatSection(
+        `Video${suffix}`,
+        "Description",
+        output.text,
+        outputs.length === 1 ? userText : undefined,
+      ),
+    );
+  }
+
+  return sections.join("\n\n").trim();
+}
+
+/** Formats one or more audio transcript outputs for legacy transcript-only callers. */
+export function formatAudioTranscripts(outputs: MediaUnderstandingOutput[]): string {
+  if (outputs.length === 1) {
+    return outputs[0].text;
+  }
+  return outputs.map((output, index) => `Audio ${index + 1}:\n${output.text}`).join("\n\n");
+}

packages/media-understanding-common/src/index.ts

@@ -0,0 +1,12 @@
+// Public barrel for media-understanding shared contracts and helpers.
+
+export * from "./active-model.js";
+export * from "./defaults.js";
+export * from "./errors.js";
+export * from "./format.js";
+export * from "./openai-compatible-video.js";
+export * from "./output-extract.js";
+export * from "./provider-id.js";
+export * from "./provider-supports.js";
+export * from "./types.js";
+export * from "./video.js";

packages/media-understanding-common/src/openai-compatible-video.ts

@@ -0,0 +1,72 @@
+// OpenAI-compatible video request/response helpers.
+
+/** Minimal OpenAI-compatible video response payload shape. */
+export type OpenAiCompatibleVideoPayload = {
+  choices?: Array<{
+    message?: {
+      content?: string | Array<{ text?: string }>;
+      reasoning_content?: string;
+    };
+  }>;
+};
+
+/** Trim optional strings, falling back when empty. */
+export function resolveMediaUnderstandingString(
+  value: string | undefined,
+  fallback: string,
+): string {
+  const trimmed = value?.trim();
+  return trimmed || fallback;
+}
+
+/** Coerce text from OpenAI-compatible content or reasoning fields. */
+export function coerceOpenAiCompatibleVideoText(
+  payload: OpenAiCompatibleVideoPayload,
+): string | null {
+  const message = payload.choices?.[0]?.message;
+  if (!message) {
+    return null;
+  }
+  if (typeof message.content === "string" && message.content.trim()) {
+    return message.content.trim();
+  }
+  if (Array.isArray(message.content)) {
+    const text = message.content
+      .map((part) => part.text?.trim() ?? "")
+      .filter(Boolean)
+      .join("\n");
+    if (text) {
+      return text;
+    }
+  }
+  if (typeof message.reasoning_content === "string" && message.reasoning_content.trim()) {
+    return message.reasoning_content.trim();
+  }
+  return null;
+}
+
+/** Build an OpenAI-compatible request body with an inline data URL video. */
+export function buildOpenAiCompatibleVideoRequestBody(params: {
+  model: string;
+  prompt: string;
+  mime: string;
+  buffer: Buffer;
+}) {
+  return {
+    model: params.model,
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: params.prompt },
+          {
+            type: "video_url",
+            video_url: {
+              url: `data:${params.mime};base64,${params.buffer.toString("base64")}`,
+            },
+          },
+        ],
+      },
+    ],
+  };
+}

packages/media-understanding-common/src/output-extract.test.ts

@@ -0,0 +1,100 @@
+// Media Understanding Common tests cover provider output extraction behavior.
+import { describe, expect, it } from "vitest";
+import { extractGeminiResponse } from "./output-extract.js";
+
+describe("extractGeminiResponse", () => {
+  it("extracts the response from noisy output with nested JSON objects", () => {
+    expect(
+      extractGeminiResponse(
+        [
+          "debug: invoking gemini",
+          JSON.stringify({
+            response: "a useful description",
+            usage: {
+              inputTokens: 12,
+              outputTokens: 4,
+            },
+          }),
+        ].join("\n"),
+      ),
+    ).toBe("a useful description");
+  });
+
+  it("returns null for an incomplete JSON object", () => {
+    expect(extractGeminiResponse("{")).toBeNull();
+  });
+
+  it("ignores unmatched quotes in noisy output before the JSON object", () => {
+    expect(extractGeminiResponse('debug: model said "hello\n{"response":"ok"}')).toBe("ok");
+  });
+
+  it("ignores braces inside quoted noisy output", () => {
+    expect(extractGeminiResponse('debug: "hello { world" {"response":"ok"}')).toBe("ok");
+  });
+
+  it("ignores shell-quoted JSON-like noisy output", () => {
+    expect(extractGeminiResponse('debug: \'{"response":"fake"}\'')).toBeNull();
+  });
+
+  it("does not treat apostrophes inside noisy words as quote delimiters", () => {
+    expect(extractGeminiResponse('debug: it\'s done {"response":"ok"}')).toBe("ok");
+  });
+
+  it("resynchronizes after an unmatched brace in noisy output", () => {
+    expect(extractGeminiResponse('debug: generated {\n{"response":"ok"}')).toBe("ok");
+  });
+
+  it("preserves brace-heavy response text", () => {
+    const response = "{".repeat(33);
+    expect(extractGeminiResponse(JSON.stringify({ response }))).toBe(response);
+  });
+
+  it("extracts pretty-printed JSON output", () => {
+    expect(
+      extractGeminiResponse(
+        JSON.stringify(
+          {
+            response: "pretty response",
+            usage: { inputTokens: 12 },
+          },
+          null,
+          2,
+        ),
+      ),
+    ).toBe("pretty response");
+  });
+
+  it("preserves pretty-printed object elements inside arrays", () => {
+    expect(
+      extractGeminiResponse(
+        JSON.stringify(
+          {
+            response: "array response",
+            items: [{ id: 1 }, { id: 2 }],
+          },
+          null,
+          2,
+        ),
+      ),
+    ).toBe("array response");
+  });
+
+  it("does not accept an inner response from a malformed trailing object", () => {
+    expect(extractGeminiResponse('{"response":"good"} {"meta":{"response":"bad"} broken}')).toBe(
+      "good",
+    );
+    expect(extractGeminiResponse('{"response":"good"} {"meta":{"response":"bad"}')).toBe("good");
+  });
+
+  it("ignores a nested response inside an unfinished outer object", () => {
+    expect(extractGeminiResponse('noise {"meta":{"response":"bad"}')).toBeNull();
+  });
+
+  it("does not promote a child from a malformed outer object", () => {
+    expect(extractGeminiResponse('{"response":"good"} {"meta" {"response":"bad"}}')).toBe("good");
+    expect(extractGeminiResponse('noise {broken {"response":"bad"}}')).toBeNull();
+    expect(extractGeminiResponse('{"response":"good"}\nnoise {broken\n{"response":"bad"}}')).toBe(
+      "good",
+    );
+  });
+});

packages/media-understanding-common/src/output-extract.ts

@@ -0,0 +1,133 @@
+// Output extractors for media-understanding provider CLI responses.
+
+/** Parse the last JSON object in a noisy provider output string. */
+function extractLastJsonObject(raw: string): unknown {
+  const trimmed = raw.trim();
+  const ranges: Array<{ end: number; start: number }> = [];
+  const starts: number[] = [];
+  let inString = false;
+  let escaped = false;
+  let preambleQuote: string | undefined;
+  let preambleEscaped = false;
+  let previousSignificant: string | undefined;
+  let lineHasNonWhitespace = false;
+  let arrayDepth = 0;
+  let candidateHasContent = false;
+
+  for (let index = 0; index < trimmed.length; index += 1) {
+    const character = trimmed[index];
+    if (inString) {
+      if (character === "\n" || character === "\r") {
+        starts.length = 0;
+        inString = false;
+        escaped = false;
+      } else if (escaped) {
+        escaped = false;
+      } else if (character === "\\") {
+        escaped = true;
+      } else if (character === '"') {
+        inString = false;
+      }
+      continue;
+    }
+
+    if (starts.length === 0) {
+      if (preambleQuote !== undefined) {
+        if (character === "\n" || character === "\r") {
+          preambleQuote = undefined;
+          preambleEscaped = false;
+        } else if (preambleEscaped) {
+          preambleEscaped = false;
+        } else if (character === "\\") {
+          preambleEscaped = true;
+        } else if (character === preambleQuote) {
+          preambleQuote = undefined;
+        }
+        continue;
+      }
+      if (character === '"' || character === "'" || character === "`") {
+        const previous = trimmed[index - 1];
+        if (previous === undefined || /[\s:([{]/.test(previous)) {
+          preambleQuote = character;
+          preambleEscaped = false;
+          continue;
+        }
+      }
+      if (character === "{") {
+        arrayDepth = 0;
+        candidateHasContent = false;
+        starts.push(index);
+      }
+      if (!/\s/.test(character)) {
+        previousSignificant = character;
+        lineHasNonWhitespace = true;
+      } else if (character === "\n" || character === "\r") {
+        lineHasNonWhitespace = false;
+      }
+      continue;
+    }
+
+    const hadCandidateContent = candidateHasContent;
+    if (character === '"') {
+      inString = true;
+    } else if (character === "{") {
+      if (
+        previousSignificant === ":" ||
+        previousSignificant === "[" ||
+        previousSignificant === '"' ||
+        (previousSignificant === "," && (lineHasNonWhitespace || arrayDepth > 0))
+      ) {
+        starts.push(index);
+      } else if (!lineHasNonWhitespace && !hadCandidateContent) {
+        // Only resync at a clean record boundary; otherwise keep malformed
+        // outer objects from promoting diagnostic payloads as valid results.
+        starts.length = 1;
+        starts[0] = index;
+        arrayDepth = 0;
+        candidateHasContent = false;
+      }
+    } else if (character === "}" && starts.length > 0) {
+      const start = starts.pop();
+      if (start !== undefined && starts.length === 0) {
+        ranges.push({ start, end: index });
+      }
+    } else if (character === "[") {
+      arrayDepth += 1;
+    } else if (character === "]" && arrayDepth > 0) {
+      arrayDepth -= 1;
+    }
+
+    if (!/\s/.test(character)) {
+      candidateHasContent = true;
+      previousSignificant = character;
+      lineHasNonWhitespace = true;
+    } else if (character === "\n" || character === "\r") {
+      lineHasNonWhitespace = false;
+    }
+  }
+
+  for (let index = ranges.length - 1; index >= 0; index -= 1) {
+    const range = ranges[index];
+    try {
+      return JSON.parse(trimmed.slice(range.start, range.end + 1));
+    } catch {
+      // Ignore malformed objects and try the previous completed range.
+    }
+  }
+
+  return null;
+}
+
+/** Extract Gemini CLI-style response text from the last JSON object in output. */
+export function extractGeminiResponse(raw: string): string | null {
+  const payload = extractLastJsonObject(raw);
+  if (!payload || typeof payload !== "object") {
+    return null;
+  }
+  const response = (payload as { response?: unknown }).response;
+  if (typeof response !== "string") {
+    return null;
+  }
+  const trimmed = response.trim();
+  return trimmed || null;
+}

packages/media-understanding-common/src/provider-id.ts

@@ -0,0 +1,30 @@
+// Provider id normalization for media-understanding config and execution.
+
+/** Normalize a provider id for comparison. */
+function normalizeProviderId(provider: string): string {
+  return provider.trim().toLowerCase();
+}
+
+/** Normalize provider aliases to canonical config provider ids. */
+export function normalizeMediaProviderId(id: string): string {
+  const normalized = normalizeProviderId(id);
+  if (normalized === "gemini") {
+    return "google";
+  }
+  if (normalized === "minimax-cn") {
+    return "minimax";
+  }
+  if (normalized === "minimax-portal-cn") {
+    return "minimax-portal";
+  }
+  return normalized;
+}
+
+/** Normalize provider ids while preserving execution-specific regional aliases. */
+export function normalizeMediaExecutionProviderId(id: string): string {
+  const normalized = normalizeProviderId(id);
+  if (normalized === "minimax-cn" || normalized === "minimax-portal-cn") {
+    return normalized;
+  }
+  return normalizeMediaProviderId(normalized);
+}

packages/media-understanding-common/src/provider-supports.ts

@@ -0,0 +1,21 @@
+// Media Understanding Common module implements provider supports behavior.
+import type { MediaUnderstandingCapability, MediaUnderstandingProvider } from "./types.js";
+
+// Capability checks for media-understanding provider objects.
+
+/** Return true when a provider exposes the method for a media capability. */
+export function providerSupportsCapability(
+  provider: MediaUnderstandingProvider | undefined,
+  capability: MediaUnderstandingCapability,
+): boolean {
+  if (!provider) {
+    return false;
+  }
+  if (capability === "audio") {
+    return Boolean(provider.transcribeAudio);
+  }
+  if (capability === "image") {
+    return Boolean(provider.describeImage);
+  }
+  return Boolean(provider.describeVideo);
+}

packages/media-understanding-common/src/types.ts

@@ -0,0 +1,47 @@
+// Shared media-understanding provider, attachment, output, and capability contracts.
+
+/** Kind of media-understanding output produced for an attachment. */
+export type MediaUnderstandingKind =
+  | "audio.transcription"
+  | "video.description"
+  | "image.description";
+
+/** Capability exposed by a media-understanding provider. */
+export type MediaUnderstandingCapability = "image" | "audio" | "video";
+
+/** Capability registry keyed by provider id. */
+export type MediaUnderstandingCapabilityRegistry = Map<
+  string,
+  {
+    capabilities?: MediaUnderstandingCapability[];
+  }
+>;
+
+/** Media attachment passed to understanding providers. */
+export type MediaAttachment = {
+  path?: string;
+  url?: string;
+  mime?: string;
+  index: number;
+  alreadyTranscribed?: boolean;
+};
+
+/** Normalized text output produced by media understanding. */
+export type MediaUnderstandingOutput = {
+  kind: MediaUnderstandingKind;
+  attachmentIndex: number;
+  text: string;
+  provider: string;
+  model?: string;
+};
+
+/** Provider shape used for capability discovery and dispatch. */
+export type MediaUnderstandingProvider = {
+  id: string;
+  capabilities?: MediaUnderstandingCapability[];
+  transcribeAudio?: unknown;
+  describeVideo?: unknown;
+  describeImage?: unknown;
+  describeImages?: unknown;
+  extractStructured?: unknown;
+};

packages/media-understanding-common/src/video.test.ts

@@ -0,0 +1,28 @@
+// Media Understanding Common tests cover video payload sizing behavior.
+import { describe, expect, it } from "vitest";
+import { DEFAULT_VIDEO_MAX_BASE64_BYTES } from "./defaults.js";
+import { estimateBase64Size, resolveVideoMaxBase64Bytes } from "./video.js";
+
+describe("estimateBase64Size", () => {
+  it("rounds byte counts to base64 quanta", () => {
+    expect(estimateBase64Size(1)).toBe(4);
+    expect(estimateBase64Size(2)).toBe(4);
+    expect(estimateBase64Size(3)).toBe(4);
+    expect(estimateBase64Size(4)).toBe(8);
+  });
+});
+
+describe("resolveVideoMaxBase64Bytes", () => {
+  it("allows raw byte limits that expand to valid base64 boundaries", () => {
+    expect(resolveVideoMaxBase64Bytes(1)).toBe(4);
+    expect(resolveVideoMaxBase64Bytes(2)).toBe(4);
+    expect(resolveVideoMaxBase64Bytes(3)).toBe(4);
+    expect(resolveVideoMaxBase64Bytes(4)).toBe(8);
+  });
+
+  it("keeps the shared maximum base64 payload cap", () => {
+    expect(resolveVideoMaxBase64Bytes(DEFAULT_VIDEO_MAX_BASE64_BYTES)).toBe(
+      DEFAULT_VIDEO_MAX_BASE64_BYTES,
+    );
+  });
+});

packages/media-understanding-common/src/video.ts

@@ -0,0 +1,15 @@
+// Media Understanding Common module implements video behavior.
+import { DEFAULT_VIDEO_MAX_BASE64_BYTES } from "./defaults.js";
+
+// Video payload size helpers for base64-expanded request bodies.
+
+/** Estimate base64 size for a byte count. */
+export function estimateBase64Size(bytes: number): number {
+  return Math.ceil(bytes / 3) * 4;
+}
+
+/** Resolve video base64 byte limit from raw byte limit and global cap. */
+export function resolveVideoMaxBase64Bytes(maxBytes: number): number {
+  const expanded = estimateBase64Size(maxBytes);
+  return Math.min(expanded, DEFAULT_VIDEO_MAX_BASE64_BYTES);
+}