Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
18
extensions/openai/api.ts
Normal file
18
extensions/openai/api.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
// Openai API module exposes the plugin public contract.
|
||||
export {
|
||||
applyOpenAIConfig,
|
||||
applyOpenAIProviderConfig,
|
||||
OPENAI_CODEX_DEFAULT_MODEL,
|
||||
OPENAI_DEFAULT_AUDIO_TRANSCRIPTION_MODEL,
|
||||
OPENAI_DEFAULT_EMBEDDING_MODEL,
|
||||
OPENAI_DEFAULT_IMAGE_MODEL,
|
||||
OPENAI_DEFAULT_MODEL,
|
||||
OPENAI_DEFAULT_TTS_MODEL,
|
||||
OPENAI_DEFAULT_TTS_VOICE,
|
||||
} from "./default-models.js";
|
||||
export { buildOpenAICodexProvider } from "./openai-chatgpt-catalog.js";
|
||||
export { loginOpenAICodexOAuth } from "./openai-chatgpt-oauth.runtime.js";
|
||||
export { refreshOpenAICodexToken } from "./openai-chatgpt-provider.runtime.js";
|
||||
export { buildOpenAICodexProviderPlugin, buildOpenAIProvider } from "./openai-provider.js";
|
||||
export { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
|
||||
export { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
|
||||
21
extensions/openai/auth-choice-copy.ts
Normal file
21
extensions/openai/auth-choice-copy.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
// Openai plugin module implements auth choice copy behavior.
|
||||
export const OPENAI_API_KEY_LABEL = "OpenAI API Key";
|
||||
export const OPENAI_CHATGPT_LOGIN_LABEL = "ChatGPT Login";
|
||||
export const OPENAI_CHATGPT_LOGIN_HINT = "Sign in with your ChatGPT or Codex subscription";
|
||||
export const OPENAI_CHATGPT_DEVICE_PAIRING_LABEL = "ChatGPT Device Pairing";
|
||||
export const OPENAI_CHATGPT_DEVICE_PAIRING_HINT =
|
||||
"Pair your ChatGPT account in browser with a device code";
|
||||
|
||||
const OPENAI_UNIFIED_GROUP_HINT = "ChatGPT/Codex sign-in or API key";
|
||||
|
||||
export const OPENAI_ACCOUNT_WIZARD_GROUP = {
|
||||
groupId: "openai",
|
||||
groupLabel: "OpenAI",
|
||||
groupHint: OPENAI_UNIFIED_GROUP_HINT,
|
||||
} as const;
|
||||
|
||||
export const OPENAI_CODEX_WIZARD_GROUP = {
|
||||
groupId: "openai",
|
||||
groupLabel: "OpenAI",
|
||||
groupHint: OPENAI_UNIFIED_GROUP_HINT,
|
||||
} as const;
|
||||
71
extensions/openai/base-url.test.ts
Normal file
71
extensions/openai/base-url.test.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
// Openai tests cover base url plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
canonicalizeCodexResponsesBaseUrl,
|
||||
isOpenAIApiBaseUrl,
|
||||
isOpenAICodexBaseUrl,
|
||||
OPENAI_API_BASE_URL,
|
||||
OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
resolveOpenAIDefaultBaseUrl,
|
||||
} from "./base-url.js";
|
||||
|
||||
describe("openai base URL helpers", () => {
|
||||
it("recognizes direct OpenAI API routes", () => {
|
||||
expect(isOpenAIApiBaseUrl("https://api.openai.com")).toBe(true);
|
||||
expect(isOpenAIApiBaseUrl("https://api.openai.com/v1")).toBe(true);
|
||||
expect(isOpenAIApiBaseUrl("https://api.openai.com/v1/")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects proxy or unrelated API routes", () => {
|
||||
expect(isOpenAIApiBaseUrl("https://proxy.example.com/v1")).toBe(false);
|
||||
expect(isOpenAIApiBaseUrl("https://chatgpt.com/backend-api")).toBe(false);
|
||||
expect(isOpenAIApiBaseUrl(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("recognizes Codex ChatGPT backend routes", () => {
|
||||
// New canonical form (includes /codex segment; OpenAI removed the
|
||||
// /backend-api/responses alias server-side on 2026-04).
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/codex")).toBe(true);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/codex/")).toBe(true);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/codex/v1")).toBe(true);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/codex/v1/")).toBe(true);
|
||||
// Legacy form still recognized as a Codex baseURL for backward
|
||||
// compatibility with existing user configs.
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api")).toBe(true);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/")).toBe(true);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/v1")).toBe(true);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/v1/")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects non-Codex backend routes", () => {
|
||||
expect(isOpenAICodexBaseUrl("https://api.openai.com/v1")).toBe(false);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com")).toBe(false);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/v2")).toBe(false);
|
||||
expect(isOpenAICodexBaseUrl("https://chatgpt.com/backend-api/codex/v2")).toBe(false);
|
||||
expect(isOpenAICodexBaseUrl(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("canonicalizes legacy Codex Responses base URLs", () => {
|
||||
expect(canonicalizeCodexResponsesBaseUrl("https://chatgpt.com/backend-api")).toBe(
|
||||
OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
);
|
||||
expect(canonicalizeCodexResponsesBaseUrl("https://chatgpt.com/backend-api/v1")).toBe(
|
||||
OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
);
|
||||
expect(canonicalizeCodexResponsesBaseUrl("https://chatgpt.com/backend-api/codex/v1")).toBe(
|
||||
OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
);
|
||||
expect(canonicalizeCodexResponsesBaseUrl("https://proxy.example.com/v1")).toBe(
|
||||
"https://proxy.example.com/v1",
|
||||
);
|
||||
expect(canonicalizeCodexResponsesBaseUrl(undefined)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("resolves default API base URL from OPENAI_BASE_URL", () => {
|
||||
expect(resolveOpenAIDefaultBaseUrl({})).toBe(OPENAI_API_BASE_URL);
|
||||
expect(resolveOpenAIDefaultBaseUrl({ OPENAI_BASE_URL: " " })).toBe(OPENAI_API_BASE_URL);
|
||||
expect(resolveOpenAIDefaultBaseUrl({ OPENAI_BASE_URL: " https://proxy.example/v1 " })).toBe(
|
||||
"https://proxy.example/v1",
|
||||
);
|
||||
});
|
||||
});
|
||||
31
extensions/openai/base-url.ts
Normal file
31
extensions/openai/base-url.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
// Openai plugin module implements base url behavior.
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
|
||||
export const OPENAI_CODEX_RESPONSES_BASE_URL = "https://chatgpt.com/backend-api/codex";
|
||||
export const OPENAI_API_BASE_URL = "https://api.openai.com/v1";
|
||||
|
||||
export function resolveOpenAIDefaultBaseUrl(
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): string {
|
||||
return normalizeOptionalString(env.OPENAI_BASE_URL) ?? OPENAI_API_BASE_URL;
|
||||
}
|
||||
|
||||
export function isOpenAIApiBaseUrl(baseUrl?: string): boolean {
|
||||
const trimmed = normalizeOptionalString(baseUrl);
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
return /^https?:\/\/api\.openai\.com(?:\/v1)?\/?$/i.test(trimmed);
|
||||
}
|
||||
|
||||
export function isOpenAICodexBaseUrl(baseUrl?: string): boolean {
|
||||
const trimmed = normalizeOptionalString(baseUrl);
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
return /^https?:\/\/chatgpt\.com\/backend-api(?:\/codex)?(?:\/v1)?\/?$/i.test(trimmed);
|
||||
}
|
||||
|
||||
export function canonicalizeCodexResponsesBaseUrl(baseUrl?: string): string | undefined {
|
||||
return isOpenAICodexBaseUrl(baseUrl) ? OPENAI_CODEX_RESPONSES_BASE_URL : baseUrl;
|
||||
}
|
||||
37
extensions/openai/default-models.test.ts
Normal file
37
extensions/openai/default-models.test.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
// Openai tests cover default models plugin behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyOpenAIConfig, applyOpenAIProviderConfig, OPENAI_DEFAULT_MODEL } from "./api.js";
|
||||
|
||||
describe("openai default models", () => {
|
||||
it("adds allowlist entry for the default model", () => {
|
||||
const next = applyOpenAIProviderConfig({});
|
||||
expect(Object.keys(next.agents?.defaults?.models ?? {})).toEqual([OPENAI_DEFAULT_MODEL]);
|
||||
expect(next.agents?.defaults?.models?.[OPENAI_DEFAULT_MODEL]).toEqual({ alias: "GPT" });
|
||||
});
|
||||
|
||||
it("preserves existing alias for the default model", () => {
|
||||
const next = applyOpenAIProviderConfig({
|
||||
agents: {
|
||||
defaults: {
|
||||
models: {
|
||||
[OPENAI_DEFAULT_MODEL]: { alias: "My GPT" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(next.agents?.defaults?.models?.[OPENAI_DEFAULT_MODEL]?.alias).toBe("My GPT");
|
||||
});
|
||||
|
||||
it("sets the default model when it is unset", () => {
|
||||
const next = applyOpenAIConfig({});
|
||||
expect(next.agents?.defaults?.model).toEqual({ primary: OPENAI_DEFAULT_MODEL });
|
||||
});
|
||||
|
||||
it("overrides model.primary while preserving fallbacks", () => {
|
||||
const next = applyOpenAIConfig({
|
||||
agents: { defaults: { model: { primary: "anthropic/claude-opus-4-6", fallbacks: [] } } },
|
||||
} as OpenClawConfig);
|
||||
expect(next.agents?.defaults?.model).toEqual({ primary: OPENAI_DEFAULT_MODEL, fallbacks: [] });
|
||||
});
|
||||
});
|
||||
41
extensions/openai/default-models.ts
Normal file
41
extensions/openai/default-models.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
// Openai plugin module implements default models behavior.
|
||||
import { ensureModelAllowlistEntry } from "openclaw/plugin-sdk/provider-onboard";
|
||||
import {
|
||||
applyAgentDefaultModelPrimary,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/provider-onboard";
|
||||
|
||||
export const OPENAI_DEFAULT_MODEL = "openai/gpt-5.5";
|
||||
export const OPENAI_CODEX_DEFAULT_MODEL = OPENAI_DEFAULT_MODEL;
|
||||
export const OPENAI_DEFAULT_IMAGE_MODEL = "gpt-image-2";
|
||||
export const OPENAI_DEFAULT_TTS_MODEL = "gpt-4o-mini-tts";
|
||||
export const OPENAI_DEFAULT_TTS_VOICE = "alloy";
|
||||
export const OPENAI_DEFAULT_AUDIO_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
||||
export const OPENAI_DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small";
|
||||
|
||||
export function applyOpenAIProviderConfig(cfg: OpenClawConfig): OpenClawConfig {
|
||||
const next = ensureModelAllowlistEntry({
|
||||
cfg,
|
||||
modelRef: OPENAI_DEFAULT_MODEL,
|
||||
});
|
||||
const models = { ...next.agents?.defaults?.models };
|
||||
models[OPENAI_DEFAULT_MODEL] = {
|
||||
...models[OPENAI_DEFAULT_MODEL],
|
||||
alias: models[OPENAI_DEFAULT_MODEL]?.alias ?? "GPT",
|
||||
};
|
||||
|
||||
return {
|
||||
...next,
|
||||
agents: {
|
||||
...next.agents,
|
||||
defaults: {
|
||||
...next.agents?.defaults,
|
||||
models,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function applyOpenAIConfig(cfg: OpenClawConfig): OpenClawConfig {
|
||||
return applyAgentDefaultModelPrimary(applyOpenAIProviderConfig(cfg), OPENAI_DEFAULT_MODEL);
|
||||
}
|
||||
640
extensions/openai/embedding-batch.test.ts
Normal file
640
extensions/openai/embedding-batch.test.ts
Normal file
@@ -0,0 +1,640 @@
|
||||
// Openai tests cover embedding batch plugin behavior.
|
||||
import { createServer } from "node:http";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { parseOpenAiBatchOutput, runOpenAiEmbeddingBatches } from "./embedding-batch.js";
|
||||
|
||||
const jsonlEncoder = new TextEncoder();
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function jsonlBytes(value: string): number {
|
||||
return jsonlEncoder.encode(value).byteLength;
|
||||
}
|
||||
|
||||
function cancelTrackedResponse(
|
||||
text: string,
|
||||
init: ResponseInit,
|
||||
): {
|
||||
response: Response;
|
||||
wasCanceled: () => boolean;
|
||||
} {
|
||||
let canceled = false;
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(text));
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
});
|
||||
return {
|
||||
response: new Response(stream, init),
|
||||
wasCanceled: () => canceled,
|
||||
};
|
||||
}
|
||||
|
||||
function fetchInputUrl(input: RequestInfo | URL): string {
|
||||
if (typeof input === "string") {
|
||||
return input;
|
||||
}
|
||||
if (input instanceof URL) {
|
||||
return input.href;
|
||||
}
|
||||
return input.url;
|
||||
}
|
||||
|
||||
function parseStringBody(init: RequestInit | undefined): unknown {
|
||||
if (typeof init?.body !== "string") {
|
||||
throw new Error("missing JSON request body");
|
||||
}
|
||||
return JSON.parse(init.body) as unknown;
|
||||
}
|
||||
|
||||
async function listenLoopbackServer(server: ReturnType<typeof createServer>): Promise<number> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
reject(new Error("expected loopback TCP address"));
|
||||
return;
|
||||
}
|
||||
resolve(address.port);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function closeServer(server: ReturnType<typeof createServer>): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe("OpenAI embedding batch output", () => {
|
||||
it("wraps malformed JSONL output", () => {
|
||||
expect(() => parseOpenAiBatchOutput('{"custom_id":"ok"}\n{not json')).toThrow(
|
||||
"OpenAI embedding batch output contained malformed JSONL",
|
||||
);
|
||||
});
|
||||
|
||||
it("splits provider uploads by serialized JSONL byte cap", async () => {
|
||||
const requests: Parameters<typeof runOpenAiEmbeddingBatches>[0]["requests"] = Array.from(
|
||||
{ length: 3 },
|
||||
(_, index) => ({
|
||||
custom_id: String(index),
|
||||
method: "POST" as const,
|
||||
url: "/v1/embeddings",
|
||||
body: {
|
||||
model: "text-embedding-3-small",
|
||||
input: `payload-${index}-${"β".repeat(8)}`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
const uploadedJsonl: string[] = [];
|
||||
const requestsByFileId = new Map<string, Array<{ custom_id?: string }>>();
|
||||
const outputByFileId = new Map<string, string>();
|
||||
let fileIndex = 0;
|
||||
let batchIndex = 0;
|
||||
const maxJsonlBytes = jsonlBytes(JSON.stringify(requests[0]));
|
||||
const fetchImpl = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url.endsWith("/files") && init?.method === "POST") {
|
||||
const form = init.body as FormData;
|
||||
const file = form.get("file");
|
||||
if (!(file instanceof Blob)) {
|
||||
throw new Error("missing batch upload file");
|
||||
}
|
||||
const jsonl = await file.text();
|
||||
const fileId = `file-${fileIndex}`;
|
||||
fileIndex += 1;
|
||||
uploadedJsonl.push(jsonl);
|
||||
requestsByFileId.set(
|
||||
fileId,
|
||||
jsonl.split("\n").map((line) => JSON.parse(line) as { custom_id?: string }),
|
||||
);
|
||||
return jsonResponse({ id: fileId });
|
||||
}
|
||||
if (url.endsWith("/batches") && init?.method === "POST") {
|
||||
const body = parseStringBody(init) as { input_file_id?: string };
|
||||
const batchId = `batch-${batchIndex}`;
|
||||
const outputFileId = `output-${batchIndex}`;
|
||||
batchIndex += 1;
|
||||
const uploadedRequests = requestsByFileId.get(body.input_file_id ?? "") ?? [];
|
||||
outputByFileId.set(
|
||||
outputFileId,
|
||||
uploadedRequests
|
||||
.map((request) =>
|
||||
JSON.stringify({
|
||||
custom_id: request.custom_id,
|
||||
response: {
|
||||
status_code: 200,
|
||||
body: { data: [{ embedding: [Number(request.custom_id) + 1] }] },
|
||||
},
|
||||
}),
|
||||
)
|
||||
.join("\n"),
|
||||
);
|
||||
return jsonResponse({ id: batchId, status: "completed", output_file_id: outputFileId });
|
||||
}
|
||||
const contentMatch = url.match(/\/files\/([^/]+)\/content$/);
|
||||
if (contentMatch) {
|
||||
return new Response(outputByFileId.get(contentMatch[1] ?? "") ?? "", { status: 200 });
|
||||
}
|
||||
return new Response("unexpected request", { status: 500 });
|
||||
});
|
||||
|
||||
const byCustomId = await runOpenAiEmbeddingBatches({
|
||||
openAi: {
|
||||
baseUrl: "https://openai-compatible.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
fetchImpl,
|
||||
},
|
||||
agentId: "main",
|
||||
requests,
|
||||
maxJsonlBytes,
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
|
||||
expect(uploadedJsonl).toHaveLength(3);
|
||||
expect(uploadedJsonl.every((jsonl) => jsonlBytes(jsonl) <= maxJsonlBytes)).toBe(true);
|
||||
expect([...byCustomId.entries()]).toEqual([
|
||||
["0", [1]],
|
||||
["1", [2]],
|
||||
["2", [3]],
|
||||
]);
|
||||
});
|
||||
|
||||
it("adapts OpenAI-compatible upload groups after payload-size rejection", async () => {
|
||||
const requests: Parameters<typeof runOpenAiEmbeddingBatches>[0]["requests"] = Array.from(
|
||||
{ length: 4 },
|
||||
(_, index) => ({
|
||||
custom_id: String(index),
|
||||
method: "POST" as const,
|
||||
url: "/v1/embeddings",
|
||||
body: {
|
||||
model: "text-embedding-3-small",
|
||||
input: `payload-${index}`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
const uploadedGroups: string[][] = [];
|
||||
const requestsByFileId = new Map<string, Array<{ custom_id?: string }>>();
|
||||
const outputByFileId = new Map<string, string>();
|
||||
const debug = vi.fn();
|
||||
let fileIndex = 0;
|
||||
let batchIndex = 0;
|
||||
const fetchImpl = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url.endsWith("/files") && init?.method === "POST") {
|
||||
const form = init.body as FormData;
|
||||
const file = form.get("file");
|
||||
if (!(file instanceof Blob)) {
|
||||
throw new Error("missing batch upload file");
|
||||
}
|
||||
const uploadedRequests = (await file.text())
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line) as { custom_id?: string });
|
||||
const customIds = uploadedRequests.map((request) => request.custom_id ?? "");
|
||||
uploadedGroups.push(customIds);
|
||||
if (uploadedRequests.length > 2) {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: {
|
||||
message: "Request body too large. Maximum allowed: 10 MB",
|
||||
type: "payload_too_large",
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
},
|
||||
},
|
||||
413,
|
||||
);
|
||||
}
|
||||
const fileId = `file-${fileIndex}`;
|
||||
fileIndex += 1;
|
||||
requestsByFileId.set(fileId, uploadedRequests);
|
||||
return jsonResponse({ id: fileId });
|
||||
}
|
||||
if (url.endsWith("/batches") && init?.method === "POST") {
|
||||
const body = parseStringBody(init) as { input_file_id?: string };
|
||||
const batchId = `batch-${batchIndex}`;
|
||||
const outputFileId = `output-${batchIndex}`;
|
||||
batchIndex += 1;
|
||||
const uploadedRequests = requestsByFileId.get(body.input_file_id ?? "") ?? [];
|
||||
outputByFileId.set(
|
||||
outputFileId,
|
||||
uploadedRequests
|
||||
.map((request) =>
|
||||
JSON.stringify({
|
||||
custom_id: request.custom_id,
|
||||
response: {
|
||||
status_code: 200,
|
||||
body: { data: [{ embedding: [Number(request.custom_id) + 1] }] },
|
||||
},
|
||||
}),
|
||||
)
|
||||
.join("\n"),
|
||||
);
|
||||
return jsonResponse({ id: batchId, status: "completed", output_file_id: outputFileId });
|
||||
}
|
||||
const contentMatch = url.match(/\/files\/([^/]+)\/content$/);
|
||||
if (contentMatch) {
|
||||
return new Response(outputByFileId.get(contentMatch[1] ?? "") ?? "", { status: 200 });
|
||||
}
|
||||
return new Response("unexpected request", { status: 500 });
|
||||
});
|
||||
|
||||
const byCustomId = await runOpenAiEmbeddingBatches({
|
||||
openAi: {
|
||||
baseUrl: "https://openai-compatible.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
fetchImpl,
|
||||
},
|
||||
agentId: "main",
|
||||
requests,
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
debug,
|
||||
});
|
||||
|
||||
expect(uploadedGroups).toEqual([
|
||||
["0", "1", "2", "3"],
|
||||
["0", "1"],
|
||||
["2", "3"],
|
||||
]);
|
||||
expect(debug).toHaveBeenCalledWith(
|
||||
"memory embeddings: openai batch upload too large; splitting group",
|
||||
expect.objectContaining({
|
||||
requests: 4,
|
||||
parts: [2, 2],
|
||||
}),
|
||||
);
|
||||
expect([...byCustomId.entries()]).toEqual([
|
||||
["0", [1]],
|
||||
["1", [2]],
|
||||
["2", [3]],
|
||||
["3", [4]],
|
||||
]);
|
||||
});
|
||||
|
||||
it("bounds batch status success body via readProviderJsonResponse", async () => {
|
||||
const chunkSize = 1024 * 1024;
|
||||
const chunkCount = 20; // 20 MiB, well over 16 MiB cap
|
||||
let readCount = 0;
|
||||
let canceled = false;
|
||||
const oversizedStatus = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (readCount >= chunkCount) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
readCount += 1;
|
||||
controller.enqueue(new Uint8Array(chunkSize));
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
let batchStatusCalled = false;
|
||||
const fetchImpl = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url.endsWith("/files") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "file-0" });
|
||||
}
|
||||
if (url.endsWith("/batches") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "batch-0", status: "in_progress" });
|
||||
}
|
||||
if (url.endsWith("/batches/batch-0") && !batchStatusCalled) {
|
||||
batchStatusCalled = true;
|
||||
return oversizedStatus;
|
||||
}
|
||||
return new Response("unexpected request", { status: 500 });
|
||||
});
|
||||
|
||||
await expect(
|
||||
runOpenAiEmbeddingBatches({
|
||||
openAi: {
|
||||
baseUrl: "https://openai-compatible.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
fetchImpl,
|
||||
},
|
||||
agentId: "main",
|
||||
requests: [
|
||||
{
|
||||
custom_id: "0",
|
||||
method: "POST",
|
||||
url: "/v1/embeddings",
|
||||
body: { model: "text-embedding-3-small", input: "payload" },
|
||||
},
|
||||
],
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
}),
|
||||
).rejects.toThrow(/openai\.batch-status/);
|
||||
expect(canceled).toBe(true);
|
||||
expect(readCount).toBeLessThan(chunkCount);
|
||||
});
|
||||
|
||||
it("streams valid batch output files larger than the provider text cap", async () => {
|
||||
const outputLineCount = 18;
|
||||
const padding = "x".repeat(1024 * 1024);
|
||||
const requests: Parameters<typeof runOpenAiEmbeddingBatches>[0]["requests"] = Array.from(
|
||||
{ length: outputLineCount },
|
||||
(_, index) => ({
|
||||
custom_id: String(index),
|
||||
method: "POST" as const,
|
||||
url: "/v1/embeddings",
|
||||
body: { model: "text-embedding-3-small", input: `payload-${index}` },
|
||||
}),
|
||||
);
|
||||
let outputLinesSent = 0;
|
||||
const outputResponse = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (outputLinesSent >= outputLineCount) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
const customId = String(outputLinesSent);
|
||||
controller.enqueue(
|
||||
jsonlEncoder.encode(
|
||||
`${JSON.stringify({
|
||||
custom_id: customId,
|
||||
response: {
|
||||
status_code: 200,
|
||||
body: { data: [{ embedding: [outputLinesSent + 1] }] },
|
||||
},
|
||||
padding,
|
||||
})}\n`,
|
||||
),
|
||||
);
|
||||
outputLinesSent += 1;
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/jsonl" } },
|
||||
);
|
||||
const fetchImpl = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url.endsWith("/files") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "file-0" });
|
||||
}
|
||||
if (url.endsWith("/batches") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "batch-0", status: "completed", output_file_id: "output-0" });
|
||||
}
|
||||
if (url.endsWith("/files/output-0/content")) {
|
||||
return outputResponse;
|
||||
}
|
||||
return new Response("unexpected request", { status: 500 });
|
||||
});
|
||||
|
||||
const byCustomId = await runOpenAiEmbeddingBatches({
|
||||
openAi: {
|
||||
baseUrl: "https://openai-compatible.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
fetchImpl,
|
||||
},
|
||||
agentId: "main",
|
||||
requests,
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
|
||||
expect(outputLinesSent).toBe(outputLineCount);
|
||||
expect([...byCustomId.entries()]).toEqual(
|
||||
requests.map((request, index) => [request.custom_id, [index + 1]]),
|
||||
);
|
||||
});
|
||||
|
||||
it("stops reading batch output after all requested custom IDs are accounted for", async () => {
|
||||
const outputLineCount = 1024;
|
||||
let outputLinesSent = 0;
|
||||
let canceled = false;
|
||||
const outputResponse = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (outputLinesSent >= outputLineCount) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
const line =
|
||||
outputLinesSent === 0
|
||||
? {
|
||||
custom_id: "0",
|
||||
response: {
|
||||
status_code: 200,
|
||||
body: { data: [{ embedding: [1] }] },
|
||||
},
|
||||
}
|
||||
: {
|
||||
custom_id: `extra-${outputLinesSent}`,
|
||||
response: {
|
||||
status_code: 200,
|
||||
body: { data: [{ embedding: [outputLinesSent] }] },
|
||||
},
|
||||
};
|
||||
controller.enqueue(jsonlEncoder.encode(`${JSON.stringify(line)}\n`));
|
||||
outputLinesSent += 1;
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/jsonl" } },
|
||||
);
|
||||
const fetchImpl = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url.endsWith("/files") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "file-0" });
|
||||
}
|
||||
if (url.endsWith("/batches") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "batch-0", status: "completed", output_file_id: "output-0" });
|
||||
}
|
||||
if (url.endsWith("/files/output-0/content")) {
|
||||
return outputResponse;
|
||||
}
|
||||
return new Response("unexpected request", { status: 500 });
|
||||
});
|
||||
|
||||
const byCustomId = await runOpenAiEmbeddingBatches({
|
||||
openAi: {
|
||||
baseUrl: "https://openai-compatible.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
fetchImpl,
|
||||
},
|
||||
agentId: "main",
|
||||
requests: [
|
||||
{
|
||||
custom_id: "0",
|
||||
method: "POST",
|
||||
url: "/v1/embeddings",
|
||||
body: { model: "text-embedding-3-small", input: "payload" },
|
||||
},
|
||||
],
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
|
||||
expect([...byCustomId.entries()]).toEqual([["0", [1]]]);
|
||||
expect(canceled).toBe(true);
|
||||
expect(outputLinesSent).toBeLessThan(outputLineCount);
|
||||
});
|
||||
|
||||
it("bounds batch output file content without buffering the whole response", async () => {
|
||||
const outputChunkCount = 1024;
|
||||
let outputChunksSent = 0;
|
||||
const server = createServer((req, res) => {
|
||||
const url = req.url ?? "";
|
||||
if (url === "/v1/files") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ id: "file-0" }));
|
||||
return;
|
||||
}
|
||||
if (url === "/v1/batches") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ id: "batch-0", status: "completed", output_file_id: "output-0" }));
|
||||
return;
|
||||
}
|
||||
if (url === "/v1/files/output-0/content") {
|
||||
res.writeHead(200, { "Content-Type": "application/jsonl" });
|
||||
const chunkSize = 1024 * 1024;
|
||||
const writeNext = () => {
|
||||
if (outputChunksSent >= outputChunkCount) {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
outputChunksSent += 1;
|
||||
if (res.write(Buffer.alloc(chunkSize))) {
|
||||
setImmediate(writeNext);
|
||||
} else {
|
||||
res.once("drain", writeNext);
|
||||
}
|
||||
};
|
||||
writeNext();
|
||||
return;
|
||||
}
|
||||
res.writeHead(500);
|
||||
res.end("unexpected request");
|
||||
});
|
||||
|
||||
const port = await listenLoopbackServer(server);
|
||||
const realFetch = globalThis.fetch.bind(globalThis);
|
||||
const fetchImpl = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const originalUrl = new URL(fetchInputUrl(input));
|
||||
const loopbackUrl = new URL(
|
||||
`${originalUrl.pathname}${originalUrl.search}`,
|
||||
`http://127.0.0.1:${port}`,
|
||||
);
|
||||
return await realFetch(loopbackUrl, init);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
runOpenAiEmbeddingBatches({
|
||||
openAi: {
|
||||
baseUrl: "https://openai-compatible.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
fetchImpl,
|
||||
},
|
||||
agentId: "main",
|
||||
requests: [
|
||||
{
|
||||
custom_id: "0",
|
||||
method: "POST",
|
||||
url: "/v1/embeddings",
|
||||
body: { model: "text-embedding-3-small", input: "payload" },
|
||||
},
|
||||
],
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
}),
|
||||
).rejects.toThrow(/openai\.batch-file-content/);
|
||||
} finally {
|
||||
await closeServer(server);
|
||||
}
|
||||
expect(outputChunksSent).toBeLessThan(outputChunkCount);
|
||||
});
|
||||
|
||||
it("bounds batch resource error bodies without using response.text()", async () => {
|
||||
const tracked = cancelTrackedResponse(`${"batch status unavailable ".repeat(1024)}tail`, {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "text/plain" },
|
||||
});
|
||||
const textSpy = vi.spyOn(tracked.response, "text").mockRejectedValue(new Error("unbounded"));
|
||||
let batchStatusReturned = false;
|
||||
const fetchImpl = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url.endsWith("/files") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "file-0" });
|
||||
}
|
||||
if (url.endsWith("/batches") && init?.method === "POST") {
|
||||
return jsonResponse({ id: "batch-0", status: "in_progress" });
|
||||
}
|
||||
if (url.endsWith("/batches/batch-0") && !batchStatusReturned) {
|
||||
batchStatusReturned = true;
|
||||
return tracked.response;
|
||||
}
|
||||
return new Response("unexpected request", { status: 500 });
|
||||
});
|
||||
|
||||
await expect(
|
||||
runOpenAiEmbeddingBatches({
|
||||
openAi: {
|
||||
baseUrl: "https://openai-compatible.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
fetchImpl,
|
||||
},
|
||||
agentId: "main",
|
||||
requests: [
|
||||
{
|
||||
custom_id: "0",
|
||||
method: "POST",
|
||||
url: "/v1/embeddings",
|
||||
body: {
|
||||
model: "text-embedding-3-small",
|
||||
input: "payload",
|
||||
},
|
||||
},
|
||||
],
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
}),
|
||||
).rejects.toThrow(/openai batch status failed: 400 batch status unavailable/);
|
||||
expect(tracked.wasCanceled()).toBe(true);
|
||||
expect(textSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
510
extensions/openai/embedding-batch.ts
Normal file
510
extensions/openai/embedding-batch.ts
Normal file
@@ -0,0 +1,510 @@
|
||||
// Openai plugin module implements embedding batch behavior.
|
||||
import {
|
||||
applyEmbeddingBatchOutputLine,
|
||||
buildBatchHeaders,
|
||||
buildEmbeddingBatchGroupOptions,
|
||||
EMBEDDING_BATCH_ENDPOINT,
|
||||
extractBatchErrorMessage,
|
||||
formatUnavailableBatchError,
|
||||
normalizeBatchBaseUrl,
|
||||
postJsonWithRetry,
|
||||
resolveBatchCompletionFromStatus,
|
||||
resolveCompletedBatchResult,
|
||||
runEmbeddingBatchGroups,
|
||||
throwIfBatchTerminalFailure,
|
||||
type EmbeddingBatchStatus,
|
||||
type BatchCompletionResult,
|
||||
type ProviderBatchOutputLine,
|
||||
uploadBatchJsonlFile,
|
||||
withRemoteHttpResponse,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import {
|
||||
readProviderJsonResponse,
|
||||
readProviderTextResponse,
|
||||
readResponseTextLimited,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { normalizeStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type { OpenAiEmbeddingClient } from "./embedding-provider.js";
|
||||
|
||||
type EmbeddingBatchExecutionParams = {
|
||||
wait: boolean;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
concurrency: number;
|
||||
debug?: (message: string, data?: Record<string, unknown>) => void;
|
||||
};
|
||||
|
||||
type OpenAiBatchRequest = {
|
||||
custom_id: string;
|
||||
method: "POST";
|
||||
url: "/v1/embeddings";
|
||||
body: {
|
||||
model: string;
|
||||
input: string;
|
||||
};
|
||||
};
|
||||
|
||||
type OpenAiBatchStatus = EmbeddingBatchStatus & {
|
||||
request_counts?: {
|
||||
total?: number;
|
||||
completed?: number;
|
||||
failed?: number;
|
||||
};
|
||||
};
|
||||
type OpenAiBatchOutputLine = ProviderBatchOutputLine;
|
||||
|
||||
export const OPENAI_BATCH_ENDPOINT = EMBEDDING_BATCH_ENDPOINT;
|
||||
const OPENAI_BATCH_COMPLETION_WINDOW = "24h";
|
||||
const OPENAI_BATCH_MAX_REQUESTS = 50000;
|
||||
// OpenAI accepts 200 MB Batch input files. Keep a safety margin so the JSONL
|
||||
// splitter avoids boundary-size uploads while preserving source-wide batching.
|
||||
const OPENAI_BATCH_MAX_JSONL_BYTES = 190 * 1024 * 1024;
|
||||
const OPENAI_BATCH_MAX_POLL_BACKOFF_MS = 5 * 60_000;
|
||||
const OPENAI_BATCH_ERROR_BODY_LIMIT_BYTES = 8 * 1024;
|
||||
const OPENAI_BATCH_OUTPUT_LINE_MAX_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
async function submitOpenAiBatch(params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
requests: OpenAiBatchRequest[];
|
||||
agentId: string;
|
||||
}): Promise<OpenAiBatchStatus> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.openAi);
|
||||
const inputFileId = await uploadBatchJsonlFile({
|
||||
client: params.openAi,
|
||||
requests: params.requests,
|
||||
errorPrefix: "openai batch file upload failed",
|
||||
});
|
||||
|
||||
return await postJsonWithRetry<OpenAiBatchStatus>({
|
||||
url: `${baseUrl}/batches`,
|
||||
headers: buildBatchHeaders(params.openAi, { json: true }),
|
||||
ssrfPolicy: params.openAi.ssrfPolicy,
|
||||
fetchImpl: params.openAi.fetchImpl,
|
||||
body: {
|
||||
input_file_id: inputFileId,
|
||||
endpoint: OPENAI_BATCH_ENDPOINT,
|
||||
completion_window: OPENAI_BATCH_COMPLETION_WINDOW,
|
||||
metadata: {
|
||||
source: "openclaw-memory",
|
||||
agent: params.agentId,
|
||||
},
|
||||
},
|
||||
errorPrefix: "openai batch create failed",
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchOpenAiBatchStatus(params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
batchId: string;
|
||||
}): Promise<OpenAiBatchStatus> {
|
||||
return await fetchOpenAiBatchResource({
|
||||
openAi: params.openAi,
|
||||
path: `/batches/${params.batchId}`,
|
||||
errorPrefix: "openai batch status",
|
||||
parse: async (res) => readProviderJsonResponse<OpenAiBatchStatus>(res, "openai.batch-status"),
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchOpenAiFileContent(params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
fileId: string;
|
||||
}): Promise<string> {
|
||||
return await fetchOpenAiBatchResource({
|
||||
openAi: params.openAi,
|
||||
path: `/files/${params.fileId}/content`,
|
||||
errorPrefix: "openai batch file content",
|
||||
parse: async (res) => await readProviderTextResponse(res, "openai.batch-file-content"),
|
||||
});
|
||||
}
|
||||
|
||||
async function readOpenAiBatchOutputLines(
|
||||
response: Response,
|
||||
params: {
|
||||
maxLines: number;
|
||||
onLine: (line: OpenAiBatchOutputLine) => boolean;
|
||||
},
|
||||
): Promise<void> {
|
||||
let lineCount = 0;
|
||||
const emitOutputLine = (line: OpenAiBatchOutputLine): boolean => {
|
||||
lineCount += 1;
|
||||
if (lineCount > params.maxLines) {
|
||||
throw new Error(`openai.batch-file-content: JSONL output exceeds ${params.maxLines} records`);
|
||||
}
|
||||
return params.onLine(line);
|
||||
};
|
||||
const emitParsedLine = (line: string): boolean =>
|
||||
emitOutputLine(parseOpenAiBatchOutputLine(line));
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
const text = await readProviderTextResponse(response, "openai.batch-file-content", {
|
||||
maxBytes: OPENAI_BATCH_OUTPUT_LINE_MAX_BYTES,
|
||||
});
|
||||
for (const line of parseOpenAiBatchOutput(text)) {
|
||||
if (!emitOutputLine(line)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
const encoder = new TextEncoder();
|
||||
let line = "";
|
||||
let lineBytes = 0;
|
||||
|
||||
const appendSegment = (segment: string) => {
|
||||
if (!segment) {
|
||||
return;
|
||||
}
|
||||
lineBytes += encoder.encode(segment).byteLength;
|
||||
if (lineBytes > OPENAI_BATCH_OUTPUT_LINE_MAX_BYTES) {
|
||||
throw new Error(
|
||||
`openai.batch-file-content: JSONL line exceeds ${OPENAI_BATCH_OUTPUT_LINE_MAX_BYTES} bytes`,
|
||||
);
|
||||
}
|
||||
line += segment;
|
||||
};
|
||||
const emitLine = (): boolean => {
|
||||
const trimmed = line.trim();
|
||||
line = "";
|
||||
lineBytes = 0;
|
||||
if (trimmed) {
|
||||
return emitParsedLine(trimmed);
|
||||
}
|
||||
return true;
|
||||
};
|
||||
const consumeText = (text: string): boolean => {
|
||||
let offset = 0;
|
||||
while (true) {
|
||||
const newline = text.indexOf("\n", offset);
|
||||
if (newline === -1) {
|
||||
appendSegment(text.slice(offset));
|
||||
return true;
|
||||
}
|
||||
appendSegment(text.slice(offset, newline));
|
||||
if (!emitLine()) {
|
||||
return false;
|
||||
}
|
||||
offset = newline + 1;
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
if (value && value.byteLength > 0) {
|
||||
if (!consumeText(decoder.decode(value, { stream: true }))) {
|
||||
await reader.cancel().catch(() => {});
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!consumeText(decoder.decode())) {
|
||||
return;
|
||||
}
|
||||
if (line.trim()) {
|
||||
emitLine();
|
||||
}
|
||||
} catch (error) {
|
||||
await reader.cancel().catch(() => {});
|
||||
throw error;
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
async function readOpenAiBatchOutputFile(params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
fileId: string;
|
||||
maxLines: number;
|
||||
onLine: (line: OpenAiBatchOutputLine) => boolean;
|
||||
}): Promise<void> {
|
||||
return await fetchOpenAiBatchResource({
|
||||
openAi: params.openAi,
|
||||
path: `/files/${params.fileId}/content`,
|
||||
errorPrefix: "openai batch file content",
|
||||
parse: async (res) =>
|
||||
await readOpenAiBatchOutputLines(res, {
|
||||
maxLines: params.maxLines,
|
||||
onLine: params.onLine,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchOpenAiBatchResource<T>(params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
path: string;
|
||||
errorPrefix: string;
|
||||
parse: (res: Response) => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.openAi);
|
||||
return await withRemoteHttpResponse({
|
||||
url: `${baseUrl}${params.path}`,
|
||||
ssrfPolicy: params.openAi.ssrfPolicy,
|
||||
fetchImpl: params.openAi.fetchImpl,
|
||||
init: {
|
||||
headers: buildBatchHeaders(params.openAi, { json: true }),
|
||||
},
|
||||
onResponse: async (res) => {
|
||||
if (!res.ok) {
|
||||
const text = await readResponseTextLimited(res, OPENAI_BATCH_ERROR_BODY_LIMIT_BYTES);
|
||||
throw new Error(`${params.errorPrefix} failed: ${res.status} ${text}`);
|
||||
}
|
||||
return await params.parse(res);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function formatOpenAiBatchError(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isOpenAiBatchUploadTooLargeError(error: unknown): boolean {
|
||||
const message = formatOpenAiBatchError(error);
|
||||
if (!/openai batch file upload failed/i.test(message)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
/\b413\b/.test(message) ||
|
||||
/payload too large/i.test(message) ||
|
||||
/request body too large/i.test(message) ||
|
||||
/file too large/i.test(message) ||
|
||||
/maximum allowed/i.test(message) ||
|
||||
/max(?:imum)? (?:body|payload|file) (?:size )?(?:exceeded|limit)/i.test(message)
|
||||
);
|
||||
}
|
||||
|
||||
export function parseOpenAiBatchOutput(text: string): OpenAiBatchOutputLine[] {
|
||||
if (!text.trim()) {
|
||||
return [];
|
||||
}
|
||||
return normalizeStringEntries(text.split("\n")).map(parseOpenAiBatchOutputLine);
|
||||
}
|
||||
|
||||
function parseOpenAiBatchOutputLine(line: string): OpenAiBatchOutputLine {
|
||||
try {
|
||||
return JSON.parse(line) as OpenAiBatchOutputLine;
|
||||
} catch {
|
||||
throw new Error("OpenAI embedding batch output contained malformed JSONL");
|
||||
}
|
||||
}
|
||||
|
||||
async function readOpenAiBatchError(params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
errorFileId: string;
|
||||
}): Promise<string | undefined> {
|
||||
try {
|
||||
const content = await fetchOpenAiFileContent({
|
||||
openAi: params.openAi,
|
||||
fileId: params.errorFileId,
|
||||
});
|
||||
const lines = parseOpenAiBatchOutput(content);
|
||||
return extractBatchErrorMessage(lines);
|
||||
} catch (err) {
|
||||
return formatUnavailableBatchError(err);
|
||||
}
|
||||
}
|
||||
|
||||
function createOpenAiBatchPollBackoff(params: { pollIntervalMs: number; timeoutMs: number }): {
|
||||
nextDelayMs: () => number;
|
||||
} {
|
||||
const maxDelayMs = Math.max(
|
||||
params.pollIntervalMs,
|
||||
Math.min(params.timeoutMs, OPENAI_BATCH_MAX_POLL_BACKOFF_MS),
|
||||
);
|
||||
let delayMs = params.pollIntervalMs;
|
||||
return {
|
||||
nextDelayMs: () => {
|
||||
const current = delayMs;
|
||||
delayMs = Math.min(maxDelayMs, current * 2);
|
||||
return current;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatOpenAiBatchProgress(status: OpenAiBatchStatus): string {
|
||||
const counts = status.request_counts;
|
||||
if (!counts || typeof counts.total !== "number") {
|
||||
return "";
|
||||
}
|
||||
const completed = typeof counts.completed === "number" ? counts.completed : 0;
|
||||
const failed = typeof counts.failed === "number" ? counts.failed : 0;
|
||||
return `; progress ${completed}/${counts.total} failed=${failed}`;
|
||||
}
|
||||
|
||||
function formatOpenAiBatchPollError(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isRetryableOpenAiBatchPollError(error: unknown): boolean {
|
||||
const message = formatOpenAiBatchPollError(error);
|
||||
return (
|
||||
/openai batch status failed: (408|409|425|429|5\d\d)\b/i.test(message) ||
|
||||
/\b(ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN)\b|fetch failed|network error/i.test(message)
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForOpenAiBatch(params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
batchId: string;
|
||||
wait: boolean;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
debug?: (message: string, data?: Record<string, unknown>) => void;
|
||||
initial?: OpenAiBatchStatus;
|
||||
}): Promise<BatchCompletionResult> {
|
||||
const start = Date.now();
|
||||
const pollBackoff = createOpenAiBatchPollBackoff(params);
|
||||
let current: OpenAiBatchStatus | undefined = params.initial;
|
||||
while (true) {
|
||||
let status: OpenAiBatchStatus;
|
||||
try {
|
||||
status =
|
||||
current ??
|
||||
(await fetchOpenAiBatchStatus({
|
||||
openAi: params.openAi,
|
||||
batchId: params.batchId,
|
||||
}));
|
||||
} catch (error) {
|
||||
if (!params.wait || !isRetryableOpenAiBatchPollError(error)) {
|
||||
throw error;
|
||||
}
|
||||
if (Date.now() - start > params.timeoutMs) {
|
||||
throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
const delayMs = pollBackoff.nextDelayMs();
|
||||
params.debug?.(
|
||||
`openai batch ${params.batchId} status check failed: ${formatOpenAiBatchPollError(
|
||||
error,
|
||||
)}; waiting ${delayMs}ms`,
|
||||
);
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
});
|
||||
current = undefined;
|
||||
continue;
|
||||
}
|
||||
const state = status.status ?? "unknown";
|
||||
if (state === "completed") {
|
||||
return resolveBatchCompletionFromStatus({
|
||||
provider: "openai",
|
||||
batchId: params.batchId,
|
||||
status,
|
||||
});
|
||||
}
|
||||
await throwIfBatchTerminalFailure({
|
||||
provider: "openai",
|
||||
status: { ...status, id: params.batchId },
|
||||
readError: async (errorFileId) =>
|
||||
await readOpenAiBatchError({
|
||||
openAi: params.openAi,
|
||||
errorFileId,
|
||||
}),
|
||||
});
|
||||
if (!params.wait) {
|
||||
throw new Error(`openai batch ${params.batchId} still ${state}; wait disabled`);
|
||||
}
|
||||
if (Date.now() - start > params.timeoutMs) {
|
||||
throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
|
||||
}
|
||||
const delayMs = pollBackoff.nextDelayMs();
|
||||
params.debug?.(
|
||||
`openai batch ${params.batchId} ${state}${formatOpenAiBatchProgress(
|
||||
status,
|
||||
)}; waiting ${delayMs}ms`,
|
||||
);
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
});
|
||||
current = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runOpenAiEmbeddingBatches(
|
||||
params: {
|
||||
openAi: OpenAiEmbeddingClient;
|
||||
agentId: string;
|
||||
requests: OpenAiBatchRequest[];
|
||||
maxJsonlBytes?: number;
|
||||
} & EmbeddingBatchExecutionParams,
|
||||
): Promise<Map<string, number[]>> {
|
||||
return await runEmbeddingBatchGroups({
|
||||
...buildEmbeddingBatchGroupOptions(params, {
|
||||
maxRequests: OPENAI_BATCH_MAX_REQUESTS,
|
||||
maxJsonlBytes: params.maxJsonlBytes ?? OPENAI_BATCH_MAX_JSONL_BYTES,
|
||||
debugLabel: "memory embeddings: openai batch submit",
|
||||
}),
|
||||
shouldSplitGroupOnError: isOpenAiBatchUploadTooLargeError,
|
||||
onSplitGroup: ({ error, group, parts, depth }) => {
|
||||
params.debug?.("memory embeddings: openai batch upload too large; splitting group", {
|
||||
requests: group.length,
|
||||
parts: parts.map((part) => part.length),
|
||||
depth,
|
||||
error: formatOpenAiBatchError(error),
|
||||
});
|
||||
},
|
||||
runGroup: async ({ group, groupIndex, groups, byCustomId, pollIntervalMs, timeoutMs }) => {
|
||||
const batchInfo = await submitOpenAiBatch({
|
||||
openAi: params.openAi,
|
||||
requests: group,
|
||||
agentId: params.agentId,
|
||||
});
|
||||
if (!batchInfo.id) {
|
||||
throw new Error("openai batch create failed: missing batch id");
|
||||
}
|
||||
const batchId = batchInfo.id;
|
||||
|
||||
params.debug?.("memory embeddings: openai batch created", {
|
||||
batchId: batchInfo.id,
|
||||
status: batchInfo.status,
|
||||
group: groupIndex + 1,
|
||||
groups,
|
||||
requests: group.length,
|
||||
});
|
||||
|
||||
const completed = await resolveCompletedBatchResult({
|
||||
provider: "openai",
|
||||
status: batchInfo,
|
||||
wait: params.wait,
|
||||
waitForBatch: async () =>
|
||||
await waitForOpenAiBatch({
|
||||
openAi: params.openAi,
|
||||
batchId,
|
||||
wait: params.wait,
|
||||
pollIntervalMs,
|
||||
timeoutMs,
|
||||
debug: params.debug,
|
||||
initial: batchInfo,
|
||||
}),
|
||||
});
|
||||
|
||||
const errors: string[] = [];
|
||||
const remaining = new Set(group.map((request) => request.custom_id));
|
||||
|
||||
await readOpenAiBatchOutputFile({
|
||||
openAi: params.openAi,
|
||||
fileId: completed.outputFileId,
|
||||
maxLines: group.length,
|
||||
onLine: (line) => {
|
||||
applyEmbeddingBatchOutputLine({ line, remaining, errors, byCustomId });
|
||||
return remaining.size > 0;
|
||||
},
|
||||
});
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`openai batch ${batchInfo.id} failed: ${errors.join("; ")}`);
|
||||
}
|
||||
if (remaining.size > 0) {
|
||||
throw new Error(
|
||||
`openai batch ${batchInfo.id} missing ${remaining.size} embedding responses`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
214
extensions/openai/embedding-provider.test.ts
Normal file
214
extensions/openai/embedding-provider.test.ts
Normal file
@@ -0,0 +1,214 @@
|
||||
// Openai tests cover embedding provider plugin behavior.
|
||||
import type { MemoryEmbeddingProviderCreateOptions } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const DEFAULT_MOCK_CLIENT = {
|
||||
baseUrl: "https://embeddings.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
model: "text-embedding-3-small",
|
||||
};
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
fetchRemoteEmbeddingVectors: vi.fn(async () => [[1, 0]]),
|
||||
resolveRemoteEmbeddingClient: vi.fn(async () => ({ ...DEFAULT_MOCK_CLIENT })),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/memory-core-host-engine-embeddings", () => ({
|
||||
fetchRemoteEmbeddingVectors: mocks.fetchRemoteEmbeddingVectors,
|
||||
resolveRemoteEmbeddingClient: mocks.resolveRemoteEmbeddingClient,
|
||||
}));
|
||||
|
||||
import { createOpenAiEmbeddingProvider } from "./embedding-provider.js";
|
||||
|
||||
function createOptions(
|
||||
overrides: Partial<MemoryEmbeddingProviderCreateOptions> = {},
|
||||
): MemoryEmbeddingProviderCreateOptions {
|
||||
return {
|
||||
config: {} as MemoryEmbeddingProviderCreateOptions["config"],
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
fallback: "none",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function expectFetchRemoteEmbeddingVectorsBody(body: Record<string, unknown>) {
|
||||
expect(mocks.fetchRemoteEmbeddingVectors).toHaveBeenCalledWith({
|
||||
url: "https://embeddings.example/v1/embeddings",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
ssrfPolicy: undefined,
|
||||
fetchImpl: undefined,
|
||||
signal: undefined,
|
||||
body,
|
||||
errorPrefix: "openai embeddings failed",
|
||||
});
|
||||
}
|
||||
|
||||
describe("OpenAI embedding provider", () => {
|
||||
beforeEach(() => {
|
||||
mocks.fetchRemoteEmbeddingVectors.mockClear();
|
||||
mocks.resolveRemoteEmbeddingClient.mockClear();
|
||||
});
|
||||
|
||||
it("sends queryInputType on query embeddings", async () => {
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({ inputType: "passage", queryInputType: "query" }),
|
||||
);
|
||||
|
||||
await provider.embedQuery("hello");
|
||||
|
||||
expectFetchRemoteEmbeddingVectorsBody({
|
||||
model: "text-embedding-3-small",
|
||||
input: ["hello"],
|
||||
input_type: "query",
|
||||
});
|
||||
});
|
||||
|
||||
it("sends documentInputType on document batch embeddings", async () => {
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({ inputType: "query", documentInputType: "document" }),
|
||||
);
|
||||
|
||||
await provider.embedBatch(["doc one", "doc two"]);
|
||||
|
||||
expectFetchRemoteEmbeddingVectorsBody({
|
||||
model: "text-embedding-3-small",
|
||||
input: ["doc one", "doc two"],
|
||||
input_type: "document",
|
||||
});
|
||||
});
|
||||
|
||||
it("omits input_type unless configured", async () => {
|
||||
const { provider } = await createOpenAiEmbeddingProvider(createOptions());
|
||||
|
||||
await provider.embedBatch(["doc"]);
|
||||
|
||||
expectFetchRemoteEmbeddingVectorsBody({
|
||||
model: "text-embedding-3-small",
|
||||
input: ["doc"],
|
||||
});
|
||||
});
|
||||
|
||||
it("sends outputDimensionality as OpenAI dimensions", async () => {
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({ outputDimensionality: 512 }),
|
||||
);
|
||||
|
||||
await provider.embedBatch(["doc"]);
|
||||
|
||||
expectFetchRemoteEmbeddingVectorsBody({
|
||||
model: "text-embedding-3-small",
|
||||
input: ["doc"],
|
||||
dimensions: 512,
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards custom provider ids to the remote embedding client", async () => {
|
||||
await createOpenAiEmbeddingProvider(createOptions({ provider: "bailian-embedding" }));
|
||||
|
||||
expect(mocks.resolveRemoteEmbeddingClient).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "bailian-embedding",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("defaults the remote embedding client lookup to openai", async () => {
|
||||
await createOpenAiEmbeddingProvider(createOptions({ provider: undefined }));
|
||||
|
||||
expect(mocks.resolveRemoteEmbeddingClient).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "openai",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
// --- openai/ prefix preservation ---
|
||||
|
||||
it("strips openai/ prefix when using native OpenAI API base URL", async () => {
|
||||
mocks.resolveRemoteEmbeddingClient.mockResolvedValueOnce({
|
||||
...DEFAULT_MOCK_CLIENT,
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "text-embedding-3-small",
|
||||
});
|
||||
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({ model: "openai/text-embedding-3-small" }),
|
||||
);
|
||||
|
||||
expect(provider.model).toBe("text-embedding-3-small");
|
||||
});
|
||||
|
||||
it("strips openai/ prefix for semantically native URLs (uppercase hostname)", async () => {
|
||||
mocks.resolveRemoteEmbeddingClient.mockResolvedValueOnce({
|
||||
...DEFAULT_MOCK_CLIENT,
|
||||
baseUrl: "https://API.OPENAI.COM/v1",
|
||||
model: "text-embedding-3-small",
|
||||
});
|
||||
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({ model: "openai/text-embedding-3-small" }),
|
||||
);
|
||||
|
||||
expect(provider.model).toBe("text-embedding-3-small");
|
||||
});
|
||||
|
||||
it("preserves openai/ prefix for non-native OpenAI base URLs", async () => {
|
||||
mocks.resolveRemoteEmbeddingClient.mockResolvedValueOnce({
|
||||
...DEFAULT_MOCK_CLIENT,
|
||||
baseUrl: "https://router.requesty.ai/v1",
|
||||
model: "text-embedding-3-small",
|
||||
});
|
||||
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({ model: "openai/text-embedding-3-small" }),
|
||||
);
|
||||
|
||||
expect(provider.model).toBe("openai/text-embedding-3-small");
|
||||
});
|
||||
|
||||
it("provides maxInputTokens for qualified model with non-native base URL", async () => {
|
||||
mocks.resolveRemoteEmbeddingClient.mockResolvedValueOnce({
|
||||
...DEFAULT_MOCK_CLIENT,
|
||||
baseUrl: "https://router.requesty.ai/v1",
|
||||
model: "text-embedding-3-small",
|
||||
});
|
||||
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({ model: "openai/text-embedding-3-small" }),
|
||||
);
|
||||
|
||||
expect(provider.maxInputTokens).toBe(8192);
|
||||
});
|
||||
|
||||
it("preserves openai/ prefix in embedding request body for non-native base URLs", async () => {
|
||||
mocks.resolveRemoteEmbeddingClient.mockResolvedValueOnce({
|
||||
...DEFAULT_MOCK_CLIENT,
|
||||
baseUrl: "https://router.requesty.ai/v1",
|
||||
model: "text-embedding-3-small",
|
||||
});
|
||||
|
||||
const { provider } = await createOpenAiEmbeddingProvider(
|
||||
createOptions({
|
||||
model: "openai/text-embedding-3-small",
|
||||
inputType: "query",
|
||||
}),
|
||||
);
|
||||
|
||||
await provider.embedQuery("test");
|
||||
|
||||
expect(mocks.fetchRemoteEmbeddingVectors).toHaveBeenCalledWith({
|
||||
url: "https://router.requesty.ai/v1/embeddings",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
ssrfPolicy: undefined,
|
||||
fetchImpl: undefined,
|
||||
signal: undefined,
|
||||
body: {
|
||||
model: "openai/text-embedding-3-small",
|
||||
input: ["test"],
|
||||
input_type: "query",
|
||||
},
|
||||
errorPrefix: "openai embeddings failed",
|
||||
});
|
||||
});
|
||||
});
|
||||
128
extensions/openai/embedding-provider.ts
Normal file
128
extensions/openai/embedding-provider.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import {
|
||||
fetchRemoteEmbeddingVectors,
|
||||
resolveRemoteEmbeddingClient,
|
||||
type MemoryEmbeddingProvider,
|
||||
type MemoryEmbeddingProviderCreateOptions,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import type { SsrFPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { OPENAI_DEFAULT_EMBEDDING_MODEL } from "./default-models.js";
|
||||
|
||||
export type OpenAiEmbeddingClient = {
|
||||
baseUrl: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
model: string;
|
||||
inputType?: string;
|
||||
queryInputType?: string;
|
||||
documentInputType?: string;
|
||||
outputDimensionality?: number;
|
||||
};
|
||||
|
||||
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
||||
export const DEFAULT_OPENAI_EMBEDDING_MODEL = OPENAI_DEFAULT_EMBEDDING_MODEL;
|
||||
const OPENAI_MAX_INPUT_TOKENS: Record<string, number> = {
|
||||
"text-embedding-3-small": 8192,
|
||||
"text-embedding-3-large": 8192,
|
||||
"text-embedding-ada-002": 8191,
|
||||
};
|
||||
|
||||
function normalizeOpenAiModel(model: string): string {
|
||||
const trimmed = model.trim();
|
||||
if (!trimmed) {
|
||||
return DEFAULT_OPENAI_EMBEDDING_MODEL;
|
||||
}
|
||||
return trimmed.startsWith("openai/") ? trimmed.slice("openai/".length) : trimmed;
|
||||
}
|
||||
|
||||
/** Whether the embedding base URL points to the native OpenAI API endpoint. */
|
||||
function isNativeOpenAiBaseUrl(baseUrl: string): boolean {
|
||||
try {
|
||||
return new URL(baseUrl).hostname.toLowerCase().replace(/\.+$/, "") === "api.openai.com";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createOpenAiEmbeddingProvider(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<{ provider: MemoryEmbeddingProvider; client: OpenAiEmbeddingClient }> {
|
||||
const client = await resolveOpenAiEmbeddingClient(options);
|
||||
const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
|
||||
|
||||
const resolveInputType = (kind: "query" | "document"): string | undefined => {
|
||||
const explicit = kind === "query" ? client.queryInputType : client.documentInputType;
|
||||
const value = explicit ?? client.inputType;
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
|
||||
};
|
||||
|
||||
const embed = async (
|
||||
input: string[],
|
||||
kind: "query" | "document",
|
||||
signal?: AbortSignal,
|
||||
): Promise<number[][]> => {
|
||||
if (input.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const inputType = resolveInputType(kind);
|
||||
return await fetchRemoteEmbeddingVectors({
|
||||
url,
|
||||
headers: client.headers,
|
||||
ssrfPolicy: client.ssrfPolicy,
|
||||
fetchImpl: client.fetchImpl,
|
||||
signal,
|
||||
body: {
|
||||
model: client.model,
|
||||
input,
|
||||
...(typeof client.outputDimensionality === "number"
|
||||
? { dimensions: client.outputDimensionality }
|
||||
: {}),
|
||||
...(inputType ? { input_type: inputType } : {}),
|
||||
},
|
||||
errorPrefix: "openai embeddings failed",
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
provider: {
|
||||
id: "openai",
|
||||
model: client.model,
|
||||
...(typeof OPENAI_MAX_INPUT_TOKENS[normalizeOpenAiModel(client.model)] === "number"
|
||||
? { maxInputTokens: OPENAI_MAX_INPUT_TOKENS[normalizeOpenAiModel(client.model)] }
|
||||
: {}),
|
||||
embedQuery: async (text, optionsValue) => {
|
||||
const [vec] = await embed([text], "query", optionsValue?.signal);
|
||||
return vec ?? [];
|
||||
},
|
||||
embedBatch: async (texts, optionsLocal) =>
|
||||
await embed(texts, "document", optionsLocal?.signal),
|
||||
},
|
||||
client,
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveOpenAiEmbeddingClient(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<OpenAiEmbeddingClient> {
|
||||
const originalModel = options.model;
|
||||
const client = await resolveRemoteEmbeddingClient({
|
||||
provider: options.provider ?? "openai",
|
||||
options,
|
||||
defaultBaseUrl: DEFAULT_OPENAI_BASE_URL,
|
||||
normalizeModel: normalizeOpenAiModel,
|
||||
});
|
||||
// Non-native OpenAI routers (e.g. Requesty) expect the provider-qualified
|
||||
// model name ("openai/text-embedding-3-small") in embedding requests.
|
||||
// Strip the prefix only when talking to the native OpenAI API.
|
||||
if (!isNativeOpenAiBaseUrl(client.baseUrl) && originalModel.startsWith("openai/")) {
|
||||
client.model = `openai/${normalizeOpenAiModel(originalModel)}`;
|
||||
}
|
||||
return {
|
||||
...client,
|
||||
inputType: options.inputType,
|
||||
queryInputType: options.queryInputType,
|
||||
documentInputType: options.documentInputType,
|
||||
outputDimensionality: options.outputDimensionality,
|
||||
};
|
||||
}
|
||||
2138
extensions/openai/image-generation-provider.test.ts
Normal file
2138
extensions/openai/image-generation-provider.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
1062
extensions/openai/image-generation-provider.ts
Normal file
1062
extensions/openai/image-generation-provider.ts
Normal file
File diff suppressed because it is too large
Load Diff
601
extensions/openai/index.test.ts
Normal file
601
extensions/openai/index.test.ts
Normal file
@@ -0,0 +1,601 @@
|
||||
// Openai tests cover index plugin behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api";
|
||||
import { requireRegisteredProvider } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import * as providerHttp from "openclaw/plugin-sdk/provider-http";
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildOpenAIImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import plugin from "./index.js";
|
||||
import {
|
||||
OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
OPENAI_GPT5_BEHAVIOR_CONTRACT,
|
||||
OPENAI_HEARTBEAT_PROMPT_OVERLAY,
|
||||
shouldApplyOpenAIPromptOverlay,
|
||||
} from "./prompt-overlay.js";
|
||||
|
||||
const runtimeMocks = vi.hoisted(() => ({
|
||||
ensureGlobalUndiciEnvProxyDispatcher: vi.fn(),
|
||||
refreshOpenAICodexToken: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/runtime-env", async () => {
|
||||
const actual = await vi.importActual<typeof import("openclaw/plugin-sdk/runtime-env")>(
|
||||
"openclaw/plugin-sdk/runtime-env",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
ensureGlobalUndiciEnvProxyDispatcher: runtimeMocks.ensureGlobalUndiciEnvProxyDispatcher,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./openai-chatgpt-oauth-flow.runtime.js", () => ({
|
||||
refreshOpenAICodexToken: runtimeMocks.refreshOpenAICodexToken,
|
||||
}));
|
||||
|
||||
import { createOpenAICodexProviderRuntime } from "./openai-chatgpt-provider.runtime.js";
|
||||
async function registerOpenAIPluginWithHook(params?: { pluginConfig?: Record<string, unknown> }) {
|
||||
const on = vi.fn();
|
||||
const providers: ProviderPlugin[] = [];
|
||||
plugin.register(
|
||||
createTestPluginApi({
|
||||
id: "openai",
|
||||
name: "OpenAI Provider",
|
||||
source: "test",
|
||||
config: {},
|
||||
runtime: {} as never,
|
||||
pluginConfig: params?.pluginConfig,
|
||||
on,
|
||||
registerProvider: (provider) => {
|
||||
providers.push(provider);
|
||||
},
|
||||
}),
|
||||
);
|
||||
return { on, providers };
|
||||
}
|
||||
|
||||
function expectOpenAIPromptContribution(
|
||||
provider: ProviderPlugin,
|
||||
sectionOverrides: Record<string, unknown>,
|
||||
contextOverrides: Partial<
|
||||
Parameters<NonNullable<ProviderPlugin["resolveSystemPromptContribution"]>>[0]
|
||||
> = {},
|
||||
) {
|
||||
expect(
|
||||
provider.resolveSystemPromptContribution?.({
|
||||
config: undefined,
|
||||
agentDir: undefined,
|
||||
workspaceDir: undefined,
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
promptMode: "full",
|
||||
runtimeChannel: undefined,
|
||||
runtimeCapabilities: undefined,
|
||||
agentId: undefined,
|
||||
...contextOverrides,
|
||||
}),
|
||||
).toEqual({
|
||||
stablePrefix: OPENAI_GPT5_BEHAVIOR_CONTRACT,
|
||||
sectionOverrides,
|
||||
});
|
||||
}
|
||||
|
||||
function mockOpenAIImageApiResponse(params: {
|
||||
finalUrl: string;
|
||||
imageData: string;
|
||||
revisedPrompt?: string;
|
||||
}) {
|
||||
const response = () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
data: [
|
||||
{
|
||||
b64_json: Buffer.from(params.imageData).toString("base64"),
|
||||
...(params.revisedPrompt ? { revised_prompt: params.revisedPrompt } : {}),
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
const resolveApiKeySpy = vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "sk-test",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
const postJsonRequestSpy = vi.spyOn(providerHttp, "postJsonRequest").mockResolvedValue({
|
||||
finalUrl: params.finalUrl,
|
||||
response: response(),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
const postMultipartRequestSpy = vi.spyOn(providerHttp, "postMultipartRequest").mockResolvedValue({
|
||||
finalUrl: params.finalUrl,
|
||||
response: response(),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
vi.spyOn(providerHttp, "assertOkOrThrowHttpError").mockResolvedValue(undefined);
|
||||
return { resolveApiKeySpy, postJsonRequestSpy, postMultipartRequestSpy };
|
||||
}
|
||||
|
||||
function firstMockArg(mocked: unknown): Record<string, unknown> {
|
||||
const arg = (mocked as { mock?: { calls?: unknown[][] } }).mock?.calls?.[0]?.[0];
|
||||
if (!arg || typeof arg !== "object") {
|
||||
throw new Error("Expected first mock argument");
|
||||
}
|
||||
return arg as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function mockCalls(mocked: unknown): unknown[][] {
|
||||
return (mocked as { mock?: { calls?: unknown[][] } }).mock?.calls ?? [];
|
||||
}
|
||||
|
||||
function expectNoBeforePromptBuildHook(on: unknown): void {
|
||||
const hasBeforePromptBuild = mockCalls(on).some((call) => call[0] === "before_prompt_build");
|
||||
expect(hasBeforePromptBuild).toBe(false);
|
||||
}
|
||||
|
||||
function expectNoRequestUrl(mocked: unknown, url: string): void {
|
||||
const hasUrl = mockCalls(mocked).some((call) => {
|
||||
const arg = call[0] as { url?: unknown } | undefined;
|
||||
return arg?.url === url;
|
||||
});
|
||||
expect(hasUrl).toBe(false);
|
||||
}
|
||||
|
||||
describe("openai plugin", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("generates PNG buffers from the OpenAI Images API", async () => {
|
||||
const { resolveApiKeySpy, postJsonRequestSpy } = mockOpenAIImageApiResponse({
|
||||
finalUrl: "https://api.openai.com/v1/images/generations",
|
||||
imageData: "png-data",
|
||||
revisedPrompt: "revised",
|
||||
});
|
||||
|
||||
const provider = buildOpenAIImageGenerationProvider();
|
||||
const authStore = { version: 1, profiles: {} };
|
||||
const result = await provider.generateImage({
|
||||
provider: "openai",
|
||||
model: "gpt-image-2",
|
||||
prompt: "draw a cat",
|
||||
cfg: {},
|
||||
authStore,
|
||||
count: 2,
|
||||
size: "2048x2048",
|
||||
});
|
||||
|
||||
const authArgs = firstMockArg(resolveApiKeySpy);
|
||||
expect(authArgs.provider).toBe("openai");
|
||||
expect(authArgs.store).toBe(authStore);
|
||||
const requestArgs = firstMockArg(postJsonRequestSpy);
|
||||
expect(requestArgs.url).toBe("https://api.openai.com/v1/images/generations");
|
||||
expect(requestArgs.body).toEqual({
|
||||
model: "gpt-image-2",
|
||||
prompt: "draw a cat",
|
||||
n: 2,
|
||||
size: "2048x2048",
|
||||
});
|
||||
expectNoRequestUrl(postJsonRequestSpy, "https://api.openai.com/v1/images/edits");
|
||||
expect(result).toEqual({
|
||||
images: [
|
||||
{
|
||||
buffer: Buffer.from("png-data"),
|
||||
mimeType: "image/png",
|
||||
fileName: "image-1.png",
|
||||
revisedPrompt: "revised",
|
||||
},
|
||||
],
|
||||
model: "gpt-image-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("submits reference-image edits to the OpenAI Images edits endpoint", async () => {
|
||||
const { resolveApiKeySpy, postJsonRequestSpy, postMultipartRequestSpy } =
|
||||
mockOpenAIImageApiResponse({
|
||||
finalUrl: "https://api.openai.com/v1/images/edits",
|
||||
imageData: "edited-image",
|
||||
});
|
||||
|
||||
const provider = buildOpenAIImageGenerationProvider();
|
||||
const authStore = { version: 1, profiles: {} };
|
||||
|
||||
const result = await provider.generateImage({
|
||||
provider: "openai",
|
||||
model: "gpt-image-2",
|
||||
prompt: "Edit this image",
|
||||
cfg: {},
|
||||
authStore,
|
||||
count: 2,
|
||||
size: "1536x1024",
|
||||
inputImages: [
|
||||
{ buffer: Buffer.from("x"), mimeType: "image/png" },
|
||||
{ buffer: Buffer.from("y"), mimeType: "image/jpeg", fileName: "ref.jpg" },
|
||||
],
|
||||
});
|
||||
|
||||
const authArgs = firstMockArg(resolveApiKeySpy);
|
||||
expect(authArgs.provider).toBe("openai");
|
||||
expect(authArgs.store).toBe(authStore);
|
||||
const multipartArgs = firstMockArg(postMultipartRequestSpy);
|
||||
expect(multipartArgs.url).toBe("https://api.openai.com/v1/images/edits");
|
||||
expect(multipartArgs.body).toBeInstanceOf(FormData);
|
||||
expect(multipartArgs.allowPrivateNetwork).toBe(false);
|
||||
expect(multipartArgs.dispatcherPolicy).toBeUndefined();
|
||||
expect(multipartArgs.fetchFn).toBe(fetch);
|
||||
const editCallArgs = multipartArgs as unknown as {
|
||||
headers: Headers;
|
||||
body: FormData;
|
||||
};
|
||||
expect(editCallArgs.headers.has("Content-Type")).toBe(false);
|
||||
const form = editCallArgs.body;
|
||||
expect(form.get("model")).toBe("gpt-image-2");
|
||||
expect(form.get("prompt")).toBe("Edit this image");
|
||||
expect(form.get("n")).toBe("2");
|
||||
expect(form.get("size")).toBe("1536x1024");
|
||||
const images = form.getAll("image[]") as File[];
|
||||
expect(images).toHaveLength(2);
|
||||
expect(images[0]?.name).toBe("image-1.png");
|
||||
expect(images[0]?.type).toBe("image/png");
|
||||
expect(images[1]?.name).toBe("ref.jpg");
|
||||
expect(images[1]?.type).toBe("image/jpeg");
|
||||
expectNoRequestUrl(postJsonRequestSpy, "https://api.openai.com/v1/images/edits");
|
||||
expect(result).toEqual({
|
||||
images: [
|
||||
{
|
||||
buffer: Buffer.from("edited-image"),
|
||||
mimeType: "image/png",
|
||||
fileName: "image-1.png",
|
||||
},
|
||||
],
|
||||
model: "gpt-image-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not allow private-network routing just because a custom base URL is configured", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "sk-test",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const provider = buildOpenAIImageGenerationProvider();
|
||||
await expect(
|
||||
provider.generateImage({
|
||||
provider: "openai",
|
||||
model: "gpt-image-2",
|
||||
prompt: "draw a cat",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "http://127.0.0.1:8080/v1",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig,
|
||||
}),
|
||||
).rejects.toThrow("Blocked hostname or private/internal/special-use IP address");
|
||||
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bootstraps the env proxy dispatcher before refreshing codex oauth credentials", async () => {
|
||||
const refreshed = {
|
||||
access: "next-access",
|
||||
refresh: "next-refresh",
|
||||
expires: Date.now() + 60_000,
|
||||
};
|
||||
runtimeMocks.refreshOpenAICodexToken.mockResolvedValue(refreshed);
|
||||
const runtime = createOpenAICodexProviderRuntime({
|
||||
ensureGlobalUndiciEnvProxyDispatcher: runtimeMocks.ensureGlobalUndiciEnvProxyDispatcher,
|
||||
getOAuthApiKey: vi.fn(),
|
||||
refreshOpenAICodexToken: runtimeMocks.refreshOpenAICodexToken,
|
||||
});
|
||||
|
||||
await expect(runtime.refreshOpenAICodexToken("refresh-token")).resolves.toBe(refreshed);
|
||||
|
||||
expect(runtimeMocks.ensureGlobalUndiciEnvProxyDispatcher).toHaveBeenCalledOnce();
|
||||
expect(runtimeMocks.refreshOpenAICodexToken).toHaveBeenCalledOnce();
|
||||
expect(
|
||||
runtimeMocks.ensureGlobalUndiciEnvProxyDispatcher.mock.invocationCallOrder[0],
|
||||
).toBeLessThan(runtimeMocks.refreshOpenAICodexToken.mock.invocationCallOrder[0]);
|
||||
});
|
||||
|
||||
it("registers provider-owned OpenAI tool compat hooks for API and Codex transports", async () => {
|
||||
const { providers } = await registerOpenAIPluginWithHook();
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
const noParamsTool = {
|
||||
name: "ping",
|
||||
description: "",
|
||||
parameters: {},
|
||||
execute: vi.fn(),
|
||||
} as never;
|
||||
|
||||
const normalizedOpenAI = openaiProvider.normalizeToolSchemas?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
modelApi: "openai-responses",
|
||||
model: {
|
||||
provider: "openai",
|
||||
api: "openai-responses",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
id: "gpt-5.4",
|
||||
} as never,
|
||||
tools: [noParamsTool],
|
||||
} as never);
|
||||
const normalizedCodex = openaiProvider.normalizeToolSchemas?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
modelApi: "openai-chatgpt-responses",
|
||||
model: {
|
||||
provider: "openai",
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
id: "gpt-5.4",
|
||||
} as never,
|
||||
tools: [noParamsTool],
|
||||
} as never);
|
||||
|
||||
expect(normalizedOpenAI?.[0]?.parameters).toEqual({
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: [],
|
||||
additionalProperties: false,
|
||||
});
|
||||
expect(normalizedCodex?.[0]?.parameters).toEqual({
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: [],
|
||||
additionalProperties: false,
|
||||
});
|
||||
expect(
|
||||
openaiProvider.inspectToolSchemas?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
modelApi: "openai-responses",
|
||||
model: {
|
||||
provider: "openai",
|
||||
api: "openai-responses",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
id: "gpt-5.4",
|
||||
} as never,
|
||||
tools: [noParamsTool],
|
||||
} as never),
|
||||
).toStrictEqual([]);
|
||||
expect(
|
||||
openaiProvider.inspectToolSchemas?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
modelApi: "openai-chatgpt-responses",
|
||||
model: {
|
||||
provider: "openai",
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api",
|
||||
id: "gpt-5.4",
|
||||
} as never,
|
||||
tools: [noParamsTool],
|
||||
} as never),
|
||||
).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("registers GPT-5 system prompt contributions when the friendly overlay is enabled", async () => {
|
||||
const { on, providers } = await registerOpenAIPluginWithHook({
|
||||
pluginConfig: { personality: "friendly" },
|
||||
});
|
||||
|
||||
expectNoBeforePromptBuildHook(on);
|
||||
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
const contributionContext: Parameters<
|
||||
NonNullable<ProviderPlugin["resolveSystemPromptContribution"]>
|
||||
>[0] = {
|
||||
config: undefined,
|
||||
agentDir: undefined,
|
||||
workspaceDir: undefined,
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
promptMode: "full",
|
||||
runtimeChannel: undefined,
|
||||
runtimeCapabilities: undefined,
|
||||
agentId: undefined,
|
||||
};
|
||||
|
||||
expect(openaiProvider.resolveSystemPromptContribution?.(contributionContext)).toEqual({
|
||||
stablePrefix: OPENAI_GPT5_BEHAVIOR_CONTRACT,
|
||||
sectionOverrides: {
|
||||
interaction_style: OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
},
|
||||
});
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).toContain("Live chat tone: short, natural, human.");
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).toContain(
|
||||
"Avoid memo voice, long preambles, walls of text, and repetitive restatement.",
|
||||
);
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).toContain("Show grounded emotional range when it fits");
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).toContain(
|
||||
"Occasional emoji are fine when they fit naturally, especially for warmth or brief celebration; keep them sparse.",
|
||||
);
|
||||
expect(
|
||||
openaiProvider.resolveSystemPromptContribution?.({
|
||||
...contributionContext,
|
||||
trigger: "heartbeat",
|
||||
}),
|
||||
).toEqual({
|
||||
stablePrefix: OPENAI_GPT5_BEHAVIOR_CONTRACT,
|
||||
sectionOverrides: {
|
||||
interaction_style: `${OPENAI_FRIENDLY_PROMPT_OVERLAY}\n\n${OPENAI_HEARTBEAT_PROMPT_OVERLAY}`,
|
||||
},
|
||||
});
|
||||
expect(
|
||||
openaiProvider.resolveSystemPromptContribution?.({
|
||||
...contributionContext,
|
||||
modelId: "openai/gpt-5.4-mini",
|
||||
}),
|
||||
).toEqual({
|
||||
stablePrefix: OPENAI_GPT5_BEHAVIOR_CONTRACT,
|
||||
sectionOverrides: {
|
||||
interaction_style: OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
},
|
||||
});
|
||||
expect(
|
||||
openaiProvider.resolveSystemPromptContribution?.({
|
||||
...contributionContext,
|
||||
modelId: "gpt-image-1",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(shouldApplyOpenAIPromptOverlay({ modelProviderId: "openai", modelId: "gpt-4.1" })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
shouldApplyOpenAIPromptOverlay({ modelProviderId: "anthropic", modelId: "gpt-5.4" }),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("includes the tagged GPT-5 behavior contract in the OpenAI prompt overlay", () => {
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).toContain("Keep progress updates concrete.");
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).toContain(
|
||||
'Brief first-person feeling language is ok when useful: "I\'m glad we caught that", "I\'m excited about this direction", "I\'m worried this will break", "that\'s frustrating".',
|
||||
);
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).not.toContain(
|
||||
"Use heartbeats to create useful proactive progress",
|
||||
);
|
||||
expect(OPENAI_HEARTBEAT_PROMPT_OVERLAY).toContain(
|
||||
"Use heartbeats to create useful proactive progress, not chatter.",
|
||||
);
|
||||
expect(OPENAI_HEARTBEAT_PROMPT_OVERLAY).toContain(
|
||||
"Treat a heartbeat as a wake-up: orient, read HEARTBEAT.md when present, then do what is actually useful now.",
|
||||
);
|
||||
expect(OPENAI_HEARTBEAT_PROMPT_OVERLAY).toContain(
|
||||
"If HEARTBEAT.md assigns concrete or ongoing work, execute its spirit with judgment.",
|
||||
);
|
||||
expect(OPENAI_HEARTBEAT_PROMPT_OVERLAY).toContain(
|
||||
"Prefer meaningful action over commentary. A good heartbeat often looks like silent progress.",
|
||||
);
|
||||
expect(OPENAI_HEARTBEAT_PROMPT_OVERLAY).toContain(
|
||||
'Do not send "same state", "no change", "still", or repetitive summaries because a problem continues.',
|
||||
);
|
||||
expect(OPENAI_HEARTBEAT_PROMPT_OVERLAY).toContain(
|
||||
"Notify only for something worth interrupting the user",
|
||||
);
|
||||
expect(OPENAI_FRIENDLY_PROMPT_OVERLAY).toContain(
|
||||
"Occasional emoji are fine when they fit naturally, especially for warmth or brief celebration; keep them sparse.",
|
||||
);
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain("<persona_latch>");
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain("<execution_policy>");
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain("<tool_discipline>");
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain("<output_contract>");
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain("<completion_contract>");
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain(
|
||||
"For irreversible, external, destructive, or privacy-sensitive actions: ask first.",
|
||||
);
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain(
|
||||
"Prefer tool evidence over recall when action, state, or mutable facts matter.",
|
||||
);
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain(
|
||||
"If more tool work would likely change the answer, do it before replying.",
|
||||
);
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain("Return requested sections/order only.");
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).toContain(
|
||||
"Treat the task as incomplete until every requested item is handled",
|
||||
);
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).not.toContain("/approve");
|
||||
expect(OPENAI_GPT5_BEHAVIOR_CONTRACT).not.toContain("GPT-5 Output Contract");
|
||||
});
|
||||
|
||||
it("defaults to the friendly OpenAI interaction-style overlay", async () => {
|
||||
const { on, providers } = await registerOpenAIPluginWithHook();
|
||||
|
||||
expectNoBeforePromptBuildHook(on);
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
expectOpenAIPromptContribution(openaiProvider, {
|
||||
interaction_style: OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
});
|
||||
});
|
||||
|
||||
it("supports opting out of the friendly prompt overlay via plugin config", async () => {
|
||||
const { on, providers } = await registerOpenAIPluginWithHook({
|
||||
pluginConfig: { personality: "off" },
|
||||
});
|
||||
|
||||
expectNoBeforePromptBuildHook(on);
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
expectOpenAIPromptContribution(openaiProvider, {});
|
||||
});
|
||||
|
||||
it("treats mixed-case off values as disabling the friendly prompt overlay", async () => {
|
||||
const { providers } = await registerOpenAIPluginWithHook({
|
||||
pluginConfig: { personality: "Off" },
|
||||
});
|
||||
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
expectOpenAIPromptContribution(openaiProvider, {});
|
||||
});
|
||||
|
||||
it("supports explicitly configuring the friendly prompt overlay", async () => {
|
||||
const { on, providers } = await registerOpenAIPluginWithHook({
|
||||
pluginConfig: { personality: "friendly" },
|
||||
});
|
||||
|
||||
expectNoBeforePromptBuildHook(on);
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
expectOpenAIPromptContribution(openaiProvider, {
|
||||
interaction_style: OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses live plugin config for GPT-5 prompt overlay mode", async () => {
|
||||
const { providers } = await registerOpenAIPluginWithHook({
|
||||
pluginConfig: { personality: "off" },
|
||||
});
|
||||
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
expect(
|
||||
openaiProvider.resolveSystemPromptContribution?.({
|
||||
config: {
|
||||
plugins: {
|
||||
entries: {
|
||||
openai: {
|
||||
config: {
|
||||
personality: "friendly",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
agentDir: undefined,
|
||||
workspaceDir: undefined,
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
promptMode: "full",
|
||||
runtimeChannel: undefined,
|
||||
runtimeCapabilities: undefined,
|
||||
agentId: undefined,
|
||||
}),
|
||||
).toEqual({
|
||||
stablePrefix: OPENAI_GPT5_BEHAVIOR_CONTRACT,
|
||||
sectionOverrides: {
|
||||
interaction_style: OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("treats on as an alias for the friendly prompt overlay", async () => {
|
||||
const { providers } = await registerOpenAIPluginWithHook({
|
||||
pluginConfig: { personality: "on" },
|
||||
});
|
||||
|
||||
const openaiProvider = requireRegisteredProvider(providers, "openai");
|
||||
expectOpenAIPromptContribution(openaiProvider, {
|
||||
interaction_style: OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
});
|
||||
});
|
||||
});
|
||||
53
extensions/openai/index.ts
Normal file
53
extensions/openai/index.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
// Openai plugin entrypoint registers its OpenClaw integration.
|
||||
import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { buildProviderToolCompatFamilyHooks } from "openclaw/plugin-sdk/provider-tools";
|
||||
import { buildOpenAIImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { openaiMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
import { openAiMemoryEmbeddingProviderAdapter } from "./memory-embedding-adapter.js";
|
||||
import { buildOpenAIProvider } from "./openai-provider.js";
|
||||
import {
|
||||
resolveOpenAIPromptOverlayMode,
|
||||
resolveOpenAISystemPromptContribution,
|
||||
} from "./prompt-overlay.js";
|
||||
import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
|
||||
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
|
||||
import { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
import { buildOpenAIVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "openai",
|
||||
name: "OpenAI Provider",
|
||||
description: "Bundled OpenAI provider plugins",
|
||||
register(api) {
|
||||
const openAIToolCompatHooks = buildProviderToolCompatFamilyHooks("openai");
|
||||
const buildProviderWithPromptContribution = <T extends ReturnType<typeof buildOpenAIProvider>>(
|
||||
provider: T,
|
||||
): T => ({
|
||||
...provider,
|
||||
...openAIToolCompatHooks,
|
||||
resolveSystemPromptContribution: (ctx) => {
|
||||
const runtimePluginConfig = resolvePluginConfigObject(ctx.config, "openai");
|
||||
const pluginConfig =
|
||||
runtimePluginConfig ??
|
||||
(ctx.config ? undefined : (api.pluginConfig as Record<string, unknown>));
|
||||
return resolveOpenAISystemPromptContribution({
|
||||
config: ctx.config,
|
||||
legacyPluginConfig: pluginConfig,
|
||||
mode: resolveOpenAIPromptOverlayMode(pluginConfig),
|
||||
modelProviderId: provider.id,
|
||||
modelId: ctx.modelId,
|
||||
trigger: ctx.trigger,
|
||||
});
|
||||
},
|
||||
});
|
||||
api.registerProvider(buildProviderWithPromptContribution(buildOpenAIProvider()));
|
||||
api.registerMemoryEmbeddingProvider(openAiMemoryEmbeddingProviderAdapter);
|
||||
api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider());
|
||||
api.registerRealtimeTranscriptionProvider(buildOpenAIRealtimeTranscriptionProvider());
|
||||
api.registerRealtimeVoiceProvider(buildOpenAIRealtimeVoiceProvider());
|
||||
api.registerSpeechProvider(buildOpenAISpeechProvider());
|
||||
api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider);
|
||||
api.registerVideoGenerationProvider(buildOpenAIVideoGenerationProvider());
|
||||
},
|
||||
});
|
||||
100
extensions/openai/media-understanding-provider.test.ts
Normal file
100
extensions/openai/media-understanding-provider.test.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
// Openai tests cover media understanding provider plugin behavior.
|
||||
import {
|
||||
createAuthCaptureJsonFetch,
|
||||
createRequestCaptureJsonFetch,
|
||||
installPinnedHostnameTestHooks,
|
||||
} from "openclaw/plugin-sdk/test-env";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
openaiMediaUnderstandingProvider,
|
||||
transcribeOpenAiAudio,
|
||||
} from "./media-understanding-provider.js";
|
||||
|
||||
installPinnedHostnameTestHooks();
|
||||
|
||||
describe("openaiMediaUnderstandingProvider", () => {
|
||||
it("declares audio support with the transcription default", () => {
|
||||
expect(openaiMediaUnderstandingProvider.capabilities).toEqual(["image", "audio"]);
|
||||
expect(openaiMediaUnderstandingProvider.defaultModels).toEqual({
|
||||
image: "gpt-5.5",
|
||||
audio: "gpt-4o-transcribe",
|
||||
});
|
||||
expect(openaiMediaUnderstandingProvider.autoPriority).toEqual({ image: 20, audio: 20 });
|
||||
expect(openaiMediaUnderstandingProvider.transcribeAudio).toBe(transcribeOpenAiAudio);
|
||||
});
|
||||
});
|
||||
|
||||
describe("transcribeOpenAiAudio", () => {
|
||||
it("respects lowercase authorization header overrides", async () => {
|
||||
const { fetchFn, getAuthHeader } = createAuthCaptureJsonFetch({ text: "ok" });
|
||||
|
||||
const result = await transcribeOpenAiAudio({
|
||||
buffer: Buffer.from("audio"),
|
||||
fileName: "note.mp3",
|
||||
apiKey: "test-key",
|
||||
timeoutMs: 1000,
|
||||
headers: { authorization: "Bearer override" },
|
||||
fetchFn,
|
||||
});
|
||||
|
||||
expect(getAuthHeader()).toBe("Bearer override");
|
||||
expect(result.text).toBe("ok");
|
||||
});
|
||||
|
||||
it("builds the expected request payload", async () => {
|
||||
const { fetchFn, getRequest } = createRequestCaptureJsonFetch({ text: "hello" });
|
||||
|
||||
const result = await transcribeOpenAiAudio({
|
||||
buffer: Buffer.from("audio-bytes"),
|
||||
fileName: "voice.wav",
|
||||
apiKey: "test-key",
|
||||
timeoutMs: 1234,
|
||||
baseUrl: "https://api.example.com/v1/",
|
||||
model: " ",
|
||||
language: " en ",
|
||||
prompt: " hello ",
|
||||
mime: "audio/wav",
|
||||
headers: { "X-Custom": "1" },
|
||||
fetchFn,
|
||||
});
|
||||
const { url: seenUrl, init: seenInit } = getRequest();
|
||||
|
||||
expect(result.model).toBe("gpt-4o-transcribe");
|
||||
expect(result.text).toBe("hello");
|
||||
expect(seenUrl).toBe("https://api.example.com/v1/audio/transcriptions");
|
||||
expect(seenInit?.method).toBe("POST");
|
||||
expect(seenInit?.signal).toBeInstanceOf(AbortSignal);
|
||||
|
||||
const headers = new Headers(seenInit?.headers);
|
||||
expect(headers.get("authorization")).toBe("Bearer test-key");
|
||||
expect(headers.get("x-custom")).toBe("1");
|
||||
|
||||
const form = seenInit?.body as FormData;
|
||||
expect(form).toBeInstanceOf(FormData);
|
||||
expect(form.get("model")).toBe("gpt-4o-transcribe");
|
||||
expect(form.get("language")).toBe("en");
|
||||
expect(form.get("prompt")).toBe("hello");
|
||||
const file = form.get("file") as Blob | { type?: string; name?: string } | null;
|
||||
if (!file) {
|
||||
throw new Error("expected OpenAI audio file");
|
||||
}
|
||||
expect(file.type).toBe("audio/wav");
|
||||
if (file && "name" in file && typeof file.name === "string") {
|
||||
expect(file.name).toBe("voice.wav");
|
||||
}
|
||||
});
|
||||
|
||||
it("throws when the provider response omits text", async () => {
|
||||
const { fetchFn } = createRequestCaptureJsonFetch({});
|
||||
|
||||
await expect(
|
||||
transcribeOpenAiAudio({
|
||||
buffer: Buffer.from("audio-bytes"),
|
||||
fileName: "voice.wav",
|
||||
apiKey: "test-key",
|
||||
timeoutMs: 1234,
|
||||
fetchFn,
|
||||
}),
|
||||
).rejects.toThrow("Audio transcription response missing text");
|
||||
});
|
||||
});
|
||||
30
extensions/openai/media-understanding-provider.ts
Normal file
30
extensions/openai/media-understanding-provider.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import {
|
||||
describeImageWithModel,
|
||||
describeImagesWithModel,
|
||||
transcribeOpenAiCompatibleAudio,
|
||||
type AudioTranscriptionRequest,
|
||||
type MediaUnderstandingProvider,
|
||||
} from "openclaw/plugin-sdk/media-understanding";
|
||||
import { OPENAI_DEFAULT_AUDIO_TRANSCRIPTION_MODEL } from "./default-models.js";
|
||||
|
||||
const DEFAULT_OPENAI_AUDIO_BASE_URL = "https://api.openai.com/v1";
|
||||
|
||||
export async function transcribeOpenAiAudio(params: AudioTranscriptionRequest) {
|
||||
return await transcribeOpenAiCompatibleAudio({
|
||||
...params,
|
||||
provider: "openai",
|
||||
defaultBaseUrl: DEFAULT_OPENAI_AUDIO_BASE_URL,
|
||||
defaultModel: OPENAI_DEFAULT_AUDIO_TRANSCRIPTION_MODEL,
|
||||
});
|
||||
}
|
||||
|
||||
export const openaiMediaUnderstandingProvider: MediaUnderstandingProvider = {
|
||||
id: "openai",
|
||||
capabilities: ["image", "audio"],
|
||||
defaultModels: { image: "gpt-5.5", audio: OPENAI_DEFAULT_AUDIO_TRANSCRIPTION_MODEL },
|
||||
autoPriority: { image: 20, audio: 20 },
|
||||
describeImage: describeImageWithModel,
|
||||
describeImages: describeImagesWithModel,
|
||||
transcribeAudio: transcribeOpenAiAudio,
|
||||
};
|
||||
115
extensions/openai/memory-embedding-adapter.test.ts
Normal file
115
extensions/openai/memory-embedding-adapter.test.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
// Openai tests cover memory embedding adapter plugin behavior.
|
||||
import type { MemoryEmbeddingProvider } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
createOpenAiEmbeddingProvider: vi.fn(),
|
||||
runOpenAiEmbeddingBatches: vi.fn(async () => new Map([["0", [1, 0]]])),
|
||||
}));
|
||||
|
||||
vi.mock("./embedding-provider.js", () => ({
|
||||
DEFAULT_OPENAI_EMBEDDING_MODEL: "text-embedding-3-small",
|
||||
createOpenAiEmbeddingProvider: mocks.createOpenAiEmbeddingProvider,
|
||||
}));
|
||||
|
||||
vi.mock("./embedding-batch.js", () => ({
|
||||
OPENAI_BATCH_ENDPOINT: "/v1/embeddings",
|
||||
runOpenAiEmbeddingBatches: mocks.runOpenAiEmbeddingBatches,
|
||||
}));
|
||||
|
||||
import { openAiMemoryEmbeddingProviderAdapter } from "./memory-embedding-adapter.js";
|
||||
|
||||
const provider: MemoryEmbeddingProvider = {
|
||||
id: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
embedQuery: async () => [1, 0],
|
||||
embedBatch: async (texts) => texts.map(() => [1, 0]),
|
||||
};
|
||||
|
||||
describe("OpenAI memory embedding adapter", () => {
|
||||
beforeEach(() => {
|
||||
mocks.createOpenAiEmbeddingProvider.mockReset();
|
||||
mocks.runOpenAiEmbeddingBatches.mockClear();
|
||||
mocks.createOpenAiEmbeddingProvider.mockResolvedValue({
|
||||
provider,
|
||||
client: {
|
||||
baseUrl: "https://embeddings.example/v1",
|
||||
headers: {},
|
||||
model: "text-embedding-3-small",
|
||||
inputType: "passage",
|
||||
documentInputType: "document",
|
||||
outputDimensionality: 512,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("sends document input_type in OpenAI batch embedding requests", async () => {
|
||||
const result = await openAiMemoryEmbeddingProviderAdapter.create({
|
||||
config: {} as never,
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
await result.runtime?.batchEmbed?.({
|
||||
agentId: "main",
|
||||
chunks: [{ text: "doc one" }],
|
||||
wait: true,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
debug: () => {},
|
||||
});
|
||||
|
||||
const batchCalls = mocks.runOpenAiEmbeddingBatches.mock.calls as unknown as Array<
|
||||
[
|
||||
{
|
||||
requests: Array<{
|
||||
body: Record<string, unknown>;
|
||||
}>;
|
||||
},
|
||||
]
|
||||
>;
|
||||
const [batchOptions] = batchCalls[0] ?? [];
|
||||
expect(batchOptions?.requests).toHaveLength(1);
|
||||
const request = batchOptions?.requests[0];
|
||||
expect(request?.body).toEqual({
|
||||
model: "text-embedding-3-small",
|
||||
input: "doc one",
|
||||
dimensions: 512,
|
||||
input_type: "document",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves the caller provider id for custom OpenAI-compatible embedding providers", async () => {
|
||||
const result = await openAiMemoryEmbeddingProviderAdapter.create({
|
||||
config: {} as never,
|
||||
provider: "bailian-embedding",
|
||||
model: "text-embedding-v3",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
expect(mocks.createOpenAiEmbeddingProvider).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "bailian-embedding",
|
||||
fallback: "none",
|
||||
model: "text-embedding-v3",
|
||||
}),
|
||||
);
|
||||
expect(result.runtime?.cacheKeyData?.provider).toBe("bailian-embedding");
|
||||
});
|
||||
|
||||
it("defaults provider id to openai when the caller leaves it unset", async () => {
|
||||
await openAiMemoryEmbeddingProviderAdapter.create({
|
||||
config: {} as never,
|
||||
model: "text-embedding-3-small",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
expect(mocks.createOpenAiEmbeddingProvider).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "openai",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
71
extensions/openai/memory-embedding-adapter.ts
Normal file
71
extensions/openai/memory-embedding-adapter.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
// Openai plugin module implements memory embedding adapter behavior.
|
||||
import {
|
||||
isMissingEmbeddingApiKeyError,
|
||||
mapBatchEmbeddingsByIndex,
|
||||
sanitizeEmbeddingCacheHeaders,
|
||||
type MemoryEmbeddingProviderAdapter,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { OPENAI_BATCH_ENDPOINT, runOpenAiEmbeddingBatches } from "./embedding-batch.js";
|
||||
import {
|
||||
createOpenAiEmbeddingProvider,
|
||||
DEFAULT_OPENAI_EMBEDDING_MODEL,
|
||||
} from "./embedding-provider.js";
|
||||
|
||||
export const openAiMemoryEmbeddingProviderAdapter: MemoryEmbeddingProviderAdapter = {
|
||||
id: "openai",
|
||||
defaultModel: DEFAULT_OPENAI_EMBEDDING_MODEL,
|
||||
transport: "remote",
|
||||
authProviderId: "openai",
|
||||
autoSelectPriority: 20,
|
||||
allowExplicitWhenConfiguredAuto: true,
|
||||
shouldContinueAutoSelection: isMissingEmbeddingApiKeyError,
|
||||
create: async (options) => {
|
||||
const resolvedProvider = options.provider ?? "openai";
|
||||
const { provider, client } = await createOpenAiEmbeddingProvider({
|
||||
...options,
|
||||
provider: resolvedProvider,
|
||||
fallback: "none",
|
||||
});
|
||||
return {
|
||||
provider,
|
||||
runtime: {
|
||||
id: "openai",
|
||||
sourceWideBatchEmbed: true,
|
||||
cacheKeyData: {
|
||||
provider: resolvedProvider,
|
||||
baseUrl: client.baseUrl,
|
||||
model: client.model,
|
||||
outputDimensionality: client.outputDimensionality,
|
||||
documentInputType: client.documentInputType ?? client.inputType,
|
||||
headers: sanitizeEmbeddingCacheHeaders(client.headers, ["authorization"]),
|
||||
},
|
||||
batchEmbed: async (batch) => {
|
||||
const inputType = client.documentInputType ?? client.inputType;
|
||||
const byCustomId = await runOpenAiEmbeddingBatches({
|
||||
openAi: client,
|
||||
agentId: batch.agentId,
|
||||
requests: batch.chunks.map((chunk, index) => ({
|
||||
custom_id: String(index),
|
||||
method: "POST",
|
||||
url: OPENAI_BATCH_ENDPOINT,
|
||||
body: {
|
||||
model: client.model,
|
||||
input: chunk.text,
|
||||
...(typeof client.outputDimensionality === "number"
|
||||
? { dimensions: client.outputDimensionality }
|
||||
: {}),
|
||||
...(inputType ? { input_type: inputType } : {}),
|
||||
},
|
||||
})),
|
||||
wait: batch.wait,
|
||||
concurrency: batch.concurrency,
|
||||
pollIntervalMs: batch.pollIntervalMs,
|
||||
timeoutMs: batch.timeoutMs,
|
||||
debug: batch.debug,
|
||||
});
|
||||
return mapBatchEmbeddingsByIndex(byCustomId, batch.chunks.length);
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
108
extensions/openai/native-web-search.ts
Normal file
108
extensions/openai/native-web-search.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
// Openai plugin module implements native web search behavior.
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { streamSimple } from "openclaw/plugin-sdk/llm";
|
||||
import { normalizeProviderId } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { streamWithPayloadPatch } from "openclaw/plugin-sdk/provider-stream-shared";
|
||||
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { isOpenAIApiBaseUrl } from "./base-url.js";
|
||||
|
||||
const OPENAI_WEB_SEARCH_TOOL = { type: "web_search" } as const;
|
||||
|
||||
type OpenAINativeWebSearchPatchResult =
|
||||
| "payload_not_object"
|
||||
| "native_tool_already_present"
|
||||
| "injected";
|
||||
|
||||
function isOpenAINativeWebSearchEligibleModel(model: {
|
||||
api?: unknown;
|
||||
provider?: unknown;
|
||||
baseUrl?: unknown;
|
||||
}): boolean {
|
||||
const provider = typeof model.provider === "string" ? model.provider : undefined;
|
||||
if (model.api !== "openai-responses" || !provider || normalizeProviderId(provider) !== "openai") {
|
||||
return false;
|
||||
}
|
||||
const baseUrl = typeof model.baseUrl === "string" ? model.baseUrl : undefined;
|
||||
return !baseUrl || isOpenAIApiBaseUrl(baseUrl);
|
||||
}
|
||||
|
||||
function shouldUseOpenAINativeWebSearchProvider(config: OpenClawConfig | undefined): boolean {
|
||||
const provider = config?.tools?.web?.search?.provider;
|
||||
if (typeof provider !== "string") {
|
||||
return true;
|
||||
}
|
||||
const normalized = provider.trim().toLowerCase();
|
||||
return normalized === "" || normalized === "auto" || normalized === "openai";
|
||||
}
|
||||
|
||||
function shouldEnableOpenAINativeWebSearch(params: {
|
||||
config?: OpenClawConfig;
|
||||
model: { api?: unknown; provider?: unknown; baseUrl?: unknown };
|
||||
}): boolean {
|
||||
return (
|
||||
params.config?.tools?.web?.search?.enabled !== false &&
|
||||
shouldUseOpenAINativeWebSearchProvider(params.config) &&
|
||||
isOpenAINativeWebSearchEligibleModel(params.model)
|
||||
);
|
||||
}
|
||||
|
||||
function isNativeWebSearchTool(tool: unknown): boolean {
|
||||
return isRecord(tool) && tool.type === OPENAI_WEB_SEARCH_TOOL.type;
|
||||
}
|
||||
|
||||
function isManagedWebSearchTool(tool: unknown): boolean {
|
||||
return isRecord(tool) && tool.type === "function" && tool.name === OPENAI_WEB_SEARCH_TOOL.type;
|
||||
}
|
||||
|
||||
function raiseMinimalReasoningForOpenAINativeWebSearch(payload: Record<string, unknown>): void {
|
||||
const reasoning = payload.reasoning;
|
||||
if (!isRecord(reasoning) || reasoning.effort !== "minimal") {
|
||||
return;
|
||||
}
|
||||
reasoning.effort = "low";
|
||||
}
|
||||
|
||||
export function patchOpenAINativeWebSearchPayload(
|
||||
payload: unknown,
|
||||
): OpenAINativeWebSearchPatchResult {
|
||||
if (!isRecord(payload)) {
|
||||
return "payload_not_object";
|
||||
}
|
||||
|
||||
const existingTools = Array.isArray(payload.tools) ? payload.tools : [];
|
||||
const filteredTools = existingTools.filter((tool) => !isManagedWebSearchTool(tool));
|
||||
if (filteredTools.some(isNativeWebSearchTool)) {
|
||||
if (filteredTools.length !== existingTools.length) {
|
||||
payload.tools = filteredTools;
|
||||
}
|
||||
raiseMinimalReasoningForOpenAINativeWebSearch(payload);
|
||||
return "native_tool_already_present";
|
||||
}
|
||||
|
||||
payload.tools = [...filteredTools, OPENAI_WEB_SEARCH_TOOL];
|
||||
raiseMinimalReasoningForOpenAINativeWebSearch(payload);
|
||||
return "injected";
|
||||
}
|
||||
|
||||
export function createOpenAINativeWebSearchWrapper(
|
||||
baseStreamFn: StreamFn | undefined,
|
||||
params: {
|
||||
config?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
nativeWebSearchAllowedByToolPolicy?: boolean;
|
||||
},
|
||||
): StreamFn {
|
||||
const underlying = baseStreamFn ?? streamSimple;
|
||||
return (model, context, options) => {
|
||||
if (!shouldEnableOpenAINativeWebSearch({ config: params.config, model })) {
|
||||
return underlying(model, context, options);
|
||||
}
|
||||
if (params.nativeWebSearchAllowedByToolPolicy === false) {
|
||||
return underlying(model, context, options);
|
||||
}
|
||||
return streamWithPayloadPatch(underlying, model, context, options, (payload) => {
|
||||
patchOpenAINativeWebSearchPayload(payload);
|
||||
});
|
||||
};
|
||||
}
|
||||
91
extensions/openai/openai-chatgpt-auth-identity.test.ts
Normal file
91
extensions/openai/openai-chatgpt-auth-identity.test.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
// Openai tests cover openai chatgpt auth identity plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js";
|
||||
|
||||
function createJwt(payload: Record<string, unknown>): string {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "none", typ: "JWT" })).toString("base64url");
|
||||
const body = Buffer.from(JSON.stringify(payload)).toString("base64url");
|
||||
return `${header}.${body}.signature`;
|
||||
}
|
||||
|
||||
describe("resolveCodexAuthIdentity", () => {
|
||||
it("prefers JWT profile email when present", () => {
|
||||
const identity = resolveCodexAuthIdentity({
|
||||
accessToken: createJwt({
|
||||
"https://api.openai.com/profile": {
|
||||
email: "jwt-user@example.com",
|
||||
},
|
||||
}),
|
||||
email: "credential@example.com",
|
||||
});
|
||||
|
||||
expect(identity).toEqual({
|
||||
email: "jwt-user@example.com",
|
||||
profileName: "jwt-user@example.com",
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts account and plan metadata from the JWT auth claim", () => {
|
||||
const identity = resolveCodexAuthIdentity({
|
||||
accessToken: createJwt({
|
||||
"https://api.openai.com/profile": {
|
||||
email: "jwt-user@example.com",
|
||||
},
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_id: "acct-123",
|
||||
chatgpt_plan_type: "prolite",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(identity).toEqual({
|
||||
accountId: "acct-123",
|
||||
chatgptPlanType: "prolite",
|
||||
email: "jwt-user@example.com",
|
||||
profileName: "jwt-user@example.com",
|
||||
});
|
||||
});
|
||||
|
||||
it("decodes URL-safe base64 JWT payloads", () => {
|
||||
const accessToken = createJwt({
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_id: "w_ébé_1fzcswWN6Pi5zL",
|
||||
},
|
||||
});
|
||||
expect(accessToken.split(".")[1]).toContain("_");
|
||||
|
||||
expect(resolveCodexAuthIdentity({ accessToken })).toEqual({
|
||||
accountId: "w_ébé_1fzcswWN6Pi5zL",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to credential email before synthetic ids", () => {
|
||||
const identity = resolveCodexAuthIdentity({
|
||||
accessToken: createJwt({}),
|
||||
email: "credential@example.com",
|
||||
});
|
||||
|
||||
expect(identity).toEqual({
|
||||
email: "credential@example.com",
|
||||
profileName: "credential@example.com",
|
||||
});
|
||||
});
|
||||
|
||||
it("derives a stable profile id when email is missing", () => {
|
||||
const identity = resolveCodexAuthIdentity({
|
||||
accessToken: createJwt({
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_user_id: "user-123__acct-456",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(identity).toEqual({
|
||||
profileName: `id-${Buffer.from("user-123__acct-456").toString("base64url")}`,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns no metadata when token parsing yields no identity", () => {
|
||||
expect(resolveCodexAuthIdentity({ accessToken: "not-a-jwt-token" })).toStrictEqual({});
|
||||
});
|
||||
});
|
||||
102
extensions/openai/openai-chatgpt-auth-identity.ts
Normal file
102
extensions/openai/openai-chatgpt-auth-identity.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
// Openai plugin module implements openai chatgpt auth identity behavior.
|
||||
import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { trimNonEmptyString } from "./openai-chatgpt-shared.js";
|
||||
|
||||
type CodexJwtPayload = {
|
||||
exp?: unknown;
|
||||
iss?: unknown;
|
||||
sub?: unknown;
|
||||
"https://api.openai.com/profile"?: {
|
||||
email?: unknown;
|
||||
};
|
||||
"https://api.openai.com/auth"?: {
|
||||
chatgpt_account_id?: unknown;
|
||||
chatgpt_account_user_id?: unknown;
|
||||
chatgpt_plan_type?: unknown;
|
||||
chatgpt_user_id?: unknown;
|
||||
user_id?: unknown;
|
||||
};
|
||||
};
|
||||
|
||||
function normalizeFutureEpochSeconds(value: unknown): number | undefined {
|
||||
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
|
||||
return Math.trunc(value);
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return parseStrictPositiveInteger(value);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function decodeCodexJwtPayload(accessToken: string): CodexJwtPayload | null {
|
||||
const parts = accessToken.split(".");
|
||||
if (parts.length !== 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const decoded = Buffer.from(parts[1], "base64url").toString("utf8");
|
||||
const parsed = JSON.parse(decoded);
|
||||
return parsed && typeof parsed === "object" ? (parsed as CodexJwtPayload) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveCodexStableSubject(payload: CodexJwtPayload | null): string | undefined {
|
||||
const auth = payload?.["https://api.openai.com/auth"];
|
||||
const accountUserId = trimNonEmptyString(auth?.chatgpt_account_user_id);
|
||||
if (accountUserId) {
|
||||
return accountUserId;
|
||||
}
|
||||
|
||||
const userId = trimNonEmptyString(auth?.chatgpt_user_id) ?? trimNonEmptyString(auth?.user_id);
|
||||
if (userId) {
|
||||
return userId;
|
||||
}
|
||||
|
||||
const iss = trimNonEmptyString(payload?.iss);
|
||||
const sub = trimNonEmptyString(payload?.sub);
|
||||
if (iss && sub) {
|
||||
return `${iss}|${sub}`;
|
||||
}
|
||||
return sub;
|
||||
}
|
||||
|
||||
export function resolveCodexAccessTokenExpiry(accessToken: string): number | undefined {
|
||||
const payload = decodeCodexJwtPayload(accessToken);
|
||||
const exp = normalizeFutureEpochSeconds(payload?.exp);
|
||||
return exp ? exp * 1000 : undefined;
|
||||
}
|
||||
|
||||
export function resolveCodexAuthIdentity(params: { accessToken: string; email?: string | null }): {
|
||||
accountId?: string;
|
||||
chatgptPlanType?: string;
|
||||
email?: string;
|
||||
profileName?: string;
|
||||
} {
|
||||
const payload = decodeCodexJwtPayload(params.accessToken);
|
||||
const auth = payload?.["https://api.openai.com/auth"];
|
||||
const accountId = trimNonEmptyString(auth?.chatgpt_account_id);
|
||||
const chatgptPlanType = trimNonEmptyString(auth?.chatgpt_plan_type);
|
||||
const email =
|
||||
trimNonEmptyString(payload?.["https://api.openai.com/profile"]?.email) ??
|
||||
trimNonEmptyString(params.email);
|
||||
const metadata = {
|
||||
...(accountId ? { accountId } : {}),
|
||||
...(chatgptPlanType ? { chatgptPlanType } : {}),
|
||||
};
|
||||
if (email) {
|
||||
return { ...metadata, email, profileName: email };
|
||||
}
|
||||
|
||||
const stableSubject = resolveCodexStableSubject(payload);
|
||||
if (!stableSubject) {
|
||||
return metadata;
|
||||
}
|
||||
|
||||
return {
|
||||
...metadata,
|
||||
profileName: `id-${Buffer.from(stableSubject).toString("base64url")}`,
|
||||
};
|
||||
}
|
||||
13
extensions/openai/openai-chatgpt-catalog.ts
Normal file
13
extensions/openai/openai-chatgpt-catalog.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
// Openai plugin module implements openai chatgpt catalog behavior.
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { OPENAI_CODEX_RESPONSES_BASE_URL } from "./base-url.js";
|
||||
|
||||
const OPENAI_CODEX_BASE_URL = OPENAI_CODEX_RESPONSES_BASE_URL;
|
||||
|
||||
export function buildOpenAICodexProvider(): ModelProviderConfig {
|
||||
return {
|
||||
baseUrl: OPENAI_CODEX_BASE_URL,
|
||||
api: "openai-chatgpt-responses",
|
||||
models: [],
|
||||
};
|
||||
}
|
||||
385
extensions/openai/openai-chatgpt-device-code.test.ts
Normal file
385
extensions/openai/openai-chatgpt-device-code.test.ts
Normal file
@@ -0,0 +1,385 @@
|
||||
// Openai tests cover openai chatgpt device code plugin behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveCodexAccessTokenExpiry } from "./openai-chatgpt-auth-identity.js";
|
||||
import { loginOpenAICodexDeviceCode } from "./openai-chatgpt-device-code.js";
|
||||
|
||||
function createJwt(payload: Record<string, unknown>): string {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "none", typ: "JWT" })).toString("base64url");
|
||||
const body = Buffer.from(JSON.stringify(payload)).toString("base64url");
|
||||
return `${header}.${body}.signature`;
|
||||
}
|
||||
|
||||
function createJsonResponse(body: unknown, init?: { status?: number }) {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: init?.status ?? 200,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function cancelTrackedResponse(
|
||||
text: string,
|
||||
init: ResponseInit,
|
||||
): {
|
||||
response: Response;
|
||||
wasCanceled: () => boolean;
|
||||
} {
|
||||
let canceled = false;
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(text));
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
});
|
||||
return {
|
||||
response: new Response(stream, init),
|
||||
wasCanceled: () => canceled,
|
||||
};
|
||||
}
|
||||
|
||||
function fetchCall(fetchMock: ReturnType<typeof vi.fn<typeof fetch>>, index: number) {
|
||||
const call = fetchMock.mock.calls[index];
|
||||
if (!call) {
|
||||
throw new Error(`expected fetch call ${index}`);
|
||||
}
|
||||
return call;
|
||||
}
|
||||
|
||||
describe("loginOpenAICodexDeviceCode", () => {
|
||||
it("requests a device code, polls for authorization, and exchanges OAuth tokens", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.stubEnv("OPENCLAW_VERSION", "2026.3.22");
|
||||
try {
|
||||
const fetchMock = vi
|
||||
.fn<typeof fetch>()
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
device_auth_id: "device-auth-123",
|
||||
user_code: "CODE-12345",
|
||||
interval: "0",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response(null, { status: 404 }))
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
authorization_code: "authorization-code-123",
|
||||
code_challenge: "ignored",
|
||||
code_verifier: "code-verifier-123",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
access_token: createJwt({
|
||||
exp: Math.floor(Date.now() / 1000) + 600,
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_id: "acct_123",
|
||||
},
|
||||
"https://api.openai.com/profile": {
|
||||
email: "codex@example.com",
|
||||
},
|
||||
}),
|
||||
refresh_token: "refresh-token-123",
|
||||
id_token: createJwt({
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_id: "acct_123",
|
||||
},
|
||||
}),
|
||||
expires_in: 600,
|
||||
}),
|
||||
);
|
||||
const onVerification = vi.fn(async () => {});
|
||||
const onProgress = vi.fn();
|
||||
|
||||
const credentialsPromise = loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification,
|
||||
onProgress,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(4_999);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
const credentials = await credentialsPromise;
|
||||
|
||||
const userCodeRequest = fetchCall(fetchMock, 0);
|
||||
expect(userCodeRequest[0]).toBe("https://auth.openai.com/api/accounts/deviceauth/usercode");
|
||||
expect(userCodeRequest[1]?.method).toBe("POST");
|
||||
expect(userCodeRequest[1]?.headers).toEqual({
|
||||
"Content-Type": "application/json",
|
||||
originator: "openclaw",
|
||||
version: "2026.3.22",
|
||||
"User-Agent": "openclaw/2026.3.22",
|
||||
});
|
||||
|
||||
const deviceTokenRequest = fetchCall(fetchMock, 1);
|
||||
expect(deviceTokenRequest[0]).toBe("https://auth.openai.com/api/accounts/deviceauth/token");
|
||||
expect(deviceTokenRequest[1]?.method).toBe("POST");
|
||||
expect(deviceTokenRequest[1]?.headers).toEqual({
|
||||
"Content-Type": "application/json",
|
||||
originator: "openclaw",
|
||||
version: "2026.3.22",
|
||||
"User-Agent": "openclaw/2026.3.22",
|
||||
});
|
||||
|
||||
const oauthTokenRequest = fetchCall(fetchMock, 3);
|
||||
expect(oauthTokenRequest[0]).toBe("https://auth.openai.com/oauth/token");
|
||||
expect(oauthTokenRequest[1]?.method).toBe("POST");
|
||||
expect(oauthTokenRequest[1]?.headers).toEqual({
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
originator: "openclaw",
|
||||
version: "2026.3.22",
|
||||
"User-Agent": "openclaw/2026.3.22",
|
||||
});
|
||||
expect(onVerification).toHaveBeenCalledWith({
|
||||
verificationUrl: "https://auth.openai.com/codex/device",
|
||||
userCode: "CODE-12345",
|
||||
expiresInMs: 900_000,
|
||||
});
|
||||
expect(onProgress).toHaveBeenNthCalledWith(1, "Requesting device code…");
|
||||
expect(onProgress).toHaveBeenNthCalledWith(2, "Waiting for device authorization…");
|
||||
expect(onProgress).toHaveBeenNthCalledWith(3, "Exchanging device code…");
|
||||
expect(typeof credentials.access).toBe("string");
|
||||
expect(credentials.access.length).toBeGreaterThan(0);
|
||||
expect(credentials.refresh).toBe("refresh-token-123");
|
||||
expect(credentials).not.toHaveProperty("accountId");
|
||||
expect(credentials.expires).toBeGreaterThan(Date.now());
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("treats JWT-derived expiry fallback as an absolute timestamp", async () => {
|
||||
const accessToken = createJwt({
|
||||
exp: Math.floor(Date.now() / 1000) + 600,
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_id: "acct_123",
|
||||
},
|
||||
});
|
||||
const expectedExpiry = resolveCodexAccessTokenExpiry(accessToken);
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
device_auth_id: "device-auth-123",
|
||||
user_code: "CODE-12345",
|
||||
interval: "0",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
authorization_code: "authorization-code-123",
|
||||
code_verifier: "code-verifier-123",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
access_token: accessToken,
|
||||
refresh_token: "refresh-token-123",
|
||||
}),
|
||||
);
|
||||
|
||||
const credentials = await loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification: async () => {},
|
||||
});
|
||||
|
||||
if (expectedExpiry === undefined) {
|
||||
throw new Error("expected device-code expiry to be calculated");
|
||||
}
|
||||
expect(credentials.expires).toBe(expectedExpiry);
|
||||
});
|
||||
|
||||
it("accepts token exchange JSON above the diagnostic preview limit", async () => {
|
||||
const accessToken = createJwt({
|
||||
exp: Math.floor(Date.now() / 1000) + 600,
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_id: "acct_123",
|
||||
},
|
||||
});
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
device_auth_id: "device-auth-123",
|
||||
user_code: "CODE-12345",
|
||||
interval: "0",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
authorization_code: "authorization-code-123",
|
||||
code_verifier: "code-verifier-123",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
access_token: accessToken,
|
||||
refresh_token: "refresh-token-123",
|
||||
id_token: "x".repeat(10_000),
|
||||
}),
|
||||
);
|
||||
|
||||
const credentials = await loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification: async () => {},
|
||||
});
|
||||
|
||||
expect(credentials.refresh).toBe("refresh-token-123");
|
||||
});
|
||||
|
||||
it("falls back when device-code intervals and token lifetimes overflow safe milliseconds", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const accessToken = createJwt({
|
||||
exp: Math.floor(Date.now() / 1000) + 600,
|
||||
"https://api.openai.com/auth": {
|
||||
chatgpt_account_id: "acct_123",
|
||||
},
|
||||
});
|
||||
const expectedExpiry = resolveCodexAccessTokenExpiry(accessToken);
|
||||
const fetchMock = vi
|
||||
.fn<typeof fetch>()
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
device_auth_id: "device-auth-123",
|
||||
user_code: "CODE-12345",
|
||||
interval: Number.MAX_SAFE_INTEGER,
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(new Response(null, { status: 404 }))
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
authorization_code: "authorization-code-123",
|
||||
code_verifier: "code-verifier-123",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
access_token: accessToken,
|
||||
refresh_token: "refresh-token-123",
|
||||
expires_in: Number.MAX_SAFE_INTEGER,
|
||||
}),
|
||||
);
|
||||
|
||||
const credentialsPromise = loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification: async () => {},
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(4_999);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
const credentials = await credentialsPromise;
|
||||
|
||||
if (expectedExpiry === undefined) {
|
||||
throw new Error("expected device-code expiry to be calculated");
|
||||
}
|
||||
expect(credentials.expires).toBe(expectedExpiry);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces user-code request failures", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValueOnce(
|
||||
new Response(`down\r\n\u001B[31mnow\u001B[0m`, {
|
||||
status: 503,
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification: async () => {},
|
||||
}),
|
||||
).rejects.toThrow("OpenAI device code request failed: HTTP 503 down now");
|
||||
});
|
||||
|
||||
it("bounds user-code error bodies without using response.text()", async () => {
|
||||
const tracked = cancelTrackedResponse(`${"device code unavailable ".repeat(1024)}tail`, {
|
||||
status: 503,
|
||||
headers: { "Content-Type": "text/plain" },
|
||||
});
|
||||
const textSpy = vi.spyOn(tracked.response, "text").mockRejectedValue(new Error("unbounded"));
|
||||
const fetchMock = vi.fn().mockResolvedValueOnce(tracked.response);
|
||||
|
||||
const error = await loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification: async () => {},
|
||||
}).catch((cause: unknown) => cause);
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as Error).message).toMatch(
|
||||
/OpenAI device code request failed: HTTP 503 device code unavailable/,
|
||||
);
|
||||
expect((error as Error).message).not.toContain("tail");
|
||||
expect(tracked.wasCanceled()).toBe(true);
|
||||
expect(textSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("surfaces device authorization failures with sanitized payload details", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
device_auth_id: "device-auth-123",
|
||||
user_code: "CODE-12345",
|
||||
interval: "0",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse(
|
||||
{
|
||||
error: "authorization_declined\r\n\u001B[31mspoofed\u001B[0m",
|
||||
error_description: "Denied\r\nnext line",
|
||||
},
|
||||
{ status: 401 },
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification: async () => {},
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"OpenAI device authorization failed: authorization_declined spoofed (Denied next line)",
|
||||
);
|
||||
});
|
||||
|
||||
it("strips C1 terminal controls from reflected device-code errors", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse({
|
||||
device_auth_id: "device-auth-123",
|
||||
user_code: "CODE-12345",
|
||||
interval: "0",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createJsonResponse(
|
||||
{
|
||||
error: `authorization_declined${String.fromCharCode(0x9b)}spoofed`,
|
||||
error_description: `Denied${String.fromCharCode(0x9d)}next line`,
|
||||
},
|
||||
{ status: 401 },
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
loginOpenAICodexDeviceCode({
|
||||
fetchFn: fetchMock as typeof fetch,
|
||||
onVerification: async () => {},
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"OpenAI device authorization failed: authorization_declined spoofed (Denied next line)",
|
||||
);
|
||||
});
|
||||
});
|
||||
305
extensions/openai/openai-chatgpt-device-code.ts
Normal file
305
extensions/openai/openai-chatgpt-device-code.ts
Normal file
@@ -0,0 +1,305 @@
|
||||
// Openai plugin module implements openai chatgpt device code behavior.
|
||||
import {
|
||||
positiveSecondsToSafeMilliseconds,
|
||||
resolveExpiresAtMsFromDurationSeconds,
|
||||
} from "openclaw/plugin-sdk/number-runtime";
|
||||
import { readResponseTextLimited } from "openclaw/plugin-sdk/provider-http";
|
||||
import { resolveCodexAccessTokenExpiry } from "./openai-chatgpt-auth-identity.js";
|
||||
import { trimNonEmptyString } from "./openai-chatgpt-shared.js";
|
||||
|
||||
const OPENAI_AUTH_BASE_URL = "https://auth.openai.com";
|
||||
const OPENAI_CODEX_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann";
|
||||
const OPENAI_CODEX_DEVICE_CODE_TIMEOUT_MS = 15 * 60_000;
|
||||
const OPENAI_CODEX_DEVICE_CODE_DEFAULT_INTERVAL_MS = 5_000;
|
||||
const OPENAI_CODEX_DEVICE_CODE_MIN_INTERVAL_MS = 1_000;
|
||||
const OPENAI_CODEX_DEVICE_CALLBACK_URL = `${OPENAI_AUTH_BASE_URL}/deviceauth/callback`;
|
||||
const OPENAI_CODEX_DEVICE_ERROR_BODY_LIMIT_BYTES = 8 * 1024;
|
||||
const OPENAI_CODEX_DEVICE_JSON_BODY_LIMIT_BYTES = 256 * 1024;
|
||||
|
||||
function resolveOpenAICodexDeviceCodeHeaders(contentType: string): Record<string, string> {
|
||||
const version = process.env.OPENCLAW_VERSION?.trim();
|
||||
return {
|
||||
"Content-Type": contentType,
|
||||
originator: "openclaw",
|
||||
...(version ? { version } : {}),
|
||||
"User-Agent": version ? `openclaw/${version}` : "openclaw",
|
||||
};
|
||||
}
|
||||
|
||||
type OpenAICodexDeviceCodePrompt = {
|
||||
verificationUrl: string;
|
||||
userCode: string;
|
||||
expiresInMs: number;
|
||||
};
|
||||
|
||||
type OpenAICodexDeviceCodeCredentials = {
|
||||
access: string;
|
||||
refresh: string;
|
||||
expires: number;
|
||||
};
|
||||
|
||||
type DeviceCodeUserCodePayload = {
|
||||
device_auth_id?: unknown;
|
||||
user_code?: unknown;
|
||||
usercode?: unknown;
|
||||
interval?: unknown;
|
||||
};
|
||||
|
||||
type DeviceCodeTokenPayload = {
|
||||
authorization_code?: unknown;
|
||||
code_challenge?: unknown;
|
||||
code_verifier?: unknown;
|
||||
};
|
||||
|
||||
type OAuthTokenPayload = {
|
||||
access_token?: unknown;
|
||||
refresh_token?: unknown;
|
||||
expires_in?: unknown;
|
||||
};
|
||||
|
||||
type RequestedDeviceCode = {
|
||||
deviceAuthId: string;
|
||||
userCode: string;
|
||||
verificationUrl: string;
|
||||
intervalMs: number;
|
||||
};
|
||||
|
||||
type DeviceCodeAuthorizationCode = {
|
||||
authorizationCode: string;
|
||||
codeVerifier: string;
|
||||
};
|
||||
|
||||
function parseJsonObject(text: string): Record<string, unknown> | null {
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeDeviceCodeErrorText(value: string): string {
|
||||
const esc = String.fromCharCode(0x1b);
|
||||
const ansiCsiRegex = new RegExp(`${esc}\\[[\\u0020-\\u003f]*[\\u0040-\\u007e]`, "g");
|
||||
const osc8Regex = new RegExp(`${esc}\\]8;;.*?${esc}\\\\|${esc}\\]8;;${esc}\\\\`, "g");
|
||||
const c0Start = String.fromCharCode(0x00);
|
||||
const c0End = String.fromCharCode(0x1f);
|
||||
const del = String.fromCharCode(0x7f);
|
||||
const c1Start = String.fromCharCode(0x80);
|
||||
const c1End = String.fromCharCode(0x9f);
|
||||
const controlCharsRegex = new RegExp(`[${c0Start}-${c0End}${del}${c1Start}-${c1End}]`, "g");
|
||||
return value
|
||||
.replace(osc8Regex, "")
|
||||
.replace(ansiCsiRegex, "")
|
||||
.replace(controlCharsRegex, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function resolveNextDeviceCodePollDelayMs(intervalMs: number, deadlineMs: number): number {
|
||||
const remainingMs = Math.max(0, deadlineMs - Date.now());
|
||||
return Math.min(Math.max(intervalMs, OPENAI_CODEX_DEVICE_CODE_MIN_INTERVAL_MS), remainingMs);
|
||||
}
|
||||
|
||||
function formatDeviceCodeError(params: {
|
||||
prefix: string;
|
||||
status: number;
|
||||
bodyText: string;
|
||||
}): string {
|
||||
const body = parseJsonObject(params.bodyText);
|
||||
const error = trimNonEmptyString(body?.error);
|
||||
const description = trimNonEmptyString(body?.error_description);
|
||||
const safeError = error ? sanitizeDeviceCodeErrorText(error) : undefined;
|
||||
const safeDescription = description ? sanitizeDeviceCodeErrorText(description) : undefined;
|
||||
if (safeError && safeDescription) {
|
||||
return `${params.prefix}: ${safeError} (${safeDescription})`;
|
||||
}
|
||||
if (safeError) {
|
||||
return `${params.prefix}: ${safeError}`;
|
||||
}
|
||||
const bodyText = sanitizeDeviceCodeErrorText(params.bodyText);
|
||||
return bodyText
|
||||
? `${params.prefix}: HTTP ${params.status} ${bodyText}`
|
||||
: `${params.prefix}: HTTP ${params.status}`;
|
||||
}
|
||||
|
||||
async function readOpenAICodexDeviceBody(response: Response): Promise<string> {
|
||||
return await readResponseTextLimited(
|
||||
response,
|
||||
response.ok
|
||||
? OPENAI_CODEX_DEVICE_JSON_BODY_LIMIT_BYTES
|
||||
: OPENAI_CODEX_DEVICE_ERROR_BODY_LIMIT_BYTES,
|
||||
);
|
||||
}
|
||||
|
||||
async function requestOpenAICodexDeviceCode(fetchFn: typeof fetch): Promise<RequestedDeviceCode> {
|
||||
const response = await fetchFn(`${OPENAI_AUTH_BASE_URL}/api/accounts/deviceauth/usercode`, {
|
||||
method: "POST",
|
||||
headers: resolveOpenAICodexDeviceCodeHeaders("application/json"),
|
||||
body: JSON.stringify({
|
||||
client_id: OPENAI_CODEX_CLIENT_ID,
|
||||
}),
|
||||
});
|
||||
|
||||
const bodyText = await readOpenAICodexDeviceBody(response);
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
throw new Error(
|
||||
"OpenAI Codex device code login is not enabled for this server. Use ChatGPT OAuth instead.",
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
formatDeviceCodeError({
|
||||
prefix: "OpenAI device code request failed",
|
||||
status: response.status,
|
||||
bodyText,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const body = parseJsonObject(bodyText) as DeviceCodeUserCodePayload | null;
|
||||
const deviceAuthId = trimNonEmptyString(body?.device_auth_id);
|
||||
const userCode = trimNonEmptyString(body?.user_code) ?? trimNonEmptyString(body?.usercode);
|
||||
if (!deviceAuthId || !userCode) {
|
||||
throw new Error("OpenAI device code response was missing the device code or user code.");
|
||||
}
|
||||
|
||||
return {
|
||||
deviceAuthId,
|
||||
userCode,
|
||||
verificationUrl: `${OPENAI_AUTH_BASE_URL}/codex/device`,
|
||||
intervalMs:
|
||||
positiveSecondsToSafeMilliseconds(body?.interval) ??
|
||||
OPENAI_CODEX_DEVICE_CODE_DEFAULT_INTERVAL_MS,
|
||||
};
|
||||
}
|
||||
|
||||
async function pollOpenAICodexDeviceCode(params: {
|
||||
fetchFn: typeof fetch;
|
||||
deviceAuthId: string;
|
||||
userCode: string;
|
||||
intervalMs: number;
|
||||
}): Promise<DeviceCodeAuthorizationCode> {
|
||||
const deadline = Date.now() + OPENAI_CODEX_DEVICE_CODE_TIMEOUT_MS;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const response = await params.fetchFn(`${OPENAI_AUTH_BASE_URL}/api/accounts/deviceauth/token`, {
|
||||
method: "POST",
|
||||
headers: resolveOpenAICodexDeviceCodeHeaders("application/json"),
|
||||
body: JSON.stringify({
|
||||
device_auth_id: params.deviceAuthId,
|
||||
user_code: params.userCode,
|
||||
}),
|
||||
});
|
||||
|
||||
const bodyText = await readOpenAICodexDeviceBody(response);
|
||||
if (response.ok) {
|
||||
const body = parseJsonObject(bodyText) as DeviceCodeTokenPayload | null;
|
||||
const authorizationCode = trimNonEmptyString(body?.authorization_code);
|
||||
const codeVerifier = trimNonEmptyString(body?.code_verifier);
|
||||
if (!authorizationCode || !codeVerifier) {
|
||||
throw new Error("OpenAI device authorization response was missing the exchange code.");
|
||||
}
|
||||
return {
|
||||
authorizationCode,
|
||||
codeVerifier,
|
||||
};
|
||||
}
|
||||
|
||||
if (response.status === 403 || response.status === 404) {
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, resolveNextDeviceCodePollDelayMs(params.intervalMs, deadline));
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
formatDeviceCodeError({
|
||||
prefix: "OpenAI device authorization failed",
|
||||
status: response.status,
|
||||
bodyText,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error("OpenAI device authorization timed out after 15 minutes.");
|
||||
}
|
||||
|
||||
async function exchangeOpenAICodexDeviceCode(params: {
|
||||
fetchFn: typeof fetch;
|
||||
authorizationCode: string;
|
||||
codeVerifier: string;
|
||||
}): Promise<OpenAICodexDeviceCodeCredentials> {
|
||||
const response = await params.fetchFn(`${OPENAI_AUTH_BASE_URL}/oauth/token`, {
|
||||
method: "POST",
|
||||
headers: resolveOpenAICodexDeviceCodeHeaders("application/x-www-form-urlencoded"),
|
||||
body: new URLSearchParams({
|
||||
grant_type: "authorization_code",
|
||||
code: params.authorizationCode,
|
||||
redirect_uri: OPENAI_CODEX_DEVICE_CALLBACK_URL,
|
||||
client_id: OPENAI_CODEX_CLIENT_ID,
|
||||
code_verifier: params.codeVerifier,
|
||||
}),
|
||||
});
|
||||
|
||||
const bodyText = await readOpenAICodexDeviceBody(response);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
formatDeviceCodeError({
|
||||
prefix: "OpenAI device token exchange failed",
|
||||
status: response.status,
|
||||
bodyText,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const body = parseJsonObject(bodyText) as OAuthTokenPayload | null;
|
||||
const access = trimNonEmptyString(body?.access_token);
|
||||
const refresh = trimNonEmptyString(body?.refresh_token);
|
||||
if (!access || !refresh) {
|
||||
throw new Error("OpenAI token exchange succeeded but did not return OAuth tokens.");
|
||||
}
|
||||
|
||||
const expires =
|
||||
resolveExpiresAtMsFromDurationSeconds(body?.expires_in) ??
|
||||
resolveCodexAccessTokenExpiry(access) ??
|
||||
Date.now();
|
||||
|
||||
return {
|
||||
access,
|
||||
refresh,
|
||||
expires,
|
||||
};
|
||||
}
|
||||
|
||||
export async function loginOpenAICodexDeviceCode(params: {
|
||||
fetchFn?: typeof fetch;
|
||||
onVerification: (prompt: OpenAICodexDeviceCodePrompt) => Promise<void> | void;
|
||||
onProgress?: (message: string) => void;
|
||||
}): Promise<OpenAICodexDeviceCodeCredentials> {
|
||||
const fetchFn = params.fetchFn ?? fetch;
|
||||
|
||||
params.onProgress?.("Requesting device code…");
|
||||
const deviceCode = await requestOpenAICodexDeviceCode(fetchFn);
|
||||
|
||||
await params.onVerification({
|
||||
verificationUrl: deviceCode.verificationUrl,
|
||||
userCode: deviceCode.userCode,
|
||||
expiresInMs: OPENAI_CODEX_DEVICE_CODE_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
params.onProgress?.("Waiting for device authorization…");
|
||||
const authorization = await pollOpenAICodexDeviceCode({
|
||||
fetchFn,
|
||||
deviceAuthId: deviceCode.deviceAuthId,
|
||||
userCode: deviceCode.userCode,
|
||||
intervalMs: deviceCode.intervalMs,
|
||||
});
|
||||
|
||||
params.onProgress?.("Exchanging device code…");
|
||||
return await exchangeOpenAICodexDeviceCode({
|
||||
fetchFn,
|
||||
authorizationCode: authorization.authorizationCode,
|
||||
codeVerifier: authorization.codeVerifier,
|
||||
});
|
||||
}
|
||||
7
extensions/openai/openai-chatgpt-oauth-abort.runtime.ts
Normal file
7
extensions/openai/openai-chatgpt-oauth-abort.runtime.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
// Openai plugin module implements openai chatgpt oauth abort behavior.
|
||||
export {
|
||||
buildOAuthRequestSignal,
|
||||
createOAuthLoginCancelledError,
|
||||
throwIfOAuthLoginAborted,
|
||||
withOAuthLoginAbort,
|
||||
} from "openclaw/plugin-sdk/provider-oauth-runtime";
|
||||
177
extensions/openai/openai-chatgpt-oauth-flow.runtime.test.ts
Normal file
177
extensions/openai/openai-chatgpt-oauth-flow.runtime.test.ts
Normal file
@@ -0,0 +1,177 @@
|
||||
// Openai tests cover openai chatgpt oauth flow plugin behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const ssrfMocks = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuard: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
|
||||
fetchWithSsrFGuard: ssrfMocks.fetchWithSsrFGuard,
|
||||
}));
|
||||
|
||||
import { openaiCodexOAuthProvider, testing } from "./openai-chatgpt-oauth-flow.runtime.js";
|
||||
|
||||
function timeoutError(): Error {
|
||||
return new DOMException("timed out", "TimeoutError");
|
||||
}
|
||||
|
||||
function mockTokenResponse(body: unknown, status = 200): void {
|
||||
mockTokenResponseText(JSON.stringify(body), status);
|
||||
}
|
||||
|
||||
function mockTokenResponseText(body: string, status = 200): void {
|
||||
ssrfMocks.fetchWithSsrFGuard.mockResolvedValueOnce({
|
||||
response: new Response(body, {
|
||||
status,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => undefined),
|
||||
});
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
ssrfMocks.fetchWithSsrFGuard.mockReset();
|
||||
});
|
||||
|
||||
describe("OpenAI Codex OAuth flow", () => {
|
||||
it("cancels provider login before opening the OAuth flow", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
openaiCodexOAuthProvider.login({
|
||||
onAuth: vi.fn(),
|
||||
onPrompt: vi.fn(async () => "unused-code"),
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow("Login cancelled");
|
||||
});
|
||||
|
||||
it("does not open the OAuth flow after cancellation during setup", async () => {
|
||||
const controller = new AbortController();
|
||||
const onAuth = vi.fn();
|
||||
const loginPromise = openaiCodexOAuthProvider.login({
|
||||
onAuth,
|
||||
onPrompt: vi.fn(async () => "unused-code"),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
controller.abort();
|
||||
|
||||
await expect(loginPromise).rejects.toThrow("Login cancelled");
|
||||
expect(onAuth).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("waits for Node OAuth runtime before creating an authorization flow", async () => {
|
||||
const flow = await testing.createAuthorizationFlow("openclaw-test");
|
||||
const url = new URL(flow.url);
|
||||
|
||||
expect(flow.state).toMatch(/^[a-f0-9]{32}$/u);
|
||||
expect(url.searchParams.get("state")).toBe(flow.state);
|
||||
expect(url.searchParams.get("originator")).toBe("openclaw-test");
|
||||
const redirectUri = url.searchParams.get("redirect_uri");
|
||||
expect(redirectUri).toBeTruthy();
|
||||
expect(flow.redirectUri).toBe(redirectUri);
|
||||
expect(testing.callbackHost).toBe(new URL(redirectUri ?? "").hostname);
|
||||
});
|
||||
|
||||
it("builds callback redirect URIs from the configured loopback host", () => {
|
||||
expect(testing.resolveRedirectUri("127.0.0.1")).toBe("http://127.0.0.1:1455/auth/callback");
|
||||
});
|
||||
|
||||
it("rejects non-loopback callback bind hosts", () => {
|
||||
expect(() => testing.resolveCallbackHost({ OPENCLAW_OAUTH_CALLBACK_HOST: "0.0.0.0" })).toThrow(
|
||||
"callback host must be localhost, 127.0.0.1, or ::1",
|
||||
);
|
||||
});
|
||||
|
||||
it("times out token exchange requests", async () => {
|
||||
ssrfMocks.fetchWithSsrFGuard.mockRejectedValueOnce(timeoutError());
|
||||
|
||||
const result = await testing.exchangeAuthorizationCode(
|
||||
"code",
|
||||
"verifier",
|
||||
testing.resolveRedirectUri("localhost"),
|
||||
{ timeoutMs: 5 },
|
||||
);
|
||||
|
||||
expect(ssrfMocks.fetchWithSsrFGuard).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
auditContext: "openai-chatgpt-oauth-token",
|
||||
timeoutMs: 5,
|
||||
}),
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
type: "failed",
|
||||
message: "OpenAI Codex token exchange timed out after 5ms",
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels token exchange requests with the caller signal", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
const result = await testing.exchangeAuthorizationCode(
|
||||
"code",
|
||||
"verifier",
|
||||
testing.resolveRedirectUri("localhost"),
|
||||
{ signal: controller.signal, timeoutMs: 5 },
|
||||
);
|
||||
|
||||
expect(ssrfMocks.fetchWithSsrFGuard).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
type: "failed",
|
||||
message: "Login cancelled",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects unsafe token exchange lifetimes", async () => {
|
||||
mockTokenResponseText(
|
||||
'{"access_token":"access-token","refresh_token":"refresh-token","expires_in":1e309}',
|
||||
);
|
||||
|
||||
const result = await testing.exchangeAuthorizationCode(
|
||||
"code",
|
||||
"verifier",
|
||||
testing.resolveRedirectUri("localhost"),
|
||||
{ timeoutMs: 5 },
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
type: "failed",
|
||||
message: "OpenAI Codex token exchange response missing fields: expires_in",
|
||||
});
|
||||
});
|
||||
|
||||
it("times out token refresh requests", async () => {
|
||||
ssrfMocks.fetchWithSsrFGuard.mockRejectedValueOnce(timeoutError());
|
||||
|
||||
const result = await testing.refreshAccessToken("old-refresh-token", { timeoutMs: 5 });
|
||||
|
||||
expect(ssrfMocks.fetchWithSsrFGuard).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
auditContext: "openai-chatgpt-oauth-token",
|
||||
timeoutMs: 5,
|
||||
}),
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
type: "failed",
|
||||
message: "OpenAI Codex token refresh timed out after 5ms",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects non-positive token refresh lifetimes", async () => {
|
||||
mockTokenResponse({
|
||||
access_token: "access-token",
|
||||
refresh_token: "refresh-token",
|
||||
expires_in: 0,
|
||||
});
|
||||
|
||||
const result = await testing.refreshAccessToken("old-refresh-token", { timeoutMs: 5 });
|
||||
|
||||
expect(result).toEqual({
|
||||
type: "failed",
|
||||
message: "OpenAI Codex token refresh response missing fields: expires_in",
|
||||
});
|
||||
});
|
||||
});
|
||||
621
extensions/openai/openai-chatgpt-oauth-flow.runtime.ts
Normal file
621
extensions/openai/openai-chatgpt-oauth-flow.runtime.ts
Normal file
@@ -0,0 +1,621 @@
|
||||
/**
|
||||
* OpenAI Codex (ChatGPT OAuth) flow
|
||||
*
|
||||
* NOTE: This module uses Node.js crypto and http for the OAuth callback.
|
||||
* It is only intended for CLI use, not browser environments.
|
||||
*/
|
||||
|
||||
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
||||
import {
|
||||
parseOAuthAuthorizationInput,
|
||||
resolveOAuthTokenExpiresAt,
|
||||
resolveOAuthTokenLifetimeMs,
|
||||
} from "openclaw/plugin-sdk/provider-oauth-runtime";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js";
|
||||
import {
|
||||
createOAuthLoginCancelledError,
|
||||
throwIfOAuthLoginAborted,
|
||||
withOAuthLoginAbort,
|
||||
} from "./openai-chatgpt-oauth-abort.runtime.js";
|
||||
import { oauthErrorHtml, oauthSuccessHtml } from "./openai-chatgpt-oauth-page.runtime.js";
|
||||
import type {
|
||||
OAuthCredentials,
|
||||
OAuthLoginCallbacks,
|
||||
OAuthPrompt,
|
||||
OAuthProviderInterface,
|
||||
} from "./openai-chatgpt-oauth-types.runtime.js";
|
||||
import { generatePKCE } from "./openai-chatgpt-pkce.runtime.js";
|
||||
|
||||
const CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann";
|
||||
const AUTHORIZE_URL = "https://auth.openai.com/oauth/authorize";
|
||||
const TOKEN_URL = "https://auth.openai.com/oauth/token";
|
||||
const CALLBACK_PORT = 1455;
|
||||
const CALLBACK_PATH = "/auth/callback";
|
||||
const DEFAULT_CALLBACK_HOST = "localhost";
|
||||
const LOOPBACK_CALLBACK_HOSTS = new Set(["localhost", "127.0.0.1", "::1"]);
|
||||
const CALLBACK_HOST = resolveCallbackHost();
|
||||
const REDIRECT_URI = resolveRedirectUri(CALLBACK_HOST);
|
||||
const MANUAL_PROMPT_FALLBACK_MS = 15_000;
|
||||
const TOKEN_REQUEST_TIMEOUT_MS = 30_000;
|
||||
const SCOPE = "openid profile email offline_access";
|
||||
|
||||
type TokenSuccess = { type: "success"; access: string; refresh: string; expires: number };
|
||||
type TokenFailure = { type: "failed"; message: string; status?: number };
|
||||
type TokenResult = TokenSuccess | TokenFailure;
|
||||
type TokenResponseJson = {
|
||||
access_token?: string;
|
||||
refresh_token?: string;
|
||||
expires_in?: number;
|
||||
};
|
||||
type NodeOAuthRuntime = {
|
||||
randomBytes: typeof import("node:crypto").randomBytes;
|
||||
http: typeof import("node:http");
|
||||
};
|
||||
type TokenRequestOptions = {
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
const loadNodeOAuthModules = createLazyRuntimeModule(() =>
|
||||
Promise.all([import("node:crypto"), import("node:http")]).then(([cryptoModule, httpModule]) => ({
|
||||
randomBytes: cryptoModule.randomBytes,
|
||||
http: httpModule,
|
||||
})),
|
||||
);
|
||||
|
||||
function loadNodeOAuthRuntime(): Promise<NodeOAuthRuntime> {
|
||||
if (typeof process === "undefined" || (!process.versions?.node && !process.versions?.bun)) {
|
||||
return Promise.reject(
|
||||
new Error("OpenAI Codex OAuth is only available in Node.js environments"),
|
||||
);
|
||||
}
|
||||
return loadNodeOAuthModules();
|
||||
}
|
||||
|
||||
function resolveCallbackHost(env: NodeJS.ProcessEnv = process.env): string {
|
||||
const host = env.OPENCLAW_OAUTH_CALLBACK_HOST?.trim() || DEFAULT_CALLBACK_HOST;
|
||||
if (!LOOPBACK_CALLBACK_HOSTS.has(host)) {
|
||||
throw new Error("OpenAI Codex OAuth callback host must be localhost, 127.0.0.1, or ::1");
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
function resolveRedirectUri(host: string = CALLBACK_HOST): string {
|
||||
const hostForUrl = host === "::1" ? "[::1]" : host;
|
||||
const url = new URL(`http://${hostForUrl}:${CALLBACK_PORT}`);
|
||||
url.pathname = CALLBACK_PATH;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function createState(randomBytes: typeof import("node:crypto").randomBytes): string {
|
||||
return randomBytes(16).toString("hex");
|
||||
}
|
||||
|
||||
function waitForManualPromptFallback(signal?: AbortSignal): Promise<null> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (signal?.aborted) {
|
||||
reject(createOAuthLoginCancelledError());
|
||||
return;
|
||||
}
|
||||
|
||||
const cleanup = () => {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
};
|
||||
const abort = () => {
|
||||
clearTimeout(timeout);
|
||||
cleanup();
|
||||
reject(createOAuthLoginCancelledError());
|
||||
};
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve(null);
|
||||
}, MANUAL_PROMPT_FALLBACK_MS);
|
||||
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
timeout.unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
async function promptForAuthorizationCode(
|
||||
onPrompt: (prompt: OAuthPrompt) => Promise<string>,
|
||||
state: string,
|
||||
): Promise<string | undefined> {
|
||||
const input = await onPrompt({
|
||||
message: "Paste the authorization code (or full redirect URL):",
|
||||
});
|
||||
const parsed = parseOAuthAuthorizationInput(input);
|
||||
if (parsed.state && parsed.state !== state) {
|
||||
throw new Error("State mismatch");
|
||||
}
|
||||
return parsed.code;
|
||||
}
|
||||
|
||||
function formatMissingTokenResponseFields(json: TokenResponseJson): string {
|
||||
const missing: string[] = [];
|
||||
if (!json.access_token) {
|
||||
missing.push("access_token");
|
||||
}
|
||||
if (!json.refresh_token) {
|
||||
missing.push("refresh_token");
|
||||
}
|
||||
if (resolveOAuthTokenLifetimeMs(json.expires_in) === undefined) {
|
||||
missing.push("expires_in");
|
||||
}
|
||||
return missing.join(", ");
|
||||
}
|
||||
|
||||
function formatTokenRequestError(
|
||||
operation: "exchange" | "refresh",
|
||||
error: unknown,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
): string {
|
||||
if (signal?.aborted) {
|
||||
return "Login cancelled";
|
||||
}
|
||||
if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")) {
|
||||
return `OpenAI Codex token ${operation} timed out after ${timeoutMs}ms`;
|
||||
}
|
||||
return `OpenAI Codex token ${operation} error: ${error instanceof Error ? error.message : String(error)}`;
|
||||
}
|
||||
|
||||
async function postTokenForm(
|
||||
body: URLSearchParams,
|
||||
options: TokenRequestOptions = {},
|
||||
): Promise<Response> {
|
||||
const timeoutMs = options.timeoutMs ?? TOKEN_REQUEST_TIMEOUT_MS;
|
||||
throwIfOAuthLoginAborted(options.signal);
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: TOKEN_URL,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
body,
|
||||
},
|
||||
timeoutMs,
|
||||
signal: options.signal,
|
||||
auditContext: "openai-chatgpt-oauth-token",
|
||||
});
|
||||
try {
|
||||
const responseBody = await response.arrayBuffer();
|
||||
return new Response(responseBody, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
async function exchangeAuthorizationCode(
|
||||
code: string,
|
||||
verifier: string,
|
||||
redirectUri: string = REDIRECT_URI,
|
||||
options: TokenRequestOptions = {},
|
||||
): Promise<TokenResult> {
|
||||
const timeoutMs = options.timeoutMs ?? TOKEN_REQUEST_TIMEOUT_MS;
|
||||
let response: Response;
|
||||
try {
|
||||
response = await postTokenForm(
|
||||
new URLSearchParams({
|
||||
grant_type: "authorization_code",
|
||||
client_id: CLIENT_ID,
|
||||
code,
|
||||
code_verifier: verifier,
|
||||
redirect_uri: redirectUri,
|
||||
}),
|
||||
{ signal: options.signal, timeoutMs },
|
||||
);
|
||||
} catch (error) {
|
||||
return {
|
||||
type: "failed",
|
||||
message: formatTokenRequestError("exchange", error, timeoutMs, options.signal),
|
||||
};
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => "");
|
||||
return {
|
||||
type: "failed",
|
||||
status: response.status,
|
||||
message: `OpenAI Codex token exchange failed (${response.status}): ${text || response.statusText}`,
|
||||
};
|
||||
}
|
||||
|
||||
const json = (await response.json()) as TokenResponseJson;
|
||||
|
||||
const expires = resolveOAuthTokenExpiresAt(json.expires_in);
|
||||
if (!json.access_token || !json.refresh_token || expires === undefined) {
|
||||
return {
|
||||
type: "failed",
|
||||
message: `OpenAI Codex token exchange response missing fields: ${formatMissingTokenResponseFields(json)}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: "success",
|
||||
access: json.access_token,
|
||||
refresh: json.refresh_token,
|
||||
expires,
|
||||
};
|
||||
}
|
||||
|
||||
async function refreshAccessToken(
|
||||
refreshToken: string,
|
||||
options: TokenRequestOptions = {},
|
||||
): Promise<TokenResult> {
|
||||
try {
|
||||
const timeoutMs = options.timeoutMs ?? TOKEN_REQUEST_TIMEOUT_MS;
|
||||
const response = await postTokenForm(
|
||||
new URLSearchParams({
|
||||
grant_type: "refresh_token",
|
||||
refresh_token: refreshToken,
|
||||
client_id: CLIENT_ID,
|
||||
}),
|
||||
{ signal: options.signal, timeoutMs },
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => "");
|
||||
return {
|
||||
type: "failed",
|
||||
status: response.status,
|
||||
message: `OpenAI Codex token refresh failed (${response.status}): ${text || response.statusText}`,
|
||||
};
|
||||
}
|
||||
|
||||
const json = (await response.json()) as TokenResponseJson;
|
||||
|
||||
const expires = resolveOAuthTokenExpiresAt(json.expires_in);
|
||||
if (!json.access_token || !json.refresh_token || expires === undefined) {
|
||||
return {
|
||||
type: "failed",
|
||||
message: `OpenAI Codex token refresh response missing fields: ${formatMissingTokenResponseFields(json)}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: "success",
|
||||
access: json.access_token,
|
||||
refresh: json.refresh_token,
|
||||
expires,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
type: "failed",
|
||||
message: formatTokenRequestError(
|
||||
"refresh",
|
||||
error,
|
||||
options.timeoutMs ?? TOKEN_REQUEST_TIMEOUT_MS,
|
||||
options.signal,
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function createAuthorizationFlow(
|
||||
originator = "openclaw",
|
||||
): Promise<{ verifier: string; redirectUri: string; state: string; url: string }> {
|
||||
const [{ verifier, challenge }, runtime] = await Promise.all([
|
||||
generatePKCE(),
|
||||
loadNodeOAuthRuntime(),
|
||||
]);
|
||||
const state = createState(runtime.randomBytes);
|
||||
|
||||
const url = new URL(AUTHORIZE_URL);
|
||||
url.searchParams.set("response_type", "code");
|
||||
url.searchParams.set("client_id", CLIENT_ID);
|
||||
const redirectUri = REDIRECT_URI;
|
||||
url.searchParams.set("redirect_uri", redirectUri);
|
||||
url.searchParams.set("scope", SCOPE);
|
||||
url.searchParams.set("code_challenge", challenge);
|
||||
url.searchParams.set("code_challenge_method", "S256");
|
||||
url.searchParams.set("state", state);
|
||||
url.searchParams.set("id_token_add_organizations", "true");
|
||||
url.searchParams.set("codex_cli_simplified_flow", "true");
|
||||
url.searchParams.set("originator", originator);
|
||||
|
||||
return { verifier, redirectUri, state, url: url.toString() };
|
||||
}
|
||||
|
||||
type OAuthServerInfo = {
|
||||
close: () => void;
|
||||
cancelWait: () => void;
|
||||
waitForCode: () => Promise<{ code: string } | null>;
|
||||
};
|
||||
|
||||
async function startLocalOAuthServer(state: string): Promise<OAuthServerInfo> {
|
||||
const { http } = await loadNodeOAuthRuntime();
|
||||
let settleWait: ((value: { code: string } | null) => void) | undefined;
|
||||
const waitForCodePromise = new Promise<{ code: string } | null>((resolve) => {
|
||||
let settled = false;
|
||||
settleWait = (value) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
resolve(value);
|
||||
};
|
||||
});
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
try {
|
||||
const url = new URL(req.url || "", "http://localhost");
|
||||
if (url.pathname !== "/auth/callback") {
|
||||
res.statusCode = 404;
|
||||
res.setHeader("Content-Type", "text/html; charset=utf-8");
|
||||
res.end(oauthErrorHtml("Callback route not found."));
|
||||
return;
|
||||
}
|
||||
if (url.searchParams.get("state") !== state) {
|
||||
res.statusCode = 400;
|
||||
res.setHeader("Content-Type", "text/html; charset=utf-8");
|
||||
res.end(oauthErrorHtml("State mismatch."));
|
||||
return;
|
||||
}
|
||||
const code = url.searchParams.get("code");
|
||||
if (!code) {
|
||||
res.statusCode = 400;
|
||||
res.setHeader("Content-Type", "text/html; charset=utf-8");
|
||||
res.end(oauthErrorHtml("Missing authorization code."));
|
||||
return;
|
||||
}
|
||||
res.statusCode = 200;
|
||||
res.setHeader("Content-Type", "text/html; charset=utf-8");
|
||||
res.end(oauthSuccessHtml("OpenAI authentication completed. You can close this window."));
|
||||
settleWait?.({ code });
|
||||
} catch {
|
||||
res.statusCode = 500;
|
||||
res.setHeader("Content-Type", "text/html; charset=utf-8");
|
||||
res.end(oauthErrorHtml("Internal error while processing OAuth callback."));
|
||||
}
|
||||
});
|
||||
|
||||
return new Promise((resolve) => {
|
||||
server
|
||||
.listen(CALLBACK_PORT, CALLBACK_HOST, () => {
|
||||
resolve({
|
||||
close: () => server.close(),
|
||||
cancelWait: () => {
|
||||
settleWait?.(null);
|
||||
},
|
||||
waitForCode: () => waitForCodePromise,
|
||||
});
|
||||
})
|
||||
.on("error", () => {
|
||||
settleWait?.(null);
|
||||
resolve({
|
||||
close: () => {
|
||||
try {
|
||||
server.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
},
|
||||
cancelWait: () => {},
|
||||
waitForCode: async () => null,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function getAccountId(accessToken: string): string | null {
|
||||
const accountId = resolveCodexAuthIdentity({ accessToken }).accountId;
|
||||
return typeof accountId === "string" && accountId.length > 0 ? accountId : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Login with OpenAI Codex OAuth
|
||||
*
|
||||
* @param options.onAuth - Called with URL and instructions when auth starts
|
||||
* @param options.onPrompt - Called to prompt user for manual code paste (fallback if no onManualCodeInput)
|
||||
* @param options.onProgress - Optional progress messages
|
||||
* @param options.onManualCodeInput - Optional promise that resolves with user-pasted code.
|
||||
* Races with browser callback - whichever completes first wins.
|
||||
* Useful for showing paste input immediately alongside browser flow.
|
||||
* @param options.originator - OAuth originator parameter (defaults to "openclaw")
|
||||
*/
|
||||
export async function loginOpenAICodex(options: {
|
||||
onAuth: (info: { url: string; instructions?: string }) => void;
|
||||
onPrompt: (prompt: OAuthPrompt) => Promise<string>;
|
||||
onProgress?: (message: string) => void;
|
||||
onManualCodeInput?: () => Promise<string>;
|
||||
originator?: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<OAuthCredentials> {
|
||||
throwIfOAuthLoginAborted(options.signal);
|
||||
const { verifier, redirectUri, state, url } = await createAuthorizationFlow(options.originator);
|
||||
const server = await startLocalOAuthServer(state);
|
||||
|
||||
let code: string | undefined;
|
||||
try {
|
||||
throwIfOAuthLoginAborted(options.signal);
|
||||
options.onAuth({
|
||||
url,
|
||||
instructions: "A browser window should open. Complete login to finish.",
|
||||
});
|
||||
throwIfOAuthLoginAborted(options.signal);
|
||||
|
||||
if (options.onManualCodeInput) {
|
||||
// Race between browser callback and manual input
|
||||
let manualCode: string | undefined;
|
||||
let manualError: Error | undefined;
|
||||
const manualPromise = options
|
||||
.onManualCodeInput()
|
||||
.then((input) => {
|
||||
manualCode = input;
|
||||
server.cancelWait();
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
manualError = err instanceof Error ? err : new Error(String(err));
|
||||
server.cancelWait();
|
||||
});
|
||||
|
||||
const result = await withOAuthLoginAbort(
|
||||
server.waitForCode(),
|
||||
options.signal,
|
||||
server.cancelWait,
|
||||
);
|
||||
|
||||
// If manual input was cancelled, throw that error
|
||||
if (manualError) {
|
||||
throw manualError;
|
||||
}
|
||||
|
||||
if (result?.code) {
|
||||
// Browser callback won
|
||||
code = result.code;
|
||||
} else if (manualCode) {
|
||||
// Manual input won (or callback timed out and user had entered code)
|
||||
const parsed = parseOAuthAuthorizationInput(manualCode);
|
||||
if (parsed.state && parsed.state !== state) {
|
||||
throw new Error("State mismatch");
|
||||
}
|
||||
code = parsed.code;
|
||||
}
|
||||
|
||||
// If still no code, wait for manual promise to complete and try that
|
||||
if (!code) {
|
||||
await withOAuthLoginAbort(manualPromise, options.signal, server.cancelWait);
|
||||
if (manualError) {
|
||||
throw toLintErrorObject(manualError, "Non-Error thrown");
|
||||
}
|
||||
if (manualCode) {
|
||||
const parsed = parseOAuthAuthorizationInput(manualCode);
|
||||
if (parsed.state && parsed.state !== state) {
|
||||
throw new Error("State mismatch");
|
||||
}
|
||||
code = parsed.code;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const callbackPromise = server.waitForCode();
|
||||
const result = await withOAuthLoginAbort(
|
||||
Promise.race([callbackPromise, waitForManualPromptFallback(options.signal)]),
|
||||
options.signal,
|
||||
server.cancelWait,
|
||||
);
|
||||
if (result?.code) {
|
||||
code = result.code;
|
||||
} else {
|
||||
const promptCodePromise = promptForAuthorizationCode(options.onPrompt, state).then(
|
||||
(promptCode) => {
|
||||
server.cancelWait();
|
||||
return promptCode;
|
||||
},
|
||||
);
|
||||
code = await withOAuthLoginAbort(
|
||||
Promise.race([callbackPromise.then((callback) => callback?.code), promptCodePromise]),
|
||||
options.signal,
|
||||
server.cancelWait,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to onPrompt if still no code
|
||||
if (!code) {
|
||||
code = await withOAuthLoginAbort(
|
||||
promptForAuthorizationCode(options.onPrompt, state),
|
||||
options.signal,
|
||||
server.cancelWait,
|
||||
);
|
||||
}
|
||||
|
||||
if (!code) {
|
||||
throw new Error("Missing authorization code");
|
||||
}
|
||||
|
||||
const tokenResult = await exchangeAuthorizationCode(code, verifier, redirectUri, {
|
||||
signal: options.signal,
|
||||
});
|
||||
if (tokenResult.type !== "success") {
|
||||
throw new Error(tokenResult.message);
|
||||
}
|
||||
|
||||
const accountId = getAccountId(tokenResult.access);
|
||||
if (!accountId) {
|
||||
throw new Error("Failed to extract accountId from token");
|
||||
}
|
||||
|
||||
return {
|
||||
access: tokenResult.access,
|
||||
refresh: tokenResult.refresh,
|
||||
expires: tokenResult.expires,
|
||||
accountId,
|
||||
};
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh OpenAI Codex OAuth token
|
||||
*/
|
||||
export async function refreshOpenAICodexToken(refreshToken: string): Promise<OAuthCredentials> {
|
||||
const result = await refreshAccessToken(refreshToken);
|
||||
if (result.type !== "success") {
|
||||
throw new Error(result.message);
|
||||
}
|
||||
|
||||
const accountId = getAccountId(result.access);
|
||||
if (!accountId) {
|
||||
throw new Error("Failed to extract accountId from token");
|
||||
}
|
||||
|
||||
return {
|
||||
access: result.access,
|
||||
refresh: result.refresh,
|
||||
expires: result.expires,
|
||||
accountId,
|
||||
};
|
||||
}
|
||||
|
||||
export const openaiCodexOAuthProvider: OAuthProviderInterface = {
|
||||
id: "openai",
|
||||
name: "ChatGPT Plus/Pro (Codex Subscription)",
|
||||
usesCallbackServer: true,
|
||||
|
||||
async login(callbacks: OAuthLoginCallbacks): Promise<OAuthCredentials> {
|
||||
return loginOpenAICodex({
|
||||
onAuth: callbacks.onAuth,
|
||||
onPrompt: callbacks.onPrompt,
|
||||
onProgress: callbacks.onProgress,
|
||||
onManualCodeInput: callbacks.onManualCodeInput,
|
||||
signal: callbacks.signal,
|
||||
});
|
||||
},
|
||||
|
||||
async refreshToken(credentials: OAuthCredentials): Promise<OAuthCredentials> {
|
||||
return refreshOpenAICodexToken(credentials.refresh);
|
||||
},
|
||||
|
||||
getApiKey(credentials: OAuthCredentials): string {
|
||||
return credentials.access;
|
||||
},
|
||||
};
|
||||
|
||||
export const testing = {
|
||||
callbackHost: CALLBACK_HOST,
|
||||
createAuthorizationFlow,
|
||||
exchangeAuthorizationCode,
|
||||
loginOpenAICodex,
|
||||
refreshAccessToken,
|
||||
resolveCallbackHost,
|
||||
resolveRedirectUri,
|
||||
};
|
||||
|
||||
function toLintErrorObject(value: unknown, fallbackMessage: string): Error {
|
||||
if (value instanceof Error) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return new Error(value);
|
||||
}
|
||||
const error = new Error(fallbackMessage, { cause: value });
|
||||
if ((typeof value === "object" && value !== null) || typeof value === "function") {
|
||||
Object.assign(error, value);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
2
extensions/openai/openai-chatgpt-oauth-page.runtime.ts
Normal file
2
extensions/openai/openai-chatgpt-oauth-page.runtime.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
// Openai plugin module implements openai chatgpt oauth page behavior.
|
||||
export { oauthErrorHtml, oauthSuccessHtml } from "openclaw/plugin-sdk/provider-oauth-runtime";
|
||||
13
extensions/openai/openai-chatgpt-oauth-types.runtime.ts
Normal file
13
extensions/openai/openai-chatgpt-oauth-types.runtime.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
// Openai plugin module implements openai chatgpt oauth types behavior.
|
||||
export type {
|
||||
OAuthAuthInfo,
|
||||
OAuthCredentials,
|
||||
OAuthLoginCallbacks,
|
||||
OAuthPrompt,
|
||||
OAuthProvider,
|
||||
OAuthProviderId,
|
||||
OAuthProviderInfo,
|
||||
OAuthProviderInterface,
|
||||
OAuthSelectOption,
|
||||
OAuthSelectPrompt,
|
||||
} from "openclaw/plugin-sdk/provider-oauth-runtime";
|
||||
40
extensions/openai/openai-chatgpt-oauth.runtime.test.ts
Normal file
40
extensions/openai/openai-chatgpt-oauth.runtime.test.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
// Openai tests cover openai chatgpt oauth plugin behavior.
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { testing } from "./openai-chatgpt-oauth.runtime.js";
|
||||
|
||||
describe("OpenAI Codex OAuth runtime", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("caps oversized TLS preflight timeouts before creating an abort signal", async () => {
|
||||
const timeoutSpy = vi.spyOn(AbortSignal, "timeout");
|
||||
const fetchImpl = vi.fn(async () => new Response(null, { status: 302 }));
|
||||
|
||||
await expect(
|
||||
testing.runOpenAIOAuthTlsPreflight({
|
||||
timeoutMs: Number.MAX_SAFE_INTEGER,
|
||||
fetchImpl,
|
||||
}),
|
||||
).resolves.toEqual({ ok: true });
|
||||
|
||||
expect(timeoutSpy).toHaveBeenCalledWith(MAX_TIMER_TIMEOUT_MS);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("cancels reachable TLS preflight response bodies", async () => {
|
||||
const response = new Response("reachable", { status: 302 });
|
||||
const cancel = vi.spyOn(response.body!, "cancel").mockResolvedValue(undefined);
|
||||
const fetchImpl = vi.fn(async () => response);
|
||||
|
||||
await expect(
|
||||
testing.runOpenAIOAuthTlsPreflight({
|
||||
timeoutMs: 20,
|
||||
fetchImpl,
|
||||
}),
|
||||
).resolves.toEqual({ ok: true });
|
||||
|
||||
expect(cancel).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
364
extensions/openai/openai-chatgpt-oauth.runtime.ts
Normal file
364
extensions/openai/openai-chatgpt-oauth.runtime.ts
Normal file
@@ -0,0 +1,364 @@
|
||||
// Openai plugin module implements openai chatgpt oauth behavior.
|
||||
import path from "node:path";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
|
||||
import type { ProviderAuthContext } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { ensureGlobalUndiciEnvProxyDispatcher } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { formatCliCommand } from "openclaw/plugin-sdk/setup-tools";
|
||||
import { loginOpenAICodex } from "./openai-chatgpt-oauth-flow.runtime.js";
|
||||
import type { OAuthCredentials } from "./openai-chatgpt-oauth-types.runtime.js";
|
||||
|
||||
const manualInputPromptMessage = "Paste the authorization code (or full redirect URL):";
|
||||
const openAICodexOAuthOriginator = "openclaw";
|
||||
const localManualFallbackDelayMs = 15_000;
|
||||
const localManualFallbackGraceMs = 1_000;
|
||||
const openAIAuthProbeUrl =
|
||||
"https://auth.openai.com/oauth/authorize?response_type=code&client_id=openclaw-preflight&redirect_uri=http%3A%2F%2Flocalhost%3A1455%2Fauth%2Fcallback&scope=openid+profile+email";
|
||||
|
||||
const tlsCertErrorCodes = new Set([
|
||||
"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
|
||||
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
|
||||
"CERT_HAS_EXPIRED",
|
||||
"DEPTH_ZERO_SELF_SIGNED_CERT",
|
||||
"SELF_SIGNED_CERT_IN_CHAIN",
|
||||
"ERR_TLS_CERT_ALTNAME_INVALID",
|
||||
]);
|
||||
|
||||
const tlsCertErrorPatterns = [
|
||||
/unable to get local issuer certificate/i,
|
||||
/unable to verify the first certificate/i,
|
||||
/self[- ]signed certificate/i,
|
||||
/certificate has expired/i,
|
||||
];
|
||||
|
||||
type OpenAICodexOAuthFailureCode =
|
||||
| "callback_timeout"
|
||||
| "callback_validation_failed"
|
||||
| "unsupported_region";
|
||||
|
||||
type PreflightFailureKind = "tls-cert" | "network";
|
||||
type OpenAIOAuthTlsPreflightResult =
|
||||
| { ok: true }
|
||||
| {
|
||||
ok: false;
|
||||
kind: PreflightFailureKind;
|
||||
code?: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
function getErrorRecord(error: unknown): Record<string, unknown> | null {
|
||||
return error && typeof error === "object" ? (error as Record<string, unknown>) : null;
|
||||
}
|
||||
|
||||
function extractFailure(error: unknown): {
|
||||
code?: string;
|
||||
message: string;
|
||||
kind: PreflightFailureKind;
|
||||
} {
|
||||
const root = getErrorRecord(error);
|
||||
const rootCause = getErrorRecord(root?.cause);
|
||||
const code = typeof rootCause?.code === "string" ? rootCause.code : undefined;
|
||||
const message =
|
||||
typeof rootCause?.message === "string"
|
||||
? rootCause.message
|
||||
: typeof root?.message === "string"
|
||||
? root.message
|
||||
: String(error);
|
||||
const isTlsCertError =
|
||||
(code ? tlsCertErrorCodes.has(code) : false) ||
|
||||
tlsCertErrorPatterns.some((pattern) => pattern.test(message));
|
||||
return {
|
||||
code,
|
||||
message,
|
||||
kind: isTlsCertError ? "tls-cert" : "network",
|
||||
};
|
||||
}
|
||||
|
||||
function resolveHomebrewPrefixFromExecPath(execPath: string): string | null {
|
||||
const marker = `${path.sep}Cellar${path.sep}`;
|
||||
const idx = execPath.indexOf(marker);
|
||||
if (idx > 0) {
|
||||
return execPath.slice(0, idx);
|
||||
}
|
||||
return process.env.HOMEBREW_PREFIX?.trim() || null;
|
||||
}
|
||||
|
||||
function resolveCertBundlePath(): string | null {
|
||||
const prefix = resolveHomebrewPrefixFromExecPath(process.execPath);
|
||||
return prefix ? path.join(prefix, "etc", "openssl@3", "cert.pem") : null;
|
||||
}
|
||||
|
||||
async function runOpenAIOAuthTlsPreflight(options?: {
|
||||
timeoutMs?: number;
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<OpenAIOAuthTlsPreflightResult> {
|
||||
const timeoutMs = resolveTimerTimeoutMs(options?.timeoutMs, 5000);
|
||||
const fetchImpl = options?.fetchImpl ?? fetch;
|
||||
let response: Response | undefined;
|
||||
try {
|
||||
response = await fetchImpl(openAIAuthProbeUrl, {
|
||||
method: "GET",
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
return { ok: true };
|
||||
} catch (error) {
|
||||
const failure = extractFailure(error);
|
||||
return {
|
||||
ok: false,
|
||||
kind: failure.kind,
|
||||
code: failure.code,
|
||||
message: failure.message,
|
||||
};
|
||||
} finally {
|
||||
if (response?.bodyUsed !== true) {
|
||||
await response?.body?.cancel().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const testing = { runOpenAIOAuthTlsPreflight };
|
||||
export { testing as __testing };
|
||||
|
||||
function formatOpenAIOAuthTlsPreflightFix(
|
||||
result: Exclude<OpenAIOAuthTlsPreflightResult, { ok: true }>,
|
||||
): string {
|
||||
if (result.kind !== "tls-cert") {
|
||||
return [
|
||||
"OpenAI OAuth prerequisites check failed due to a network error before the browser flow.",
|
||||
`Cause: ${result.message}`,
|
||||
"Verify DNS/firewall/proxy access to auth.openai.com and retry.",
|
||||
].join("\n");
|
||||
}
|
||||
const certBundlePath = resolveCertBundlePath();
|
||||
const lines = [
|
||||
"OpenAI OAuth prerequisites check failed: Node/OpenSSL cannot validate TLS certificates.",
|
||||
`Cause: ${result.code ? `${result.code} (${result.message})` : result.message}`,
|
||||
"",
|
||||
"Fix (Homebrew Node/OpenSSL):",
|
||||
`- ${formatCliCommand("brew postinstall ca-certificates")}`,
|
||||
`- ${formatCliCommand("brew postinstall openssl@3")}`,
|
||||
];
|
||||
if (certBundlePath) {
|
||||
lines.push(`- Verify cert bundle exists: ${certBundlePath}`);
|
||||
}
|
||||
lines.push("- Retry the OAuth login flow.");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function settleAfterDelay(params: {
|
||||
delayMs: number;
|
||||
waitForLoginToSettle: Promise<void>;
|
||||
}): Promise<"delay" | "settled"> {
|
||||
return new Promise((resolve) => {
|
||||
let done = false;
|
||||
const complete = (outcome: "delay" | "settled") => {
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
done = true;
|
||||
clearTimeout(timer);
|
||||
resolve(outcome);
|
||||
};
|
||||
const timer = setTimeout(() => complete("delay"), params.delayMs);
|
||||
params.waitForLoginToSettle.then(
|
||||
() => complete("settled"),
|
||||
() => complete("settled"),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function waitForeverForPromptInput(): Promise<string> {
|
||||
return new Promise<string>(() => {});
|
||||
}
|
||||
|
||||
function createOpenAICodexOAuthError(
|
||||
code: OpenAICodexOAuthFailureCode,
|
||||
message: string,
|
||||
cause?: unknown,
|
||||
): Error & { code: OpenAICodexOAuthFailureCode } {
|
||||
return Object.assign(new Error(`OpenAI Codex OAuth failed (${code}): ${message}`, { cause }), {
|
||||
code,
|
||||
});
|
||||
}
|
||||
|
||||
function rewriteOpenAICodexOAuthError(error: unknown): Error {
|
||||
const message = formatErrorMessage(error);
|
||||
if (/unsupported_country_region_territory/i.test(message)) {
|
||||
return createOpenAICodexOAuthError(
|
||||
"unsupported_region",
|
||||
[
|
||||
"OpenAI rejected the token exchange for this country, region, or network route.",
|
||||
"If you normally use a proxy, verify HTTPS_PROXY, HTTP_PROXY, or ALL_PROXY is set for the OpenClaw process and then retry `openclaw models auth login --provider openai`.",
|
||||
].join(" "),
|
||||
error,
|
||||
);
|
||||
}
|
||||
if (/state mismatch|missing authorization code/i.test(message)) {
|
||||
return createOpenAICodexOAuthError("callback_validation_failed", message, error);
|
||||
}
|
||||
return error instanceof Error ? error : new Error(message);
|
||||
}
|
||||
|
||||
function createManualCodeInputHandler(params: {
|
||||
isRemote: boolean;
|
||||
onPrompt: (prompt: { message: string }) => Promise<string>;
|
||||
runtime: ProviderAuthContext["runtime"];
|
||||
updateProgress: (message: string) => void;
|
||||
stopProgress: (message?: string) => void;
|
||||
waitForLoginToSettle: Promise<void>;
|
||||
hasBrowserAuthStarted: () => boolean;
|
||||
}): (() => Promise<string>) | undefined {
|
||||
let manualFallbackPromise: Promise<string> | undefined;
|
||||
const promptForManualCode = () => params.onPrompt({ message: manualInputPromptMessage });
|
||||
if (params.isRemote) {
|
||||
return async () => {
|
||||
manualFallbackPromise ??= promptForManualCode();
|
||||
return await manualFallbackPromise;
|
||||
};
|
||||
}
|
||||
|
||||
const switchToManualEntry = async (progressMessage: string, logMessage?: string) => {
|
||||
params.updateProgress(progressMessage);
|
||||
if (logMessage) {
|
||||
params.runtime.log(logMessage);
|
||||
}
|
||||
params.stopProgress("Manual OAuth entry required");
|
||||
return await promptForManualCode();
|
||||
};
|
||||
|
||||
const runLocalManualFallback = async () => {
|
||||
if (!params.hasBrowserAuthStarted()) {
|
||||
return await switchToManualEntry(
|
||||
"Local OAuth callback was unavailable. Paste the redirect URL to continue...",
|
||||
"OpenAI Codex OAuth local callback did not start; switching to manual entry immediately.",
|
||||
);
|
||||
}
|
||||
|
||||
const firstWait = await settleAfterDelay({
|
||||
delayMs: localManualFallbackDelayMs,
|
||||
waitForLoginToSettle: params.waitForLoginToSettle,
|
||||
});
|
||||
if (firstWait === "settled") {
|
||||
return await waitForeverForPromptInput();
|
||||
}
|
||||
const graceWait = await settleAfterDelay({
|
||||
delayMs: localManualFallbackGraceMs,
|
||||
waitForLoginToSettle: params.waitForLoginToSettle,
|
||||
});
|
||||
if (graceWait === "settled") {
|
||||
return await waitForeverForPromptInput();
|
||||
}
|
||||
return await switchToManualEntry(
|
||||
"Browser callback did not finish. Paste the redirect URL to continue...",
|
||||
`OpenAI Codex OAuth callback did not arrive within ${localManualFallbackDelayMs}ms; switching to manual entry (callback_timeout).`,
|
||||
);
|
||||
};
|
||||
|
||||
return async () => {
|
||||
manualFallbackPromise ??= runLocalManualFallback();
|
||||
return await manualFallbackPromise;
|
||||
};
|
||||
}
|
||||
|
||||
export async function loginOpenAICodexOAuth(params: {
|
||||
prompter: ProviderAuthContext["prompter"];
|
||||
runtime: ProviderAuthContext["runtime"];
|
||||
oauth: ProviderAuthContext["oauth"];
|
||||
isRemote: boolean;
|
||||
openUrl: (url: string) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
onManualCodeInput?: () => Promise<string>;
|
||||
localBrowserMessage?: string;
|
||||
}): Promise<OAuthCredentials | null> {
|
||||
const { prompter, runtime, isRemote, openUrl, localBrowserMessage } = params;
|
||||
|
||||
ensureGlobalUndiciEnvProxyDispatcher();
|
||||
|
||||
const preflight = await runOpenAIOAuthTlsPreflight();
|
||||
if (!preflight.ok && preflight.kind === "tls-cert") {
|
||||
const hint = formatOpenAIOAuthTlsPreflightFix(preflight);
|
||||
await prompter.note(hint, "OAuth prerequisites");
|
||||
runtime.error(hint);
|
||||
throw new Error(`OpenAI Codex OAuth prerequisites failed: ${preflight.message}`);
|
||||
}
|
||||
|
||||
await prompter.note(
|
||||
isRemote
|
||||
? [
|
||||
"You are running in a remote/VPS environment.",
|
||||
"A URL will be shown for you to open in your LOCAL browser.",
|
||||
"Open it, sign in, then paste the redirect URL here.",
|
||||
"If this OpenClaw process can receive the browser callback, sign-in may finish automatically before you paste.",
|
||||
].join("\n")
|
||||
: [
|
||||
"Browser will open for OpenAI authentication.",
|
||||
"If the callback doesn't auto-complete, paste the redirect URL.",
|
||||
"OpenAI OAuth uses localhost:1455 for the callback.",
|
||||
].join("\n"),
|
||||
"OpenAI Codex OAuth",
|
||||
);
|
||||
|
||||
const spin = prompter.progress("Starting OAuth flow...");
|
||||
let progressActive = true;
|
||||
const updateProgress = (message: string) => {
|
||||
if (progressActive) {
|
||||
spin.update(message);
|
||||
}
|
||||
};
|
||||
const stopProgress = (message?: string) => {
|
||||
if (progressActive) {
|
||||
progressActive = false;
|
||||
spin.stop(message);
|
||||
}
|
||||
};
|
||||
let browserAuthStarted = false;
|
||||
let markLoginSettled!: () => void;
|
||||
const waitForLoginToSettle = new Promise<void>((resolve) => {
|
||||
markLoginSettled = resolve;
|
||||
});
|
||||
try {
|
||||
const { onAuth: baseOnAuth, onPrompt } = params.oauth.createVpsAwareHandlers({
|
||||
isRemote,
|
||||
prompter,
|
||||
runtime,
|
||||
spin,
|
||||
openUrl,
|
||||
localBrowserMessage: localBrowserMessage ?? "Complete sign-in in browser...",
|
||||
manualPromptMessage: manualInputPromptMessage,
|
||||
});
|
||||
const onAuth = (event: Parameters<typeof baseOnAuth>[0]) => {
|
||||
browserAuthStarted = true;
|
||||
void baseOnAuth(event);
|
||||
};
|
||||
|
||||
const creds = await loginOpenAICodex({
|
||||
onAuth,
|
||||
onPrompt,
|
||||
originator: openAICodexOAuthOriginator,
|
||||
onManualCodeInput:
|
||||
params.onManualCodeInput ??
|
||||
createManualCodeInputHandler({
|
||||
isRemote,
|
||||
onPrompt,
|
||||
runtime,
|
||||
updateProgress,
|
||||
stopProgress,
|
||||
waitForLoginToSettle,
|
||||
hasBrowserAuthStarted: () => browserAuthStarted,
|
||||
}),
|
||||
onProgress: (msg: string) => updateProgress(msg),
|
||||
signal: params.signal,
|
||||
});
|
||||
stopProgress("OpenAI OAuth complete");
|
||||
return creds ?? null;
|
||||
} catch (err) {
|
||||
stopProgress("OpenAI OAuth failed");
|
||||
const rewrittenError = rewriteOpenAICodexOAuthError(err);
|
||||
runtime.error(String(rewrittenError));
|
||||
await prompter.note("Trouble with OAuth? See https://docs.openclaw.ai/start/faq", "OAuth help");
|
||||
throw rewrittenError;
|
||||
} finally {
|
||||
markLoginSettled();
|
||||
}
|
||||
}
|
||||
2
extensions/openai/openai-chatgpt-pkce.runtime.ts
Normal file
2
extensions/openai/openai-chatgpt-pkce.runtime.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
// Openai plugin module implements openai chatgpt pkce behavior.
|
||||
export { generateOAuthState, generatePKCE } from "openclaw/plugin-sdk/provider-oauth-runtime";
|
||||
61
extensions/openai/openai-chatgpt-provider.runtime.ts
Normal file
61
extensions/openai/openai-chatgpt-provider.runtime.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import { ensureGlobalUndiciEnvProxyDispatcher } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { refreshOpenAICodexToken as refreshOpenAICodexTokenFromFlow } from "./openai-chatgpt-oauth-flow.runtime.js";
|
||||
import type { OAuthCredentials } from "./openai-chatgpt-oauth-types.runtime.js";
|
||||
|
||||
type OpenAICodexProviderRuntimeDeps = {
|
||||
ensureGlobalUndiciEnvProxyDispatcher: typeof ensureGlobalUndiciEnvProxyDispatcher;
|
||||
getOAuthApiKey: typeof getOpenAICodexOAuthApiKey;
|
||||
refreshOpenAICodexToken: typeof refreshOpenAICodexTokenFromFlow;
|
||||
};
|
||||
|
||||
export function createOpenAICodexProviderRuntime(deps: OpenAICodexProviderRuntimeDeps): {
|
||||
getOAuthApiKey: typeof getOAuthApiKey;
|
||||
refreshOpenAICodexToken: typeof refreshOpenAICodexToken;
|
||||
} {
|
||||
return {
|
||||
async getOAuthApiKey(...args) {
|
||||
deps.ensureGlobalUndiciEnvProxyDispatcher();
|
||||
return await deps.getOAuthApiKey(...args);
|
||||
},
|
||||
async refreshOpenAICodexToken(...args) {
|
||||
deps.ensureGlobalUndiciEnvProxyDispatcher();
|
||||
return await deps.refreshOpenAICodexToken(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const runtime = createOpenAICodexProviderRuntime({
|
||||
ensureGlobalUndiciEnvProxyDispatcher,
|
||||
getOAuthApiKey: getOpenAICodexOAuthApiKey,
|
||||
refreshOpenAICodexToken: refreshOpenAICodexTokenFromFlow,
|
||||
});
|
||||
|
||||
export async function getOAuthApiKey(
|
||||
...args: Parameters<typeof getOpenAICodexOAuthApiKey>
|
||||
): Promise<Awaited<ReturnType<typeof getOpenAICodexOAuthApiKey>>> {
|
||||
return await runtime.getOAuthApiKey(...args);
|
||||
}
|
||||
|
||||
export async function refreshOpenAICodexToken(
|
||||
...args: Parameters<typeof refreshOpenAICodexTokenFromFlow>
|
||||
): Promise<Awaited<ReturnType<typeof refreshOpenAICodexTokenFromFlow>>> {
|
||||
return await runtime.refreshOpenAICodexToken(...args);
|
||||
}
|
||||
|
||||
async function getOpenAICodexOAuthApiKey(
|
||||
providerId: string,
|
||||
credentials: Record<string, OAuthCredentials>,
|
||||
): Promise<{ newCredentials: OAuthCredentials; apiKey: string } | null> {
|
||||
if (providerId !== "openai") {
|
||||
throw new Error(`Unknown OAuth provider: ${providerId}`);
|
||||
}
|
||||
let creds = credentials[providerId];
|
||||
if (!creds) {
|
||||
return null;
|
||||
}
|
||||
if (Date.now() >= creds.expires) {
|
||||
creds = await refreshOpenAICodexTokenFromFlow(creds.refresh);
|
||||
}
|
||||
return { newCredentials: creds, apiKey: creds.access };
|
||||
}
|
||||
237
extensions/openai/openai-chatgpt-provider.test.ts
Normal file
237
extensions/openai/openai-chatgpt-provider.test.ts
Normal file
@@ -0,0 +1,237 @@
|
||||
// Openai tests cover openai chatgpt provider plugin behavior.
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const refreshOpenAICodexTokenMock = vi.hoisted(() => vi.fn());
|
||||
const loginOpenAICodexDeviceCodeMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./openai-chatgpt-provider.runtime.js", () => ({
|
||||
refreshOpenAICodexToken: refreshOpenAICodexTokenMock,
|
||||
}));
|
||||
|
||||
vi.mock("./openai-chatgpt-device-code.js", () => ({
|
||||
loginOpenAICodexDeviceCode: loginOpenAICodexDeviceCodeMock,
|
||||
}));
|
||||
|
||||
let buildOpenAIProvider: typeof import("./openai-provider.js").buildOpenAIProvider;
|
||||
|
||||
describe("OpenAI provider Codex transport hooks", () => {
|
||||
beforeAll(async () => {
|
||||
({ buildOpenAIProvider } = await import("./openai-provider.js"));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
refreshOpenAICodexTokenMock.mockReset();
|
||||
loginOpenAICodexDeviceCodeMock.mockReset();
|
||||
});
|
||||
|
||||
it("exposes ChatGPT OAuth on the canonical OpenAI provider", () => {
|
||||
const provider = buildOpenAIProvider();
|
||||
|
||||
expect(provider.id).toBe("openai");
|
||||
expect(provider.aliases).toBeUndefined();
|
||||
expect(provider.hookAliases).toEqual(["azure-openai", "azure-openai-responses"]);
|
||||
expect(provider.auth?.map((method) => method.id)).toEqual(["oauth", "device-code", "api-key"]);
|
||||
expect(provider.auth?.map((method) => method.wizard?.choiceId)).toEqual([
|
||||
"openai",
|
||||
"openai-device-code",
|
||||
"openai-api-key",
|
||||
]);
|
||||
expect(provider.oauthProfileIdRepairs).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stores device-code logins as OpenAI OAuth profiles", async () => {
|
||||
const provider = buildOpenAIProvider();
|
||||
const deviceCodeMethod = provider.auth?.find((method) => method.id === "device-code");
|
||||
loginOpenAICodexDeviceCodeMock.mockResolvedValueOnce({
|
||||
access: "access-token",
|
||||
refresh: "refresh-token",
|
||||
expires: 1_700_000_000_000,
|
||||
});
|
||||
|
||||
const result = await deviceCodeMethod?.run({
|
||||
isRemote: false,
|
||||
openUrl: vi.fn(async () => {}),
|
||||
prompter: {
|
||||
note: vi.fn(async () => {}),
|
||||
progress: vi.fn(() => ({ update: vi.fn(), stop: vi.fn() })),
|
||||
},
|
||||
runtime: { log: vi.fn(), error: vi.fn() },
|
||||
config: {},
|
||||
oauth: {},
|
||||
} as never);
|
||||
|
||||
expect(result?.profiles?.[0]).toMatchObject({
|
||||
profileId: "openai:default",
|
||||
credential: {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "access-token",
|
||||
refresh: "refresh-token",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("routes Codex-backed OpenAI models through the Codex Responses transport", () => {
|
||||
const provider = buildOpenAIProvider();
|
||||
|
||||
const model = provider.resolveDynamicModel?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
providerConfig: { api: "openai-chatgpt-responses" },
|
||||
modelRegistry: { find: () => null },
|
||||
} as never);
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "openai",
|
||||
id: "gpt-5.4",
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])(
|
||||
"resolves %s through the Codex Responses transport without live catalog metadata",
|
||||
(modelId) => {
|
||||
const provider = buildOpenAIProvider();
|
||||
|
||||
const model = provider.resolveDynamicModel?.({
|
||||
provider: "openai",
|
||||
modelId,
|
||||
authProfileMode: "oauth",
|
||||
modelRegistry: { find: () => null },
|
||||
} as never);
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "openai",
|
||||
id: modelId,
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
input: ["text", "image"],
|
||||
contextWindow: 372_000,
|
||||
contextTokens: 372_000,
|
||||
maxTokens: 128_000,
|
||||
thinkingLevelMap: { off: null, xhigh: "xhigh", max: "max" },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ name: "fills a missing map", thinkingLevelMap: undefined, expectedOff: null },
|
||||
{ name: "preserves explicit overrides", thinkingLevelMap: { off: "low" }, expectedOff: "low" },
|
||||
])("$name on registry-backed GPT-5.6 models", ({ thinkingLevelMap, expectedOff }) => {
|
||||
const provider = buildOpenAIProvider();
|
||||
const model = provider.resolveDynamicModel?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.6-luna",
|
||||
authProfileMode: "oauth",
|
||||
modelRegistry: {
|
||||
find: () => ({
|
||||
id: "gpt-5.6-luna",
|
||||
name: "GPT-5.6 Luna",
|
||||
provider: "openai",
|
||||
api: "openai-responses",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 1, output: 6, cacheRead: 0.1, cacheWrite: 1.25 },
|
||||
contextWindow: 372_000,
|
||||
maxTokens: 128_000,
|
||||
...(thinkingLevelMap ? { thinkingLevelMap } : {}),
|
||||
}),
|
||||
},
|
||||
} as never);
|
||||
|
||||
expect(model).toMatchObject({
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
input: ["text", "image"],
|
||||
thinkingLevelMap: { off: expectedOff, xhigh: "xhigh", max: "max" },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps default Codex-backed OpenAI catalog models on the Codex Responses transport", () => {
|
||||
const provider = buildOpenAIProvider();
|
||||
|
||||
const model = provider.resolveDynamicModel?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.5",
|
||||
providerConfig: { api: "openai-chatgpt-responses" },
|
||||
modelRegistry: {
|
||||
find: () => ({
|
||||
provider: "openai",
|
||||
id: "gpt-5.5",
|
||||
name: "gpt-5.5",
|
||||
api: "openai-responses",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 1, output: 1, cacheRead: 1, cacheWrite: 1 },
|
||||
contextWindow: 400_000,
|
||||
maxTokens: 128_000,
|
||||
}),
|
||||
},
|
||||
} as never);
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "openai",
|
||||
id: "gpt-5.5",
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps cloned Codex-backed OpenAI models on the Codex Responses transport", () => {
|
||||
const provider = buildOpenAIProvider();
|
||||
|
||||
const model = provider.resolveDynamicModel?.({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
providerConfig: { api: "openai-chatgpt-responses" },
|
||||
modelRegistry: {
|
||||
find: () => ({
|
||||
provider: "openai",
|
||||
id: "gpt-5.4",
|
||||
name: "gpt-5.4",
|
||||
api: "openai-responses",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 1, output: 1, cacheRead: 1, cacheWrite: 1 },
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 16_384,
|
||||
}),
|
||||
},
|
||||
} as never);
|
||||
|
||||
expect(model).toMatchObject({
|
||||
provider: "openai",
|
||||
id: "gpt-5.4",
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
});
|
||||
});
|
||||
|
||||
it("refreshes ChatGPT OAuth credentials under the OpenAI provider", async () => {
|
||||
const provider = buildOpenAIProvider();
|
||||
refreshOpenAICodexTokenMock.mockResolvedValueOnce({
|
||||
access: "new-access",
|
||||
refresh: "new-refresh",
|
||||
expires: 1_700_000_000_000,
|
||||
});
|
||||
|
||||
await expect(
|
||||
provider.refreshOAuth?.({
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "old-access",
|
||||
refresh: "old-refresh",
|
||||
expires: Date.now() - 60_000,
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "new-access",
|
||||
refresh: "new-refresh",
|
||||
});
|
||||
});
|
||||
});
|
||||
727
extensions/openai/openai-chatgpt-provider.ts
Normal file
727
extensions/openai/openai-chatgpt-provider.ts
Normal file
@@ -0,0 +1,727 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import type {
|
||||
ProviderAuthContext,
|
||||
ProviderAuthMethod,
|
||||
ProviderAuthResult,
|
||||
ProviderResolveDynamicModelContext,
|
||||
ProviderRuntimeModel,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { CODEX_CLI_PROFILE_ID, type OAuthCredential } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { buildOauthProviderAuthResult } from "openclaw/plugin-sdk/provider-auth";
|
||||
import {
|
||||
DEFAULT_CONTEXT_TOKENS,
|
||||
normalizeModelCompat,
|
||||
normalizeProviderId,
|
||||
type ProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { fetchCodexUsage } from "openclaw/plugin-sdk/provider-usage";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
readStringValue,
|
||||
uniqueValues,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
OPENAI_CHATGPT_LOGIN_HINT,
|
||||
OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
OPENAI_CODEX_WIZARD_GROUP,
|
||||
} from "./auth-choice-copy.js";
|
||||
import {
|
||||
isOpenAIApiBaseUrl,
|
||||
isOpenAICodexBaseUrl,
|
||||
OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
} from "./base-url.js";
|
||||
import { OPENAI_CODEX_DEFAULT_MODEL } from "./default-models.js";
|
||||
import { resolveCodexAuthIdentity } from "./openai-chatgpt-auth-identity.js";
|
||||
import { loginOpenAICodexDeviceCode } from "./openai-chatgpt-device-code.js";
|
||||
import { loginOpenAICodexOAuth } from "./openai-chatgpt-oauth.runtime.js";
|
||||
import {
|
||||
buildOpenAIResponsesProviderHooks,
|
||||
buildOpenAISyntheticCatalogEntry,
|
||||
cloneFirstTemplateModel,
|
||||
findCatalogTemplate,
|
||||
matchesExactOrPrefix,
|
||||
} from "./shared.js";
|
||||
import { resolveOpenAICodexThinkingProfile } from "./thinking-policy.js";
|
||||
|
||||
const PROVIDER_ID = "openai";
|
||||
const OPENAI_CODEX_BASE_URL = OPENAI_CODEX_RESPONSES_BASE_URL;
|
||||
const OPENAI_CODEX_LOGIN_ASSISTANT_PRIORITY = -30;
|
||||
const OPENAI_CODEX_DEVICE_PAIRING_ASSISTANT_PRIORITY = -10;
|
||||
const OPENAI_CODEX_GPT_56_MODEL_IDS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
|
||||
const OPENAI_CODEX_GPT_56_THINKING_LEVEL_MAP = {
|
||||
off: null,
|
||||
xhigh: "xhigh",
|
||||
max: "max",
|
||||
} as const;
|
||||
const OPENAI_CODEX_GPT_55_MODEL_ID = "gpt-5.5";
|
||||
const OPENAI_CODEX_GPT_55_PRO_MODEL_ID = "gpt-5.5-pro";
|
||||
const OPENAI_CODEX_GPT_54_MODEL_ID = "gpt-5.4";
|
||||
const OPENAI_CODEX_GPT_54_LEGACY_MODEL_ID = "gpt-5.4-codex";
|
||||
const OPENAI_CODEX_GPT_54_MINI_MODEL_ID = "gpt-5.4-mini";
|
||||
const OPENAI_CODEX_GPT_54_PRO_MODEL_ID = "gpt-5.4-pro";
|
||||
const OPENAI_CODEX_GPT_53_SPARK_MODEL_ID = "gpt-5.3-codex-spark";
|
||||
const OPENAI_CODEX_GPT_56_CONTEXT_TOKENS = 372_000;
|
||||
const OPENAI_CODEX_GPT_55_CODEX_CONTEXT_TOKENS = 400_000;
|
||||
const OPENAI_CODEX_GPT_55_DEFAULT_RUNTIME_CONTEXT_TOKENS = 272_000;
|
||||
const OPENAI_CODEX_GPT_55_PRO_NATIVE_CONTEXT_TOKENS = 1_000_000;
|
||||
const OPENAI_CODEX_GPT_55_PRO_DEFAULT_CONTEXT_TOKENS = 272_000;
|
||||
const OPENAI_CODEX_GPT_54_NATIVE_CONTEXT_TOKENS = 1_050_000;
|
||||
const OPENAI_CODEX_GPT_54_DEFAULT_CONTEXT_TOKENS = 272_000;
|
||||
const OPENAI_CODEX_GPT_54_MINI_NATIVE_CONTEXT_TOKENS = 400_000;
|
||||
const OPENAI_CODEX_GPT_53_SPARK_CONTEXT_TOKENS = 128_000;
|
||||
const OPENAI_CODEX_GPT_54_MAX_TOKENS = 128_000;
|
||||
const OPENAI_CODEX_GPT_55_PRO_COST = {
|
||||
input: 30,
|
||||
output: 180,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
} as const;
|
||||
const OPENAI_CODEX_GPT_54_COST = {
|
||||
input: 2.5,
|
||||
output: 15,
|
||||
cacheRead: 0.25,
|
||||
cacheWrite: 0,
|
||||
} as const;
|
||||
const OPENAI_CODEX_GPT_54_PRO_COST = {
|
||||
input: 30,
|
||||
output: 180,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
} as const;
|
||||
const OPENAI_CODEX_GPT_54_MINI_COST = {
|
||||
input: 0.75,
|
||||
output: 4.5,
|
||||
cacheRead: 0.075,
|
||||
cacheWrite: 0,
|
||||
} as const;
|
||||
const OPENAI_CODEX_GPT_54_TEMPLATE_MODEL_IDS = ["gpt-5.3-codex"] as const;
|
||||
/** Legacy codex rows first; fall back to catalog `gpt-5.4` when the API omits 5.3/5.2. */
|
||||
const OPENAI_CODEX_GPT_54_CATALOG_SYNTH_TEMPLATE_MODEL_IDS = [
|
||||
...OPENAI_CODEX_GPT_54_TEMPLATE_MODEL_IDS,
|
||||
OPENAI_CODEX_GPT_54_MODEL_ID,
|
||||
] as const;
|
||||
const OPENAI_CODEX_GPT_55_PRO_TEMPLATE_MODEL_IDS = [
|
||||
OPENAI_CODEX_GPT_54_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_PRO_MODEL_ID,
|
||||
...OPENAI_CODEX_GPT_54_TEMPLATE_MODEL_IDS,
|
||||
] as const;
|
||||
const OPENAI_CODEX_MODERN_MODEL_IDS = [
|
||||
...OPENAI_CODEX_GPT_56_MODEL_IDS,
|
||||
OPENAI_CODEX_GPT_55_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_55_PRO_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_PRO_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_MINI_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_53_SPARK_MODEL_ID,
|
||||
] as const;
|
||||
const OPENAI_CODEX_IMAGE_CAPABLE_MODEL_IDS = [
|
||||
...OPENAI_CODEX_GPT_56_MODEL_IDS,
|
||||
OPENAI_CODEX_GPT_55_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_55_PRO_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_PRO_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_MINI_MODEL_ID,
|
||||
] as const;
|
||||
|
||||
function isOpenAIOrLegacyCodexProvider(provider: string | undefined): boolean {
|
||||
const normalized = normalizeProviderId(provider ?? "");
|
||||
return normalized === PROVIDER_ID;
|
||||
}
|
||||
|
||||
function isLegacyCodexCompatBaseUrl(baseUrl?: string): boolean {
|
||||
const trimmed = baseUrl?.trim();
|
||||
return (
|
||||
trimmed !== undefined && /^https?:\/\/api\.githubcopilot\.com(?:\/v1)?\/?$/iu.test(trimmed)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeCodexTransportFields(params: {
|
||||
api?: ProviderRuntimeModel["api"] | null;
|
||||
baseUrl?: string;
|
||||
}): {
|
||||
api?: ProviderRuntimeModel["api"];
|
||||
baseUrl?: string;
|
||||
} {
|
||||
const useCodexTransport =
|
||||
!params.baseUrl ||
|
||||
isOpenAIApiBaseUrl(params.baseUrl) ||
|
||||
isOpenAICodexBaseUrl(params.baseUrl) ||
|
||||
isLegacyCodexCompatBaseUrl(params.baseUrl);
|
||||
const api =
|
||||
useCodexTransport &&
|
||||
(!params.api || params.api === "openai-responses" || params.api === "openai-completions")
|
||||
? "openai-chatgpt-responses"
|
||||
: (params.api ?? undefined);
|
||||
const baseUrl =
|
||||
api === "openai-chatgpt-responses" && useCodexTransport
|
||||
? OPENAI_CODEX_BASE_URL
|
||||
: params.baseUrl;
|
||||
return { api, baseUrl };
|
||||
}
|
||||
|
||||
function hasImageInput(input: unknown): boolean {
|
||||
return Array.isArray(input) && input.includes("image");
|
||||
}
|
||||
|
||||
function matchesOpenAICodexImageCapableModel(modelId: string, modelName?: string): boolean {
|
||||
return [modelId, modelName]
|
||||
.filter((value): value is string => typeof value === "string")
|
||||
.some((candidate) => matchesExactOrPrefix(candidate, OPENAI_CODEX_IMAGE_CAPABLE_MODEL_IDS));
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore native `["text", "image"]` input capability on resolved Codex rows
|
||||
* for known image-capable modern model IDs (GPT-5.4 through GPT-5.6).
|
||||
* Persisted/configured model rows can omit the `input` field
|
||||
* entirely when they were written by older OpenClaw versions. When that row wins
|
||||
* the catalog merge, `modelSupportsInput(entry, "image")` returns false and the
|
||||
* gateway's `chat.send` handler offloads inbound images as `media://inbound/<id>`
|
||||
* claim-check URIs instead of inlining them.
|
||||
*
|
||||
* Mirrors the Anthropic precedent set by upstream #83756.
|
||||
*/
|
||||
function applyOpenAICodexImageInputCapability(params: {
|
||||
modelId: string;
|
||||
model: ProviderRuntimeModel;
|
||||
}): ProviderRuntimeModel | undefined {
|
||||
if (hasImageInput(params.model.input)) {
|
||||
return undefined;
|
||||
}
|
||||
if (!matchesOpenAICodexImageCapableModel(params.modelId, params.model.name)) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
...params.model,
|
||||
input: ["text", "image"],
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeCodexTransport(model: ProviderRuntimeModel): ProviderRuntimeModel {
|
||||
const lowerModelId = normalizeLowercaseStringOrEmpty(model.id);
|
||||
const canonicalModelId =
|
||||
lowerModelId === OPENAI_CODEX_GPT_54_LEGACY_MODEL_ID ? OPENAI_CODEX_GPT_54_MODEL_ID : model.id;
|
||||
const canonicalName =
|
||||
normalizeLowercaseStringOrEmpty(model.name) === OPENAI_CODEX_GPT_54_LEGACY_MODEL_ID
|
||||
? OPENAI_CODEX_GPT_54_MODEL_ID
|
||||
: model.name;
|
||||
const normalizedTransport = normalizeCodexTransportFields({
|
||||
api: model.api,
|
||||
baseUrl: model.baseUrl,
|
||||
});
|
||||
const api = normalizedTransport.api ?? model.api;
|
||||
const baseUrl = normalizedTransport.baseUrl ?? model.baseUrl;
|
||||
if (
|
||||
api === model.api &&
|
||||
baseUrl === model.baseUrl &&
|
||||
canonicalModelId === model.id &&
|
||||
canonicalName === model.name
|
||||
) {
|
||||
return model;
|
||||
}
|
||||
return {
|
||||
...model,
|
||||
id: canonicalModelId,
|
||||
name: canonicalName,
|
||||
api,
|
||||
baseUrl,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCodexForwardCompatModel(ctx: ProviderResolveDynamicModelContext) {
|
||||
const trimmedModelId = ctx.modelId.trim();
|
||||
const lower = normalizeLowercaseStringOrEmpty(trimmedModelId);
|
||||
const synthBaseUrl = ctx.providerConfig?.baseUrl ?? OPENAI_CODEX_BASE_URL;
|
||||
|
||||
if (OPENAI_CODEX_GPT_56_MODEL_IDS.some((modelId) => modelId === lower)) {
|
||||
const model = ctx.modelRegistry.find(PROVIDER_ID, trimmedModelId) as
|
||||
| ProviderRuntimeModel
|
||||
| undefined;
|
||||
const registeredModel = withDefaultCodexContextMetadata({
|
||||
model: withCodexTransport(model, synthBaseUrl),
|
||||
contextWindow: OPENAI_CODEX_GPT_56_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_56_CONTEXT_TOKENS,
|
||||
});
|
||||
if (registeredModel) {
|
||||
return normalizeModelCompat({
|
||||
...registeredModel,
|
||||
thinkingLevelMap: {
|
||||
...OPENAI_CODEX_GPT_56_THINKING_LEVEL_MAP,
|
||||
...registeredModel.thinkingLevelMap,
|
||||
},
|
||||
} as ProviderRuntimeModel);
|
||||
}
|
||||
return normalizeModelCompat({
|
||||
id: trimmedModelId,
|
||||
name: trimmedModelId,
|
||||
api: "openai-chatgpt-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: synthBaseUrl,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: OPENAI_CODEX_GPT_56_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_56_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_CODEX_GPT_54_MAX_TOKENS,
|
||||
thinkingLevelMap: OPENAI_CODEX_GPT_56_THINKING_LEVEL_MAP,
|
||||
} as ProviderRuntimeModel);
|
||||
}
|
||||
|
||||
if (lower === OPENAI_CODEX_GPT_55_MODEL_ID) {
|
||||
const model = ctx.modelRegistry.find(PROVIDER_ID, trimmedModelId) as
|
||||
| ProviderRuntimeModel
|
||||
| undefined;
|
||||
return (
|
||||
withDefaultCodexContextMetadata({
|
||||
model: withCodexTransport(model, synthBaseUrl),
|
||||
contextWindow: OPENAI_CODEX_GPT_55_CODEX_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_55_DEFAULT_RUNTIME_CONTEXT_TOKENS,
|
||||
}) ??
|
||||
normalizeModelCompat({
|
||||
id: trimmedModelId,
|
||||
name: trimmedModelId,
|
||||
api: "openai-chatgpt-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: synthBaseUrl,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: OPENAI_CODEX_GPT_55_CODEX_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_55_DEFAULT_RUNTIME_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_CODEX_GPT_54_MAX_TOKENS,
|
||||
} as ProviderRuntimeModel)
|
||||
);
|
||||
}
|
||||
|
||||
let templateIds: readonly string[];
|
||||
let patch: Parameters<typeof cloneFirstTemplateModel>[0]["patch"];
|
||||
if (lower === OPENAI_CODEX_GPT_55_PRO_MODEL_ID) {
|
||||
templateIds = OPENAI_CODEX_GPT_55_PRO_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
contextWindow: OPENAI_CODEX_GPT_55_PRO_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_55_PRO_DEFAULT_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_CODEX_GPT_54_MAX_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_55_PRO_COST,
|
||||
};
|
||||
} else if (
|
||||
lower === OPENAI_CODEX_GPT_54_MODEL_ID ||
|
||||
lower === OPENAI_CODEX_GPT_54_LEGACY_MODEL_ID
|
||||
) {
|
||||
templateIds = OPENAI_CODEX_GPT_54_CATALOG_SYNTH_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
contextWindow: OPENAI_CODEX_GPT_54_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_54_DEFAULT_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_CODEX_GPT_54_MAX_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_54_COST,
|
||||
};
|
||||
} else if (lower === OPENAI_CODEX_GPT_54_PRO_MODEL_ID) {
|
||||
templateIds = OPENAI_CODEX_GPT_54_CATALOG_SYNTH_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
contextWindow: OPENAI_CODEX_GPT_54_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_54_DEFAULT_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_CODEX_GPT_54_MAX_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_54_PRO_COST,
|
||||
};
|
||||
} else if (lower === OPENAI_CODEX_GPT_54_MINI_MODEL_ID) {
|
||||
templateIds = OPENAI_CODEX_GPT_54_CATALOG_SYNTH_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
contextWindow: OPENAI_CODEX_GPT_54_MINI_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_54_DEFAULT_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_CODEX_GPT_54_MAX_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_54_MINI_COST,
|
||||
};
|
||||
} else if (lower === OPENAI_CODEX_GPT_53_SPARK_MODEL_ID) {
|
||||
templateIds = OPENAI_CODEX_GPT_54_CATALOG_SYNTH_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
input: ["text"],
|
||||
contextWindow: OPENAI_CODEX_GPT_53_SPARK_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_53_SPARK_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_CODEX_GPT_54_MAX_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_54_MINI_COST,
|
||||
};
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
patch = {
|
||||
...patch,
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: synthBaseUrl,
|
||||
};
|
||||
|
||||
return (
|
||||
cloneFirstTemplateModel({
|
||||
providerId: PROVIDER_ID,
|
||||
modelId:
|
||||
lower === OPENAI_CODEX_GPT_54_LEGACY_MODEL_ID
|
||||
? OPENAI_CODEX_GPT_54_MODEL_ID
|
||||
: trimmedModelId,
|
||||
templateIds,
|
||||
ctx,
|
||||
patch,
|
||||
}) ??
|
||||
normalizeModelCompat({
|
||||
id:
|
||||
lower === OPENAI_CODEX_GPT_54_LEGACY_MODEL_ID
|
||||
? OPENAI_CODEX_GPT_54_MODEL_ID
|
||||
: trimmedModelId,
|
||||
name:
|
||||
lower === OPENAI_CODEX_GPT_54_LEGACY_MODEL_ID
|
||||
? OPENAI_CODEX_GPT_54_MODEL_ID
|
||||
: trimmedModelId,
|
||||
api: "openai-chatgpt-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: synthBaseUrl,
|
||||
reasoning: true,
|
||||
input: patch?.input ?? ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: patch?.contextWindow ?? DEFAULT_CONTEXT_TOKENS,
|
||||
contextTokens: patch?.contextTokens,
|
||||
maxTokens: patch?.maxTokens ?? DEFAULT_CONTEXT_TOKENS,
|
||||
} as ProviderRuntimeModel)
|
||||
);
|
||||
}
|
||||
|
||||
function withDefaultCodexContextMetadata(params: {
|
||||
model: ProviderRuntimeModel | undefined;
|
||||
contextWindow: number;
|
||||
contextTokens: number;
|
||||
}): ProviderRuntimeModel | undefined {
|
||||
if (!params.model) {
|
||||
return undefined;
|
||||
}
|
||||
const contextTokens =
|
||||
typeof params.model.contextTokens === "number"
|
||||
? params.model.contextTokens
|
||||
: typeof params.model.contextWindow === "number" && params.model.contextWindow > 0
|
||||
? Math.min(params.contextTokens, params.model.contextWindow)
|
||||
: params.contextTokens;
|
||||
const input = params.model.input?.includes("image")
|
||||
? params.model.input
|
||||
: uniqueValues<"text" | "image">([...(params.model.input ?? ["text"]), "image"]);
|
||||
return {
|
||||
...params.model,
|
||||
input,
|
||||
contextWindow: params.contextWindow,
|
||||
contextTokens,
|
||||
};
|
||||
}
|
||||
|
||||
function withCodexTransport(
|
||||
model: ProviderRuntimeModel | undefined,
|
||||
baseUrl: string,
|
||||
): ProviderRuntimeModel | undefined {
|
||||
if (!model) {
|
||||
return undefined;
|
||||
}
|
||||
return normalizeModelCompat({
|
||||
...model,
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl,
|
||||
} as ProviderRuntimeModel);
|
||||
}
|
||||
|
||||
function buildCodexCredentialExtra(identity: {
|
||||
accountId?: string;
|
||||
chatgptPlanType?: string;
|
||||
}): Record<string, unknown> | undefined {
|
||||
const extra = {
|
||||
...(identity.accountId ? { accountId: identity.accountId } : {}),
|
||||
...(identity.chatgptPlanType ? { chatgptPlanType: identity.chatgptPlanType } : {}),
|
||||
};
|
||||
return Object.keys(extra).length > 0 ? extra : undefined;
|
||||
}
|
||||
|
||||
function buildOpenAICodexAuthConfigPatch(): NonNullable<ProviderAuthResult["configPatch"]> {
|
||||
return {
|
||||
agents: {
|
||||
defaults: {
|
||||
models: {
|
||||
[OPENAI_CODEX_DEFAULT_MODEL]: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function refreshOpenAICodexOAuthCredential(cred: OAuthCredential) {
|
||||
try {
|
||||
const { refreshOpenAICodexToken } = await import("./openai-chatgpt-provider.runtime.js");
|
||||
const refreshed = await refreshOpenAICodexToken(cred.refresh);
|
||||
const identity = resolveCodexAuthIdentity({
|
||||
accessToken: refreshed.access,
|
||||
email: cred.email,
|
||||
});
|
||||
return {
|
||||
...cred,
|
||||
...refreshed,
|
||||
type: "oauth" as const,
|
||||
provider: PROVIDER_ID,
|
||||
email: identity.email ?? cred.email,
|
||||
displayName: cred.displayName,
|
||||
...buildCodexCredentialExtra(identity),
|
||||
};
|
||||
} catch (error) {
|
||||
const message = formatErrorMessage(error);
|
||||
if (
|
||||
/extract\s+accountid\s+from\s+token/i.test(message) &&
|
||||
typeof cred.access === "string" &&
|
||||
cred.access.trim().length > 0
|
||||
) {
|
||||
return cred;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
type OpenAICodexOAuthContext = ProviderAuthContext & {
|
||||
signal?: AbortSignal;
|
||||
onManualCodeInput?: () => Promise<string>;
|
||||
};
|
||||
|
||||
async function runOpenAICodexOAuth(ctx: OpenAICodexOAuthContext) {
|
||||
const creds = await loginOpenAICodexOAuth({
|
||||
prompter: ctx.prompter,
|
||||
runtime: ctx.runtime,
|
||||
oauth: ctx.oauth,
|
||||
isRemote: ctx.isRemote,
|
||||
openUrl: ctx.openUrl,
|
||||
signal: ctx.signal,
|
||||
onManualCodeInput: ctx.onManualCodeInput,
|
||||
localBrowserMessage: "Complete sign-in in browser…",
|
||||
});
|
||||
if (!creds) {
|
||||
return { profiles: [] };
|
||||
}
|
||||
|
||||
const identity = resolveCodexAuthIdentity({
|
||||
accessToken: creds.access,
|
||||
email: readStringValue(creds.email),
|
||||
});
|
||||
|
||||
return buildOauthProviderAuthResult({
|
||||
providerId: PROVIDER_ID,
|
||||
defaultModel: OPENAI_CODEX_DEFAULT_MODEL,
|
||||
configPatch: buildOpenAICodexAuthConfigPatch(),
|
||||
access: creds.access,
|
||||
refresh: creds.refresh,
|
||||
expires: creds.expires,
|
||||
email: identity.email,
|
||||
profileName: identity.profileName,
|
||||
credentialExtra: buildCodexCredentialExtra(identity),
|
||||
});
|
||||
}
|
||||
|
||||
async function runOpenAICodexDeviceCode(ctx: ProviderAuthContext) {
|
||||
const spin = ctx.prompter.progress("Starting device code flow…");
|
||||
try {
|
||||
const creds = await loginOpenAICodexDeviceCode({
|
||||
onProgress: (message) => spin.update(message),
|
||||
onVerification: async ({ verificationUrl, userCode, expiresInMs }) => {
|
||||
const expiresInMinutes = Math.max(1, Math.round(expiresInMs / 60_000));
|
||||
// The prompter note is the user-facing TTY surface, so remote/headless
|
||||
// users need the code there; keep the persistent runtime log URL-only.
|
||||
await ctx.prompter.note(
|
||||
[
|
||||
ctx.isRemote
|
||||
? "Open this URL in your LOCAL browser and enter the code below."
|
||||
: "Open this URL in your browser and enter the code below.",
|
||||
`URL: ${verificationUrl}`,
|
||||
`Code: ${userCode}`,
|
||||
`Code expires in ${expiresInMinutes} minutes. Never share it.`,
|
||||
].join("\n"),
|
||||
"OpenAI Codex device code",
|
||||
);
|
||||
if (ctx.isRemote) {
|
||||
ctx.runtime.log(`\nOpen this URL in your LOCAL browser:\n\n${verificationUrl}\n`);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await ctx.openUrl(verificationUrl);
|
||||
ctx.runtime.log(`Open: ${verificationUrl}`);
|
||||
} catch {
|
||||
ctx.runtime.log(`Open manually: ${verificationUrl}`);
|
||||
}
|
||||
},
|
||||
});
|
||||
spin.stop("OpenAI device code complete");
|
||||
|
||||
const identity = resolveCodexAuthIdentity({
|
||||
accessToken: creds.access,
|
||||
});
|
||||
|
||||
return buildOauthProviderAuthResult({
|
||||
providerId: PROVIDER_ID,
|
||||
defaultModel: OPENAI_CODEX_DEFAULT_MODEL,
|
||||
configPatch: buildOpenAICodexAuthConfigPatch(),
|
||||
access: creds.access,
|
||||
refresh: creds.refresh,
|
||||
expires: creds.expires,
|
||||
email: identity.email,
|
||||
profileName: identity.profileName,
|
||||
credentialExtra: buildCodexCredentialExtra(identity),
|
||||
});
|
||||
} catch (error) {
|
||||
spin.stop("OpenAI device code failed");
|
||||
ctx.runtime.error(formatErrorMessage(error));
|
||||
await ctx.prompter.note(
|
||||
"Trouble with device code login? See https://docs.openclaw.ai/start/faq",
|
||||
"OAuth help",
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function buildOpenAICodexAuthDoctorHint(ctx: { profileId?: string }) {
|
||||
if (ctx.profileId !== CODEX_CLI_PROFILE_ID) {
|
||||
return undefined;
|
||||
}
|
||||
return "Deprecated profile. Run `openclaw models auth login --provider openai` or `openclaw configure`.";
|
||||
}
|
||||
|
||||
export function buildOpenAIChatGPTAuthMethods(): ProviderAuthMethod[] {
|
||||
return [
|
||||
{
|
||||
id: "oauth",
|
||||
label: OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
hint: OPENAI_CHATGPT_LOGIN_HINT,
|
||||
kind: "oauth",
|
||||
wizard: {
|
||||
choiceId: "openai",
|
||||
choiceLabel: OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
choiceHint: OPENAI_CHATGPT_LOGIN_HINT,
|
||||
assistantPriority: OPENAI_CODEX_LOGIN_ASSISTANT_PRIORITY,
|
||||
onboardingFeatured: true,
|
||||
...OPENAI_CODEX_WIZARD_GROUP,
|
||||
},
|
||||
run: async (ctx) => await runOpenAICodexOAuth(ctx),
|
||||
},
|
||||
{
|
||||
id: "device-code",
|
||||
label: OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
hint: OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
kind: "device_code",
|
||||
wizard: {
|
||||
choiceId: "openai-device-code",
|
||||
choiceLabel: OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
choiceHint: OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
assistantPriority: OPENAI_CODEX_DEVICE_PAIRING_ASSISTANT_PRIORITY,
|
||||
...OPENAI_CODEX_WIZARD_GROUP,
|
||||
},
|
||||
run: async (ctx) => await runOpenAICodexDeviceCode(ctx),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
export function buildOpenAICodexProviderHooks(): Pick<
|
||||
ProviderPlugin,
|
||||
| "resolveDynamicModel"
|
||||
| "buildAuthDoctorHint"
|
||||
| "resolveThinkingProfile"
|
||||
| "isModernModelRef"
|
||||
| "preferRuntimeResolvedModel"
|
||||
| "normalizeResolvedModel"
|
||||
| "normalizeTransport"
|
||||
| "resolveUsageAuth"
|
||||
| "fetchUsageSnapshot"
|
||||
| "refreshOAuth"
|
||||
| "augmentModelCatalog"
|
||||
| "resolveReasoningOutputMode"
|
||||
> {
|
||||
return {
|
||||
resolveDynamicModel: (ctx) => resolveCodexForwardCompatModel(ctx),
|
||||
buildAuthDoctorHint: (ctx) => buildOpenAICodexAuthDoctorHint(ctx),
|
||||
resolveThinkingProfile: ({ modelId }) => resolveOpenAICodexThinkingProfile(modelId),
|
||||
isModernModelRef: ({ modelId }) => matchesExactOrPrefix(modelId, OPENAI_CODEX_MODERN_MODEL_IDS),
|
||||
preferRuntimeResolvedModel: (ctx) => {
|
||||
if (!isOpenAIOrLegacyCodexProvider(ctx.provider)) {
|
||||
return false;
|
||||
}
|
||||
const id = ctx.modelId.trim().toLowerCase();
|
||||
return [
|
||||
...OPENAI_CODEX_GPT_56_MODEL_IDS,
|
||||
OPENAI_CODEX_GPT_55_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_55_PRO_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_PRO_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_54_MINI_MODEL_ID,
|
||||
OPENAI_CODEX_GPT_53_SPARK_MODEL_ID,
|
||||
].includes(id);
|
||||
},
|
||||
...buildOpenAIResponsesProviderHooks(),
|
||||
resolveReasoningOutputMode: () => "native",
|
||||
normalizeResolvedModel: (ctx) => {
|
||||
if (!isOpenAIOrLegacyCodexProvider(ctx.provider)) {
|
||||
return undefined;
|
||||
}
|
||||
const transportNormalized = normalizeCodexTransport(ctx.model);
|
||||
const imageCapable =
|
||||
applyOpenAICodexImageInputCapability({
|
||||
modelId: ctx.modelId,
|
||||
model: transportNormalized,
|
||||
}) ?? transportNormalized;
|
||||
return imageCapable === ctx.model ? undefined : imageCapable;
|
||||
},
|
||||
normalizeTransport: ({ provider, api, baseUrl }) => {
|
||||
if (!isOpenAIOrLegacyCodexProvider(provider)) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeCodexTransportFields({ api, baseUrl });
|
||||
if (normalized.api === api && normalized.baseUrl === baseUrl) {
|
||||
return undefined;
|
||||
}
|
||||
return normalized;
|
||||
},
|
||||
resolveUsageAuth: async (ctx) => await ctx.resolveOAuthToken(),
|
||||
fetchUsageSnapshot: async (ctx) =>
|
||||
await fetchCodexUsage(ctx.token, ctx.accountId, ctx.timeoutMs, ctx.fetchFn),
|
||||
refreshOAuth: async (cred) => await refreshOpenAICodexOAuthCredential(cred),
|
||||
augmentModelCatalog: (ctx) => {
|
||||
const gpt54Template = findCatalogTemplate({
|
||||
entries: ctx.entries,
|
||||
providerId: PROVIDER_ID,
|
||||
templateIds: OPENAI_CODEX_GPT_54_CATALOG_SYNTH_TEMPLATE_MODEL_IDS,
|
||||
});
|
||||
const gpt55ProTemplate = findCatalogTemplate({
|
||||
entries: ctx.entries,
|
||||
providerId: PROVIDER_ID,
|
||||
templateIds: OPENAI_CODEX_GPT_55_PRO_TEMPLATE_MODEL_IDS,
|
||||
});
|
||||
return [
|
||||
buildOpenAISyntheticCatalogEntry(gpt55ProTemplate, {
|
||||
id: OPENAI_CODEX_GPT_55_PRO_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_CODEX_GPT_55_PRO_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_55_PRO_DEFAULT_CONTEXT_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_55_PRO_COST,
|
||||
}),
|
||||
buildOpenAISyntheticCatalogEntry(gpt54Template, {
|
||||
id: OPENAI_CODEX_GPT_54_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_CODEX_GPT_54_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_54_DEFAULT_CONTEXT_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_54_COST,
|
||||
}),
|
||||
buildOpenAISyntheticCatalogEntry(gpt54Template, {
|
||||
id: OPENAI_CODEX_GPT_54_PRO_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_CODEX_GPT_54_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_54_DEFAULT_CONTEXT_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_54_PRO_COST,
|
||||
}),
|
||||
buildOpenAISyntheticCatalogEntry(gpt54Template, {
|
||||
id: OPENAI_CODEX_GPT_54_MINI_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_CODEX_GPT_54_MINI_NATIVE_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_CODEX_GPT_54_DEFAULT_CONTEXT_TOKENS,
|
||||
cost: OPENAI_CODEX_GPT_54_MINI_COST,
|
||||
}),
|
||||
].filter((entry): entry is NonNullable<typeof entry> => entry !== undefined);
|
||||
},
|
||||
};
|
||||
}
|
||||
4
extensions/openai/openai-chatgpt-shared.ts
Normal file
4
extensions/openai/openai-chatgpt-shared.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
// Openai plugin module implements openai chatgpt shared behavior.
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
|
||||
export const trimNonEmptyString = normalizeOptionalString;
|
||||
188
extensions/openai/openai-provider.live.test.ts
Normal file
188
extensions/openai/openai-provider.live.test.ts
Normal file
@@ -0,0 +1,188 @@
|
||||
// Openai tests cover openai provider plugin behavior.
|
||||
import OpenAI from "openai";
|
||||
import type { ProviderRuntimeModel } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildOpenAIProvider } from "./openai-provider.js";
|
||||
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY ?? "";
|
||||
const DEFAULT_LIVE_MODEL_IDS = ["chat-latest", "gpt-5.5", "gpt-5.4-mini", "gpt-5.4-nano"] as const;
|
||||
const liveEnabled = OPENAI_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1";
|
||||
const describeLive = liveEnabled ? describe : describe.skip;
|
||||
|
||||
type LiveModelCase = {
|
||||
modelId: string;
|
||||
templateId: string;
|
||||
templateName: string;
|
||||
cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
||||
contextWindow: number;
|
||||
maxTokens: number;
|
||||
reasoning: boolean;
|
||||
textVerbosity: "low" | "medium";
|
||||
};
|
||||
|
||||
function resolveLiveModelCase(modelId: string): LiveModelCase {
|
||||
switch (modelId) {
|
||||
case "chat-latest":
|
||||
return {
|
||||
modelId,
|
||||
templateId: "gpt-5.5",
|
||||
templateName: "GPT-5.5",
|
||||
cost: { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 },
|
||||
contextWindow: 400_000,
|
||||
maxTokens: 128_000,
|
||||
reasoning: false,
|
||||
textVerbosity: "medium",
|
||||
};
|
||||
case "gpt-5.5":
|
||||
return {
|
||||
modelId,
|
||||
templateId: "gpt-5.5",
|
||||
templateName: "GPT-5.5",
|
||||
cost: { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 },
|
||||
contextWindow: 1_000_000,
|
||||
maxTokens: 128_000,
|
||||
reasoning: true,
|
||||
textVerbosity: "low",
|
||||
};
|
||||
case "gpt-5.5-pro":
|
||||
return {
|
||||
modelId,
|
||||
templateId: "gpt-5.4-pro",
|
||||
templateName: "GPT-5.4 Pro",
|
||||
cost: { input: 30, output: 180, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1_000_000,
|
||||
maxTokens: 128_000,
|
||||
reasoning: true,
|
||||
textVerbosity: "low",
|
||||
};
|
||||
case "gpt-5.4":
|
||||
return {
|
||||
modelId,
|
||||
templateId: "gpt-5.2",
|
||||
templateName: "GPT-5.2",
|
||||
cost: { input: 1.75, output: 14, cacheRead: 0.175, cacheWrite: 0 },
|
||||
contextWindow: 400_000,
|
||||
maxTokens: 128_000,
|
||||
reasoning: true,
|
||||
textVerbosity: "low",
|
||||
};
|
||||
case "gpt-5.4-pro":
|
||||
return {
|
||||
modelId,
|
||||
templateId: "gpt-5.2-pro",
|
||||
templateName: "GPT-5.2 Pro",
|
||||
cost: { input: 21, output: 168, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400_000,
|
||||
maxTokens: 128_000,
|
||||
reasoning: true,
|
||||
textVerbosity: "low",
|
||||
};
|
||||
case "gpt-5.4-mini":
|
||||
return {
|
||||
modelId,
|
||||
templateId: "gpt-5-mini",
|
||||
templateName: "GPT-5 mini",
|
||||
cost: { input: 0.25, output: 2, cacheRead: 0.025, cacheWrite: 0 },
|
||||
contextWindow: 400_000,
|
||||
maxTokens: 128_000,
|
||||
reasoning: true,
|
||||
textVerbosity: "low",
|
||||
};
|
||||
case "gpt-5.4-nano":
|
||||
return {
|
||||
modelId,
|
||||
templateId: "gpt-5-nano",
|
||||
templateName: "GPT-5 nano",
|
||||
cost: { input: 0.05, output: 0.4, cacheRead: 0.005, cacheWrite: 0 },
|
||||
contextWindow: 400_000,
|
||||
maxTokens: 128_000,
|
||||
reasoning: true,
|
||||
textVerbosity: "low",
|
||||
};
|
||||
default:
|
||||
throw new Error(`Unsupported live OpenAI model: ${modelId}`);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveLiveModelCases(raw?: string): LiveModelCase[] {
|
||||
const requested: string[] = [];
|
||||
for (const value of raw?.split(",") ?? []) {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed.length > 0) {
|
||||
requested.push(trimmed);
|
||||
}
|
||||
}
|
||||
const modelIds = requested.length ? requested : [...DEFAULT_LIVE_MODEL_IDS];
|
||||
return [...new Set(modelIds)].map((modelId) => resolveLiveModelCase(modelId));
|
||||
}
|
||||
|
||||
describeLive("buildOpenAIProvider live", () => {
|
||||
it.each(resolveLiveModelCases(process.env.OPENCLAW_LIVE_OPENAI_MODELS))(
|
||||
"resolves %s and completes through the OpenAI responses API",
|
||||
async (liveCase) => {
|
||||
const provider = buildOpenAIProvider();
|
||||
const registry = {
|
||||
find(providerId: string, id: string) {
|
||||
if (providerId !== "openai") {
|
||||
return null;
|
||||
}
|
||||
if (id === liveCase.templateId) {
|
||||
return {
|
||||
id: liveCase.templateId,
|
||||
name: liveCase.templateName,
|
||||
provider: "openai",
|
||||
api: "openai-completions",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
reasoning: liveCase.reasoning,
|
||||
input: ["text", "image"],
|
||||
cost: liveCase.cost,
|
||||
contextWindow: liveCase.contextWindow,
|
||||
maxTokens: liveCase.maxTokens,
|
||||
} satisfies ProviderRuntimeModel;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
const resolved =
|
||||
registry.find("openai", liveCase.modelId) ??
|
||||
provider.resolveDynamicModel?.({
|
||||
provider: "openai",
|
||||
modelId: liveCase.modelId,
|
||||
modelRegistry: registry as never,
|
||||
});
|
||||
if (!resolved) {
|
||||
throw new Error(`openai provider did not resolve ${liveCase.modelId}`);
|
||||
}
|
||||
|
||||
const normalized = provider.normalizeResolvedModel?.({
|
||||
provider: "openai",
|
||||
modelId: resolved.id,
|
||||
model: resolved,
|
||||
});
|
||||
|
||||
expect(normalized?.provider).toBe("openai");
|
||||
expect(normalized?.id).toBe(liveCase.modelId);
|
||||
expect(normalized?.api).toBe("openai-responses");
|
||||
expect(normalized?.baseUrl).toBe("https://api.openai.com/v1");
|
||||
expect(normalized?.reasoning).toEqual(liveCase.reasoning);
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: OPENAI_API_KEY,
|
||||
baseURL: normalized?.baseUrl,
|
||||
});
|
||||
|
||||
const response = await client.responses.create({
|
||||
model: normalized?.id ?? liveCase.modelId,
|
||||
instructions: "Return exactly OK and no other text.",
|
||||
input: "Return exactly OK.",
|
||||
max_output_tokens: 64,
|
||||
...(liveCase.reasoning ? { reasoning: { effort: "none" as const } } : {}),
|
||||
text: { verbosity: liveCase.textVerbosity },
|
||||
});
|
||||
|
||||
expect(response.output_text.trim()).toMatch(/^OK[.!]?$/);
|
||||
},
|
||||
180_000,
|
||||
);
|
||||
});
|
||||
1656
extensions/openai/openai-provider.test.ts
Normal file
1656
extensions/openai/openai-provider.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
949
extensions/openai/openai-provider.ts
Normal file
949
extensions/openai/openai-provider.ts
Normal file
@@ -0,0 +1,949 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import type {
|
||||
ProviderResolveDynamicModelContext,
|
||||
ProviderRuntimeModel,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key";
|
||||
import {
|
||||
buildLiveModelProviderConfig,
|
||||
getCachedLiveProviderModelRows,
|
||||
type LiveModelCatalogFetchGuard,
|
||||
} from "openclaw/plugin-sdk/provider-catalog-live-runtime";
|
||||
import { buildManifestModelProviderConfig } from "openclaw/plugin-sdk/provider-catalog-shared";
|
||||
import {
|
||||
DEFAULT_CONTEXT_TOKENS,
|
||||
normalizeModelCompat,
|
||||
normalizeProviderId,
|
||||
type ModelDefinitionConfig,
|
||||
type ModelProviderConfig,
|
||||
type ProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { OPENAI_ACCOUNT_WIZARD_GROUP, OPENAI_API_KEY_LABEL } from "./auth-choice-copy.js";
|
||||
import {
|
||||
OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
isOpenAIApiBaseUrl,
|
||||
isOpenAICodexBaseUrl,
|
||||
resolveOpenAIDefaultBaseUrl,
|
||||
} from "./base-url.js";
|
||||
import { applyOpenAIConfig, OPENAI_DEFAULT_MODEL } from "./default-models.js";
|
||||
import {
|
||||
buildOpenAIChatGPTAuthMethods,
|
||||
buildOpenAICodexProviderHooks,
|
||||
} from "./openai-chatgpt-provider.js";
|
||||
import manifest from "./openclaw.plugin.json" with { type: "json" };
|
||||
import {
|
||||
buildOpenAIResponsesProviderHooks,
|
||||
buildOpenAISyntheticCatalogEntry,
|
||||
cloneFirstTemplateModel,
|
||||
findCatalogTemplate,
|
||||
matchesExactOrPrefix,
|
||||
} from "./shared.js";
|
||||
import { resolveUnifiedOpenAIThinkingProfile } from "./thinking-policy.js";
|
||||
|
||||
const PROVIDER_ID = "openai";
|
||||
const OPENAI_MODELS_ENDPOINT = "https://api.openai.com/v1/models";
|
||||
const OPENAI_CODEX_MODELS_ENDPOINT = `${OPENAI_CODEX_RESPONSES_BASE_URL}/models?client_version=1.0.0`;
|
||||
const OPENAI_MODELS_CACHE_TTL_MS = 60_000;
|
||||
const OPENAI_CODEX_MODELS_CACHE_TTL_MS = 60_000;
|
||||
const OPENAI_CHAT_LATEST_MODEL_ID = "chat-latest";
|
||||
const OPENAI_GPT_56_SOL_MODEL_ID = "gpt-5.6-sol";
|
||||
const OPENAI_GPT_56_TERRA_MODEL_ID = "gpt-5.6-terra";
|
||||
const OPENAI_GPT_56_LUNA_MODEL_ID = "gpt-5.6-luna";
|
||||
const OPENAI_GPT_55_MODEL_ID = "gpt-5.5";
|
||||
const OPENAI_GPT_55_PRO_MODEL_ID = "gpt-5.5-pro";
|
||||
const OPENAI_GPT_54_MODEL_ID = "gpt-5.4";
|
||||
const OPENAI_GPT_54_PRO_MODEL_ID = "gpt-5.4-pro";
|
||||
const OPENAI_GPT_54_MINI_MODEL_ID = "gpt-5.4-mini";
|
||||
const OPENAI_GPT_54_NANO_MODEL_ID = "gpt-5.4-nano";
|
||||
const OPENAI_GPT_53_CODEX_SPARK_MODEL_ID = "gpt-5.3-codex-spark";
|
||||
const OPENAI_GPT_56_CONTEXT_TOKENS = 372_000;
|
||||
const OPENAI_GPT_55_CONTEXT_WINDOW = 1_000_000;
|
||||
const OPENAI_GPT_55_CONTEXT_TOKENS = 272_000;
|
||||
const OPENAI_GPT_55_PRO_CONTEXT_TOKENS = 1_000_000;
|
||||
const OPENAI_GPT_54_CONTEXT_TOKENS = 1_050_000;
|
||||
const OPENAI_GPT_54_PRO_CONTEXT_TOKENS = 1_050_000;
|
||||
const OPENAI_GPT_54_MINI_CONTEXT_TOKENS = 400_000;
|
||||
const OPENAI_GPT_54_NANO_CONTEXT_TOKENS = 400_000;
|
||||
const OPENAI_GPT_54_MAX_TOKENS = 128_000;
|
||||
const OPENAI_CHAT_LATEST_COST = { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 } as const;
|
||||
const OPENAI_GPT_56_SOL_COST = {
|
||||
input: 5,
|
||||
output: 30,
|
||||
cacheRead: 0.5,
|
||||
cacheWrite: 6.25,
|
||||
} as const;
|
||||
const OPENAI_GPT_56_TERRA_COST = {
|
||||
input: 2.5,
|
||||
output: 15,
|
||||
cacheRead: 0.25,
|
||||
cacheWrite: 3.125,
|
||||
} as const;
|
||||
const OPENAI_GPT_56_LUNA_COST = {
|
||||
input: 1,
|
||||
output: 6,
|
||||
cacheRead: 0.1,
|
||||
cacheWrite: 1.25,
|
||||
} as const;
|
||||
const OPENAI_GPT_55_COST = { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 } as const;
|
||||
const OPENAI_GPT_55_PRO_COST = { input: 30, output: 180, cacheRead: 0, cacheWrite: 0 } as const;
|
||||
const OPENAI_GPT_54_COST = { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 0 } as const;
|
||||
const OPENAI_GPT_54_PRO_COST = { input: 30, output: 180, cacheRead: 0, cacheWrite: 0 } as const;
|
||||
const OPENAI_GPT_54_MINI_COST = {
|
||||
input: 0.75,
|
||||
output: 4.5,
|
||||
cacheRead: 0.075,
|
||||
cacheWrite: 0,
|
||||
} as const;
|
||||
const OPENAI_GPT_54_NANO_COST = {
|
||||
input: 0.2,
|
||||
output: 1.25,
|
||||
cacheRead: 0.02,
|
||||
cacheWrite: 0,
|
||||
} as const;
|
||||
const OPENAI_GPT_55_PRO_TEMPLATE_MODEL_IDS = [
|
||||
OPENAI_GPT_54_PRO_MODEL_ID,
|
||||
OPENAI_GPT_54_MODEL_ID,
|
||||
] as const;
|
||||
const OPENAI_GPT_55_MEDIA_INPUT = {
|
||||
image: { maxSidePx: 6000, preferredSidePx: 2048, tokenMode: "detail" },
|
||||
} as const satisfies ProviderRuntimeModel["mediaInput"];
|
||||
const OPENAI_GPT_54_TEMPLATE_MODEL_IDS = [OPENAI_GPT_55_MODEL_ID] as const;
|
||||
const OPENAI_GPT_54_PRO_TEMPLATE_MODEL_IDS = [OPENAI_GPT_55_PRO_MODEL_ID] as const;
|
||||
const OPENAI_GPT_54_MINI_TEMPLATE_MODEL_IDS = ["gpt-5-mini"] as const;
|
||||
const OPENAI_GPT_54_NANO_TEMPLATE_MODEL_IDS = ["gpt-5-nano", "gpt-5-mini"] as const;
|
||||
const OPENAI_CHAT_LATEST_TEMPLATE_MODEL_IDS = [
|
||||
OPENAI_GPT_55_MODEL_ID,
|
||||
OPENAI_GPT_54_MODEL_ID,
|
||||
] as const;
|
||||
const OPENAI_GPT_56_TEMPLATE_MODEL_IDS = [OPENAI_GPT_55_MODEL_ID] as const;
|
||||
const OPENAI_GPT_56_THINKING_LEVEL_MAP = {
|
||||
off: null,
|
||||
xhigh: "xhigh",
|
||||
max: "max",
|
||||
} as const;
|
||||
const OPENAI_MODERN_MODEL_IDS = [
|
||||
OPENAI_CHAT_LATEST_MODEL_ID,
|
||||
OPENAI_GPT_56_SOL_MODEL_ID,
|
||||
OPENAI_GPT_56_TERRA_MODEL_ID,
|
||||
OPENAI_GPT_56_LUNA_MODEL_ID,
|
||||
OPENAI_GPT_55_MODEL_ID,
|
||||
OPENAI_GPT_55_PRO_MODEL_ID,
|
||||
OPENAI_GPT_54_MODEL_ID,
|
||||
OPENAI_GPT_54_PRO_MODEL_ID,
|
||||
OPENAI_GPT_54_MINI_MODEL_ID,
|
||||
OPENAI_GPT_54_NANO_MODEL_ID,
|
||||
OPENAI_GPT_53_CODEX_SPARK_MODEL_ID,
|
||||
] as const;
|
||||
const OPENAI_UNKNOWN_MODEL_COST = {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
} satisfies ModelDefinitionConfig["cost"];
|
||||
|
||||
const OPENAI_MANIFEST_PROVIDER = buildManifestModelProviderConfig({
|
||||
providerId: PROVIDER_ID,
|
||||
catalog: manifest.modelCatalog.providers.openai,
|
||||
});
|
||||
|
||||
type BuildOpenAILiveProviderConfigParams = {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
discoveryApiKey?: string;
|
||||
env?: Record<string, string | undefined>;
|
||||
fetchGuard?: LiveModelCatalogFetchGuard;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
function shouldFetchOpenAILiveModels(baseUrl: string): boolean {
|
||||
return /^https:/i.test(baseUrl) && isOpenAIApiBaseUrl(baseUrl);
|
||||
}
|
||||
|
||||
function buildOpenAIManifestModelsForBaseUrl(baseUrl: string): ModelDefinitionConfig[] {
|
||||
return OPENAI_MANIFEST_PROVIDER.models.map((model) =>
|
||||
model.api === "openai-chatgpt-responses" || isOpenAICodexBaseUrl(model.baseUrl)
|
||||
? { ...model }
|
||||
: { ...model, baseUrl },
|
||||
);
|
||||
}
|
||||
|
||||
export async function buildOpenAILiveProviderConfig(
|
||||
params: BuildOpenAILiveProviderConfigParams,
|
||||
): Promise<ModelProviderConfig> {
|
||||
const baseUrl =
|
||||
normalizeOptionalString(params.baseUrl) ?? resolveOpenAIDefaultBaseUrl(params.env);
|
||||
const models = buildOpenAIManifestModelsForBaseUrl(baseUrl);
|
||||
if (!shouldFetchOpenAILiveModels(baseUrl)) {
|
||||
return {
|
||||
baseUrl,
|
||||
api: "openai-responses",
|
||||
apiKey: params.apiKey,
|
||||
models,
|
||||
};
|
||||
}
|
||||
return await buildLiveModelProviderConfig({
|
||||
providerId: PROVIDER_ID,
|
||||
endpoint: OPENAI_MODELS_ENDPOINT,
|
||||
providerConfig: {
|
||||
baseUrl,
|
||||
api: "openai-responses",
|
||||
},
|
||||
models,
|
||||
apiKey: params.apiKey,
|
||||
discoveryApiKey: params.discoveryApiKey,
|
||||
fetchGuard: params.fetchGuard,
|
||||
signal: params.signal,
|
||||
ttlMs: OPENAI_MODELS_CACHE_TTL_MS,
|
||||
auditContext: "openai-model-discovery",
|
||||
});
|
||||
}
|
||||
|
||||
function readCodexModelString(row: unknown, key: string): string | undefined {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
||||
return undefined;
|
||||
}
|
||||
const value = (row as Record<string, unknown>)[key];
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function readCodexModelPositiveInteger(row: unknown, keys: readonly string[]): number | undefined {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
||||
return undefined;
|
||||
}
|
||||
const record = row as Record<string, unknown>;
|
||||
for (const key of keys) {
|
||||
const value = record[key];
|
||||
if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readCodexModelStringArray(row: unknown, keys: readonly string[]): readonly string[] {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
||||
return [];
|
||||
}
|
||||
const record = row as Record<string, unknown>;
|
||||
for (const key of keys) {
|
||||
const value = record[key];
|
||||
if (Array.isArray(value)) {
|
||||
return value.filter((entry): entry is string => typeof entry === "string");
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function readCodexReasoningLevels(row: unknown): readonly string[] {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
||||
return [];
|
||||
}
|
||||
const record = row as Record<string, unknown>;
|
||||
const value = record.supported_reasoning_levels ?? record.supportedReasoningLevels;
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((entry) => {
|
||||
if (typeof entry === "string" && entry.trim().length > 0) {
|
||||
return [entry.trim()];
|
||||
}
|
||||
if (entry && typeof entry === "object" && !Array.isArray(entry)) {
|
||||
const effort = (entry as { effort?: unknown }).effort;
|
||||
return typeof effort === "string" && effort.trim().length > 0 ? [effort.trim()] : [];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
}
|
||||
|
||||
function readCodexModelBoolean(row: unknown, key: string): boolean | undefined {
|
||||
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
||||
return undefined;
|
||||
}
|
||||
const value = (row as Record<string, unknown>)[key];
|
||||
return typeof value === "boolean" ? value : undefined;
|
||||
}
|
||||
|
||||
function readCodexModelRows(body: unknown): readonly unknown[] {
|
||||
if (!body || typeof body !== "object" || Array.isArray(body)) {
|
||||
throw new Error("OpenAI Codex model discovery response must be { models: [] }");
|
||||
}
|
||||
const models = (body as { models?: unknown }).models;
|
||||
if (!Array.isArray(models)) {
|
||||
throw new Error("OpenAI Codex model discovery response must be { models: [] }");
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
function shouldIncludeCodexModelRow(row: unknown): boolean {
|
||||
const visibility = normalizeLowercaseStringOrEmpty(readCodexModelString(row, "visibility") ?? "");
|
||||
if (visibility && visibility !== "list") {
|
||||
return false;
|
||||
}
|
||||
const showInPicker =
|
||||
readCodexModelBoolean(row, "show_in_picker") ?? readCodexModelBoolean(row, "showInPicker");
|
||||
return showInPicker !== false;
|
||||
}
|
||||
|
||||
function resolveCodexModelInput(
|
||||
row: unknown,
|
||||
fallback: ModelDefinitionConfig | undefined,
|
||||
): ModelDefinitionConfig["input"] {
|
||||
const rawModalities = readCodexModelStringArray(row, ["input_modalities", "inputModalities"]);
|
||||
if (rawModalities.length === 0) {
|
||||
return fallback?.input ?? ["text", "image"];
|
||||
}
|
||||
const modalities = new Set(
|
||||
rawModalities.map((modality) => normalizeLowercaseStringOrEmpty(modality)),
|
||||
);
|
||||
const input = new Set<ModelDefinitionConfig["input"][number]>();
|
||||
if (modalities.has("text")) {
|
||||
input.add("text");
|
||||
}
|
||||
if (modalities.has("image") || modalities.has("vision")) {
|
||||
input.add("image");
|
||||
}
|
||||
if (modalities.has("audio")) {
|
||||
input.add("audio");
|
||||
}
|
||||
if (modalities.has("video")) {
|
||||
input.add("video");
|
||||
}
|
||||
return input.size > 0 ? [...input] : (fallback?.input ?? ["text", "image"]);
|
||||
}
|
||||
|
||||
function resolveCodexModelFallback(modelId: string): ModelDefinitionConfig | undefined {
|
||||
return OPENAI_MANIFEST_PROVIDER.models.find(
|
||||
(model) =>
|
||||
normalizeLowercaseStringOrEmpty(model.id) === normalizeLowercaseStringOrEmpty(modelId),
|
||||
);
|
||||
}
|
||||
|
||||
function buildOpenAICodexModelFromLiveRow(row: unknown): ModelDefinitionConfig | undefined {
|
||||
if (!shouldIncludeCodexModelRow(row)) {
|
||||
return undefined;
|
||||
}
|
||||
const modelId = readCodexModelString(row, "slug") ?? readCodexModelString(row, "id");
|
||||
if (!modelId) {
|
||||
return undefined;
|
||||
}
|
||||
const fallback = resolveCodexModelFallback(modelId);
|
||||
const reasoningLevels = readCodexReasoningLevels(row);
|
||||
const contextTokens = readCodexModelPositiveInteger(row, ["context_window", "contextWindow"]);
|
||||
const contextWindow =
|
||||
readCodexModelPositiveInteger(row, ["max_context_window", "maxContextWindow"]) ??
|
||||
fallback?.contextWindow ??
|
||||
contextTokens ??
|
||||
DEFAULT_CONTEXT_TOKENS;
|
||||
const maxTokens =
|
||||
readCodexModelPositiveInteger(row, [
|
||||
"max_output_tokens",
|
||||
"maxOutputTokens",
|
||||
"max_completion_tokens",
|
||||
"maxCompletionTokens",
|
||||
]) ??
|
||||
fallback?.maxTokens ??
|
||||
OPENAI_GPT_54_MAX_TOKENS;
|
||||
const compat =
|
||||
reasoningLevels.length > 0
|
||||
? {
|
||||
...fallback?.compat,
|
||||
supportsReasoningEffort: true,
|
||||
supportedReasoningEfforts: [...reasoningLevels],
|
||||
}
|
||||
: fallback?.compat;
|
||||
const thinkingLevelMap = {
|
||||
...fallback?.thinkingLevelMap,
|
||||
...(normalizeLowercaseStringOrEmpty(modelId).startsWith("gpt-5.6") ? { off: null } : {}),
|
||||
...(reasoningLevels.includes("xhigh") ? { xhigh: "xhigh" as const } : {}),
|
||||
...(reasoningLevels.includes("max") ? { max: "max" as const } : {}),
|
||||
};
|
||||
|
||||
return {
|
||||
id: modelId,
|
||||
name: readCodexModelString(row, "display_name") ?? fallback?.name ?? modelId,
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
reasoning: reasoningLevels.length > 0 || fallback?.reasoning || false,
|
||||
input: resolveCodexModelInput(row, fallback),
|
||||
cost: fallback?.cost ?? OPENAI_UNKNOWN_MODEL_COST,
|
||||
contextWindow,
|
||||
maxTokens,
|
||||
...((contextTokens ?? fallback?.contextTokens)
|
||||
? { contextTokens: contextTokens ?? fallback?.contextTokens }
|
||||
: {}),
|
||||
...(fallback?.mediaInput ? { mediaInput: fallback.mediaInput } : {}),
|
||||
...(compat ? { compat } : {}),
|
||||
...(Object.keys(thinkingLevelMap).length > 0 ? { thinkingLevelMap } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function buildOpenAICodexStaticProviderConfig(): ModelProviderConfig {
|
||||
return {
|
||||
baseUrl: OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
api: "openai-chatgpt-responses",
|
||||
auth: "oauth",
|
||||
models: OPENAI_MANIFEST_PROVIDER.models,
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildOpenAICodexLiveProviderConfig(params: {
|
||||
discoveryApiKey: string;
|
||||
accountId?: string;
|
||||
fetchGuard?: LiveModelCatalogFetchGuard;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ModelProviderConfig> {
|
||||
try {
|
||||
const rows = await getCachedLiveProviderModelRows({
|
||||
providerId: PROVIDER_ID,
|
||||
endpoint: OPENAI_CODEX_MODELS_ENDPOINT,
|
||||
discoveryApiKey: params.discoveryApiKey,
|
||||
fetchGuard: params.fetchGuard,
|
||||
signal: params.signal,
|
||||
ttlMs: OPENAI_CODEX_MODELS_CACHE_TTL_MS,
|
||||
auditContext: "openai-codex-model-discovery",
|
||||
readRows: readCodexModelRows,
|
||||
buildRequestHeaders: ({ discoveryApiKey }) => ({
|
||||
Accept: "application/json",
|
||||
...(discoveryApiKey ? { Authorization: `Bearer ${discoveryApiKey}` } : {}),
|
||||
...(params.accountId ? { "ChatGPT-Account-ID": params.accountId } : {}),
|
||||
}),
|
||||
cacheKeyParts: [
|
||||
PROVIDER_ID,
|
||||
"codex-model-rows",
|
||||
OPENAI_CODEX_MODELS_ENDPOINT,
|
||||
params.discoveryApiKey,
|
||||
params.accountId ?? "",
|
||||
],
|
||||
});
|
||||
const models = rows
|
||||
.map(buildOpenAICodexModelFromLiveRow)
|
||||
.filter((model): model is ModelDefinitionConfig => Boolean(model));
|
||||
if (models.length > 0) {
|
||||
return {
|
||||
baseUrl: OPENAI_CODEX_RESPONSES_BASE_URL,
|
||||
api: "openai-chatgpt-responses",
|
||||
auth: "oauth",
|
||||
models,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Codex/ChatGPT discovery is advisory. Static OpenAI rows stay available
|
||||
// when OAuth refresh or the remote model list is unavailable.
|
||||
}
|
||||
return buildOpenAICodexStaticProviderConfig();
|
||||
}
|
||||
|
||||
function isCodexCatalogAuthMode(mode: string): boolean {
|
||||
return mode === "oauth" || mode === "token";
|
||||
}
|
||||
|
||||
function resolveOpenAICatalogBaseUrl(ctx: {
|
||||
config?: { models?: { providers?: Record<string, { baseUrl?: string } | undefined> } };
|
||||
env?: Record<string, string | undefined>;
|
||||
}): string {
|
||||
const configuredProvider = Object.entries(ctx.config?.models?.providers ?? {}).find(
|
||||
([providerId]) => normalizeProviderId(providerId) === PROVIDER_ID,
|
||||
)?.[1];
|
||||
return (
|
||||
normalizeOptionalString(configuredProvider?.baseUrl) ??
|
||||
resolveOpenAIDefaultBaseUrl(ctx.env ?? process.env)
|
||||
);
|
||||
}
|
||||
|
||||
function shouldUseOpenAIResponsesTransport(params: {
|
||||
provider: string;
|
||||
api?: string | null;
|
||||
baseUrl?: string;
|
||||
}): boolean {
|
||||
if (params.api !== "openai-completions") {
|
||||
return false;
|
||||
}
|
||||
const isOwnerProvider = normalizeProviderId(params.provider) === PROVIDER_ID;
|
||||
if (isOwnerProvider) {
|
||||
return !params.baseUrl || isOpenAIApiBaseUrl(params.baseUrl);
|
||||
}
|
||||
return typeof params.baseUrl === "string" && isOpenAIApiBaseUrl(params.baseUrl);
|
||||
}
|
||||
|
||||
function isOpenAIProvider(provider: string | undefined): boolean {
|
||||
const normalized = normalizeProviderId(provider ?? "");
|
||||
return normalized === PROVIDER_ID;
|
||||
}
|
||||
|
||||
function normalizeOpenAITransport(model: ProviderRuntimeModel): ProviderRuntimeModel {
|
||||
const useResponsesTransport = shouldUseOpenAIResponsesTransport({
|
||||
provider: model.provider,
|
||||
api: model.api,
|
||||
baseUrl: model.baseUrl,
|
||||
});
|
||||
|
||||
if (!useResponsesTransport) {
|
||||
return model;
|
||||
}
|
||||
|
||||
return {
|
||||
...model,
|
||||
api: "openai-responses",
|
||||
};
|
||||
}
|
||||
|
||||
function shouldUseCodexResponsesHooks(params: {
|
||||
provider?: string;
|
||||
api?: ProviderRuntimeModel["api"] | null;
|
||||
baseUrl?: string;
|
||||
}): boolean {
|
||||
if (params.api === "openai-chatgpt-responses") {
|
||||
return true;
|
||||
}
|
||||
return typeof params.baseUrl === "string" && isOpenAICodexBaseUrl(params.baseUrl);
|
||||
}
|
||||
|
||||
function resolveConfiguredAuthTransport(
|
||||
ctx: Pick<
|
||||
ProviderResolveDynamicModelContext,
|
||||
"authProfileId" | "authProfileMode" | "config" | "providerConfig"
|
||||
>,
|
||||
) {
|
||||
if (ctx.authProfileMode === "oauth" || ctx.authProfileMode === "token") {
|
||||
return "codex";
|
||||
}
|
||||
if (ctx.authProfileMode === "api_key" || ctx.authProfileMode === "aws-sdk") {
|
||||
return "responses";
|
||||
}
|
||||
const authMode = ctx.providerConfig?.auth;
|
||||
if (authMode === "oauth" || authMode === "token") {
|
||||
return "codex";
|
||||
}
|
||||
if (authMode === "api-key") {
|
||||
return "responses";
|
||||
}
|
||||
|
||||
const auth = ctx.config?.auth;
|
||||
const profiles = auth?.profiles ?? {};
|
||||
const orderedProfileIds = auth?.order?.[PROVIDER_ID] ?? [];
|
||||
for (const profileId of orderedProfileIds) {
|
||||
const mode = profiles[profileId]?.mode;
|
||||
if (mode === "oauth" || mode === "token") {
|
||||
return "codex";
|
||||
}
|
||||
if (mode === "api_key") {
|
||||
return "responses";
|
||||
}
|
||||
}
|
||||
|
||||
const providerModes = Object.values(profiles)
|
||||
.filter((profile) => normalizeProviderId(profile.provider) === PROVIDER_ID)
|
||||
.map((profile) => profile.mode);
|
||||
if (providerModes.some((mode) => mode === "oauth" || mode === "token")) {
|
||||
return "codex";
|
||||
}
|
||||
if (providerModes.includes("api_key")) {
|
||||
return "responses";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function shouldResolveDynamicModelThroughCodex(ctx: ProviderResolveDynamicModelContext): boolean {
|
||||
if (
|
||||
shouldUseCodexResponsesHooks({
|
||||
provider: ctx.provider,
|
||||
api: ctx.providerConfig?.api,
|
||||
baseUrl: ctx.providerConfig?.baseUrl,
|
||||
})
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (ctx.providerConfig?.baseUrl && !isOpenAIApiBaseUrl(ctx.providerConfig.baseUrl)) {
|
||||
return false;
|
||||
}
|
||||
const authTransport = resolveConfiguredAuthTransport(ctx);
|
||||
if (authTransport) {
|
||||
return authTransport === "codex";
|
||||
}
|
||||
return ctx.agentRuntimeId === "codex";
|
||||
}
|
||||
|
||||
function buildOpenAIUnknownModelHint(modelId: string): string | undefined {
|
||||
const normalized = normalizeLowercaseStringOrEmpty(modelId);
|
||||
if (normalized !== OPENAI_GPT_53_CODEX_SPARK_MODEL_ID) {
|
||||
return undefined;
|
||||
}
|
||||
return "gpt-5.3-codex-spark is available only through ChatGPT/Codex OAuth. Run `openclaw models auth login --provider openai` and use openai/gpt-5.3-codex-spark with that OAuth profile; OpenAI API-key auth cannot use this model.";
|
||||
}
|
||||
|
||||
function resolveOpenAIGptForwardCompatModel(ctx: ProviderResolveDynamicModelContext) {
|
||||
const trimmedModelId = ctx.modelId.trim();
|
||||
const lower = normalizeLowercaseStringOrEmpty(trimmedModelId);
|
||||
let templateIds: readonly string[];
|
||||
let patch: Partial<ProviderRuntimeModel>;
|
||||
if (lower === OPENAI_CHAT_LATEST_MODEL_ID) {
|
||||
templateIds = OPENAI_CHAT_LATEST_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: OPENAI_CHAT_LATEST_COST,
|
||||
contextWindow: 400_000,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
};
|
||||
} else if (
|
||||
lower === OPENAI_GPT_56_SOL_MODEL_ID ||
|
||||
lower === OPENAI_GPT_56_TERRA_MODEL_ID ||
|
||||
lower === OPENAI_GPT_56_LUNA_MODEL_ID
|
||||
) {
|
||||
templateIds = OPENAI_GPT_56_TEMPLATE_MODEL_IDS;
|
||||
const cost =
|
||||
lower === OPENAI_GPT_56_SOL_MODEL_ID
|
||||
? OPENAI_GPT_56_SOL_COST
|
||||
: lower === OPENAI_GPT_56_TERRA_MODEL_ID
|
||||
? OPENAI_GPT_56_TERRA_COST
|
||||
: OPENAI_GPT_56_LUNA_COST;
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost,
|
||||
contextWindow: OPENAI_GPT_56_CONTEXT_TOKENS,
|
||||
contextTokens: OPENAI_GPT_56_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
thinkingLevelMap: OPENAI_GPT_56_THINKING_LEVEL_MAP,
|
||||
};
|
||||
} else if (lower === OPENAI_GPT_55_MODEL_ID) {
|
||||
templateIds = [OPENAI_GPT_55_MODEL_ID, OPENAI_GPT_54_MODEL_ID];
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
mediaInput: OPENAI_GPT_55_MEDIA_INPUT,
|
||||
cost: OPENAI_GPT_55_COST,
|
||||
contextWindow: OPENAI_GPT_55_CONTEXT_WINDOW,
|
||||
contextTokens: OPENAI_GPT_55_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
};
|
||||
} else if (lower === OPENAI_GPT_55_PRO_MODEL_ID) {
|
||||
templateIds = OPENAI_GPT_55_PRO_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: OPENAI_GPT_55_PRO_COST,
|
||||
contextWindow: OPENAI_GPT_55_PRO_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
};
|
||||
} else if (lower === OPENAI_GPT_54_MODEL_ID) {
|
||||
templateIds = OPENAI_GPT_54_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: OPENAI_GPT_54_COST,
|
||||
contextWindow: OPENAI_GPT_54_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
};
|
||||
} else if (lower === OPENAI_GPT_54_PRO_MODEL_ID) {
|
||||
templateIds = OPENAI_GPT_54_PRO_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: OPENAI_GPT_54_PRO_COST,
|
||||
contextWindow: OPENAI_GPT_54_PRO_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
};
|
||||
} else if (lower === OPENAI_GPT_54_MINI_MODEL_ID) {
|
||||
templateIds = OPENAI_GPT_54_MINI_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: OPENAI_GPT_54_MINI_COST,
|
||||
contextWindow: OPENAI_GPT_54_MINI_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
};
|
||||
} else if (lower === OPENAI_GPT_54_NANO_MODEL_ID) {
|
||||
templateIds = OPENAI_GPT_54_NANO_TEMPLATE_MODEL_IDS;
|
||||
patch = {
|
||||
api: "openai-responses",
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: resolveOpenAIDefaultBaseUrl(),
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: OPENAI_GPT_54_NANO_COST,
|
||||
contextWindow: OPENAI_GPT_54_NANO_CONTEXT_TOKENS,
|
||||
maxTokens: OPENAI_GPT_54_MAX_TOKENS,
|
||||
};
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return (
|
||||
cloneFirstTemplateModel({
|
||||
providerId: PROVIDER_ID,
|
||||
modelId: trimmedModelId,
|
||||
templateIds,
|
||||
ctx,
|
||||
patch,
|
||||
}) ??
|
||||
normalizeModelCompat({
|
||||
id: trimmedModelId,
|
||||
name: trimmedModelId,
|
||||
...patch,
|
||||
cost: patch.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: patch.contextWindow ?? DEFAULT_CONTEXT_TOKENS,
|
||||
maxTokens: patch.maxTokens ?? DEFAULT_CONTEXT_TOKENS,
|
||||
} as ProviderRuntimeModel)
|
||||
);
|
||||
}
|
||||
|
||||
export function buildOpenAIProvider(): ProviderPlugin {
|
||||
const codexHooks = buildOpenAICodexProviderHooks();
|
||||
const codexResponsesHooks = buildOpenAIResponsesProviderHooks();
|
||||
const responsesHooks = buildOpenAIResponsesProviderHooks({ transport: "sse" });
|
||||
return {
|
||||
id: PROVIDER_ID,
|
||||
label: "OpenAI",
|
||||
hookAliases: ["azure-openai", "azure-openai-responses"],
|
||||
docsPath: "/providers/models",
|
||||
envVars: ["OPENAI_API_KEY"],
|
||||
auth: [
|
||||
...buildOpenAIChatGPTAuthMethods(),
|
||||
createProviderApiKeyAuthMethod({
|
||||
providerId: PROVIDER_ID,
|
||||
methodId: "api-key",
|
||||
label: OPENAI_API_KEY_LABEL,
|
||||
hint: "Use your OpenAI API key directly",
|
||||
optionKey: "openaiApiKey",
|
||||
flagName: "--openai-api-key",
|
||||
envVar: "OPENAI_API_KEY",
|
||||
promptMessage: "Enter OpenAI API key",
|
||||
profileId: "openai:api-key",
|
||||
defaultModel: OPENAI_DEFAULT_MODEL,
|
||||
expectedProviders: ["openai"],
|
||||
applyConfig: (cfg) => applyOpenAIConfig(cfg),
|
||||
wizard: {
|
||||
choiceId: "openai-api-key",
|
||||
choiceLabel: OPENAI_API_KEY_LABEL,
|
||||
choiceHint: "Use your OpenAI API key directly",
|
||||
assistantPriority: 5,
|
||||
...OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
},
|
||||
}),
|
||||
],
|
||||
catalog: {
|
||||
order: "simple",
|
||||
run: async (ctx) => {
|
||||
const auth = ctx.resolveProviderAuth(PROVIDER_ID);
|
||||
try {
|
||||
const { resolveApiKeyForProvider, resolveProviderAuthProfileMetadata } =
|
||||
await import("openclaw/plugin-sdk/provider-auth-runtime");
|
||||
const runtimeAuth = await resolveApiKeyForProvider({
|
||||
provider: PROVIDER_ID,
|
||||
cfg: ctx.config,
|
||||
...(ctx.agentDir ? { agentDir: ctx.agentDir } : {}),
|
||||
...(ctx.workspaceDir ? { workspaceDir: ctx.workspaceDir } : {}),
|
||||
...(auth.profileId
|
||||
? {
|
||||
profileId: auth.profileId,
|
||||
lockedProfile: true,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (runtimeAuth && isCodexCatalogAuthMode(runtimeAuth.mode) && runtimeAuth.apiKey) {
|
||||
const metadata = resolveProviderAuthProfileMetadata({
|
||||
provider: PROVIDER_ID,
|
||||
cfg: ctx.config,
|
||||
...(ctx.agentDir ? { agentDir: ctx.agentDir } : {}),
|
||||
...((runtimeAuth.profileId ?? auth.profileId)
|
||||
? { profileId: runtimeAuth.profileId ?? auth.profileId }
|
||||
: {}),
|
||||
});
|
||||
const provider = await buildOpenAICodexLiveProviderConfig({
|
||||
discoveryApiKey: runtimeAuth.apiKey,
|
||||
accountId: metadata.accountId,
|
||||
});
|
||||
return { providers: { [PROVIDER_ID]: provider } };
|
||||
}
|
||||
} catch {
|
||||
// OAuth discovery is advisory; fall through so configured API-key
|
||||
// auth can still publish the standard OpenAI catalog.
|
||||
}
|
||||
if (auth.mode === "api_key" && auth.apiKey) {
|
||||
return {
|
||||
providers: {
|
||||
[PROVIDER_ID]: await buildOpenAILiveProviderConfig({
|
||||
apiKey: auth.apiKey,
|
||||
baseUrl: resolveOpenAICatalogBaseUrl(ctx),
|
||||
discoveryApiKey: auth.discoveryApiKey,
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
const apiKey = ctx.resolveProviderApiKey(PROVIDER_ID);
|
||||
if (!apiKey.apiKey) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
providers: {
|
||||
[PROVIDER_ID]: await buildOpenAILiveProviderConfig({
|
||||
apiKey: apiKey.apiKey,
|
||||
baseUrl: resolveOpenAICatalogBaseUrl(ctx),
|
||||
discoveryApiKey: apiKey.discoveryApiKey,
|
||||
}),
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
staticCatalog: {
|
||||
order: "simple",
|
||||
run: async () => ({ providers: { [PROVIDER_ID]: OPENAI_MANIFEST_PROVIDER } }),
|
||||
},
|
||||
resolveDynamicModel: (ctx) =>
|
||||
shouldResolveDynamicModelThroughCodex(ctx)
|
||||
? codexHooks.resolveDynamicModel?.(ctx)
|
||||
: resolveOpenAIGptForwardCompatModel(ctx),
|
||||
preferRuntimeResolvedModel: (ctx) => codexHooks.preferRuntimeResolvedModel?.(ctx) ?? false,
|
||||
normalizeResolvedModel: (ctx) => {
|
||||
if (!isOpenAIProvider(ctx.provider)) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
shouldUseCodexResponsesHooks({
|
||||
provider: ctx.provider,
|
||||
api: ctx.model.api,
|
||||
baseUrl: ctx.model.baseUrl,
|
||||
})
|
||||
) {
|
||||
return codexHooks.normalizeResolvedModel?.(ctx);
|
||||
}
|
||||
return normalizeOpenAITransport(ctx.model);
|
||||
},
|
||||
normalizeTransport: (ctx) => {
|
||||
if (shouldUseCodexResponsesHooks(ctx)) {
|
||||
return codexHooks.normalizeTransport?.(ctx);
|
||||
}
|
||||
return shouldUseOpenAIResponsesTransport(ctx)
|
||||
? { api: "openai-responses", baseUrl: ctx.baseUrl }
|
||||
: undefined;
|
||||
},
|
||||
...responsesHooks,
|
||||
prepareExtraParams: (ctx) => {
|
||||
const providerConfig = ctx.config?.models?.providers?.[PROVIDER_ID];
|
||||
const useCodexTransport =
|
||||
shouldUseCodexResponsesHooks({
|
||||
provider: ctx.provider,
|
||||
api: ctx.model?.api,
|
||||
baseUrl: ctx.model?.baseUrl,
|
||||
}) ||
|
||||
(normalizeProviderId(ctx.provider) === PROVIDER_ID &&
|
||||
(!providerConfig?.baseUrl || isOpenAIApiBaseUrl(providerConfig.baseUrl)) &&
|
||||
resolveConfiguredAuthTransport({
|
||||
config: ctx.config,
|
||||
providerConfig,
|
||||
}) === "codex");
|
||||
return (useCodexTransport ? codexResponsesHooks : responsesHooks).prepareExtraParams?.(ctx);
|
||||
},
|
||||
resolveUsageAuth: codexHooks.resolveUsageAuth,
|
||||
fetchUsageSnapshot: codexHooks.fetchUsageSnapshot,
|
||||
refreshOAuth: codexHooks.refreshOAuth,
|
||||
buildUnknownModelHint: ({ modelId }) => buildOpenAIUnknownModelHint(modelId),
|
||||
buildMissingAuthMessage: (ctx) => {
|
||||
if (normalizeProviderId(ctx.provider) !== PROVIDER_ID) {
|
||||
return undefined;
|
||||
}
|
||||
if (ctx.listProfileIds(PROVIDER_ID).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return 'No API key found for provider "openai". You are authenticated with OpenAI ChatGPT/Codex OAuth. Use openai/gpt-5.5 with the ChatGPT/Codex OAuth profile, or set OPENAI_API_KEY for direct OpenAI API access.';
|
||||
},
|
||||
matchesContextOverflowError: ({ errorMessage }) =>
|
||||
/content_filter.*(?:prompt|input).*(?:too long|exceed)/i.test(errorMessage),
|
||||
resolveReasoningOutputMode: () => "native",
|
||||
resolveThinkingProfile: ({ provider, modelId }) =>
|
||||
normalizeProviderId(provider) === PROVIDER_ID
|
||||
? resolveUnifiedOpenAIThinkingProfile(modelId)
|
||||
: null,
|
||||
isModernModelRef: ({ modelId }) => matchesExactOrPrefix(modelId, OPENAI_MODERN_MODEL_IDS),
|
||||
augmentModelCatalog: (ctx) => {
|
||||
const openAiGpt55ProTemplate = findCatalogTemplate({
|
||||
entries: ctx.entries,
|
||||
providerId: PROVIDER_ID,
|
||||
templateIds: OPENAI_GPT_55_PRO_TEMPLATE_MODEL_IDS,
|
||||
});
|
||||
const openAiGpt54Template = findCatalogTemplate({
|
||||
entries: ctx.entries,
|
||||
providerId: PROVIDER_ID,
|
||||
templateIds: OPENAI_GPT_54_TEMPLATE_MODEL_IDS,
|
||||
});
|
||||
const openAiGpt54ProTemplate = findCatalogTemplate({
|
||||
entries: ctx.entries,
|
||||
providerId: PROVIDER_ID,
|
||||
templateIds: OPENAI_GPT_54_PRO_TEMPLATE_MODEL_IDS,
|
||||
});
|
||||
const openAiGpt54MiniTemplate = findCatalogTemplate({
|
||||
entries: ctx.entries,
|
||||
providerId: PROVIDER_ID,
|
||||
templateIds: OPENAI_GPT_54_MINI_TEMPLATE_MODEL_IDS,
|
||||
});
|
||||
const openAiGpt54NanoTemplate = findCatalogTemplate({
|
||||
entries: ctx.entries,
|
||||
providerId: PROVIDER_ID,
|
||||
templateIds: OPENAI_GPT_54_NANO_TEMPLATE_MODEL_IDS,
|
||||
});
|
||||
return [
|
||||
buildOpenAISyntheticCatalogEntry(openAiGpt55ProTemplate, {
|
||||
id: OPENAI_GPT_55_PRO_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_GPT_55_PRO_CONTEXT_TOKENS,
|
||||
}),
|
||||
buildOpenAISyntheticCatalogEntry(openAiGpt54Template, {
|
||||
id: OPENAI_GPT_54_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_GPT_54_CONTEXT_TOKENS,
|
||||
}),
|
||||
buildOpenAISyntheticCatalogEntry(openAiGpt54ProTemplate, {
|
||||
id: OPENAI_GPT_54_PRO_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_GPT_54_PRO_CONTEXT_TOKENS,
|
||||
}),
|
||||
buildOpenAISyntheticCatalogEntry(openAiGpt54MiniTemplate, {
|
||||
id: OPENAI_GPT_54_MINI_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_GPT_54_MINI_CONTEXT_TOKENS,
|
||||
}),
|
||||
buildOpenAISyntheticCatalogEntry(openAiGpt54NanoTemplate, {
|
||||
id: OPENAI_GPT_54_NANO_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: OPENAI_GPT_54_NANO_CONTEXT_TOKENS,
|
||||
}),
|
||||
].filter((entry): entry is NonNullable<typeof entry> => entry !== undefined);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** @deprecated Use buildOpenAIProvider; OpenAI Codex is now an OpenAI auth/transport mode. */
|
||||
export function buildOpenAICodexProviderPlugin(): ProviderPlugin {
|
||||
return buildOpenAIProvider();
|
||||
}
|
||||
45
extensions/openai/openai-tts.live.test.ts
Normal file
45
extensions/openai/openai-tts.live.test.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
// Openai tests cover openai tts plugin behavior.
|
||||
import { isLiveTestEnabled } from "openclaw/plugin-sdk/test-env";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY?.trim() ?? "";
|
||||
const LIVE = isLiveTestEnabled() && OPENAI_API_KEY.length > 0;
|
||||
const describeLive = LIVE ? describe : describe.skip;
|
||||
|
||||
describeLive("openai tts live", () => {
|
||||
it("synthesizes audio through the speech provider", async () => {
|
||||
const speechProvider = buildOpenAISpeechProvider();
|
||||
|
||||
const voices = await speechProvider.listVoices?.({});
|
||||
expect(voices?.some((voice) => voice.id === "alloy")).toBe(true);
|
||||
|
||||
const providerConfig = {
|
||||
apiKey: OPENAI_API_KEY,
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
};
|
||||
|
||||
const audioFile = await speechProvider.synthesize({
|
||||
text: "OpenClaw OpenAI text to speech integration test OK.",
|
||||
cfg: { plugins: { enabled: true } } as never,
|
||||
providerConfig,
|
||||
target: "audio-file",
|
||||
timeoutMs: 45_000,
|
||||
});
|
||||
expect(audioFile.outputFormat).toBe("mp3");
|
||||
expect(audioFile.fileExtension).toBe(".mp3");
|
||||
expect(audioFile.audioBuffer.byteLength).toBeGreaterThan(512);
|
||||
|
||||
const telephony = await speechProvider.synthesizeTelephony?.({
|
||||
text: "OpenClaw OpenAI telephony integration test OK.",
|
||||
cfg: { plugins: { enabled: true } } as never,
|
||||
providerConfig,
|
||||
timeoutMs: 45_000,
|
||||
});
|
||||
expect(telephony?.outputFormat).toBe("pcm");
|
||||
expect(telephony?.sampleRate).toBe(24_000);
|
||||
expect(telephony?.audioBuffer.byteLength).toBeGreaterThan(512);
|
||||
}, 60_000);
|
||||
});
|
||||
474
extensions/openai/openai.live.test.ts
Normal file
474
extensions/openai/openai.live.test.ts
Normal file
@@ -0,0 +1,474 @@
|
||||
// Openai tests cover openai plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import OpenAI from "openai";
|
||||
import type { ResolvedTtsConfig } from "openclaw/plugin-sdk/agent-runtime";
|
||||
import { AuthStorage, ModelRegistry } from "openclaw/plugin-sdk/agent-sessions";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { encodePngRgba, fillPixel } from "openclaw/plugin-sdk/media-runtime";
|
||||
import {
|
||||
registerProviderPlugin,
|
||||
requireRegisteredProvider,
|
||||
} from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { runRealtimeSttLiveTest } from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import { getRuntimeConfig } from "openclaw/plugin-sdk/runtime-config-snapshot";
|
||||
import {
|
||||
isOverloadedErrorMessage,
|
||||
isServerErrorMessage,
|
||||
isTimeoutErrorMessage,
|
||||
} from "openclaw/plugin-sdk/test-env";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import plugin from "./index.js";
|
||||
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY ?? "";
|
||||
const LIVE_MODEL_ID = process.env.OPENCLAW_LIVE_OPENAI_PLUGIN_MODEL?.trim() || "gpt-5.5";
|
||||
const LIVE_IMAGE_MODEL = process.env.OPENCLAW_LIVE_OPENAI_IMAGE_MODEL?.trim() || "gpt-image-2";
|
||||
const LIVE_VISION_MODEL = process.env.OPENCLAW_LIVE_OPENAI_VISION_MODEL?.trim() || "gpt-5.4-mini";
|
||||
const liveEnabled = OPENAI_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1";
|
||||
const describeLive = liveEnabled ? describe : describe.skip;
|
||||
const EMPTY_AUTH_STORE = { version: 1, profiles: {} } as const;
|
||||
const LIVE_TTS_TIMEOUT_MS = 60_000;
|
||||
const LIVE_STT_FIXTURE_TTS_TIMEOUT_MS = 120_000;
|
||||
const ModelRegistryCtor = ModelRegistry as unknown as {
|
||||
new (authStorage: AuthStorage, modelsJsonPath?: string): ModelRegistry;
|
||||
};
|
||||
|
||||
function createLiveModelRegistry(modelId: string): ModelRegistry {
|
||||
const registry = new ModelRegistryCtor(AuthStorage.inMemory());
|
||||
registry.registerProvider("openai", {
|
||||
apiKey: "test",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
models: [
|
||||
{
|
||||
id: modelId,
|
||||
name: modelId,
|
||||
api: "openai-responses",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400_000,
|
||||
maxTokens: 128_000,
|
||||
},
|
||||
],
|
||||
});
|
||||
return registry;
|
||||
}
|
||||
|
||||
const registerOpenAIPlugin = () =>
|
||||
registerProviderPlugin({
|
||||
plugin,
|
||||
id: "openai",
|
||||
name: "OpenAI Provider",
|
||||
});
|
||||
|
||||
function createReferencePng(): Buffer {
|
||||
const width = 96;
|
||||
const height = 96;
|
||||
const buf = Buffer.alloc(width * height * 4, 255);
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
fillPixel(buf, x, y, width, 225, 242, 255, 255);
|
||||
}
|
||||
}
|
||||
|
||||
for (let y = 24; y < 72; y += 1) {
|
||||
for (let x = 24; x < 72; x += 1) {
|
||||
fillPixel(buf, x, y, width, 255, 153, 51, 255);
|
||||
}
|
||||
}
|
||||
|
||||
return encodePngRgba(buf, width, height);
|
||||
}
|
||||
|
||||
function formatLiveOpenAIError(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function resolveLiveOpenAISkipReason(error: unknown): string | null {
|
||||
const message = formatLiveOpenAIError(error);
|
||||
if (isTimeoutErrorMessage(message) || /timed out|operation was aborted/i.test(message)) {
|
||||
return "provider timeout";
|
||||
}
|
||||
if (isOverloadedErrorMessage(message) || isServerErrorMessage(message)) {
|
||||
return "provider outage";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function createLiveConfig(): OpenClawConfig {
|
||||
const cfg = getRuntimeConfig();
|
||||
return {
|
||||
...cfg,
|
||||
models: {
|
||||
...cfg.models,
|
||||
providers: {
|
||||
...cfg.models?.providers,
|
||||
openai: {
|
||||
...cfg.models?.providers?.openai,
|
||||
apiKey: OPENAI_API_KEY,
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
|
||||
function createLiveTtsConfig(): ResolvedTtsConfig {
|
||||
return {
|
||||
auto: "off",
|
||||
mode: "final",
|
||||
provider: "openai",
|
||||
providerSource: "config",
|
||||
modelOverrides: {
|
||||
enabled: true,
|
||||
allowText: true,
|
||||
allowProvider: true,
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
allowNormalization: true,
|
||||
allowSeed: true,
|
||||
},
|
||||
providerConfigs: {
|
||||
openai: {
|
||||
apiKey: OPENAI_API_KEY,
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
},
|
||||
},
|
||||
personas: {},
|
||||
maxTextLength: 4_000,
|
||||
timeoutMs: LIVE_TTS_TIMEOUT_MS,
|
||||
};
|
||||
}
|
||||
|
||||
async function createTempAgentDir(): Promise<string> {
|
||||
return await fs.mkdtemp(path.join(os.tmpdir(), "openai-plugin-live-"));
|
||||
}
|
||||
|
||||
async function removeTempAgentDir(agentDir: string): Promise<void> {
|
||||
await fs.rm(agentDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
}
|
||||
|
||||
function normalizeTranscriptForMatch(value: string): string {
|
||||
return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
|
||||
}
|
||||
|
||||
function linearToMulaw(sample: number): number {
|
||||
const bias = 132;
|
||||
const clip = 32635;
|
||||
let next = Math.max(-clip, Math.min(clip, sample));
|
||||
const sign = next < 0 ? 0x80 : 0;
|
||||
if (next < 0) {
|
||||
next = -next;
|
||||
}
|
||||
|
||||
next += bias;
|
||||
let exponent = 7;
|
||||
for (let expMask = 0x4000; (next & expMask) === 0 && exponent > 0; exponent -= 1) {
|
||||
expMask >>= 1;
|
||||
}
|
||||
|
||||
const mantissa = (next >> (exponent + 3)) & 0x0f;
|
||||
return ~(sign | (exponent << 4) | mantissa) & 0xff;
|
||||
}
|
||||
|
||||
function convertPcm24kToMulaw8k(pcm: Buffer): Buffer {
|
||||
const inputSamples = Math.floor(pcm.length / 2);
|
||||
const outputSamples = Math.floor(inputSamples / 3);
|
||||
const mulaw = Buffer.alloc(outputSamples);
|
||||
|
||||
for (let i = 0; i < outputSamples; i += 1) {
|
||||
mulaw[i] = linearToMulaw(pcm.readInt16LE(i * 3 * 2));
|
||||
}
|
||||
|
||||
return mulaw;
|
||||
}
|
||||
|
||||
describeLive("openai plugin live", () => {
|
||||
it("registers an OpenAI provider that can complete a live request", async () => {
|
||||
const { providers } = await registerOpenAIPlugin();
|
||||
const provider = requireRegisteredProvider(providers, "openai");
|
||||
const modelRegistry = createLiveModelRegistry(LIVE_MODEL_ID);
|
||||
|
||||
const resolved =
|
||||
modelRegistry.find("openai", LIVE_MODEL_ID) ??
|
||||
provider.resolveDynamicModel?.({
|
||||
provider: "openai",
|
||||
modelId: LIVE_MODEL_ID,
|
||||
modelRegistry,
|
||||
});
|
||||
|
||||
if (!resolved) {
|
||||
throw new Error("openai provider did not resolve the live model");
|
||||
}
|
||||
|
||||
const normalized = provider.normalizeResolvedModel?.({
|
||||
provider: "openai",
|
||||
modelId: resolved.id,
|
||||
model: resolved,
|
||||
});
|
||||
|
||||
expect(normalized?.provider).toBe("openai");
|
||||
expect(normalized?.id).toBe(LIVE_MODEL_ID);
|
||||
expect(normalized?.api).toBe("openai-responses");
|
||||
expect(normalized?.baseUrl).toBe("https://api.openai.com/v1");
|
||||
|
||||
const client = new OpenAI({
|
||||
apiKey: OPENAI_API_KEY,
|
||||
baseURL: normalized?.baseUrl,
|
||||
});
|
||||
const response = await client.responses.create({
|
||||
model: normalized?.id ?? LIVE_MODEL_ID,
|
||||
instructions: "Return exactly OK and no other text.",
|
||||
input: "Return exactly OK.",
|
||||
max_output_tokens: 64,
|
||||
reasoning: { effort: "none" },
|
||||
text: { verbosity: "low" },
|
||||
});
|
||||
|
||||
expect(response.output_text.trim()).toMatch(/^OK[.!]?$/);
|
||||
}, 30_000);
|
||||
|
||||
it("lists voices and synthesizes audio through the registered speech provider", async () => {
|
||||
const { speechProviders } = await registerOpenAIPlugin();
|
||||
const speechProvider = requireRegisteredProvider(speechProviders, "openai");
|
||||
|
||||
const voices = await speechProvider.listVoices?.({});
|
||||
if (!voices) {
|
||||
throw new Error("openai speech provider did not return voices");
|
||||
}
|
||||
expect(voices.some((voice) => voice.id === "alloy")).toBe(true);
|
||||
|
||||
const cfg = createLiveConfig();
|
||||
const ttsConfig = createLiveTtsConfig();
|
||||
|
||||
const audioFile = await speechProvider.synthesize({
|
||||
text: "OpenClaw integration test OK.",
|
||||
cfg,
|
||||
providerConfig: ttsConfig.providerConfigs.openai ?? {},
|
||||
target: "audio-file",
|
||||
timeoutMs: ttsConfig.timeoutMs,
|
||||
});
|
||||
expect(audioFile.outputFormat).toBe("mp3");
|
||||
expect(audioFile.fileExtension).toBe(".mp3");
|
||||
expect(audioFile.audioBuffer.byteLength).toBeGreaterThan(512);
|
||||
|
||||
const telephony = await speechProvider.synthesizeTelephony?.({
|
||||
text: "Telephony check OK.",
|
||||
cfg,
|
||||
providerConfig: ttsConfig.providerConfigs.openai ?? {},
|
||||
timeoutMs: ttsConfig.timeoutMs,
|
||||
});
|
||||
expect(telephony?.outputFormat).toBe("pcm");
|
||||
expect(telephony?.sampleRate).toBe(24_000);
|
||||
expect(telephony?.audioBuffer.byteLength).toBeGreaterThan(512);
|
||||
}, 150_000);
|
||||
|
||||
it("transcribes synthesized speech through the registered media provider", async () => {
|
||||
const { speechProviders, mediaProviders } = await registerOpenAIPlugin();
|
||||
const speechProvider = requireRegisteredProvider(speechProviders, "openai");
|
||||
const mediaProvider = requireRegisteredProvider(mediaProviders, "openai");
|
||||
|
||||
const cfg = createLiveConfig();
|
||||
const ttsConfig = createLiveTtsConfig();
|
||||
|
||||
const synthesized = await speechProvider.synthesize({
|
||||
text: "Speech transcription check okay.",
|
||||
cfg,
|
||||
providerConfig: ttsConfig.providerConfigs.openai ?? {},
|
||||
target: "audio-file",
|
||||
timeoutMs: ttsConfig.timeoutMs,
|
||||
});
|
||||
|
||||
const transcription = await mediaProvider.transcribeAudio?.({
|
||||
buffer: synthesized.audioBuffer,
|
||||
fileName: "openai-plugin-live.mp3",
|
||||
mime: "audio/mpeg",
|
||||
apiKey: OPENAI_API_KEY,
|
||||
language: "en",
|
||||
prompt: "Speech transcription check okay.",
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
|
||||
const text = (transcription?.text ?? "").toLowerCase();
|
||||
const collapsedText = text.replace(/[\s-]+/g, "");
|
||||
expect(text.length).toBeGreaterThan(0);
|
||||
expect(collapsedText).toContain("speech");
|
||||
expect(collapsedText).toMatch(/(?:check|okay|ok|transcription)/);
|
||||
}, 120_000);
|
||||
|
||||
it("opens OpenAI realtime STT before sending audio", async () => {
|
||||
const { realtimeTranscriptionProviders } = await registerOpenAIPlugin();
|
||||
const realtimeProvider = requireRegisteredProvider(realtimeTranscriptionProviders, "openai");
|
||||
const errors: Error[] = [];
|
||||
const session = realtimeProvider.createSession({
|
||||
providerConfig: {
|
||||
apiKey: OPENAI_API_KEY,
|
||||
language: "en",
|
||||
},
|
||||
onError: (error) => errors.push(error),
|
||||
});
|
||||
|
||||
try {
|
||||
await session.connect();
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 1_000);
|
||||
});
|
||||
expect(errors).toStrictEqual([]);
|
||||
expect(session.isConnected()).toBe(true);
|
||||
} finally {
|
||||
session.close();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("streams realtime STT through the registered transcription provider", async () => {
|
||||
const { realtimeTranscriptionProviders, speechProviders } = await registerOpenAIPlugin();
|
||||
const realtimeProvider = requireRegisteredProvider(realtimeTranscriptionProviders, "openai");
|
||||
const speechProvider = requireRegisteredProvider(speechProviders, "openai");
|
||||
const cfg = createLiveConfig();
|
||||
const ttsConfig = createLiveTtsConfig();
|
||||
const phrase = "Testing OpenClaw OpenAI realtime transcription integration test OK.";
|
||||
|
||||
const telephony = await speechProvider.synthesizeTelephony?.({
|
||||
text: phrase,
|
||||
cfg,
|
||||
providerConfig: ttsConfig.providerConfigs.openai ?? {},
|
||||
timeoutMs: LIVE_STT_FIXTURE_TTS_TIMEOUT_MS,
|
||||
});
|
||||
if (!telephony) {
|
||||
throw new Error("OpenAI telephony synthesis did not return audio");
|
||||
}
|
||||
expect(telephony.outputFormat).toBe("pcm");
|
||||
expect(telephony.sampleRate).toBe(24_000);
|
||||
|
||||
const speech = convertPcm24kToMulaw8k(telephony.audioBuffer);
|
||||
const silence = Buffer.alloc(8_000, 0xff);
|
||||
const audio = Buffer.concat([silence.subarray(0, 4_000), speech, silence]);
|
||||
const { transcripts, partials } = await runRealtimeSttLiveTest({
|
||||
provider: realtimeProvider,
|
||||
providerConfig: {
|
||||
apiKey: OPENAI_API_KEY,
|
||||
language: "en",
|
||||
silenceDurationMs: 500,
|
||||
},
|
||||
audio,
|
||||
expectedNormalizedText: /openai.*realtime.*transcription/,
|
||||
});
|
||||
const normalized = transcripts.join(" ").toLowerCase();
|
||||
const compact = normalizeTranscriptForMatch(normalized);
|
||||
expect(compact).toContain("openai");
|
||||
expect(normalized).toContain("transcription");
|
||||
expect(partials.length + transcripts.length).toBeGreaterThan(0);
|
||||
}, 240_000);
|
||||
|
||||
it("generates an image through the registered image provider", async () => {
|
||||
const { imageProviders } = await registerOpenAIPlugin();
|
||||
const imageProvider = requireRegisteredProvider(imageProviders, "openai");
|
||||
|
||||
const cfg = createLiveConfig();
|
||||
const agentDir = await createTempAgentDir();
|
||||
|
||||
try {
|
||||
const generated = await imageProvider.generateImage({
|
||||
provider: "openai",
|
||||
model: LIVE_IMAGE_MODEL,
|
||||
prompt: "Create a minimal flat orange square centered on a white background.",
|
||||
cfg,
|
||||
agentDir,
|
||||
authStore: EMPTY_AUTH_STORE,
|
||||
timeoutMs: 180_000,
|
||||
count: 1,
|
||||
size: "1536x1024",
|
||||
});
|
||||
|
||||
expect(generated.model).toBe(LIVE_IMAGE_MODEL);
|
||||
expect(generated.images.length).toBeGreaterThan(0);
|
||||
expect(generated.images[0]?.mimeType).toBe("image/png");
|
||||
expect(generated.images[0]?.buffer.byteLength).toBeGreaterThan(1_000);
|
||||
} finally {
|
||||
await removeTempAgentDir(agentDir);
|
||||
}
|
||||
}, 240_000);
|
||||
|
||||
it("edits a reference image through the registered image provider", async () => {
|
||||
const { imageProviders } = await registerOpenAIPlugin();
|
||||
const imageProvider = requireRegisteredProvider(imageProviders, "openai");
|
||||
|
||||
const cfg = createLiveConfig();
|
||||
const agentDir = await createTempAgentDir();
|
||||
|
||||
try {
|
||||
const edited = await imageProvider.generateImage({
|
||||
provider: "openai",
|
||||
model: LIVE_IMAGE_MODEL,
|
||||
prompt:
|
||||
"Edit this image: remove the orange square in the center and keep the background clean and light blue.",
|
||||
cfg,
|
||||
agentDir,
|
||||
authStore: EMPTY_AUTH_STORE,
|
||||
timeoutMs: 240_000,
|
||||
count: 1,
|
||||
size: "1024x1024",
|
||||
inputImages: [
|
||||
{
|
||||
buffer: createReferencePng(),
|
||||
mimeType: "image/png",
|
||||
fileName: "reference.png",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(edited.model).toBe(LIVE_IMAGE_MODEL);
|
||||
expect(edited.images.length).toBeGreaterThan(0);
|
||||
expect(edited.images[0]?.mimeType).toBe("image/png");
|
||||
expect(edited.images[0]?.buffer.byteLength).toBeGreaterThan(1_000);
|
||||
} finally {
|
||||
await removeTempAgentDir(agentDir);
|
||||
}
|
||||
}, 300_000);
|
||||
|
||||
it("describes a deterministic image through the registered media provider", async () => {
|
||||
const { mediaProviders } = await registerOpenAIPlugin();
|
||||
const mediaProvider = requireRegisteredProvider(mediaProviders, "openai");
|
||||
|
||||
const cfg = createLiveConfig();
|
||||
const agentDir = await createTempAgentDir();
|
||||
|
||||
try {
|
||||
let description:
|
||||
| Awaited<ReturnType<NonNullable<typeof mediaProvider.describeImage>>>
|
||||
| undefined;
|
||||
try {
|
||||
description = await mediaProvider.describeImage?.({
|
||||
buffer: createReferencePng(),
|
||||
fileName: "reference.png",
|
||||
mime: "image/png",
|
||||
prompt: "Reply with one lowercase word for the dominant center color.",
|
||||
timeoutMs: 45_000,
|
||||
agentDir,
|
||||
cfg,
|
||||
authStore: EMPTY_AUTH_STORE,
|
||||
model: LIVE_VISION_MODEL,
|
||||
provider: "openai",
|
||||
});
|
||||
} catch (err) {
|
||||
const skipReason = resolveLiveOpenAISkipReason(err);
|
||||
if (skipReason) {
|
||||
console.warn(
|
||||
`[live:openai] image description skipped: ${skipReason}: ${formatLiveOpenAIError(err)}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
expect((description?.text ?? "").toLowerCase()).toContain("orange");
|
||||
} finally {
|
||||
await removeTempAgentDir(agentDir);
|
||||
}
|
||||
}, 240_000);
|
||||
});
|
||||
374
extensions/openai/openclaw.plugin.json
Normal file
374
extensions/openai/openclaw.plugin.json
Normal file
@@ -0,0 +1,374 @@
|
||||
{
|
||||
"id": "openai",
|
||||
"activation": {
|
||||
"onStartup": false
|
||||
},
|
||||
"enabledByDefault": true,
|
||||
"providers": ["openai"],
|
||||
"modelSupport": {
|
||||
"modelPrefixes": ["gpt-", "o1", "o3", "o4"]
|
||||
},
|
||||
"providerEndpoints": [
|
||||
{
|
||||
"endpointClass": "openai-public",
|
||||
"hosts": ["api.openai.com"],
|
||||
"hostSuffixes": [".api.openai.com"]
|
||||
},
|
||||
{
|
||||
"endpointClass": "openai",
|
||||
"hosts": ["chatgpt.com"]
|
||||
},
|
||||
{
|
||||
"endpointClass": "azure-openai",
|
||||
"hostSuffixes": [".openai.azure.com"]
|
||||
}
|
||||
],
|
||||
"providerRequest": {
|
||||
"providers": {
|
||||
"openai": {
|
||||
"family": "openai-family"
|
||||
}
|
||||
}
|
||||
},
|
||||
"modelCatalog": {
|
||||
"providers": {
|
||||
"openai": {
|
||||
"baseUrl": "https://api.openai.com/v1",
|
||||
"api": "openai-responses",
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-5.3-chat-latest",
|
||||
"name": "GPT-5.3 Chat (latest)",
|
||||
"reasoning": false,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 128000,
|
||||
"maxTokens": 16384,
|
||||
"cost": { "input": 1.75, "output": 14, "cacheRead": 0.175, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.3-codex",
|
||||
"name": "GPT-5.3 Codex",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 400000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 1.75, "output": 14, "cacheRead": 0.175, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.4",
|
||||
"name": "GPT-5.4",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 2048, "preferredSidePx": 2048, "tokenMode": "detail" }
|
||||
},
|
||||
"contextWindow": 272000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 2.5, "output": 15, "cacheRead": 0.25, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.4-mini",
|
||||
"name": "GPT-5.4 mini",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 2048, "preferredSidePx": 2048, "tokenMode": "detail" }
|
||||
},
|
||||
"contextWindow": 400000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 0.75, "output": 4.5, "cacheRead": 0.075, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.4-nano",
|
||||
"name": "GPT-5.4 nano",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 2048, "preferredSidePx": 2048, "tokenMode": "detail" }
|
||||
},
|
||||
"contextWindow": 400000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 0.2, "output": 1.25, "cacheRead": 0.02, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.4-pro",
|
||||
"name": "GPT-5.4 Pro",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 2048, "preferredSidePx": 2048, "tokenMode": "detail" }
|
||||
},
|
||||
"contextWindow": 1050000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 30, "output": 180, "cacheRead": 0, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-sol",
|
||||
"name": "GPT-5.6 Sol",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 372000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 5, "output": 30, "cacheRead": 0.5, "cacheWrite": 6.25 },
|
||||
"thinkingLevelMap": { "off": null, "xhigh": "xhigh", "max": "max" },
|
||||
"compat": {
|
||||
"supportsReasoningEffort": true,
|
||||
"supportedReasoningEfforts": ["low", "medium", "high", "xhigh", "max"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-terra",
|
||||
"name": "GPT-5.6 Terra",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 372000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 2.5, "output": 15, "cacheRead": 0.25, "cacheWrite": 3.125 },
|
||||
"thinkingLevelMap": { "off": null, "xhigh": "xhigh", "max": "max" },
|
||||
"compat": {
|
||||
"supportsReasoningEffort": true,
|
||||
"supportedReasoningEfforts": ["low", "medium", "high", "xhigh", "max"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-luna",
|
||||
"name": "GPT-5.6 Luna",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 372000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 1, "output": 6, "cacheRead": 0.1, "cacheWrite": 1.25 },
|
||||
"thinkingLevelMap": { "off": null, "xhigh": "xhigh", "max": "max" },
|
||||
"compat": {
|
||||
"supportsReasoningEffort": true,
|
||||
"supportedReasoningEfforts": ["low", "medium", "high", "xhigh", "max"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.5",
|
||||
"name": "GPT-5.5",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 6000, "preferredSidePx": 2048, "tokenMode": "detail" }
|
||||
},
|
||||
"contextWindow": 1000000,
|
||||
"contextTokens": 272000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 5, "output": 30, "cacheRead": 0.5, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o1",
|
||||
"name": "o1",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 15, "output": 60, "cacheRead": 7.5, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o1-pro",
|
||||
"name": "o1-pro",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 150, "output": 600, "cacheRead": 0, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o3",
|
||||
"name": "o3",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 2, "output": 8, "cacheRead": 0.5, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o3-deep-research",
|
||||
"name": "o3-deep-research",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 10, "output": 40, "cacheRead": 2.5, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o3-mini",
|
||||
"name": "o3-mini",
|
||||
"reasoning": true,
|
||||
"input": ["text"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 1.1, "output": 4.4, "cacheRead": 0.55, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o3-pro",
|
||||
"name": "o3-pro",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 20, "output": 80, "cacheRead": 0, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o4-mini",
|
||||
"name": "o4-mini",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 1.1, "output": 4.4, "cacheRead": 0.28, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "o4-mini-deep-research",
|
||||
"name": "o4-mini-deep-research",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"contextWindow": 200000,
|
||||
"maxTokens": 100000,
|
||||
"cost": { "input": 2, "output": 8, "cacheRead": 0.5, "cacheWrite": 0 }
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.5-pro",
|
||||
"name": "gpt-5.5-pro",
|
||||
"reasoning": true,
|
||||
"input": ["text", "image"],
|
||||
"mediaInput": {
|
||||
"image": { "maxSidePx": 6000, "preferredSidePx": 2048, "tokenMode": "detail" }
|
||||
},
|
||||
"contextWindow": 1000000,
|
||||
"maxTokens": 128000,
|
||||
"cost": { "input": 30, "output": 180, "cacheRead": 0, "cacheWrite": 0 }
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"aliases": {
|
||||
"azure-openai-responses": {
|
||||
"provider": "openai",
|
||||
"api": "azure-openai-responses"
|
||||
}
|
||||
},
|
||||
"discovery": {
|
||||
"openai": "runtime"
|
||||
},
|
||||
"suppressions": [
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "gpt-5.3-codex-spark",
|
||||
"reason": "gpt-5.3-codex-spark is available only through ChatGPT/Codex OAuth. Run `openclaw models auth login --provider openai` and use openai/gpt-5.3-codex-spark with that OAuth profile; OpenAI API-key auth cannot use this model.",
|
||||
"when": {
|
||||
"baseUrlHosts": ["api.openai.com"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"provider": "azure-openai-responses",
|
||||
"model": "gpt-5.3-codex-spark",
|
||||
"reason": "gpt-5.3-codex-spark is available only through ChatGPT/Codex OAuth. Run `openclaw models auth login --provider openai` and use openai/gpt-5.3-codex-spark with that OAuth profile; Azure/OpenAI API-key auth cannot use this model."
|
||||
}
|
||||
]
|
||||
},
|
||||
"setup": {
|
||||
"providers": [
|
||||
{
|
||||
"id": "openai",
|
||||
"envVars": ["OPENAI_API_KEY"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"providerAuthChoices": [
|
||||
{
|
||||
"provider": "openai",
|
||||
"method": "oauth",
|
||||
"choiceId": "openai",
|
||||
"choiceLabel": "ChatGPT Login",
|
||||
"choiceHint": "Sign in with your ChatGPT or Codex subscription",
|
||||
"assistantPriority": -40,
|
||||
"groupId": "openai",
|
||||
"groupLabel": "OpenAI",
|
||||
"groupHint": "ChatGPT/Codex sign-in or API key",
|
||||
"onboardingFeatured": true
|
||||
},
|
||||
{
|
||||
"provider": "openai",
|
||||
"method": "device-code",
|
||||
"choiceId": "openai-device-code",
|
||||
"choiceLabel": "ChatGPT Device Pairing",
|
||||
"choiceHint": "Pair your ChatGPT account in browser with a device code",
|
||||
"assistantPriority": -10,
|
||||
"assistantVisibility": "manual-only",
|
||||
"groupId": "openai",
|
||||
"groupLabel": "OpenAI",
|
||||
"groupHint": "ChatGPT/Codex sign-in or API key"
|
||||
},
|
||||
{
|
||||
"provider": "openai",
|
||||
"method": "api-key",
|
||||
"choiceId": "openai-api-key",
|
||||
"choiceLabel": "OpenAI API Key",
|
||||
"choiceHint": "Use your OpenAI API key directly",
|
||||
"assistantPriority": 5,
|
||||
"groupId": "openai",
|
||||
"groupLabel": "OpenAI",
|
||||
"groupHint": "ChatGPT/Codex sign-in or API key",
|
||||
"onboardingFeatured": true,
|
||||
"optionKey": "openaiApiKey",
|
||||
"cliFlag": "--openai-api-key",
|
||||
"cliOption": "--openai-api-key <key>",
|
||||
"cliDescription": "OpenAI API Key"
|
||||
}
|
||||
],
|
||||
"contracts": {
|
||||
"speechProviders": ["openai"],
|
||||
"realtimeTranscriptionProviders": ["openai"],
|
||||
"realtimeVoiceProviders": ["openai"],
|
||||
"memoryEmbeddingProviders": ["openai"],
|
||||
"mediaUnderstandingProviders": ["openai"],
|
||||
"imageGenerationProviders": ["openai"],
|
||||
"videoGenerationProviders": ["openai"]
|
||||
},
|
||||
"imageGenerationProviderMetadata": {
|
||||
"openai": {
|
||||
"authSignals": [
|
||||
{
|
||||
"provider": "openai"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"videoGenerationProviderMetadata": {
|
||||
"openai": {
|
||||
"authSignals": [
|
||||
{
|
||||
"provider": "openai"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"mediaUnderstandingProviderMetadata": {
|
||||
"openai": {
|
||||
"capabilities": ["image", "audio"],
|
||||
"defaultModels": {
|
||||
"image": "gpt-5.5",
|
||||
"audio": "gpt-4o-transcribe"
|
||||
},
|
||||
"autoPriority": {
|
||||
"image": 20,
|
||||
"audio": 20
|
||||
}
|
||||
}
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"personality": {
|
||||
"type": "string",
|
||||
"enum": ["friendly", "on", "off"],
|
||||
"default": "friendly",
|
||||
"description": "Legacy compatibility fallback for the shared GPT-5 friendly interaction-style overlay. Prefer agents.defaults.promptOverlays.gpt5.personality. `friendly` and `on` enable the style overlay; `off` disables only that style layer."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
207
extensions/openai/openclaw.plugin.test.ts
Normal file
207
extensions/openai/openclaw.plugin.test.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
// Openai tests cover openclaw.plugin plugin behavior.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildOpenAIProvider } from "./openai-provider.js";
|
||||
import { buildOpenAISetupProvider } from "./setup-api.js";
|
||||
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(new URL("./openclaw.plugin.json", import.meta.url), "utf8"),
|
||||
) as {
|
||||
mediaUnderstandingProviderMetadata?: Record<
|
||||
string,
|
||||
{
|
||||
capabilities?: string[];
|
||||
defaultModels?: Record<string, string>;
|
||||
autoPriority?: Record<string, number>;
|
||||
}
|
||||
>;
|
||||
providerAuthChoices?: Array<{
|
||||
provider?: string;
|
||||
method?: string;
|
||||
choiceLabel?: string;
|
||||
choiceHint?: string;
|
||||
choiceId?: string;
|
||||
deprecatedChoiceIds?: string[];
|
||||
assistantVisibility?: string;
|
||||
groupId?: string;
|
||||
groupLabel?: string;
|
||||
groupHint?: string;
|
||||
}>;
|
||||
setup?: {
|
||||
providers?: Array<{ id: string }>;
|
||||
};
|
||||
modelCatalog?: {
|
||||
suppressions?: Array<{
|
||||
provider?: string;
|
||||
model?: string;
|
||||
when?: {
|
||||
baseUrlHosts?: string[];
|
||||
providerConfigApiIn?: string[];
|
||||
};
|
||||
}>;
|
||||
};
|
||||
providerEndpoints?: Array<{
|
||||
endpointClass?: string;
|
||||
hosts?: string[];
|
||||
hostSuffixes?: string[];
|
||||
}>;
|
||||
providerAuthAliases?: Record<string, string>;
|
||||
legacyPluginIds?: string[];
|
||||
};
|
||||
|
||||
const packageJson = JSON.parse(
|
||||
readFileSync(new URL("./package.json", import.meta.url), "utf8"),
|
||||
) as {
|
||||
dependencies?: Record<string, string>;
|
||||
devDependencies?: Record<string, string>;
|
||||
};
|
||||
|
||||
function manifestComparableWizardFields(choice: {
|
||||
choiceId?: string;
|
||||
choiceLabel?: string;
|
||||
choiceHint?: string;
|
||||
assistantVisibility?: string;
|
||||
groupId?: string;
|
||||
groupLabel?: string;
|
||||
groupHint?: string;
|
||||
}) {
|
||||
return Object.fromEntries(
|
||||
Object.entries({
|
||||
choiceId: choice.choiceId,
|
||||
choiceLabel: choice.choiceLabel,
|
||||
choiceHint: choice.choiceHint,
|
||||
assistantVisibility: choice.assistantVisibility,
|
||||
groupId: choice.groupId,
|
||||
groupLabel: choice.groupLabel,
|
||||
groupHint: choice.groupHint,
|
||||
}).filter(([, value]) => value !== undefined),
|
||||
);
|
||||
}
|
||||
|
||||
function providerWizardByKey() {
|
||||
const providers = [buildOpenAIProvider(), buildOpenAISetupProvider()];
|
||||
const wizards = new Map<string, Record<string, unknown>>();
|
||||
|
||||
for (const provider of providers) {
|
||||
for (const authMethod of provider.auth ?? []) {
|
||||
if (authMethod.wizard) {
|
||||
wizards.set(`${provider.id}:${authMethod.id}`, authMethod.wizard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return wizards;
|
||||
}
|
||||
|
||||
function expectWizardFields(
|
||||
wizard: Record<string, unknown> | undefined,
|
||||
choice: ReturnType<typeof manifestComparableWizardFields>,
|
||||
key: string,
|
||||
) {
|
||||
if (!wizard) {
|
||||
throw new Error(`Missing wizard for ${key}`);
|
||||
}
|
||||
for (const [field, value] of Object.entries(choice)) {
|
||||
expect(wizard[field], `${key}.${field}`).toBe(value);
|
||||
}
|
||||
}
|
||||
|
||||
describe("OpenAI plugin manifest", () => {
|
||||
it("keeps runtime dependencies in the package manifest", () => {
|
||||
expect(packageJson.devDependencies?.["@openclaw/plugin-sdk"]).toBe("workspace:*");
|
||||
expect(packageJson.dependencies?.ws).toBe("8.21.0");
|
||||
});
|
||||
|
||||
it("exposes only current OpenAI login choices", () => {
|
||||
const openAiLogin = manifest.providerAuthChoices?.find(
|
||||
(choice) => choice.choiceId === "openai",
|
||||
);
|
||||
|
||||
expect(openAiLogin?.deprecatedChoiceIds).toBeUndefined();
|
||||
});
|
||||
|
||||
it("routes setup through the OpenAI setup runtime", () => {
|
||||
expect(manifest.legacyPluginIds).toBeUndefined();
|
||||
expect(manifest.setup?.providers?.map((provider) => provider.id)).toEqual(["openai"]);
|
||||
expect(manifest.providerAuthAliases).toBeUndefined();
|
||||
});
|
||||
|
||||
it("classifies ChatGPT backend traffic with the supported OpenAI endpoint class", () => {
|
||||
const chatGptEndpoint = manifest.providerEndpoints?.find((endpoint) =>
|
||||
endpoint.hosts?.includes("chatgpt.com"),
|
||||
);
|
||||
expect(chatGptEndpoint?.endpointClass).toBe("openai");
|
||||
});
|
||||
|
||||
it("classifies regional API hosts as public OpenAI endpoints", () => {
|
||||
const publicEndpoint = manifest.providerEndpoints?.find((endpoint) =>
|
||||
endpoint.hosts?.includes("api.openai.com"),
|
||||
);
|
||||
expect(publicEndpoint?.endpointClass).toBe("openai-public");
|
||||
expect(publicEndpoint?.hostSuffixes).toContain(".api.openai.com");
|
||||
});
|
||||
|
||||
it("keeps OpenAI media-understanding manifest metadata aligned with runtime audio support", () => {
|
||||
const metadata = manifest.mediaUnderstandingProviderMetadata?.openai;
|
||||
expect(metadata?.capabilities).toEqual(["image", "audio"]);
|
||||
expect(metadata?.defaultModels?.image).toBe("gpt-5.5");
|
||||
expect(metadata?.defaultModels?.audio).toBe("gpt-4o-transcribe");
|
||||
expect(metadata?.autoPriority?.image).toBe(20);
|
||||
expect(metadata?.autoPriority?.audio).toBe(20);
|
||||
});
|
||||
|
||||
it("labels OpenAI API key and Codex auth choices without stale mixed OAuth wording", () => {
|
||||
const choices = manifest.providerAuthChoices ?? [];
|
||||
const openAiLogin = choices.find((choice) => choice.choiceId === "openai");
|
||||
const openAiDeviceCode = choices.find((choice) => choice.choiceId === "openai-device-code");
|
||||
const apiKey = choices.find(
|
||||
(choice) => choice.provider === "openai" && choice.method === "api-key",
|
||||
);
|
||||
|
||||
expect(openAiLogin?.choiceLabel).toBe("ChatGPT Login");
|
||||
expect(openAiLogin?.choiceHint).toBe("Sign in with your ChatGPT or Codex subscription");
|
||||
expect(openAiLogin?.assistantVisibility).toBeUndefined();
|
||||
expect(openAiLogin?.groupId).toBe("openai");
|
||||
expect(openAiLogin?.groupLabel).toBe("OpenAI");
|
||||
expect(openAiLogin?.groupHint).toBe("ChatGPT/Codex sign-in or API key");
|
||||
expect(openAiDeviceCode?.choiceLabel).toBe("ChatGPT Device Pairing");
|
||||
expect(openAiDeviceCode?.choiceHint).toBe(
|
||||
"Pair your ChatGPT account in browser with a device code",
|
||||
);
|
||||
expect(openAiDeviceCode?.assistantVisibility).toBe("manual-only");
|
||||
expect(openAiDeviceCode?.groupId).toBe("openai");
|
||||
expect(openAiDeviceCode?.groupLabel).toBe("OpenAI");
|
||||
expect(openAiDeviceCode?.groupHint).toBe("ChatGPT/Codex sign-in or API key");
|
||||
expect(apiKey?.choiceLabel).toBe("OpenAI API Key");
|
||||
expect(apiKey?.choiceHint).toBe("Use your OpenAI API key directly");
|
||||
expect(apiKey?.groupId).toBe("openai");
|
||||
expect(apiKey?.groupLabel).toBe("OpenAI");
|
||||
expect(apiKey?.groupHint).toBe("ChatGPT/Codex sign-in or API key");
|
||||
expect(choices.map((choice) => choice.choiceLabel)).not.toContain(
|
||||
"OpenAI Codex (ChatGPT OAuth)",
|
||||
);
|
||||
expect(choices.map((choice) => choice.groupHint)).not.toContain("Codex OAuth + API key");
|
||||
expect(choices.map((choice) => choice.groupHint)).not.toContain("API key or Codex sign-in");
|
||||
});
|
||||
|
||||
it("keeps Spark suppression conditional on direct OpenAI API rows", () => {
|
||||
const sparkSuppression = manifest.modelCatalog?.suppressions?.find(
|
||||
(suppression) =>
|
||||
suppression.provider === "openai" && suppression.model === "gpt-5.3-codex-spark",
|
||||
);
|
||||
|
||||
expect(sparkSuppression?.when).toEqual({
|
||||
baseUrlHosts: ["api.openai.com"],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps auth choice copy aligned with provider wizard metadata", () => {
|
||||
const wizards = providerWizardByKey();
|
||||
|
||||
for (const choice of manifest.providerAuthChoices ?? []) {
|
||||
const key = `${choice.provider}:${choice.method}`;
|
||||
|
||||
expectWizardFields(wizards.get(key), manifestComparableWizardFields(choice), key);
|
||||
}
|
||||
});
|
||||
});
|
||||
18
extensions/openai/package.json
Normal file
18
extensions/openai/package.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "@openclaw/openai-provider",
|
||||
"version": "2026.6.11",
|
||||
"private": true,
|
||||
"description": "OpenClaw OpenAI provider plugins",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"ws": "8.21.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
52
extensions/openai/prompt-overlay.ts
Normal file
52
extensions/openai/prompt-overlay.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
// Openai plugin module implements prompt overlay behavior.
|
||||
import {
|
||||
GPT5_BEHAVIOR_CONTRACT,
|
||||
GPT5_FRIENDLY_CHAT_PROMPT_OVERLAY,
|
||||
GPT5_HEARTBEAT_PROMPT_OVERLAY,
|
||||
isGpt5ModelId,
|
||||
resolveGpt5PromptOverlayMode,
|
||||
resolveGpt5SystemPromptContribution,
|
||||
type Gpt5PromptOverlayMode,
|
||||
} from "openclaw/plugin-sdk/provider-model-shared";
|
||||
|
||||
const OPENAI_PROVIDER_IDS = new Set(["openai"]);
|
||||
|
||||
export const OPENAI_FRIENDLY_PROMPT_OVERLAY = GPT5_FRIENDLY_CHAT_PROMPT_OVERLAY;
|
||||
export const OPENAI_HEARTBEAT_PROMPT_OVERLAY = GPT5_HEARTBEAT_PROMPT_OVERLAY;
|
||||
export const OPENAI_GPT5_BEHAVIOR_CONTRACT = GPT5_BEHAVIOR_CONTRACT;
|
||||
|
||||
type OpenAIPromptOverlayMode = Gpt5PromptOverlayMode;
|
||||
|
||||
export function resolveOpenAIPromptOverlayMode(
|
||||
pluginConfig?: Record<string, unknown>,
|
||||
): OpenAIPromptOverlayMode {
|
||||
return resolveGpt5PromptOverlayMode(undefined, pluginConfig);
|
||||
}
|
||||
|
||||
export function shouldApplyOpenAIPromptOverlay(params: {
|
||||
modelProviderId?: string;
|
||||
modelId?: string;
|
||||
}): boolean {
|
||||
return OPENAI_PROVIDER_IDS.has(params.modelProviderId ?? "") && isGpt5ModelId(params.modelId);
|
||||
}
|
||||
|
||||
export function resolveOpenAISystemPromptContribution(params: {
|
||||
config?: Parameters<typeof resolveGpt5SystemPromptContribution>[0]["config"];
|
||||
legacyPluginConfig?: Record<string, unknown>;
|
||||
mode?: OpenAIPromptOverlayMode;
|
||||
modelProviderId?: string;
|
||||
modelId?: string;
|
||||
trigger?: Parameters<typeof resolveGpt5SystemPromptContribution>[0]["trigger"];
|
||||
}) {
|
||||
return resolveGpt5SystemPromptContribution({
|
||||
config: params.config,
|
||||
legacyPluginConfig:
|
||||
params.mode === undefined ? params.legacyPluginConfig : { personality: params.mode },
|
||||
modelId: params.modelId,
|
||||
trigger: params.trigger,
|
||||
enabled: shouldApplyOpenAIPromptOverlay({
|
||||
modelProviderId: params.modelProviderId,
|
||||
modelId: params.modelId,
|
||||
}),
|
||||
});
|
||||
}
|
||||
13
extensions/openai/provider-auth.contract.test.ts
Normal file
13
extensions/openai/provider-auth.contract.test.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
// Openai tests cover provider auth.contract plugin behavior.
|
||||
import { describeOpenAICodexProviderAuthContract } from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import { vi } from "vitest";
|
||||
|
||||
const loginOpenAICodexOAuthMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./openai-chatgpt-oauth.runtime.js", () => ({
|
||||
loginOpenAICodexOAuth: loginOpenAICodexOAuthMock,
|
||||
}));
|
||||
|
||||
describeOpenAICodexProviderAuthContract(() => import("./index.js"), {
|
||||
loginOpenAICodexOAuthMock,
|
||||
});
|
||||
4
extensions/openai/provider-catalog.contract.test.ts
Normal file
4
extensions/openai/provider-catalog.contract.test.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
// Openai tests cover provider catalog.contract plugin behavior.
|
||||
import { describeOpenAIProviderCatalogContract } from "./test-support/provider-catalog.contract-test-support.js";
|
||||
|
||||
describeOpenAIProviderCatalogContract();
|
||||
68
extensions/openai/provider-contract-api.ts
Normal file
68
extensions/openai/provider-contract-api.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
// Openai API module exposes the plugin public contract.
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import {
|
||||
OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
OPENAI_API_KEY_LABEL,
|
||||
OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
OPENAI_CHATGPT_LOGIN_HINT,
|
||||
OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
} from "./auth-choice-copy.js";
|
||||
|
||||
const noopAuth = async () => ({ profiles: [] });
|
||||
|
||||
export function createOpenAIProvider(): ProviderPlugin {
|
||||
return {
|
||||
id: "openai",
|
||||
label: "OpenAI",
|
||||
hookAliases: ["azure-openai", "azure-openai-responses"],
|
||||
docsPath: "/providers/models",
|
||||
envVars: ["OPENAI_API_KEY"],
|
||||
auth: [
|
||||
{
|
||||
id: "oauth",
|
||||
kind: "oauth",
|
||||
label: OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
hint: OPENAI_CHATGPT_LOGIN_HINT,
|
||||
run: noopAuth,
|
||||
wizard: {
|
||||
choiceId: "openai",
|
||||
choiceLabel: OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
choiceHint: OPENAI_CHATGPT_LOGIN_HINT,
|
||||
assistantPriority: -40,
|
||||
onboardingFeatured: true,
|
||||
...OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "device-code",
|
||||
kind: "device_code",
|
||||
label: OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
hint: OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
run: noopAuth,
|
||||
wizard: {
|
||||
choiceId: "openai-device-code",
|
||||
choiceLabel: OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
choiceHint: OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
assistantPriority: -10,
|
||||
assistantVisibility: "manual-only",
|
||||
...OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "api-key",
|
||||
kind: "api_key",
|
||||
label: OPENAI_API_KEY_LABEL,
|
||||
hint: "Use your OpenAI API key directly",
|
||||
run: noopAuth,
|
||||
wizard: {
|
||||
choiceId: "openai-api-key",
|
||||
choiceLabel: OPENAI_API_KEY_LABEL,
|
||||
choiceHint: "Use your OpenAI API key directly",
|
||||
assistantPriority: 5,
|
||||
...OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
45
extensions/openai/provider-policy-api.test.ts
Normal file
45
extensions/openai/provider-policy-api.test.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
// Openai tests cover provider policy api plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveThinkingProfile } from "./provider-policy-api.js";
|
||||
|
||||
describe("OpenAI provider policy artifact", () => {
|
||||
it("keeps OpenAI thinking policy for openai refs", () => {
|
||||
const codexProfile = resolveThinkingProfile({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.3-codex-spark",
|
||||
});
|
||||
const openaiProfile = resolveThinkingProfile({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.3",
|
||||
});
|
||||
const openaiMiniProfile = resolveThinkingProfile({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4-mini",
|
||||
});
|
||||
|
||||
expect(codexProfile?.levels.map((level) => level.id)).toContain("xhigh");
|
||||
expect(openaiProfile?.levels.map((level) => level.id)).not.toContain("xhigh");
|
||||
expect(openaiMiniProfile?.levels.map((level) => level.id)).toContain("xhigh");
|
||||
});
|
||||
|
||||
it("exposes max for the GPT-5.6 series", () => {
|
||||
const solLevels = resolveThinkingProfile({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.6-sol",
|
||||
})?.levels.map((level) => level.id);
|
||||
const terraLevels = resolveThinkingProfile({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.6-terra",
|
||||
})?.levels.map((level) => level.id);
|
||||
const lunaLevels = resolveThinkingProfile({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.6-luna",
|
||||
})?.levels.map((level) => level.id);
|
||||
|
||||
expect(solLevels).toContain("max");
|
||||
expect(terraLevels).toContain("xhigh");
|
||||
expect(terraLevels).toContain("max");
|
||||
expect(lunaLevels).toContain("xhigh");
|
||||
expect(lunaLevels).toContain("max");
|
||||
});
|
||||
});
|
||||
16
extensions/openai/provider-policy-api.ts
Normal file
16
extensions/openai/provider-policy-api.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
// Openai API module exposes the plugin public contract.
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-types";
|
||||
import { resolveUnifiedOpenAIThinkingProfile } from "./thinking-policy.js";
|
||||
|
||||
export function normalizeConfig(params: { provider: string; providerConfig: ModelProviderConfig }) {
|
||||
return params.providerConfig;
|
||||
}
|
||||
|
||||
export function resolveThinkingProfile(params: { provider: string; modelId: string }) {
|
||||
switch (params.provider.trim().toLowerCase()) {
|
||||
case "openai":
|
||||
return resolveUnifiedOpenAIThinkingProfile(params.modelId);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
4
extensions/openai/provider-runtime.contract.test.ts
Normal file
4
extensions/openai/provider-runtime.contract.test.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
// Openai tests cover provider runtime.contract plugin behavior.
|
||||
import { describeOpenAIProviderRuntimeContract } from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
|
||||
describeOpenAIProviderRuntimeContract(() => import("./index.js"));
|
||||
100
extensions/openai/realtime-provider-shared.test.ts
Normal file
100
extensions/openai/realtime-provider-shared.test.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
// Openai tests cover realtime session secret creation behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createOpenAIRealtimeClientSecret,
|
||||
createOpenAIRealtimeTranscriptionClientSecret,
|
||||
} from "./realtime-provider-shared.js";
|
||||
|
||||
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuardMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
|
||||
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
|
||||
}));
|
||||
|
||||
function makeStreamingResponse(params: { chunkCount: number; chunkSize: number }): {
|
||||
response: Response;
|
||||
getReadCount: () => number;
|
||||
wasCanceled: () => boolean;
|
||||
} {
|
||||
let readCount = 0;
|
||||
let canceled = false;
|
||||
const chunk = new Uint8Array(params.chunkSize);
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (readCount >= params.chunkCount) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
readCount += 1;
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
return { response, getReadCount: () => readCount, wasCanceled: () => canceled };
|
||||
}
|
||||
|
||||
function guardedFetch(response: Response): void {
|
||||
fetchWithSsrFGuardMock.mockResolvedValue({ response, release: vi.fn() });
|
||||
}
|
||||
|
||||
describe("createOpenAIRealtimeClientSecret", () => {
|
||||
it("returns client secret from a well-formed response", async () => {
|
||||
guardedFetch(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
client_secret: { value: "eph-secret-abc" },
|
||||
expires_at: Math.floor(Date.now() / 1000) + 60,
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
),
|
||||
);
|
||||
|
||||
const result = await createOpenAIRealtimeClientSecret({
|
||||
authToken: "sk-test",
|
||||
auditContext: "test",
|
||||
session: { model: "gpt-4o-realtime-preview" },
|
||||
});
|
||||
|
||||
expect(result.value).toBe("eph-secret-abc");
|
||||
expect(typeof result.expiresAt).toBe("number");
|
||||
});
|
||||
|
||||
it("bounds oversized success response and cancels the stream", async () => {
|
||||
// 20 MiB in 1 MiB chunks — well over the 16 MiB cap
|
||||
const streamed = makeStreamingResponse({ chunkCount: 20, chunkSize: 1024 * 1024 });
|
||||
guardedFetch(streamed.response);
|
||||
|
||||
await expect(
|
||||
createOpenAIRealtimeClientSecret({
|
||||
authToken: "sk-test",
|
||||
auditContext: "test",
|
||||
session: { model: "gpt-4o-realtime-preview" },
|
||||
}),
|
||||
).rejects.toThrow(/openai\.realtime-session/);
|
||||
|
||||
expect(streamed.wasCanceled()).toBe(true);
|
||||
expect(streamed.getReadCount()).toBeLessThan(20);
|
||||
});
|
||||
|
||||
it("throws the provider error label on oversized body", async () => {
|
||||
const streamed = makeStreamingResponse({ chunkCount: 20, chunkSize: 1024 * 1024 });
|
||||
guardedFetch(streamed.response);
|
||||
|
||||
await expect(
|
||||
createOpenAIRealtimeTranscriptionClientSecret({
|
||||
authToken: "sk-test",
|
||||
auditContext: "test",
|
||||
session: { model: "gpt-4o-transcribe" },
|
||||
}),
|
||||
).rejects.toThrow(/openai\.realtime-session/);
|
||||
|
||||
expect(streamed.wasCanceled()).toBe(true);
|
||||
});
|
||||
});
|
||||
167
extensions/openai/realtime-provider-shared.ts
Normal file
167
extensions/openai/realtime-provider-shared.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import { resolveExpiresAtMsFromEpochSeconds } from "openclaw/plugin-sdk/number-runtime";
|
||||
import {
|
||||
createProviderHttpError,
|
||||
readProviderJsonResponse,
|
||||
resolveProviderRequestHeaders,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { captureWsEvent } from "openclaw/plugin-sdk/proxy-capture";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
asFiniteNumber,
|
||||
asOptionalRecord as asObjectRecord,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
|
||||
export const trimToUndefined = normalizeOptionalString;
|
||||
export { asFiniteNumber, asObjectRecord };
|
||||
|
||||
export function readRealtimeErrorDetail(error: unknown): string {
|
||||
if (typeof error === "string" && error) {
|
||||
return error;
|
||||
}
|
||||
const message = asObjectRecord(error)?.message;
|
||||
if (typeof message === "string" && message) {
|
||||
return message;
|
||||
}
|
||||
return "Unknown error";
|
||||
}
|
||||
|
||||
export function resolveOpenAIProviderConfigRecord(
|
||||
config: Record<string, unknown>,
|
||||
): Record<string, unknown> | undefined {
|
||||
const providers = asObjectRecord(config.providers);
|
||||
return (
|
||||
asObjectRecord(providers?.openai) ?? asObjectRecord(config.openai) ?? asObjectRecord(config)
|
||||
);
|
||||
}
|
||||
|
||||
export function captureOpenAIRealtimeWsClose(params: {
|
||||
url: string;
|
||||
flowId: string;
|
||||
capability: "realtime-transcription" | "realtime-voice";
|
||||
code: unknown;
|
||||
reasonBuffer: unknown;
|
||||
}): void {
|
||||
captureWsEvent({
|
||||
url: params.url,
|
||||
direction: "local",
|
||||
kind: "ws-close",
|
||||
flowId: params.flowId,
|
||||
closeCode: typeof params.code === "number" ? params.code : undefined,
|
||||
meta: {
|
||||
provider: "openai",
|
||||
capability: params.capability,
|
||||
reason:
|
||||
Buffer.isBuffer(params.reasonBuffer) && params.reasonBuffer.length > 0
|
||||
? params.reasonBuffer.toString("utf8")
|
||||
: undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export type OpenAIRealtimeClientSecretResult = {
|
||||
value: string;
|
||||
expiresAt?: number;
|
||||
};
|
||||
|
||||
type OpenAIRealtimeSecretRequest = {
|
||||
authToken: string;
|
||||
auditContext: string;
|
||||
url: string;
|
||||
body: unknown;
|
||||
errorMessage: string;
|
||||
missingValueMessage: string;
|
||||
};
|
||||
|
||||
function readStringField(value: unknown, key: string): string | undefined {
|
||||
if (!value || typeof value !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const raw = (value as Record<string, unknown>)[key];
|
||||
return typeof raw === "string" && raw.trim() ? raw.trim() : undefined;
|
||||
}
|
||||
|
||||
async function createOpenAIRealtimeSecret(
|
||||
params: OpenAIRealtimeSecretRequest,
|
||||
): Promise<OpenAIRealtimeClientSecretResult> {
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: params.url,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: resolveProviderRequestHeaders({
|
||||
provider: "openai",
|
||||
baseUrl: params.url,
|
||||
capability: "audio",
|
||||
transport: "http",
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${params.authToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}) ?? {
|
||||
Authorization: `Bearer ${params.authToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(params.body),
|
||||
},
|
||||
auditContext: params.auditContext,
|
||||
});
|
||||
const payload = await (async () => {
|
||||
try {
|
||||
if (!response.ok) {
|
||||
throw await createProviderHttpError(response, params.errorMessage);
|
||||
}
|
||||
return await readProviderJsonResponse<unknown>(response, "openai.realtime-session");
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
})();
|
||||
const nestedSecret =
|
||||
payload && typeof payload === "object"
|
||||
? (payload as Record<string, unknown>).client_secret
|
||||
: undefined;
|
||||
const clientSecret = readStringField(payload, "value") ?? readStringField(nestedSecret, "value");
|
||||
if (!clientSecret) {
|
||||
throw new Error(params.missingValueMessage);
|
||||
}
|
||||
const expiresAt =
|
||||
payload && typeof payload === "object"
|
||||
? (payload as Record<string, unknown>).expires_at
|
||||
: undefined;
|
||||
const expiresAtMs = resolveExpiresAtMsFromEpochSeconds(expiresAt);
|
||||
return {
|
||||
value: clientSecret,
|
||||
...(expiresAtMs === undefined ? {} : { expiresAt: expiresAtMs }),
|
||||
};
|
||||
}
|
||||
|
||||
export async function createOpenAIRealtimeClientSecret(params: {
|
||||
authToken: string;
|
||||
auditContext: string;
|
||||
session: Record<string, unknown>;
|
||||
}): Promise<OpenAIRealtimeClientSecretResult> {
|
||||
const url = "https://api.openai.com/v1/realtime/client_secrets";
|
||||
return createOpenAIRealtimeSecret({
|
||||
...params,
|
||||
url,
|
||||
body: { session: params.session },
|
||||
errorMessage: "OpenAI Realtime client secret failed",
|
||||
missingValueMessage: "OpenAI Realtime client secret response did not include a value",
|
||||
});
|
||||
}
|
||||
|
||||
export async function createOpenAIRealtimeTranscriptionClientSecret(params: {
|
||||
authToken: string;
|
||||
auditContext: string;
|
||||
session: Record<string, unknown>;
|
||||
}): Promise<OpenAIRealtimeClientSecretResult> {
|
||||
const url = "https://api.openai.com/v1/realtime/transcription_sessions";
|
||||
return createOpenAIRealtimeSecret({
|
||||
...params,
|
||||
url,
|
||||
body: params.session,
|
||||
errorMessage: "OpenAI Realtime transcription client secret failed",
|
||||
missingValueMessage:
|
||||
"OpenAI Realtime transcription client secret response did not include a value",
|
||||
});
|
||||
}
|
||||
375
extensions/openai/realtime-transcription-provider.test.ts
Normal file
375
extensions/openai/realtime-transcription-provider.test.ts
Normal file
@@ -0,0 +1,375 @@
|
||||
// Openai tests cover realtime transcription provider plugin behavior.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
|
||||
|
||||
const { FakeWebSocket, providerAuthMocks, ssrfMocks } = vi.hoisted(() => {
|
||||
type Listener = (...args: unknown[]) => void;
|
||||
|
||||
class MockWebSocket {
|
||||
static readonly OPEN = 1;
|
||||
static readonly CLOSED = 3;
|
||||
static instances: MockWebSocket[] = [];
|
||||
|
||||
readonly listeners = new Map<string, Listener[]>();
|
||||
readonly headers?: Record<string, string>;
|
||||
readonly url?: string;
|
||||
readyState = 0;
|
||||
sent: string[] = [];
|
||||
closed = false;
|
||||
|
||||
constructor(url?: string, options?: { headers?: Record<string, string> }) {
|
||||
this.url = url;
|
||||
this.headers = options?.headers;
|
||||
MockWebSocket.instances.push(this);
|
||||
}
|
||||
|
||||
on(event: string, listener: Listener): this {
|
||||
const listeners = this.listeners.get(event) ?? [];
|
||||
listeners.push(listener);
|
||||
this.listeners.set(event, listeners);
|
||||
return this;
|
||||
}
|
||||
|
||||
emit(event: string, ...args: unknown[]): void {
|
||||
for (const listener of this.listeners.get(event) ?? []) {
|
||||
listener(...args);
|
||||
}
|
||||
}
|
||||
|
||||
send(payload: string): void {
|
||||
this.sent.push(payload);
|
||||
}
|
||||
|
||||
close(code?: number, reason?: string): void {
|
||||
this.closed = true;
|
||||
this.readyState = MockWebSocket.CLOSED;
|
||||
this.emit("close", code ?? 1000, Buffer.from(reason ?? ""));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
FakeWebSocket: MockWebSocket,
|
||||
providerAuthMocks: {
|
||||
isProviderAuthProfileConfigured: vi.fn(),
|
||||
resolveProviderAuthProfileApiKey: vi.fn(),
|
||||
},
|
||||
ssrfMocks: {
|
||||
fetchWithSsrFGuard: vi.fn(),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("ws", () => ({
|
||||
default: FakeWebSocket,
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
|
||||
isProviderAuthProfileConfigured: providerAuthMocks.isProviderAuthProfileConfigured,
|
||||
resolveProviderAuthProfileApiKey: providerAuthMocks.resolveProviderAuthProfileApiKey,
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
|
||||
fetchWithSsrFGuard: ssrfMocks.fetchWithSsrFGuard,
|
||||
}));
|
||||
|
||||
type FakeWebSocketInstance = InstanceType<typeof FakeWebSocket>;
|
||||
type SentRealtimeEvent = {
|
||||
type: string;
|
||||
audio?: string;
|
||||
session?: unknown;
|
||||
};
|
||||
|
||||
function parseSent(socket: FakeWebSocketInstance): SentRealtimeEvent[] {
|
||||
return socket.sent.map((payload) => JSON.parse(payload) as SentRealtimeEvent);
|
||||
}
|
||||
|
||||
async function waitForFakeSocket(): Promise<FakeWebSocketInstance> {
|
||||
let socket: FakeWebSocketInstance | undefined;
|
||||
await vi.waitFor(() => {
|
||||
socket = FakeWebSocket.instances[0];
|
||||
if (!socket) {
|
||||
throw new Error("expected session to create a websocket");
|
||||
}
|
||||
});
|
||||
if (!socket) {
|
||||
throw new Error("expected session to create a websocket");
|
||||
}
|
||||
return socket;
|
||||
}
|
||||
|
||||
function mockCallArg(mock: { mock: { calls: unknown[][] } }, index = 0): Record<string, unknown> {
|
||||
const call = mock.mock.calls[index];
|
||||
if (!call) {
|
||||
throw new Error(`expected mock call ${index}`);
|
||||
}
|
||||
return call[0] as Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("buildOpenAIRealtimeTranscriptionProvider", () => {
|
||||
beforeEach(() => {
|
||||
FakeWebSocket.instances = [];
|
||||
providerAuthMocks.isProviderAuthProfileConfigured.mockReset();
|
||||
providerAuthMocks.resolveProviderAuthProfileApiKey.mockReset();
|
||||
ssrfMocks.fetchWithSsrFGuard.mockReset();
|
||||
vi.stubEnv("OPENAI_API_KEY", "");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("normalizes OpenAI config defaults", () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
cfg: {} as never,
|
||||
rawConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "sk-test", // pragma: allowlist secret
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolved).toEqual({
|
||||
apiKey: "sk-test",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps provider-owned transcription settings configurable via raw provider config", () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
cfg: {} as never,
|
||||
rawConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
language: "en",
|
||||
model: "gpt-4o-transcribe",
|
||||
prompt: "expect OpenClaw product names",
|
||||
silenceDurationMs: 900,
|
||||
vadThreshold: 0.45,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolved).toEqual({
|
||||
language: "en",
|
||||
model: "gpt-4o-transcribe",
|
||||
prompt: "expect OpenClaw product names",
|
||||
silenceDurationMs: 900,
|
||||
vadThreshold: 0.45,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit zero-valued VAD settings", () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
cfg: {} as never,
|
||||
rawConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
silenceDurationMs: 0,
|
||||
vadThreshold: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolved?.silenceDurationMs).toBe(0);
|
||||
expect(resolved?.vadThreshold).toBe(0);
|
||||
});
|
||||
|
||||
it("drops malformed VAD timing settings", () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
cfg: {} as never,
|
||||
rawConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
silenceDurationMs: -1,
|
||||
vadThreshold: 1.5,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolved?.silenceDurationMs).toBeUndefined();
|
||||
expect(resolved?.vadThreshold).toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts the legacy openai-realtime alias", () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
expect(provider.aliases).toContain("openai-realtime");
|
||||
});
|
||||
|
||||
it("treats a Codex OAuth profile as configured when no API key is present", () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
const cfg = { auth: { order: { openai: ["openai:default"] } } };
|
||||
providerAuthMocks.isProviderAuthProfileConfigured.mockReturnValue(true);
|
||||
|
||||
expect(provider.isConfigured({ cfg: cfg as never, providerConfig: {} })).toBe(true);
|
||||
expect(providerAuthMocks.isProviderAuthProfileConfigured).toHaveBeenCalledWith({
|
||||
provider: "openai",
|
||||
cfg,
|
||||
});
|
||||
});
|
||||
|
||||
it("mints a Codex OAuth client secret for realtime transcription sockets", async () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
const release = vi.fn();
|
||||
providerAuthMocks.resolveProviderAuthProfileApiKey.mockResolvedValue("oauth-token");
|
||||
ssrfMocks.fetchWithSsrFGuard.mockResolvedValue({
|
||||
response: new Response(JSON.stringify({ value: "ek-test" }), { status: 200 }),
|
||||
release,
|
||||
});
|
||||
const cfg = { auth: { order: { openai: ["openai:default"] } } };
|
||||
const session = provider.createSession({
|
||||
cfg: cfg as never,
|
||||
providerConfig: {},
|
||||
});
|
||||
|
||||
const connecting = session.connect();
|
||||
const socket = await waitForFakeSocket();
|
||||
|
||||
expect(socket.headers?.Authorization).toBe("Bearer ek-test");
|
||||
expect(providerAuthMocks.resolveProviderAuthProfileApiKey).toHaveBeenCalledWith({
|
||||
provider: "openai",
|
||||
cfg,
|
||||
});
|
||||
const request = mockCallArg(ssrfMocks.fetchWithSsrFGuard);
|
||||
expect(request.auditContext).toBe("openai-realtime-transcription-session");
|
||||
expect(request.url).toBe("https://api.openai.com/v1/realtime/transcription_sessions");
|
||||
const init = request.init as {
|
||||
method?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: unknown;
|
||||
};
|
||||
expect(init.method).toBe("POST");
|
||||
expect(init.headers?.Authorization).toBe("Bearer oauth-token");
|
||||
expect(init.headers?.["Content-Type"]).toBe("application/json");
|
||||
expect(typeof init.body).toBe("string");
|
||||
expect(JSON.parse(init.body as string)).toEqual({
|
||||
type: "transcription",
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcmu" },
|
||||
transcription: { model: "gpt-4o-transcribe" },
|
||||
turn_detection: {
|
||||
type: "server_vad",
|
||||
threshold: 0.5,
|
||||
prefix_padding_ms: 300,
|
||||
silence_duration_ms: 800,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
socket.readyState = FakeWebSocket.OPEN;
|
||||
socket.emit("open");
|
||||
socket.emit("message", Buffer.from(JSON.stringify({ type: "transcription_session.updated" })));
|
||||
await connecting;
|
||||
|
||||
expect(release).toHaveBeenCalled();
|
||||
expect(parseSent(socket)[0]).toEqual({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "transcription",
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcmu" },
|
||||
transcription: { model: "gpt-4o-transcribe" },
|
||||
turn_detection: {
|
||||
type: "server_vad",
|
||||
threshold: 0.5,
|
||||
prefix_padding_ms: 300,
|
||||
silence_duration_ms: 800,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
session.close();
|
||||
});
|
||||
|
||||
it("waits for the OpenAI session update before draining audio", async () => {
|
||||
const provider = buildOpenAIRealtimeTranscriptionProvider();
|
||||
const session = provider.createSession({
|
||||
providerConfig: {
|
||||
apiKey: "sk-test", // pragma: allowlist secret
|
||||
language: "en",
|
||||
model: "gpt-4o-transcribe",
|
||||
prompt: "expect OpenClaw product names",
|
||||
silenceDurationMs: 900,
|
||||
vadThreshold: 0.45,
|
||||
},
|
||||
});
|
||||
|
||||
const connecting = session.connect();
|
||||
const socket = await waitForFakeSocket();
|
||||
|
||||
socket.readyState = FakeWebSocket.OPEN;
|
||||
socket.emit("open");
|
||||
session.sendAudio(Buffer.from("before-ready"));
|
||||
|
||||
expect(session.isConnected()).toBe(false);
|
||||
expect(parseSent(socket)).toEqual([
|
||||
{
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "transcription",
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcmu" },
|
||||
transcription: {
|
||||
model: "gpt-4o-transcribe",
|
||||
language: "en",
|
||||
prompt: "expect OpenClaw product names",
|
||||
},
|
||||
turn_detection: {
|
||||
type: "server_vad",
|
||||
threshold: 0.45,
|
||||
prefix_padding_ms: 300,
|
||||
silence_duration_ms: 900,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" })));
|
||||
await connecting;
|
||||
|
||||
expect(session.isConnected()).toBe(true);
|
||||
expect(parseSent(socket)).toEqual([
|
||||
{
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "transcription",
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcmu" },
|
||||
transcription: {
|
||||
model: "gpt-4o-transcribe",
|
||||
language: "en",
|
||||
prompt: "expect OpenClaw product names",
|
||||
},
|
||||
turn_detection: {
|
||||
type: "server_vad",
|
||||
threshold: 0.45,
|
||||
prefix_padding_ms: 300,
|
||||
silence_duration_ms: 900,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "input_audio_buffer.append",
|
||||
audio: Buffer.from("before-ready").toString("base64"),
|
||||
},
|
||||
]);
|
||||
session.close();
|
||||
});
|
||||
});
|
||||
278
extensions/openai/realtime-transcription-provider.ts
Normal file
278
extensions/openai/realtime-transcription-provider.ts
Normal file
@@ -0,0 +1,278 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import {
|
||||
isProviderAuthProfileConfigured,
|
||||
resolveProviderAuthProfileApiKey,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveProviderRequestHeaders } from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
createRealtimeTranscriptionWebSocketSession,
|
||||
type RealtimeTranscriptionProviderConfig,
|
||||
type RealtimeTranscriptionProviderPlugin,
|
||||
type RealtimeTranscriptionSession,
|
||||
type RealtimeTranscriptionSessionCreateRequest,
|
||||
type RealtimeTranscriptionWebSocketTransport,
|
||||
} from "openclaw/plugin-sdk/realtime-transcription";
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import {
|
||||
asFiniteNumber,
|
||||
createOpenAIRealtimeTranscriptionClientSecret,
|
||||
readRealtimeErrorDetail,
|
||||
resolveOpenAIProviderConfigRecord,
|
||||
trimToUndefined,
|
||||
} from "./realtime-provider-shared.js";
|
||||
|
||||
type OpenAIRealtimeTranscriptionProviderConfig = {
|
||||
apiKey?: string;
|
||||
language?: string;
|
||||
model?: string;
|
||||
prompt?: string;
|
||||
silenceDurationMs?: number;
|
||||
vadThreshold?: number;
|
||||
};
|
||||
|
||||
type OpenAIRealtimeTranscriptionSessionConfig = RealtimeTranscriptionSessionCreateRequest & {
|
||||
apiKey?: string;
|
||||
cfg?: OpenClawConfig;
|
||||
language?: string;
|
||||
model: string;
|
||||
prompt?: string;
|
||||
silenceDurationMs: number;
|
||||
vadThreshold: number;
|
||||
};
|
||||
|
||||
type RealtimeEvent = {
|
||||
type: string;
|
||||
delta?: string;
|
||||
transcript?: string;
|
||||
error?: unknown;
|
||||
};
|
||||
|
||||
type OpenAIRealtimeTranscriptionSessionPayload = {
|
||||
type: "transcription";
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcmu" };
|
||||
transcription: {
|
||||
model: string;
|
||||
language?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
turn_detection: {
|
||||
type: "server_vad";
|
||||
threshold: number;
|
||||
prefix_padding_ms: number;
|
||||
silence_duration_ms: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const OPENAI_REALTIME_TRANSCRIPTION_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||
const OPENAI_REALTIME_TRANSCRIPTION_CONNECT_TIMEOUT_MS = 10_000;
|
||||
const OPENAI_REALTIME_TRANSCRIPTION_MAX_RECONNECT_ATTEMPTS = 5;
|
||||
const OPENAI_REALTIME_TRANSCRIPTION_RECONNECT_DELAY_MS = 1000;
|
||||
const OPENAI_REALTIME_TRANSCRIPTION_DEFAULT_MODEL = "gpt-4o-transcribe";
|
||||
|
||||
function normalizeProviderConfig(
|
||||
config: RealtimeTranscriptionProviderConfig,
|
||||
): OpenAIRealtimeTranscriptionProviderConfig {
|
||||
const raw = resolveOpenAIProviderConfigRecord(config);
|
||||
return {
|
||||
apiKey:
|
||||
normalizeResolvedSecretInputString({
|
||||
value: raw?.apiKey,
|
||||
path: "plugins.entries.voice-call.config.streaming.providers.openai.apiKey",
|
||||
}) ??
|
||||
normalizeResolvedSecretInputString({
|
||||
value: raw?.openaiApiKey,
|
||||
path: "plugins.entries.voice-call.config.streaming.openaiApiKey",
|
||||
}),
|
||||
language: trimToUndefined(raw?.language),
|
||||
model: trimToUndefined(raw?.model) ?? trimToUndefined(raw?.sttModel),
|
||||
prompt: trimToUndefined(raw?.prompt),
|
||||
silenceDurationMs: normalizeNonNegativeInteger(raw?.silenceDurationMs),
|
||||
vadThreshold: normalizeVadThreshold(raw?.vadThreshold),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeNonNegativeInteger(value: unknown): number | undefined {
|
||||
const number = asFiniteNumber(value);
|
||||
if (number === undefined || !Number.isSafeInteger(number) || number < 0) {
|
||||
return undefined;
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function normalizeVadThreshold(value: unknown): number | undefined {
|
||||
const number = asFiniteNumber(value);
|
||||
if (number === undefined || number < 0 || number > 1) {
|
||||
return undefined;
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function buildOpenAIRealtimeTranscriptionSessionPayload(
|
||||
config: OpenAIRealtimeTranscriptionSessionConfig,
|
||||
): OpenAIRealtimeTranscriptionSessionPayload {
|
||||
return {
|
||||
type: "transcription",
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcmu" },
|
||||
transcription: {
|
||||
model: config.model,
|
||||
...(config.language ? { language: config.language } : {}),
|
||||
...(config.prompt ? { prompt: config.prompt } : {}),
|
||||
},
|
||||
turn_detection: {
|
||||
type: "server_vad",
|
||||
threshold: config.vadThreshold,
|
||||
prefix_padding_ms: 300,
|
||||
silence_duration_ms: config.silenceDurationMs,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveOpenAIRealtimeTranscriptionAuthorization(
|
||||
config: OpenAIRealtimeTranscriptionSessionConfig,
|
||||
): Promise<string> {
|
||||
const apiKey = config.apiKey || process.env.OPENAI_API_KEY;
|
||||
if (apiKey) {
|
||||
return apiKey;
|
||||
}
|
||||
const authToken = await resolveProviderAuthProfileApiKey({
|
||||
provider: "openai",
|
||||
cfg: config.cfg,
|
||||
});
|
||||
if (!authToken) {
|
||||
throw new Error("OpenAI API key or Codex OAuth missing");
|
||||
}
|
||||
const clientSecret = await createOpenAIRealtimeTranscriptionClientSecret({
|
||||
authToken,
|
||||
auditContext: "openai-realtime-transcription-session",
|
||||
session: buildOpenAIRealtimeTranscriptionSessionPayload(config),
|
||||
});
|
||||
return clientSecret.value;
|
||||
}
|
||||
|
||||
function createOpenAIRealtimeTranscriptionSession(
|
||||
config: OpenAIRealtimeTranscriptionSessionConfig,
|
||||
): RealtimeTranscriptionSession {
|
||||
let pendingTranscript = "";
|
||||
|
||||
const handleEvent = (
|
||||
event: RealtimeEvent,
|
||||
transport: RealtimeTranscriptionWebSocketTransport,
|
||||
) => {
|
||||
switch (event.type) {
|
||||
case "session.updated":
|
||||
case "transcription_session.updated":
|
||||
transport.markReady();
|
||||
return;
|
||||
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
if (event.delta) {
|
||||
pendingTranscript += event.delta;
|
||||
config.onPartial?.(pendingTranscript);
|
||||
}
|
||||
return;
|
||||
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
if (event.transcript) {
|
||||
config.onTranscript?.(event.transcript);
|
||||
}
|
||||
pendingTranscript = "";
|
||||
return;
|
||||
|
||||
case "input_audio_buffer.speech_started":
|
||||
pendingTranscript = "";
|
||||
config.onSpeechStart?.();
|
||||
return;
|
||||
|
||||
case "error": {
|
||||
const detail = readRealtimeErrorDetail(event.error);
|
||||
const error = new Error(detail);
|
||||
if (!transport.isReady()) {
|
||||
transport.failConnect(error);
|
||||
} else {
|
||||
config.onError?.(error);
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
}
|
||||
};
|
||||
|
||||
return createRealtimeTranscriptionWebSocketSession<RealtimeEvent>({
|
||||
providerId: "openai",
|
||||
callbacks: config,
|
||||
url: OPENAI_REALTIME_TRANSCRIPTION_URL,
|
||||
headers: async () => {
|
||||
const bearer = await resolveOpenAIRealtimeTranscriptionAuthorization(config);
|
||||
return (
|
||||
resolveProviderRequestHeaders({
|
||||
provider: "openai",
|
||||
baseUrl: OPENAI_REALTIME_TRANSCRIPTION_URL,
|
||||
capability: "audio",
|
||||
transport: "websocket",
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${bearer}`,
|
||||
},
|
||||
}) ?? {
|
||||
Authorization: `Bearer ${bearer}`,
|
||||
}
|
||||
);
|
||||
},
|
||||
connectTimeoutMs: OPENAI_REALTIME_TRANSCRIPTION_CONNECT_TIMEOUT_MS,
|
||||
maxReconnectAttempts: OPENAI_REALTIME_TRANSCRIPTION_MAX_RECONNECT_ATTEMPTS,
|
||||
reconnectDelayMs: OPENAI_REALTIME_TRANSCRIPTION_RECONNECT_DELAY_MS,
|
||||
connectTimeoutMessage: "OpenAI realtime transcription connection timeout",
|
||||
connectClosedBeforeReadyMessage: "OpenAI realtime transcription connection closed before ready",
|
||||
reconnectLimitMessage: "OpenAI realtime transcription reconnect limit reached",
|
||||
sendAudio: (audio, transport) => {
|
||||
transport.sendJson({
|
||||
type: "input_audio_buffer.append",
|
||||
audio: audio.toString("base64"),
|
||||
});
|
||||
},
|
||||
onOpen: (transport: RealtimeTranscriptionWebSocketTransport) => {
|
||||
transport.sendJson({
|
||||
type: "session.update",
|
||||
session: buildOpenAIRealtimeTranscriptionSessionPayload(config),
|
||||
});
|
||||
},
|
||||
onMessage: handleEvent,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildOpenAIRealtimeTranscriptionProvider(): RealtimeTranscriptionProviderPlugin {
|
||||
return {
|
||||
id: "openai",
|
||||
label: "OpenAI Realtime Transcription",
|
||||
aliases: ["openai-realtime"],
|
||||
defaultModel: OPENAI_REALTIME_TRANSCRIPTION_DEFAULT_MODEL,
|
||||
autoSelectOrder: 10,
|
||||
resolveConfig: ({ rawConfig }) => normalizeProviderConfig(rawConfig),
|
||||
isConfigured: ({ cfg, providerConfig }) =>
|
||||
Boolean(
|
||||
normalizeProviderConfig(providerConfig).apiKey ||
|
||||
process.env.OPENAI_API_KEY ||
|
||||
isProviderAuthProfileConfigured({ provider: "openai", cfg }),
|
||||
),
|
||||
createSession: (req) => {
|
||||
const config = normalizeProviderConfig(req.providerConfig);
|
||||
return createOpenAIRealtimeTranscriptionSession({
|
||||
...req,
|
||||
apiKey: config.apiKey,
|
||||
language: config.language,
|
||||
model: config.model ?? OPENAI_REALTIME_TRANSCRIPTION_DEFAULT_MODEL,
|
||||
prompt: config.prompt,
|
||||
silenceDurationMs: config.silenceDurationMs ?? 800,
|
||||
vadThreshold: config.vadThreshold ?? 0.5,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
2162
extensions/openai/realtime-voice-provider.test.ts
Normal file
2162
extensions/openai/realtime-voice-provider.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
1416
extensions/openai/realtime-voice-provider.ts
Normal file
1416
extensions/openai/realtime-voice-provider.ts
Normal file
File diff suppressed because it is too large
Load Diff
12
extensions/openai/register.runtime.ts
Normal file
12
extensions/openai/register.runtime.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
// Openai plugin module implements register behavior.
|
||||
export { buildOpenAIImageGenerationProvider } from "./image-generation-provider.js";
|
||||
export { openaiMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
export { buildOpenAIProvider } from "./openai-provider.js";
|
||||
export {
|
||||
OPENAI_FRIENDLY_PROMPT_OVERLAY,
|
||||
resolveOpenAIPromptOverlayMode,
|
||||
shouldApplyOpenAIPromptOverlay,
|
||||
} from "./prompt-overlay.js";
|
||||
export { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
|
||||
export { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
|
||||
export { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
33
extensions/openai/replay-policy.ts
Normal file
33
extensions/openai/replay-policy.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
// Openai plugin module implements replay policy behavior.
|
||||
import type {
|
||||
ProviderReplayPolicy,
|
||||
ProviderReplayPolicyContext,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
|
||||
const RESPONSES_FAMILY_APIS = new Set([
|
||||
"openai-responses",
|
||||
"openai-chatgpt-responses",
|
||||
"azure-openai-responses",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Returns the provider-owned replay policy for OpenAI-family transports.
|
||||
*/
|
||||
export function buildOpenAIReplayPolicy(ctx: ProviderReplayPolicyContext): ProviderReplayPolicy {
|
||||
const isResponsesFamily = RESPONSES_FAMILY_APIS.has(ctx.modelApi ?? "");
|
||||
return {
|
||||
sanitizeMode: "images-only",
|
||||
applyAssistantFirstOrderingFix: false,
|
||||
validateGeminiTurns: false,
|
||||
validateAnthropicTurns: false,
|
||||
...(isResponsesFamily ? { allowSyntheticToolResults: true } : {}),
|
||||
...(ctx.modelApi === "openai-completions"
|
||||
? {
|
||||
sanitizeToolCallIds: true,
|
||||
toolCallIdMode: "strict" as const,
|
||||
}
|
||||
: {
|
||||
sanitizeToolCallIds: false,
|
||||
}),
|
||||
};
|
||||
}
|
||||
23
extensions/openai/setup-api.test.ts
Normal file
23
extensions/openai/setup-api.test.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
// Openai tests cover setup api plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildOpenAISetupProvider } from "./setup-api.js";
|
||||
|
||||
function authMethodIds(provider: ReturnType<typeof buildOpenAISetupProvider>) {
|
||||
return provider.auth.map((method) => method.id);
|
||||
}
|
||||
|
||||
describe("OpenAI setup auth provider", () => {
|
||||
it("offers ChatGPT login as the default OpenAI auth path while keeping API key explicit", () => {
|
||||
const provider = buildOpenAISetupProvider();
|
||||
const oauth = provider.auth.find((method) => method.id === "oauth");
|
||||
const apiKey = provider.auth.find((method) => method.id === "api-key");
|
||||
|
||||
expect(provider.id).toBe("openai");
|
||||
expect(authMethodIds(provider)).toEqual(["oauth", "device-code", "api-key"]);
|
||||
expect(oauth?.label).toBe("ChatGPT Login");
|
||||
expect(oauth?.wizard?.choiceId).toBe("openai");
|
||||
expect(oauth?.wizard?.assistantVisibility).toBe("manual-only");
|
||||
expect(apiKey?.label).toBe("OpenAI API Key");
|
||||
expect(apiKey?.wizard?.choiceId).toBe("openai-api-key");
|
||||
});
|
||||
});
|
||||
91
extensions/openai/setup-api.ts
Normal file
91
extensions/openai/setup-api.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
// Openai API module exposes the plugin public contract.
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { ProviderAuthContext, ProviderAuthResult } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { ProviderAuthMethod } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import {
|
||||
OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
OPENAI_API_KEY_LABEL,
|
||||
OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
OPENAI_CHATGPT_LOGIN_HINT,
|
||||
OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
} from "./auth-choice-copy.js";
|
||||
|
||||
async function runOpenAIProviderAuthMethod(
|
||||
methodId: string,
|
||||
ctx: ProviderAuthContext,
|
||||
): Promise<ProviderAuthResult> {
|
||||
const { buildOpenAIProvider } = await import("./openai-provider.js");
|
||||
const method = buildOpenAIProvider().auth.find((entry) => entry.id === methodId);
|
||||
if (!method) {
|
||||
return { profiles: [] };
|
||||
}
|
||||
return method.run(ctx);
|
||||
}
|
||||
|
||||
export function buildOpenAISetupProvider(): ProviderPlugin {
|
||||
const oauthMethod = {
|
||||
id: "oauth",
|
||||
label: OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
hint: OPENAI_CHATGPT_LOGIN_HINT,
|
||||
kind: "oauth",
|
||||
wizard: {
|
||||
choiceId: "openai",
|
||||
choiceLabel: OPENAI_CHATGPT_LOGIN_LABEL,
|
||||
choiceHint: OPENAI_CHATGPT_LOGIN_HINT,
|
||||
assistantPriority: -40,
|
||||
assistantVisibility: "manual-only",
|
||||
...OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
},
|
||||
run: async (ctx) => runOpenAIProviderAuthMethod("oauth", ctx),
|
||||
} satisfies ProviderAuthMethod;
|
||||
|
||||
const deviceCodeMethod = {
|
||||
id: "device-code",
|
||||
label: OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
hint: OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
kind: "device_code",
|
||||
wizard: {
|
||||
choiceId: "openai-device-code",
|
||||
choiceLabel: OPENAI_CHATGPT_DEVICE_PAIRING_LABEL,
|
||||
choiceHint: OPENAI_CHATGPT_DEVICE_PAIRING_HINT,
|
||||
assistantPriority: -10,
|
||||
assistantVisibility: "manual-only",
|
||||
...OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
},
|
||||
run: async (ctx) => runOpenAIProviderAuthMethod("device-code", ctx),
|
||||
} satisfies ProviderAuthMethod;
|
||||
|
||||
const apiKeyMethod = {
|
||||
id: "api-key",
|
||||
label: OPENAI_API_KEY_LABEL,
|
||||
hint: "Use your OpenAI API key directly",
|
||||
kind: "api_key",
|
||||
wizard: {
|
||||
choiceId: "openai-api-key",
|
||||
choiceLabel: OPENAI_API_KEY_LABEL,
|
||||
choiceHint: "Use your OpenAI API key directly",
|
||||
assistantPriority: 5,
|
||||
...OPENAI_ACCOUNT_WIZARD_GROUP,
|
||||
},
|
||||
run: async (ctx) => runOpenAIProviderAuthMethod("api-key", ctx),
|
||||
} satisfies ProviderAuthMethod;
|
||||
|
||||
return {
|
||||
id: "openai",
|
||||
label: "OpenAI",
|
||||
docsPath: "/providers/models",
|
||||
envVars: ["OPENAI_API_KEY"],
|
||||
auth: [oauthMethod, deviceCodeMethod, apiKeyMethod],
|
||||
};
|
||||
}
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "openai",
|
||||
name: "OpenAI Setup",
|
||||
description: "Lightweight OpenAI setup hooks",
|
||||
register(api) {
|
||||
api.registerProvider(buildOpenAISetupProvider());
|
||||
},
|
||||
});
|
||||
130
extensions/openai/shared.ts
Normal file
130
extensions/openai/shared.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
// Openai plugin module implements shared behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { findCatalogTemplate } from "openclaw/plugin-sdk/provider-catalog-shared";
|
||||
import {
|
||||
cloneFirstTemplateModel,
|
||||
matchesExactOrPrefix,
|
||||
type ProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { OPENAI_RESPONSES_STREAM_HOOKS } from "openclaw/plugin-sdk/provider-stream-family";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { createOpenAINativeWebSearchWrapper } from "./native-web-search.js";
|
||||
import { buildOpenAIReplayPolicy } from "./replay-policy.js";
|
||||
import {
|
||||
resolveOpenAITransportTurnState,
|
||||
resolveOpenAIWebSocketSessionPolicy,
|
||||
} from "./transport-policy.js";
|
||||
|
||||
type SyntheticOpenAIModelCatalogCost = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
};
|
||||
|
||||
type SyntheticOpenAIModelCatalogEntry = {
|
||||
provider: string;
|
||||
id: string;
|
||||
name: string;
|
||||
reasoning?: boolean;
|
||||
input?: ("text" | "image")[];
|
||||
contextWindow?: number;
|
||||
contextTokens?: number;
|
||||
cost?: SyntheticOpenAIModelCatalogCost;
|
||||
};
|
||||
|
||||
const OPENAI_API_BASE_URL = "https://api.openai.com/v1";
|
||||
|
||||
export function resolveConfiguredOpenAIBaseUrl(cfg: OpenClawConfig | undefined): string {
|
||||
return normalizeOptionalString(cfg?.models?.providers?.openai?.baseUrl) ?? OPENAI_API_BASE_URL;
|
||||
}
|
||||
|
||||
function hasSupportedOpenAIResponsesTransport(
|
||||
transport: unknown,
|
||||
): transport is "auto" | "sse" | "websocket" {
|
||||
return transport === "auto" || transport === "sse" || transport === "websocket";
|
||||
}
|
||||
|
||||
function defaultOpenAIResponsesExtraParams(
|
||||
extraParams: Record<string, unknown> | undefined,
|
||||
options?: { transport?: "auto" | "sse" | "websocket" },
|
||||
): Record<string, unknown> | undefined {
|
||||
const hasSupportedTransport = hasSupportedOpenAIResponsesTransport(extraParams?.transport);
|
||||
const defaultTransport = options?.transport ?? "auto";
|
||||
if (hasSupportedTransport) {
|
||||
return extraParams;
|
||||
}
|
||||
|
||||
return {
|
||||
...extraParams,
|
||||
transport: defaultTransport,
|
||||
};
|
||||
}
|
||||
|
||||
type OpenAIResponsesProviderHooks = Pick<
|
||||
ProviderPlugin,
|
||||
| "buildReplayPolicy"
|
||||
| "prepareExtraParams"
|
||||
| "wrapStreamFn"
|
||||
| "resolveTransportTurnState"
|
||||
| "resolveWebSocketSessionPolicy"
|
||||
>;
|
||||
|
||||
const resolveOpenAIResponsesTransportTurnState: NonNullable<
|
||||
OpenAIResponsesProviderHooks["resolveTransportTurnState"]
|
||||
> = (ctx) => resolveOpenAITransportTurnState(ctx);
|
||||
|
||||
const resolveOpenAIResponsesWebSocketSessionPolicy: NonNullable<
|
||||
OpenAIResponsesProviderHooks["resolveWebSocketSessionPolicy"]
|
||||
> = (ctx) => resolveOpenAIWebSocketSessionPolicy(ctx);
|
||||
|
||||
const wrapOpenAIResponsesStreamFn = OPENAI_RESPONSES_STREAM_HOOKS.wrapStreamFn;
|
||||
const wrapOpenAIResponsesProviderStreamFn: NonNullable<
|
||||
OpenAIResponsesProviderHooks["wrapStreamFn"]
|
||||
> = (ctx) =>
|
||||
createOpenAINativeWebSearchWrapper(wrapOpenAIResponsesStreamFn?.(ctx) ?? ctx.streamFn, {
|
||||
config: ctx.config,
|
||||
agentId: ctx.agentId,
|
||||
nativeWebSearchAllowedByToolPolicy: ctx.nativeWebSearchAllowedByToolPolicy,
|
||||
});
|
||||
|
||||
export function buildOpenAIResponsesProviderHooks(options?: {
|
||||
transport?: "auto" | "sse" | "websocket";
|
||||
}): OpenAIResponsesProviderHooks {
|
||||
return {
|
||||
buildReplayPolicy: buildOpenAIReplayPolicy,
|
||||
prepareExtraParams: (ctx) => defaultOpenAIResponsesExtraParams(ctx.extraParams, options),
|
||||
...OPENAI_RESPONSES_STREAM_HOOKS,
|
||||
wrapStreamFn: wrapOpenAIResponsesProviderStreamFn,
|
||||
resolveTransportTurnState: resolveOpenAIResponsesTransportTurnState,
|
||||
resolveWebSocketSessionPolicy: resolveOpenAIResponsesWebSocketSessionPolicy,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildOpenAISyntheticCatalogEntry(
|
||||
template: ReturnType<typeof findCatalogTemplate>,
|
||||
entry: {
|
||||
id: string;
|
||||
reasoning: boolean;
|
||||
input: readonly ("text" | "image")[];
|
||||
contextWindow: number;
|
||||
contextTokens?: number;
|
||||
cost?: SyntheticOpenAIModelCatalogCost;
|
||||
},
|
||||
): SyntheticOpenAIModelCatalogEntry | undefined {
|
||||
if (!template) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
...template,
|
||||
id: entry.id,
|
||||
name: entry.id,
|
||||
reasoning: entry.reasoning,
|
||||
input: [...entry.input],
|
||||
contextWindow: entry.contextWindow,
|
||||
...(entry.contextTokens === undefined ? {} : { contextTokens: entry.contextTokens }),
|
||||
...(entry.cost === undefined ? {} : { cost: entry.cost }),
|
||||
};
|
||||
}
|
||||
|
||||
export { cloneFirstTemplateModel, findCatalogTemplate, matchesExactOrPrefix };
|
||||
593
extensions/openai/speech-provider.test.ts
Normal file
593
extensions/openai/speech-provider.test.ts
Normal file
@@ -0,0 +1,593 @@
|
||||
// Openai tests cover speech provider plugin behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
|
||||
fetchWithSsrFGuard: async ({
|
||||
url,
|
||||
init,
|
||||
}: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
}): Promise<{ response: Response; release: () => Promise<void> }> => ({
|
||||
response: await globalThis.fetch(url, init),
|
||||
release: vi.fn(async () => {}),
|
||||
}),
|
||||
ssrfPolicyFromHttpBaseUrlAllowedHostname: () => undefined,
|
||||
}));
|
||||
|
||||
function isSpeechRequestBody(value: unknown): value is {
|
||||
[key: string]: unknown;
|
||||
model?: string;
|
||||
voice?: string;
|
||||
speed?: number;
|
||||
response_format?: string;
|
||||
} {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function parseRequestBody(init: RequestInit | undefined): {
|
||||
[key: string]: unknown;
|
||||
model?: string;
|
||||
voice?: string;
|
||||
speed?: number;
|
||||
response_format?: string;
|
||||
} {
|
||||
if (typeof init?.body !== "string") {
|
||||
throw new Error("expected string request body");
|
||||
}
|
||||
const body: unknown = JSON.parse(init.body);
|
||||
if (!isSpeechRequestBody(body)) {
|
||||
throw new Error("expected OpenAI speech request body");
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function mockSpeechFetchExpectingFormat(responseFormat: string) {
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
const body = parseRequestBody(init);
|
||||
expect(body.response_format).toBe(responseFormat);
|
||||
return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
|
||||
});
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
return fetchMock;
|
||||
}
|
||||
|
||||
describe("buildOpenAISpeechProvider", () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("normalizes provider-owned speech config from raw provider config", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
cfg: {} as never,
|
||||
timeoutMs: 30_000,
|
||||
rawConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "sk-test",
|
||||
baseUrl: "https://example.com/v1/",
|
||||
model: "tts-1",
|
||||
voice: "alloy",
|
||||
speed: 1.25,
|
||||
instructions: " Speak warmly ",
|
||||
responseFormat: " WAV ",
|
||||
extraBody: {
|
||||
lang: "en-US",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolved).toEqual({
|
||||
apiKey: "sk-test",
|
||||
baseUrl: "https://example.com/v1",
|
||||
model: "tts-1",
|
||||
voice: "alloy",
|
||||
speed: 1.25,
|
||||
instructions: "Speak warmly",
|
||||
responseFormat: "wav",
|
||||
extraBody: {
|
||||
lang: "en-US",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("drops malformed speech speed values", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
cfg: {} as never,
|
||||
timeoutMs: 30_000,
|
||||
rawConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
speed: 4.5,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolved?.speed).toBeUndefined();
|
||||
});
|
||||
|
||||
it("passes custom endpoint speech speeds through", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
cfg: {} as never,
|
||||
timeoutMs: 30_000,
|
||||
rawConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://tts.example.com/v1",
|
||||
speed: 4.5,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolved?.speed).toBe(4.5);
|
||||
});
|
||||
|
||||
it("uses talk base url overrides when validating speech speed", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
const resolvedConfig = provider.resolveTalkConfig?.({
|
||||
cfg: {} as never,
|
||||
timeoutMs: 30_000,
|
||||
baseTtsConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "sk-base",
|
||||
},
|
||||
},
|
||||
},
|
||||
talkProviderConfig: {
|
||||
baseUrl: "https://tts.example.com/v1",
|
||||
speed: 4.5,
|
||||
},
|
||||
});
|
||||
|
||||
expect(resolvedConfig?.baseUrl).toBe("https://tts.example.com/v1");
|
||||
expect(resolvedConfig?.speed).toBe(4.5);
|
||||
});
|
||||
|
||||
it("parses OpenAI directive tokens against the resolved base url", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "voice",
|
||||
value: "alloy",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
overrides: { voice: "alloy" },
|
||||
});
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "model",
|
||||
value: "kokoro-custom-model",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses preferred-OpenAI speed directive within the supported range", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "speed",
|
||||
value: "1.5",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
overrides: { speed: 1.5 },
|
||||
});
|
||||
});
|
||||
|
||||
it("parses explicit openai_speed alias", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "openai_speed",
|
||||
value: "0.75",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
overrides: { speed: 0.75 },
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores OpenAI speed directives when allowVoiceSettings is disabled", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "speed",
|
||||
value: "1.5",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: false,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("warns on non-numeric OpenAI speed values", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "speed",
|
||||
value: "fast",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
warnings: ['invalid OpenAI speed "fast" (0.25-4.0)'],
|
||||
});
|
||||
});
|
||||
|
||||
it("warns on partial OpenAI speed values", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "speed",
|
||||
value: "1.5abc",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
warnings: ['invalid OpenAI speed "1.5abc" (0.25-4.0)'],
|
||||
});
|
||||
});
|
||||
|
||||
it("warns on OpenAI speed values outside the supported 0.25..4 range", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "speed",
|
||||
value: "5",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
warnings: ['invalid OpenAI speed "5" (0.25-4.0)'],
|
||||
});
|
||||
});
|
||||
|
||||
it("passes custom endpoint OpenAI-compatible speed directives through", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "speed",
|
||||
value: "4.5",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
},
|
||||
providerConfig: {
|
||||
baseUrl: "https://tts.example.com/v1",
|
||||
},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
overrides: { speed: 4.5 },
|
||||
});
|
||||
});
|
||||
|
||||
it("uses OPENAI_TTS_BASE_URL when parsing OpenAI-compatible speed directives", () => {
|
||||
const previousBaseUrl = process.env.OPENAI_TTS_BASE_URL;
|
||||
process.env.OPENAI_TTS_BASE_URL = "https://tts.example.com/v1";
|
||||
try {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.parseDirectiveToken?.({
|
||||
key: "speed",
|
||||
value: "4.5",
|
||||
policy: {
|
||||
allowVoice: true,
|
||||
allowModelId: true,
|
||||
allowVoiceSettings: true,
|
||||
},
|
||||
providerConfig: {},
|
||||
} as never),
|
||||
).toEqual({
|
||||
handled: true,
|
||||
overrides: { speed: 4.5 },
|
||||
});
|
||||
} finally {
|
||||
if (previousBaseUrl === undefined) {
|
||||
delete process.env.OPENAI_TTS_BASE_URL;
|
||||
} else {
|
||||
process.env.OPENAI_TTS_BASE_URL = previousBaseUrl;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves talk responseFormat overrides", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
const resolvedConfig = provider.resolveTalkConfig?.({
|
||||
cfg: {} as never,
|
||||
timeoutMs: 30_000,
|
||||
baseTtsConfig: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "sk-base",
|
||||
responseFormat: "mp3",
|
||||
},
|
||||
},
|
||||
},
|
||||
talkProviderConfig: {
|
||||
apiKey: "sk-talk",
|
||||
responseFormat: " WAV ",
|
||||
},
|
||||
});
|
||||
expect(resolvedConfig?.apiKey).toBe("sk-talk");
|
||||
expect(resolvedConfig?.responseFormat).toBe("wav");
|
||||
});
|
||||
|
||||
it("maps Talk speak params onto OpenAI speech overrides", () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
expect(
|
||||
provider.resolveTalkOverrides?.({
|
||||
talkProviderConfig: {},
|
||||
params: {
|
||||
text: "Hello from talk mode.",
|
||||
voiceId: "nova",
|
||||
modelId: "tts-1",
|
||||
speed: 218 / 175,
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
voice: "nova",
|
||||
model: "tts-1",
|
||||
speed: 218 / 175,
|
||||
});
|
||||
});
|
||||
|
||||
it("maps persona prompt fields to instructions when instructions are unset", async () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
|
||||
const prepared = await provider.prepareSynthesis?.({
|
||||
text: "hello",
|
||||
cfg: {} as never,
|
||||
providerConfig: {
|
||||
apiKey: "sk-test",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "cedar",
|
||||
},
|
||||
persona: {
|
||||
id: "alfred",
|
||||
label: "Alfred",
|
||||
prompt: {
|
||||
profile: "A brilliant British butler.",
|
||||
scene: "A quiet late-night study.",
|
||||
sampleContext: "The speaker is answering a trusted operator.",
|
||||
style: "Refined and lightly amused.",
|
||||
accent: "British English.",
|
||||
pacing: "Measured.",
|
||||
constraints: ["Do not read configuration values aloud."],
|
||||
},
|
||||
},
|
||||
target: "audio-file",
|
||||
timeoutMs: 1_000,
|
||||
});
|
||||
|
||||
expect(prepared?.providerConfig?.instructions).toContain("Persona: Alfred");
|
||||
expect(prepared?.providerConfig?.instructions).toContain(
|
||||
"Constraint: Do not read configuration values aloud.",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses wav for Groq-compatible OpenAI TTS endpoints", async () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
mockSpeechFetchExpectingFormat("wav");
|
||||
|
||||
const result = await provider.synthesize({
|
||||
text: "hello",
|
||||
cfg: {} as never,
|
||||
providerConfig: {
|
||||
apiKey: "sk-test",
|
||||
baseUrl: "https://api.groq.com/openai/v1",
|
||||
model: "canopylabs/orpheus-v1-english",
|
||||
voice: "daniel",
|
||||
},
|
||||
target: "audio-file",
|
||||
timeoutMs: 1_000,
|
||||
});
|
||||
|
||||
expect(result.outputFormat).toBe("wav");
|
||||
expect(result.fileExtension).toBe(".wav");
|
||||
expect(result.voiceCompatible).toBe(false);
|
||||
});
|
||||
|
||||
it("applies the configured media byte cap to synthesized audio", async () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
globalThis.fetch = vi.fn(
|
||||
async () => new Response(new Uint8Array(2048), { status: 200 }),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await expect(
|
||||
provider.synthesize({
|
||||
text: "hello",
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
mediaMaxMb: 0.001,
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
providerConfig: {
|
||||
apiKey: "sk-test",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
},
|
||||
target: "audio-file",
|
||||
timeoutMs: 1_000,
|
||||
}),
|
||||
).rejects.toThrow("OpenAI TTS audio response exceeds");
|
||||
});
|
||||
|
||||
it("applies provider overrides to telephony synthesis", async () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
const body = parseRequestBody(init);
|
||||
expect(body.model).toBe("tts-1");
|
||||
expect(body.voice).toBe("nova");
|
||||
expect(body.speed).toBe(1.25);
|
||||
expect(body.response_format).toBe("pcm");
|
||||
return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
|
||||
});
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const result = await provider.synthesizeTelephony?.({
|
||||
text: "hello",
|
||||
cfg: {} as never,
|
||||
providerConfig: {
|
||||
apiKey: "sk-test",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
speed: 1,
|
||||
},
|
||||
providerOverrides: {
|
||||
model: "tts-1",
|
||||
voice: "nova",
|
||||
speed: 1.25,
|
||||
},
|
||||
timeoutMs: 1_000,
|
||||
});
|
||||
|
||||
expect(result?.outputFormat).toBe("pcm");
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("honors explicit responseFormat overrides and clears voice-note compatibility when not opus", async () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
mockSpeechFetchExpectingFormat("wav");
|
||||
|
||||
const result = await provider.synthesize({
|
||||
text: "hello",
|
||||
cfg: {} as never,
|
||||
providerConfig: {
|
||||
apiKey: "sk-test",
|
||||
baseUrl: "https://proxy.example.com/openai/v1",
|
||||
model: "canopylabs/orpheus-v1-english",
|
||||
voice: "daniel",
|
||||
responseFormat: "wav",
|
||||
},
|
||||
target: "voice-note",
|
||||
timeoutMs: 1_000,
|
||||
});
|
||||
|
||||
expect(result.outputFormat).toBe("wav");
|
||||
expect(result.fileExtension).toBe(".wav");
|
||||
expect(result.voiceCompatible).toBe(false);
|
||||
});
|
||||
|
||||
it("passes extra_body config through to OpenAI-compatible speech requests", async () => {
|
||||
const provider = buildOpenAISpeechProvider();
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
const body = parseRequestBody(init);
|
||||
expect(body.model).toBe("custom-tts");
|
||||
expect(body.voice).toBe("custom-voice");
|
||||
expect(body.lang).toBe("en-US");
|
||||
expect(body.response_format).toBe("mp3");
|
||||
return new Response(new Uint8Array([1, 2, 3]), { status: 200 });
|
||||
});
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const result = await provider.synthesize({
|
||||
text: "hello",
|
||||
cfg: {} as never,
|
||||
providerConfig: {
|
||||
apiKey: "sk-test",
|
||||
baseUrl: "https://proxy.example.com/openai/v1",
|
||||
model: "custom-tts",
|
||||
voice: "custom-voice",
|
||||
responseFormat: "mp3",
|
||||
extra_body: {
|
||||
lang: "en-US",
|
||||
},
|
||||
},
|
||||
target: "audio-file",
|
||||
timeoutMs: 1_000,
|
||||
});
|
||||
|
||||
expect(result.outputFormat).toBe("mp3");
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
399
extensions/openai/speech-provider.ts
Normal file
399
extensions/openai/speech-provider.ts
Normal file
@@ -0,0 +1,399 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
||||
import type {
|
||||
SpeechDirectiveTokenParseContext,
|
||||
SpeechProviderConfig,
|
||||
SpeechProviderOverrides,
|
||||
SpeechProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/speech-core";
|
||||
import { parseSpeechDirectiveNumberOverride } from "openclaw/plugin-sdk/speech-core";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalLowercaseString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import {
|
||||
asFiniteNumber,
|
||||
asObjectRecord,
|
||||
resolveOpenAIProviderConfigRecord,
|
||||
trimToUndefined,
|
||||
} from "./realtime-provider-shared.js";
|
||||
import {
|
||||
DEFAULT_OPENAI_BASE_URL,
|
||||
isValidOpenAIModel,
|
||||
isValidOpenAIVoice,
|
||||
normalizeOpenAITtsBaseUrl,
|
||||
OPENAI_TTS_MODELS,
|
||||
OPENAI_TTS_VOICES,
|
||||
openaiTTS,
|
||||
} from "./tts.js";
|
||||
|
||||
const OPENAI_SPEECH_RESPONSE_FORMATS = ["mp3", "opus", "wav"] as const;
|
||||
const DEFAULT_GENERATED_AUDIO_MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
type OpenAiSpeechResponseFormat = (typeof OPENAI_SPEECH_RESPONSE_FORMATS)[number];
|
||||
|
||||
type OpenAITtsProviderConfig = {
|
||||
apiKey?: string;
|
||||
baseUrl: string;
|
||||
model: string;
|
||||
voice: string;
|
||||
speed?: number;
|
||||
instructions?: string;
|
||||
responseFormat?: OpenAiSpeechResponseFormat;
|
||||
extraBody?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type OpenAITtsProviderOverrides = {
|
||||
model?: string;
|
||||
voice?: string;
|
||||
speed?: number;
|
||||
};
|
||||
|
||||
function normalizeOpenAISpeechResponseFormat(
|
||||
value: unknown,
|
||||
): OpenAiSpeechResponseFormat | undefined {
|
||||
const next = normalizeOptionalLowercaseString(value);
|
||||
if (!next) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
OPENAI_SPEECH_RESPONSE_FORMATS.includes(next as (typeof OPENAI_SPEECH_RESPONSE_FORMATS)[number])
|
||||
) {
|
||||
return next as OpenAiSpeechResponseFormat;
|
||||
}
|
||||
throw new Error(`Invalid OpenAI speech responseFormat: ${next}`);
|
||||
}
|
||||
|
||||
function isGroqSpeechBaseUrl(baseUrl: string): boolean {
|
||||
try {
|
||||
const hostname = normalizeLowercaseStringOrEmpty(new URL(baseUrl).hostname);
|
||||
return hostname === "groq.com" || hostname.endsWith(".groq.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveSpeechResponseFormat(
|
||||
baseUrl: string,
|
||||
target: "audio-file" | "voice-note" | "telephony",
|
||||
configuredFormat?: OpenAiSpeechResponseFormat,
|
||||
): OpenAiSpeechResponseFormat {
|
||||
if (configuredFormat) {
|
||||
return configuredFormat;
|
||||
}
|
||||
if (isGroqSpeechBaseUrl(baseUrl)) {
|
||||
return "wav";
|
||||
}
|
||||
return target === "voice-note" ? "opus" : "mp3";
|
||||
}
|
||||
|
||||
function responseFormatToFileExtension(
|
||||
format: OpenAiSpeechResponseFormat,
|
||||
): ".mp3" | ".opus" | ".wav" {
|
||||
switch (format) {
|
||||
case "opus":
|
||||
return ".opus";
|
||||
case "wav":
|
||||
return ".wav";
|
||||
default:
|
||||
return ".mp3";
|
||||
}
|
||||
}
|
||||
|
||||
function readExtraBody(value: unknown): Record<string, unknown> | undefined {
|
||||
const body = asObjectRecord(value);
|
||||
if (!body || Object.keys(body).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function normalizeOpenAISpeechSpeed(value: unknown, baseUrl?: string): number | undefined {
|
||||
const speed = asFiniteNumber(value);
|
||||
if (speed === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (baseUrl !== undefined && normalizeOpenAITtsBaseUrl(baseUrl) !== DEFAULT_OPENAI_BASE_URL) {
|
||||
return speed;
|
||||
}
|
||||
return speed >= 0.25 && speed <= 4 ? speed : undefined;
|
||||
}
|
||||
|
||||
function normalizeOpenAIProviderConfig(
|
||||
rawConfig: Record<string, unknown>,
|
||||
): OpenAITtsProviderConfig {
|
||||
const raw = resolveOpenAIProviderConfigRecord(rawConfig);
|
||||
const extraBody = readExtraBody(raw?.extraBody) ?? readExtraBody(raw?.extra_body);
|
||||
const baseUrl = normalizeOpenAITtsBaseUrl(
|
||||
trimToUndefined(raw?.baseUrl) ??
|
||||
trimToUndefined(process.env.OPENAI_TTS_BASE_URL) ??
|
||||
DEFAULT_OPENAI_BASE_URL,
|
||||
);
|
||||
return {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: raw?.apiKey,
|
||||
path: "messages.tts.providers.openai.apiKey",
|
||||
}),
|
||||
baseUrl,
|
||||
model: trimToUndefined(raw?.model) ?? "gpt-4o-mini-tts",
|
||||
voice: trimToUndefined(raw?.voice) ?? "coral",
|
||||
speed: normalizeOpenAISpeechSpeed(raw?.speed, baseUrl),
|
||||
instructions: trimToUndefined(raw?.instructions),
|
||||
responseFormat: normalizeOpenAISpeechResponseFormat(raw?.responseFormat),
|
||||
extraBody,
|
||||
};
|
||||
}
|
||||
|
||||
function readOpenAIProviderConfig(config: SpeechProviderConfig): OpenAITtsProviderConfig {
|
||||
const normalized = normalizeOpenAIProviderConfig({});
|
||||
return {
|
||||
apiKey: trimToUndefined(config.apiKey) ?? normalized.apiKey,
|
||||
baseUrl: trimToUndefined(config.baseUrl) ?? normalized.baseUrl,
|
||||
model: trimToUndefined(config.model) ?? normalized.model,
|
||||
voice: trimToUndefined(config.voice) ?? normalized.voice,
|
||||
speed:
|
||||
normalizeOpenAISpeechSpeed(
|
||||
config.speed,
|
||||
trimToUndefined(config.baseUrl) ?? normalized.baseUrl,
|
||||
) ?? normalized.speed,
|
||||
instructions: trimToUndefined(config.instructions) ?? normalized.instructions,
|
||||
responseFormat:
|
||||
normalizeOpenAISpeechResponseFormat(config.responseFormat) ?? normalized.responseFormat,
|
||||
extraBody: readExtraBody(config.extraBody) ?? readExtraBody(config.extra_body),
|
||||
};
|
||||
}
|
||||
|
||||
function readOpenAIOverrides(
|
||||
overrides: SpeechProviderOverrides | undefined,
|
||||
baseUrl: string,
|
||||
): OpenAITtsProviderOverrides {
|
||||
if (!overrides) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
model: trimToUndefined(overrides.model),
|
||||
voice: trimToUndefined(overrides.voice),
|
||||
speed: normalizeOpenAISpeechSpeed(overrides.speed, baseUrl),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveGeneratedAudioMaxBytes(req: {
|
||||
cfg: { agents?: { defaults?: { mediaMaxMb?: number } } };
|
||||
}): number {
|
||||
const configured = req.cfg.agents?.defaults?.mediaMaxMb;
|
||||
if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) {
|
||||
return Math.floor(configured * 1024 * 1024);
|
||||
}
|
||||
return DEFAULT_GENERATED_AUDIO_MAX_BYTES;
|
||||
}
|
||||
|
||||
function renderOpenAITtsPersonaInstructions(req: {
|
||||
label?: string;
|
||||
prompt?: {
|
||||
profile?: string;
|
||||
scene?: string;
|
||||
sampleContext?: string;
|
||||
style?: string;
|
||||
accent?: string;
|
||||
pacing?: string;
|
||||
constraints?: string[];
|
||||
};
|
||||
}): string | undefined {
|
||||
const prompt = req.prompt;
|
||||
if (!prompt) {
|
||||
return undefined;
|
||||
}
|
||||
const lines = [
|
||||
req.label ? `Persona: ${req.label}` : undefined,
|
||||
prompt.profile ? `Profile: ${prompt.profile}` : undefined,
|
||||
prompt.scene ? `Scene: ${prompt.scene}` : undefined,
|
||||
prompt.style ? `Style: ${prompt.style}` : undefined,
|
||||
prompt.accent ? `Accent: ${prompt.accent}` : undefined,
|
||||
prompt.pacing ? `Pacing: ${prompt.pacing}` : undefined,
|
||||
prompt.sampleContext ? `Sample context: ${prompt.sampleContext}` : undefined,
|
||||
...(prompt.constraints ?? []).map((constraint) => `Constraint: ${constraint}`),
|
||||
]
|
||||
.map((line) => trimToUndefined(line))
|
||||
.filter((line): line is string => Boolean(line));
|
||||
return lines.length > 0 ? lines.join("\n") : undefined;
|
||||
}
|
||||
|
||||
function isCustomOpenAITtsBaseUrl(baseUrl: string | undefined): boolean {
|
||||
if (baseUrl !== undefined) {
|
||||
return normalizeOpenAITtsBaseUrl(baseUrl) !== DEFAULT_OPENAI_BASE_URL;
|
||||
}
|
||||
return normalizeOpenAITtsBaseUrl(process.env.OPENAI_TTS_BASE_URL) !== DEFAULT_OPENAI_BASE_URL;
|
||||
}
|
||||
|
||||
function parseDirectiveToken(ctx: SpeechDirectiveTokenParseContext): {
|
||||
handled: boolean;
|
||||
overrides?: SpeechProviderOverrides;
|
||||
warnings?: string[];
|
||||
} {
|
||||
const baseUrl = trimToUndefined(asObjectRecord(ctx.providerConfig)?.baseUrl);
|
||||
switch (ctx.key) {
|
||||
case "voice":
|
||||
case "openai_voice":
|
||||
case "openaivoice":
|
||||
if (!ctx.policy.allowVoice) {
|
||||
return { handled: true };
|
||||
}
|
||||
if (!isValidOpenAIVoice(ctx.value, baseUrl)) {
|
||||
return { handled: true, warnings: [`invalid OpenAI voice "${ctx.value}"`] };
|
||||
}
|
||||
return { handled: true, overrides: { voice: ctx.value } };
|
||||
case "model":
|
||||
case "openai_model":
|
||||
case "openaimodel":
|
||||
if (!ctx.policy.allowModelId) {
|
||||
return { handled: true };
|
||||
}
|
||||
if (!isValidOpenAIModel(ctx.value, baseUrl)) {
|
||||
return { handled: false };
|
||||
}
|
||||
return { handled: true, overrides: { model: ctx.value } };
|
||||
case "speed":
|
||||
case "openai_speed":
|
||||
case "openaispeed": {
|
||||
const customBaseUrl = isCustomOpenAITtsBaseUrl(baseUrl);
|
||||
return parseSpeechDirectiveNumberOverride({
|
||||
ctx,
|
||||
overrideKey: "speed",
|
||||
range: customBaseUrl ? {} : { min: 0.25, max: 4 },
|
||||
warning: (value) =>
|
||||
customBaseUrl
|
||||
? `invalid OpenAI-compatible speed "${value}"`
|
||||
: `invalid OpenAI speed "${value}" (0.25-4.0)`,
|
||||
});
|
||||
}
|
||||
default:
|
||||
return { handled: false };
|
||||
}
|
||||
}
|
||||
|
||||
export function buildOpenAISpeechProvider(): SpeechProviderPlugin {
|
||||
return {
|
||||
id: "openai",
|
||||
label: "OpenAI",
|
||||
autoSelectOrder: 10,
|
||||
defaultModel: OPENAI_TTS_MODELS[0],
|
||||
models: OPENAI_TTS_MODELS,
|
||||
voices: OPENAI_TTS_VOICES,
|
||||
resolveConfig: ({ rawConfig }) => normalizeOpenAIProviderConfig(rawConfig),
|
||||
parseDirectiveToken,
|
||||
resolveTalkConfig: ({ baseTtsConfig, talkProviderConfig }) => {
|
||||
const base = normalizeOpenAIProviderConfig(baseTtsConfig);
|
||||
const responseFormat = normalizeOpenAISpeechResponseFormat(talkProviderConfig.responseFormat);
|
||||
const baseUrl = trimToUndefined(talkProviderConfig.baseUrl) ?? base.baseUrl;
|
||||
const speed = normalizeOpenAISpeechSpeed(talkProviderConfig.speed, baseUrl);
|
||||
return {
|
||||
...base,
|
||||
...(talkProviderConfig.apiKey === undefined
|
||||
? {}
|
||||
: {
|
||||
apiKey: normalizeResolvedSecretInputString({
|
||||
value: talkProviderConfig.apiKey,
|
||||
path: "talk.providers.openai.apiKey",
|
||||
}),
|
||||
}),
|
||||
...(trimToUndefined(talkProviderConfig.baseUrl) == null ? {} : { baseUrl }),
|
||||
...(trimToUndefined(talkProviderConfig.modelId) == null
|
||||
? {}
|
||||
: { model: trimToUndefined(talkProviderConfig.modelId) }),
|
||||
...(trimToUndefined(talkProviderConfig.voiceId) == null
|
||||
? {}
|
||||
: { voice: trimToUndefined(talkProviderConfig.voiceId) }),
|
||||
...(speed == null ? {} : { speed }),
|
||||
...(trimToUndefined(talkProviderConfig.instructions) == null
|
||||
? {}
|
||||
: { instructions: trimToUndefined(talkProviderConfig.instructions) }),
|
||||
...(responseFormat == null ? {} : { responseFormat }),
|
||||
};
|
||||
},
|
||||
resolveTalkOverrides: ({ params }) => ({
|
||||
...(trimToUndefined(params.voiceId) == null
|
||||
? {}
|
||||
: { voice: trimToUndefined(params.voiceId) }),
|
||||
...(trimToUndefined(params.modelId) == null
|
||||
? {}
|
||||
: { model: trimToUndefined(params.modelId) }),
|
||||
...(asFiniteNumber(params.speed) == null ? {} : { speed: asFiniteNumber(params.speed) }),
|
||||
}),
|
||||
listVoices: async () => OPENAI_TTS_VOICES.map((voice) => ({ id: voice, name: voice })),
|
||||
isConfigured: ({ providerConfig }) =>
|
||||
Boolean(readOpenAIProviderConfig(providerConfig).apiKey || process.env.OPENAI_API_KEY),
|
||||
prepareSynthesis: (ctx) => {
|
||||
const config = readOpenAIProviderConfig(ctx.providerConfig);
|
||||
if (config.instructions) {
|
||||
return undefined;
|
||||
}
|
||||
const instructions = renderOpenAITtsPersonaInstructions({
|
||||
label: ctx.persona?.label ?? ctx.persona?.id,
|
||||
prompt: ctx.persona?.prompt,
|
||||
});
|
||||
return instructions
|
||||
? {
|
||||
providerConfig: {
|
||||
instructions,
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
},
|
||||
synthesize: async (req) => {
|
||||
const config = readOpenAIProviderConfig(req.providerConfig);
|
||||
const overrides = readOpenAIOverrides(req.providerOverrides, config.baseUrl);
|
||||
const apiKey = config.apiKey || process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error("OpenAI API key missing");
|
||||
}
|
||||
const responseFormat = resolveSpeechResponseFormat(
|
||||
config.baseUrl,
|
||||
req.target,
|
||||
config.responseFormat,
|
||||
);
|
||||
const audioBuffer = await openaiTTS({
|
||||
text: req.text,
|
||||
apiKey,
|
||||
baseUrl: config.baseUrl,
|
||||
model: overrides.model ?? config.model,
|
||||
voice: overrides.voice ?? config.voice,
|
||||
speed: overrides.speed ?? config.speed,
|
||||
instructions: config.instructions,
|
||||
responseFormat,
|
||||
extraBody: config.extraBody,
|
||||
timeoutMs: req.timeoutMs,
|
||||
maxBytes: resolveGeneratedAudioMaxBytes(req),
|
||||
});
|
||||
return {
|
||||
audioBuffer,
|
||||
outputFormat: responseFormat,
|
||||
fileExtension: responseFormatToFileExtension(responseFormat),
|
||||
voiceCompatible: req.target === "voice-note" && responseFormat === "opus",
|
||||
};
|
||||
},
|
||||
synthesizeTelephony: async (req) => {
|
||||
const config = readOpenAIProviderConfig(req.providerConfig);
|
||||
const overrides = readOpenAIOverrides(req.providerOverrides, config.baseUrl);
|
||||
const apiKey = config.apiKey || process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error("OpenAI API key missing");
|
||||
}
|
||||
const outputFormat = "pcm";
|
||||
const sampleRate = 24_000;
|
||||
const audioBuffer = await openaiTTS({
|
||||
text: req.text,
|
||||
apiKey,
|
||||
baseUrl: config.baseUrl,
|
||||
model: overrides.model ?? config.model,
|
||||
voice: overrides.voice ?? config.voice,
|
||||
speed: overrides.speed ?? config.speed,
|
||||
instructions: config.instructions,
|
||||
responseFormat: outputFormat,
|
||||
extraBody: config.extraBody,
|
||||
timeoutMs: req.timeoutMs,
|
||||
maxBytes: resolveGeneratedAudioMaxBytes(req),
|
||||
});
|
||||
return { audioBuffer, outputFormat, sampleRate };
|
||||
},
|
||||
};
|
||||
}
|
||||
7
extensions/openai/test-api.ts
Normal file
7
extensions/openai/test-api.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
// Openai API module exposes the plugin public contract.
|
||||
export { buildOpenAIImageGenerationProvider } from "./image-generation-provider.js";
|
||||
export { openaiMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
export { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
|
||||
export { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
|
||||
export { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
export { buildOpenAIVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
@@ -0,0 +1,134 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import {
|
||||
registerProviderPlugin,
|
||||
requireRegisteredProvider,
|
||||
} from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import {
|
||||
expectAugmentedCodexCatalog,
|
||||
expectedOpenaiPluginCodexCatalogEntriesWithGpt55,
|
||||
expectCodexMissingAuthHint,
|
||||
importProviderRuntimeCatalogModule,
|
||||
loadBundledPluginPublicSurface,
|
||||
} from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import { beforeEach, describe, it, vi } from "vitest";
|
||||
|
||||
const PROVIDER_CATALOG_CONTRACT_TIMEOUT_MS = 300_000;
|
||||
|
||||
type ResolvePluginProviders = (params?: { onlyPluginIds?: string[] }) => ProviderPlugin[];
|
||||
type ResolveOwningPluginIdsForProvider = (params: { provider: string }) => string[] | undefined;
|
||||
type ResolveCatalogHookProviderPluginIds = (params: unknown) => string[];
|
||||
|
||||
const resolvePluginProvidersMock = vi.hoisted(() => vi.fn<ResolvePluginProviders>(() => []));
|
||||
const resolveOwningPluginIdsForProviderMock = vi.hoisted(() =>
|
||||
vi.fn<ResolveOwningPluginIdsForProvider>(() => undefined),
|
||||
);
|
||||
const resolveCatalogHookProviderPluginIdsMock = vi.hoisted(() =>
|
||||
vi.fn<ResolveCatalogHookProviderPluginIds>((_) => [] as string[]),
|
||||
);
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-catalog-runtime", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("openclaw/plugin-sdk/provider-catalog-runtime")
|
||||
>("openclaw/plugin-sdk/provider-catalog-runtime");
|
||||
const resolveCatalogHookProviders = (params: unknown) =>
|
||||
resolvePluginProvidersMock({
|
||||
onlyPluginIds: resolveCatalogHookProviderPluginIdsMock(params),
|
||||
});
|
||||
return {
|
||||
...actual,
|
||||
augmentModelCatalogWithProviderPlugins: async (params: {
|
||||
context: Parameters<NonNullable<ProviderPlugin["augmentModelCatalog"]>>[0];
|
||||
}) => {
|
||||
const supplemental = [];
|
||||
for (const provider of resolveCatalogHookProviders(params)) {
|
||||
const entries = await provider.augmentModelCatalog?.(params.context);
|
||||
if (entries?.length) {
|
||||
supplemental.push(...entries);
|
||||
}
|
||||
}
|
||||
return supplemental;
|
||||
},
|
||||
resolveOwningPluginIdsForProvider: (params: unknown) =>
|
||||
resolveOwningPluginIdsForProviderMock(params as never),
|
||||
resolveCatalogHookProviderPluginIds: (params: unknown) =>
|
||||
resolveCatalogHookProviderPluginIdsMock(params as never),
|
||||
isPluginProvidersLoadInFlight: () => false,
|
||||
resolvePluginProviders: (params: unknown) => resolvePluginProvidersMock(params as never),
|
||||
};
|
||||
});
|
||||
|
||||
export function describeOpenAIProviderCatalogContract() {
|
||||
const contractDepsPromise = (async () => {
|
||||
vi.resetModules();
|
||||
const openaiPlugin = await loadBundledPluginPublicSurface<{
|
||||
default: Parameters<typeof registerProviderPlugin>[0]["plugin"];
|
||||
}>({
|
||||
pluginId: "openai",
|
||||
artifactBasename: "index.js",
|
||||
});
|
||||
const openaiProviders = (
|
||||
await registerProviderPlugin({
|
||||
plugin: openaiPlugin.default,
|
||||
id: "openai",
|
||||
name: "OpenAI",
|
||||
})
|
||||
).providers;
|
||||
const openaiProvider = requireRegisteredProvider(openaiProviders, "openai", "provider");
|
||||
const { augmentModelCatalogWithProviderPlugins } = await importProviderRuntimeCatalogModule();
|
||||
return {
|
||||
augmentModelCatalogWithProviderPlugins,
|
||||
openaiProviders,
|
||||
openaiProvider,
|
||||
};
|
||||
})();
|
||||
|
||||
describe(
|
||||
"openai provider catalog contract",
|
||||
{ timeout: PROVIDER_CATALOG_CONTRACT_TIMEOUT_MS },
|
||||
() => {
|
||||
beforeEach(async () => {
|
||||
const { openaiProviders } = await contractDepsPromise;
|
||||
|
||||
resolvePluginProvidersMock.mockReset();
|
||||
resolvePluginProvidersMock.mockImplementation((params?: { onlyPluginIds?: string[] }) => {
|
||||
const onlyPluginIds = params?.onlyPluginIds;
|
||||
if (!onlyPluginIds || onlyPluginIds.length === 0) {
|
||||
return openaiProviders;
|
||||
}
|
||||
return onlyPluginIds.includes("openai") ? openaiProviders : [];
|
||||
});
|
||||
|
||||
resolveOwningPluginIdsForProviderMock.mockReset();
|
||||
resolveOwningPluginIdsForProviderMock.mockImplementation((params) => {
|
||||
switch (params.provider) {
|
||||
case "azure-openai-responses":
|
||||
case "openai":
|
||||
return ["openai"];
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
});
|
||||
|
||||
resolveCatalogHookProviderPluginIdsMock.mockReset();
|
||||
resolveCatalogHookProviderPluginIdsMock.mockReturnValue(["openai"]);
|
||||
});
|
||||
|
||||
it("keeps codex-only missing-auth hints wired through the provider runtime", async () => {
|
||||
const { openaiProvider } = await contractDepsPromise;
|
||||
expectCodexMissingAuthHint(
|
||||
(params) => openaiProvider.buildMissingAuthMessage?.(params.context) ?? undefined,
|
||||
"openai/gpt-5.5",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps bundled model augmentation wired through the provider runtime", async () => {
|
||||
const { augmentModelCatalogWithProviderPlugins } = await contractDepsPromise;
|
||||
await expectAugmentedCodexCatalog(
|
||||
augmentModelCatalogWithProviderPlugins,
|
||||
expectedOpenaiPluginCodexCatalogEntriesWithGpt55,
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
61
extensions/openai/thinking-policy.ts
Normal file
61
extensions/openai/thinking-policy.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
// Openai plugin module implements thinking policy behavior.
|
||||
import type { ProviderThinkingProfile } from "openclaw/plugin-sdk/plugin-entry";
|
||||
|
||||
const OPENAI_THINKING_BASE_LEVELS = [
|
||||
{ id: "off" },
|
||||
{ id: "minimal" },
|
||||
{ id: "low" },
|
||||
{ id: "medium" },
|
||||
{ id: "high" },
|
||||
] as const satisfies ProviderThinkingProfile["levels"];
|
||||
|
||||
const OPENAI_CODEX_XHIGH_MODEL_IDS = [
|
||||
"gpt-5.6",
|
||||
"gpt-5.5",
|
||||
"gpt-5.5-pro",
|
||||
"gpt-5.4",
|
||||
"gpt-5.4-pro",
|
||||
"gpt-5.3-codex-spark",
|
||||
] as const;
|
||||
|
||||
const OPENAI_UNIFIED_XHIGH_MODEL_IDS = [
|
||||
...OPENAI_CODEX_XHIGH_MODEL_IDS,
|
||||
"gpt-5.4-mini",
|
||||
"gpt-5.4-nano",
|
||||
] as const;
|
||||
|
||||
function normalizeModelId(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function matchesExactOrPrefix(id: string, values: readonly string[]): boolean {
|
||||
const normalizedId = normalizeModelId(id);
|
||||
return values.some((value) => {
|
||||
const normalizedValue = normalizeModelId(value);
|
||||
return normalizedId === normalizedValue || normalizedId.startsWith(normalizedValue);
|
||||
});
|
||||
}
|
||||
|
||||
function buildOpenAIThinkingProfile(params: {
|
||||
modelId: string;
|
||||
xhighModelIds: readonly string[];
|
||||
}): ProviderThinkingProfile {
|
||||
const supportsMax = normalizeModelId(params.modelId).startsWith("gpt-5.6");
|
||||
return {
|
||||
levels: [
|
||||
...OPENAI_THINKING_BASE_LEVELS,
|
||||
...(matchesExactOrPrefix(params.modelId, params.xhighModelIds)
|
||||
? [{ id: "xhigh" as const }]
|
||||
: []),
|
||||
...(supportsMax ? [{ id: "max" as const }] : []),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveOpenAICodexThinkingProfile(modelId: string): ProviderThinkingProfile {
|
||||
return buildOpenAIThinkingProfile({ modelId, xhighModelIds: OPENAI_CODEX_XHIGH_MODEL_IDS });
|
||||
}
|
||||
|
||||
export function resolveUnifiedOpenAIThinkingProfile(modelId: string): ProviderThinkingProfile {
|
||||
return buildOpenAIThinkingProfile({ modelId, xhighModelIds: OPENAI_UNIFIED_XHIGH_MODEL_IDS });
|
||||
}
|
||||
129
extensions/openai/transport-policy.test.ts
Normal file
129
extensions/openai/transport-policy.test.ts
Normal file
@@ -0,0 +1,129 @@
|
||||
// Openai tests cover transport policy plugin behavior.
|
||||
import type { ProviderRuntimeModel } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveOpenAITransportTurnState,
|
||||
resolveOpenAIWebSocketSessionPolicy,
|
||||
} from "./transport-policy.js";
|
||||
|
||||
describe("openai transport policy", () => {
|
||||
const nativeModel = {
|
||||
id: "gpt-5.4",
|
||||
name: "GPT-5.4",
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
} satisfies ProviderRuntimeModel;
|
||||
|
||||
const proxyModel = {
|
||||
...nativeModel,
|
||||
id: "proxy-model",
|
||||
name: "Proxy Model",
|
||||
baseUrl: "https://proxy.example.com/v1",
|
||||
} satisfies ProviderRuntimeModel;
|
||||
|
||||
it("builds native turn state for direct OpenAI routes", () => {
|
||||
const state = resolveOpenAITransportTurnState({
|
||||
provider: "openai",
|
||||
modelId: nativeModel.id,
|
||||
model: nativeModel,
|
||||
sessionId: "session-123",
|
||||
turnId: "turn-123",
|
||||
attempt: 2,
|
||||
transport: "websocket",
|
||||
});
|
||||
expect(state?.headers?.["x-client-request-id"]).toBe("session-123");
|
||||
expect(state?.headers?.["x-openclaw-session-id"]).toBe("session-123");
|
||||
expect(state?.headers?.["x-openclaw-turn-id"]).toBe("turn-123");
|
||||
expect(state?.headers?.["x-openclaw-turn-attempt"]).toBe("2");
|
||||
expect(state?.metadata?.openclaw_session_id).toBe("session-123");
|
||||
expect(state?.metadata?.openclaw_turn_id).toBe("turn-123");
|
||||
expect(state?.metadata?.openclaw_turn_attempt).toBe("2");
|
||||
expect(state?.metadata?.openclaw_transport).toBe("websocket");
|
||||
});
|
||||
|
||||
it("skips turn state for proxy-like OpenAI routes", () => {
|
||||
expect(
|
||||
resolveOpenAITransportTurnState({
|
||||
provider: "openai",
|
||||
modelId: proxyModel.id,
|
||||
model: proxyModel,
|
||||
sessionId: "session-123",
|
||||
turnId: "turn-123",
|
||||
attempt: 1,
|
||||
transport: "stream",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps Codex request identity session-scoped while adding turn metadata", () => {
|
||||
const state = resolveOpenAITransportTurnState({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
model: {
|
||||
...nativeModel,
|
||||
provider: "openai",
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
},
|
||||
sessionId: "session-123",
|
||||
turnId: "turn-123",
|
||||
attempt: 2,
|
||||
transport: "stream",
|
||||
});
|
||||
expect(state?.headers?.["x-client-request-id"]).toBe("session-123");
|
||||
expect(state?.headers?.["x-openclaw-session-id"]).toBe("session-123");
|
||||
expect(state?.headers?.["x-openclaw-turn-id"]).toBe("turn-123");
|
||||
expect(state?.headers?.["x-openclaw-turn-attempt"]).toBe("2");
|
||||
});
|
||||
|
||||
it("returns websocket session headers and cooldown for native routes", () => {
|
||||
const policy = resolveOpenAIWebSocketSessionPolicy({
|
||||
provider: "openai",
|
||||
modelId: nativeModel.id,
|
||||
model: nativeModel,
|
||||
sessionId: "session-123",
|
||||
});
|
||||
expect(policy?.headers?.["x-client-request-id"]).toBe("session-123");
|
||||
expect(policy?.headers?.["x-openclaw-session-id"]).toBe("session-123");
|
||||
expect(policy?.degradeCooldownMs).toBe(60_000);
|
||||
});
|
||||
|
||||
it("treats Azure routes as native OpenAI-family transports", () => {
|
||||
const policy = resolveOpenAIWebSocketSessionPolicy({
|
||||
provider: "azure-openai-responses",
|
||||
modelId: "gpt-5.4",
|
||||
model: {
|
||||
...nativeModel,
|
||||
provider: "azure-openai-responses",
|
||||
baseUrl: "https://demo.openai.azure.com/openai/v1",
|
||||
},
|
||||
sessionId: "session-123",
|
||||
});
|
||||
expect(policy?.headers?.["x-client-request-id"]).toBe("session-123");
|
||||
expect(policy?.headers?.["x-openclaw-session-id"]).toBe("session-123");
|
||||
expect(policy?.degradeCooldownMs).toBe(60_000);
|
||||
});
|
||||
|
||||
it("treats ChatGPT Codex backend routes as native OpenAI-family transports", () => {
|
||||
const policy = resolveOpenAIWebSocketSessionPolicy({
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
model: {
|
||||
...nativeModel,
|
||||
provider: "openai",
|
||||
api: "openai-chatgpt-responses",
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
},
|
||||
sessionId: "session-123",
|
||||
});
|
||||
expect(policy?.headers?.["x-client-request-id"]).toBe("session-123");
|
||||
expect(policy?.headers?.["x-openclaw-session-id"]).toBe("session-123");
|
||||
expect(policy?.degradeCooldownMs).toBe(60_000);
|
||||
});
|
||||
});
|
||||
108
extensions/openai/transport-policy.ts
Normal file
108
extensions/openai/transport-policy.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
// Openai plugin module implements transport policy behavior.
|
||||
import type {
|
||||
ProviderResolveTransportTurnStateContext,
|
||||
ProviderResolveWebSocketSessionPolicyContext,
|
||||
ProviderTransportTurnState,
|
||||
ProviderWebSocketSessionPolicy,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { normalizeProviderId } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { isOpenAIApiBaseUrl, isOpenAICodexBaseUrl } from "./base-url.js";
|
||||
|
||||
const DEFAULT_OPENAI_WS_DEGRADE_COOLDOWN_MS = 60_000;
|
||||
const AZURE_PROVIDER_IDS = new Set(["azure-openai", "azure-openai-responses"]);
|
||||
|
||||
function isAzureOpenAIBaseUrl(baseUrl?: string): boolean {
|
||||
const trimmed = baseUrl?.trim();
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return normalizeLowercaseStringOrEmpty(new URL(trimmed).hostname).endsWith(".openai.azure.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeIdentityValue(value: string, maxLength = 160): string {
|
||||
const trimmed = value.trim().replace(/[\r\n]+/g, " ");
|
||||
return trimmed.length > maxLength ? trimmed.slice(0, maxLength) : trimmed;
|
||||
}
|
||||
|
||||
function usesKnownNativeOpenAIRoute(provider: string, baseUrl?: string): boolean {
|
||||
const normalizedProvider = normalizeProviderId(provider);
|
||||
if (!normalizedProvider) {
|
||||
return false;
|
||||
}
|
||||
if (normalizedProvider === "openai") {
|
||||
return !baseUrl || isOpenAIApiBaseUrl(baseUrl) || isOpenAICodexBaseUrl(baseUrl);
|
||||
}
|
||||
if (AZURE_PROVIDER_IDS.has(normalizedProvider)) {
|
||||
return !baseUrl || isAzureOpenAIBaseUrl(baseUrl);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function resolveSessionHeaders(params: {
|
||||
provider: string;
|
||||
baseUrl?: string;
|
||||
sessionId?: string;
|
||||
}): Record<string, string> | undefined {
|
||||
if (!params.sessionId || !usesKnownNativeOpenAIRoute(params.provider, params.baseUrl)) {
|
||||
return undefined;
|
||||
}
|
||||
const sessionId = normalizeIdentityValue(params.sessionId);
|
||||
if (!sessionId) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
"x-client-request-id": sessionId,
|
||||
"x-openclaw-session-id": sessionId,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveOpenAITransportTurnState(
|
||||
ctx: ProviderResolveTransportTurnStateContext,
|
||||
): ProviderTransportTurnState | undefined {
|
||||
const sessionHeaders = resolveSessionHeaders({
|
||||
provider: ctx.provider,
|
||||
baseUrl: ctx.model?.baseUrl,
|
||||
sessionId: ctx.sessionId,
|
||||
});
|
||||
if (!sessionHeaders) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const turnId = normalizeIdentityValue(ctx.turnId);
|
||||
const attempt = String(Math.max(1, ctx.attempt));
|
||||
|
||||
return {
|
||||
headers: {
|
||||
...sessionHeaders,
|
||||
"x-openclaw-turn-id": turnId,
|
||||
"x-openclaw-turn-attempt": attempt,
|
||||
},
|
||||
metadata: {
|
||||
openclaw_session_id: sessionHeaders["x-openclaw-session-id"] ?? "",
|
||||
openclaw_turn_id: turnId,
|
||||
openclaw_turn_attempt: attempt,
|
||||
openclaw_transport: ctx.transport,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveOpenAIWebSocketSessionPolicy(
|
||||
ctx: ProviderResolveWebSocketSessionPolicyContext,
|
||||
): ProviderWebSocketSessionPolicy | undefined {
|
||||
if (!usesKnownNativeOpenAIRoute(ctx.provider, ctx.model?.baseUrl)) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
headers: resolveSessionHeaders({
|
||||
provider: ctx.provider,
|
||||
baseUrl: ctx.model?.baseUrl,
|
||||
sessionId: ctx.sessionId,
|
||||
}),
|
||||
degradeCooldownMs: DEFAULT_OPENAI_WS_DEGRADE_COOLDOWN_MS,
|
||||
};
|
||||
}
|
||||
16
extensions/openai/tsconfig.json
Normal file
16
extensions/openai/tsconfig.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"extends": "../tsconfig.package-boundary.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["./*.ts", "./src/**/*.ts"],
|
||||
"exclude": [
|
||||
"./**/*.test.ts",
|
||||
"./dist/**",
|
||||
"./node_modules/**",
|
||||
"./src/test-support/**",
|
||||
"./src/**/*test-helpers.ts",
|
||||
"./src/**/*test-harness.ts",
|
||||
"./src/**/*test-support.ts"
|
||||
]
|
||||
}
|
||||
461
extensions/openai/tts.test.ts
Normal file
461
extensions/openai/tts.test.ts
Normal file
@@ -0,0 +1,461 @@
|
||||
// Openai tests cover tts plugin behavior.
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
finalizeDebugProxyCapture,
|
||||
getDebugProxyCaptureStore,
|
||||
initializeDebugProxyCapture,
|
||||
} from "openclaw/plugin-sdk/proxy-capture";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { installDebugProxyTestResetHooks } from "../test-support/debug-proxy-env-test-helpers.js";
|
||||
import { createStreamingErrorResponse } from "../test-support/streaming-error-response.js";
|
||||
import {
|
||||
isValidOpenAIModel,
|
||||
isValidOpenAIVoice,
|
||||
OPENAI_TTS_MODELS,
|
||||
OPENAI_TTS_VOICES,
|
||||
openaiTTS,
|
||||
resolveOpenAITtsInstructions,
|
||||
} from "./tts.js";
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
|
||||
fetchWithSsrFGuard: async ({
|
||||
url,
|
||||
init,
|
||||
}: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
}): Promise<{ response: Response; release: () => Promise<void> }> => ({
|
||||
response: await globalThis.fetch(url, init),
|
||||
release: vi.fn(async () => {}),
|
||||
}),
|
||||
ssrfPolicyFromHttpBaseUrlAllowedHostname: () => undefined,
|
||||
}));
|
||||
|
||||
const officialEndpointValidationCases = [
|
||||
{
|
||||
label: "voice validator",
|
||||
isAccepted: () => isValidOpenAIVoice("kokoro-custom-voice", "https://api.openai.com/v1/"),
|
||||
},
|
||||
{
|
||||
label: "model validator",
|
||||
isAccepted: () => isValidOpenAIModel("kokoro-custom-model", "https://api.openai.com/v1/"),
|
||||
},
|
||||
];
|
||||
|
||||
function firstFetchCall(fetchMock: ReturnType<typeof vi.fn>): unknown[] {
|
||||
const call = fetchMock.mock.calls[0];
|
||||
if (!call) {
|
||||
throw new Error("expected fetch call");
|
||||
}
|
||||
return call;
|
||||
}
|
||||
|
||||
function firstFetchInit(fetchMock: ReturnType<typeof vi.fn>): RequestInit {
|
||||
const init = firstFetchCall(fetchMock)[1];
|
||||
if (!init || typeof init !== "object") {
|
||||
throw new Error("expected fetch init");
|
||||
}
|
||||
return init as RequestInit;
|
||||
}
|
||||
|
||||
describe("openai tts", () => {
|
||||
const proxyReset = installDebugProxyTestResetHooks();
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("isValidOpenAIVoice", () => {
|
||||
it("accepts all valid OpenAI voices including newer additions", () => {
|
||||
for (const voice of OPENAI_TTS_VOICES) {
|
||||
expect(isValidOpenAIVoice(voice)).toBe(true);
|
||||
}
|
||||
for (const newerVoice of ["ballad", "cedar", "juniper", "marin", "verse"]) {
|
||||
expect(isValidOpenAIVoice(newerVoice), newerVoice).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects invalid voice names", () => {
|
||||
expect(isValidOpenAIVoice("invalid")).toBe(false);
|
||||
expect(isValidOpenAIVoice("")).toBe(false);
|
||||
expect(isValidOpenAIVoice("ALLOY")).toBe(false);
|
||||
expect(isValidOpenAIVoice("alloy ")).toBe(false);
|
||||
expect(isValidOpenAIVoice(" alloy")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isValidOpenAIModel", () => {
|
||||
it("matches the supported model set and rejects unsupported values", () => {
|
||||
expect(OPENAI_TTS_MODELS).toContain("gpt-4o-mini-tts");
|
||||
expect(OPENAI_TTS_MODELS).toContain("tts-1");
|
||||
expect(OPENAI_TTS_MODELS).toContain("tts-1-hd");
|
||||
expect(OPENAI_TTS_MODELS).toHaveLength(3);
|
||||
expect(Array.isArray(OPENAI_TTS_MODELS)).toBe(true);
|
||||
expect(OPENAI_TTS_MODELS.length).toBeGreaterThan(0);
|
||||
const cases = [
|
||||
{ model: "gpt-4o-mini-tts", expected: true },
|
||||
{ model: "tts-1", expected: true },
|
||||
{ model: "tts-1-hd", expected: true },
|
||||
{ model: "invalid", expected: false },
|
||||
{ model: "", expected: false },
|
||||
{ model: "gpt-4", expected: false },
|
||||
] as const;
|
||||
for (const testCase of cases) {
|
||||
expect(isValidOpenAIModel(testCase.model), testCase.model).toBe(testCase.expected);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("official OpenAI TTS endpoint validation", () => {
|
||||
it.each(officialEndpointValidationCases)(
|
||||
"$label treats the default endpoint with trailing slash as the default endpoint",
|
||||
({ isAccepted }) => {
|
||||
expect(isAccepted()).toBe(false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("resolveOpenAITtsInstructions", () => {
|
||||
it("keeps instructions only for gpt-4o-mini-tts variants", () => {
|
||||
expect(resolveOpenAITtsInstructions("gpt-4o-mini-tts", " Speak warmly ")).toBe(
|
||||
"Speak warmly",
|
||||
);
|
||||
expect(resolveOpenAITtsInstructions("gpt-4o-mini-tts-2025-12-15", "Speak warmly")).toBe(
|
||||
"Speak warmly",
|
||||
);
|
||||
expect(resolveOpenAITtsInstructions("tts-1", "Speak warmly")).toBeUndefined();
|
||||
expect(resolveOpenAITtsInstructions("tts-1-hd", "Speak warmly")).toBeUndefined();
|
||||
expect(resolveOpenAITtsInstructions("gpt-4o-mini-tts", " ")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves instructions for custom OpenAI-compatible TTS endpoints", () => {
|
||||
expect(
|
||||
resolveOpenAITtsInstructions("tts-1", " Speak warmly ", "https://tts.example.com/v1"),
|
||||
).toBe("Speak warmly");
|
||||
expect(
|
||||
resolveOpenAITtsInstructions("tts-1", " Speak warmly ", "https://api.openai.com/v1/"),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
resolveOpenAITtsInstructions("tts-1", " ", "https://tts.example.com/v1"),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("openaiTTS diagnostics", () => {
|
||||
it("adds OpenClaw attribution headers to native OpenAI speech requests", async () => {
|
||||
vi.stubEnv("OPENCLAW_VERSION", "2026.3.22");
|
||||
const fetchMock = vi.fn(
|
||||
async (_url: string | URL, _init?: RequestInit) =>
|
||||
new Response(Buffer.from("audio-bytes"), { status: 200 }),
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
const url = firstFetchCall(fetchMock)[0];
|
||||
const init = firstFetchInit(fetchMock);
|
||||
const headers = init?.headers as Record<string, string> | undefined;
|
||||
expect(url).toBe("https://api.openai.com/v1/audio/speech");
|
||||
expect(headers?.originator).toBe("openclaw");
|
||||
expect(headers?.version).toBe("2026.3.22");
|
||||
expect(headers?.["User-Agent"]).toBe("openclaw/2026.3.22");
|
||||
});
|
||||
|
||||
it("sends instructions to custom OpenAI-compatible endpoints", async () => {
|
||||
const fetchMock = vi.fn(
|
||||
async (_url: string | URL, _init?: RequestInit) =>
|
||||
new Response(Buffer.from("audio-bytes"), { status: 200 }),
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://tts.example.com/v1",
|
||||
model: "tts-1",
|
||||
voice: "custom-voice",
|
||||
instructions: " Speak warmly ",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
const init = firstFetchInit(fetchMock);
|
||||
if (typeof init?.body !== "string") {
|
||||
throw new Error("expected JSON request body");
|
||||
}
|
||||
const body = JSON.parse(init.body) as Record<string, unknown>;
|
||||
expect(body.instructions).toBe("Speak warmly");
|
||||
expect(body.model).toBe("tts-1");
|
||||
expect(body.voice).toBe("custom-voice");
|
||||
});
|
||||
|
||||
it("merges sanitized extraBody fields into TTS requests", async () => {
|
||||
const fetchMock = vi.fn(
|
||||
async (_url: string | URL, _init?: RequestInit) =>
|
||||
new Response(Buffer.from("audio-bytes"), { status: 200 }),
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
const extraBody = JSON.parse(
|
||||
'{"lang":"e","speed":1.2,"__proto__":{"polluted":true},"constructor":"bad","prototype":"bad"}',
|
||||
) as Record<string, unknown>;
|
||||
|
||||
await openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://tts.example.com/v1",
|
||||
model: "tts-1",
|
||||
voice: "custom-voice",
|
||||
speed: 1,
|
||||
responseFormat: "mp3",
|
||||
extraBody,
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
const init = firstFetchInit(fetchMock);
|
||||
if (typeof init?.body !== "string") {
|
||||
throw new Error("expected JSON request body");
|
||||
}
|
||||
const body = JSON.parse(init.body) as Record<string, unknown>;
|
||||
expect(body.model).toBe("tts-1");
|
||||
expect(body.input).toBe("hello");
|
||||
expect(body.voice).toBe("custom-voice");
|
||||
expect(body.response_format).toBe("mp3");
|
||||
expect(body.lang).toBe("e");
|
||||
expect(body.speed).toBe(1.2);
|
||||
expect(Object.hasOwn(body, "__proto__")).toBe(false);
|
||||
expect(Object.hasOwn(body, "constructor")).toBe(false);
|
||||
expect(Object.hasOwn(body, "prototype")).toBe(false);
|
||||
expect((Object.prototype as Record<string, unknown>).polluted).toBeUndefined();
|
||||
});
|
||||
|
||||
it("omits instructions for unsupported models on the official OpenAI endpoint", async () => {
|
||||
const fetchMock = vi.fn(
|
||||
async (_url: string | URL, _init?: RequestInit) =>
|
||||
new Response(Buffer.from("audio-bytes"), { status: 200 }),
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://api.openai.com/v1/",
|
||||
model: "tts-1",
|
||||
voice: "alloy",
|
||||
instructions: "Speak warmly",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
const init = firstFetchInit(fetchMock);
|
||||
if (typeof init?.body !== "string") {
|
||||
throw new Error("expected JSON request body");
|
||||
}
|
||||
const body = JSON.parse(init.body) as Record<string, unknown>;
|
||||
expect(body.instructions).toBeUndefined();
|
||||
});
|
||||
|
||||
it("includes parsed provider detail and request id for JSON API errors", async () => {
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
message: "Invalid API key",
|
||||
type: "invalid_request_error",
|
||||
code: "invalid_api_key",
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 401,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"x-request-id": "req_123",
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await expect(
|
||||
openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "bad-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"OpenAI TTS API error (401): Invalid API key [type=invalid_request_error, code=invalid_api_key] [request_id=req_123]",
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to raw body text when the error body is non-JSON", async () => {
|
||||
const fetchMock = vi.fn(
|
||||
async () => new Response("temporary upstream outage", { status: 503 }),
|
||||
);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await expect(
|
||||
openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
}),
|
||||
).rejects.toThrow("OpenAI TTS API error (503): temporary upstream outage");
|
||||
});
|
||||
|
||||
it("caps streamed audio responses instead of buffering oversized TTS output", async () => {
|
||||
const streamed = createStreamingErrorResponse({
|
||||
status: 200,
|
||||
chunkCount: 20,
|
||||
chunkSize: 1024,
|
||||
byte: 121,
|
||||
});
|
||||
const fetchMock = vi.fn(async () => streamed.response);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await expect(
|
||||
openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
maxBytes: 2048,
|
||||
}),
|
||||
).rejects.toThrow("OpenAI TTS audio response exceeds 2048 bytes");
|
||||
|
||||
expect(streamed.getReadCount()).toBeLessThan(20);
|
||||
});
|
||||
|
||||
it("caps streamed non-JSON error reads instead of consuming full response bodies", async () => {
|
||||
const streamed = createStreamingErrorResponse({
|
||||
status: 503,
|
||||
chunkCount: 200,
|
||||
chunkSize: 1024,
|
||||
byte: 120,
|
||||
});
|
||||
const fetchMock = vi.fn(async () => streamed.response);
|
||||
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await expect(
|
||||
openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
}),
|
||||
).rejects.toThrow("OpenAI TTS API error (503)");
|
||||
|
||||
expect(streamed.getReadCount()).toBeLessThan(200);
|
||||
});
|
||||
|
||||
it("records TTS exchanges in debug proxy capture mode", async () => {
|
||||
const tempDir = mkdtempSync(path.join(os.tmpdir(), "openai-tts-capture-"));
|
||||
proxyReset.captureProxyEnv();
|
||||
process.env.OPENCLAW_DEBUG_PROXY_ENABLED = "1";
|
||||
process.env.OPENCLAW_STATE_DIR = tempDir;
|
||||
process.env.OPENCLAW_DEBUG_PROXY_SESSION_ID = "tts-session";
|
||||
|
||||
globalThis.fetch = vi
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
new Response(Buffer.from("audio-bytes"), { status: 200 }),
|
||||
) as unknown as typeof globalThis.fetch;
|
||||
|
||||
const store = getDebugProxyCaptureStore();
|
||||
store.upsertSession({
|
||||
id: "tts-session",
|
||||
startedAt: Date.now(),
|
||||
mode: "test",
|
||||
sourceScope: "openclaw",
|
||||
sourceProcess: "openclaw",
|
||||
});
|
||||
|
||||
await openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
const events = store.getSessionEvents("tts-session", 10);
|
||||
expect(
|
||||
events.some((event) => event.kind === "request" && event.host === "api.openai.com"),
|
||||
).toBe(true);
|
||||
expect(
|
||||
events.some((event) => event.kind === "response" && event.host === "api.openai.com"),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
it("does not double-capture TTS exchanges when the global fetch patch is installed", async () => {
|
||||
const tempDir = mkdtempSync(path.join(os.tmpdir(), "openai-tts-patched-capture-"));
|
||||
proxyReset.captureProxyEnv();
|
||||
process.env.OPENCLAW_DEBUG_PROXY_ENABLED = "1";
|
||||
process.env.OPENCLAW_STATE_DIR = tempDir;
|
||||
process.env.OPENCLAW_DEBUG_PROXY_SESSION_ID = "tts-patched-session";
|
||||
|
||||
globalThis.fetch = vi
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
new Response(Buffer.from("audio-bytes"), { status: 200 }),
|
||||
) as unknown as typeof globalThis.fetch;
|
||||
|
||||
initializeDebugProxyCapture("test");
|
||||
|
||||
await openaiTTS({
|
||||
text: "hello",
|
||||
apiKey: "test-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
model: "gpt-4o-mini-tts",
|
||||
voice: "alloy",
|
||||
responseFormat: "mp3",
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
const store = getDebugProxyCaptureStore();
|
||||
let events: Array<Record<string, unknown>> = [];
|
||||
try {
|
||||
await vi.waitFor(() => {
|
||||
events = store
|
||||
.getSessionEvents("tts-patched-session", 10)
|
||||
.filter((event) => event.host === "api.openai.com");
|
||||
expect(events).toHaveLength(2);
|
||||
});
|
||||
const kinds = events.map((event) => String(event.kind)).toSorted();
|
||||
expect(kinds).toEqual(["request", "response"]);
|
||||
} finally {
|
||||
finalizeDebugProxyCapture();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
192
extensions/openai/tts.ts
Normal file
192
extensions/openai/tts.ts
Normal file
@@ -0,0 +1,192 @@
|
||||
// Openai plugin module implements tts behavior.
|
||||
import {
|
||||
assertOkOrThrowProviderError,
|
||||
resolveProviderRequestHeaders,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
captureHttpExchange,
|
||||
isDebugProxyGlobalFetchPatchInstalled,
|
||||
} from "openclaw/plugin-sdk/proxy-capture";
|
||||
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
|
||||
import {
|
||||
fetchWithSsrFGuard,
|
||||
ssrfPolicyFromHttpBaseUrlAllowedHostname,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
|
||||
export const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
||||
const DEFAULT_TTS_MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
export const OPENAI_TTS_MODELS = ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"] as const;
|
||||
|
||||
export const OPENAI_TTS_VOICES = [
|
||||
"alloy",
|
||||
"ash",
|
||||
"ballad",
|
||||
"cedar",
|
||||
"coral",
|
||||
"echo",
|
||||
"fable",
|
||||
"juniper",
|
||||
"marin",
|
||||
"onyx",
|
||||
"nova",
|
||||
"sage",
|
||||
"shimmer",
|
||||
"verse",
|
||||
] as const;
|
||||
|
||||
type OpenAiTtsVoice = (typeof OPENAI_TTS_VOICES)[number];
|
||||
|
||||
export function normalizeOpenAITtsBaseUrl(baseUrl?: string): string {
|
||||
const trimmed = baseUrl?.trim();
|
||||
if (!trimmed) {
|
||||
return DEFAULT_OPENAI_BASE_URL;
|
||||
}
|
||||
return trimmed.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function isCustomOpenAIEndpoint(baseUrl?: string): boolean {
|
||||
if (baseUrl != null) {
|
||||
return normalizeOpenAITtsBaseUrl(baseUrl) !== DEFAULT_OPENAI_BASE_URL;
|
||||
}
|
||||
return normalizeOpenAITtsBaseUrl(process.env.OPENAI_TTS_BASE_URL) !== DEFAULT_OPENAI_BASE_URL;
|
||||
}
|
||||
|
||||
export function isValidOpenAIModel(model: string, baseUrl?: string): boolean {
|
||||
if (isCustomOpenAIEndpoint(baseUrl)) {
|
||||
return true;
|
||||
}
|
||||
return OPENAI_TTS_MODELS.includes(model as (typeof OPENAI_TTS_MODELS)[number]);
|
||||
}
|
||||
|
||||
export function isValidOpenAIVoice(voice: string, baseUrl?: string): voice is OpenAiTtsVoice {
|
||||
if (isCustomOpenAIEndpoint(baseUrl)) {
|
||||
return true;
|
||||
}
|
||||
return OPENAI_TTS_VOICES.includes(voice as OpenAiTtsVoice);
|
||||
}
|
||||
|
||||
export function resolveOpenAITtsInstructions(
|
||||
model: string,
|
||||
instructions?: string,
|
||||
baseUrl?: string,
|
||||
): string | undefined {
|
||||
const next = instructions?.trim();
|
||||
if (!next) {
|
||||
return undefined;
|
||||
}
|
||||
if (baseUrl !== undefined && isCustomOpenAIEndpoint(baseUrl)) {
|
||||
return next;
|
||||
}
|
||||
return model.includes("gpt-4o-mini-tts") ? next : undefined;
|
||||
}
|
||||
|
||||
function sanitizeExtraBodyRecord(value: Record<string, unknown>): Record<string, unknown> {
|
||||
const sanitized: Record<string, unknown> = {};
|
||||
for (const [key, entry] of Object.entries(value)) {
|
||||
if (key === "__proto__" || key === "constructor" || key === "prototype") {
|
||||
continue;
|
||||
}
|
||||
sanitized[key] = entry;
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
export async function openaiTTS(params: {
|
||||
text: string;
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
model: string;
|
||||
voice: string;
|
||||
speed?: number;
|
||||
instructions?: string;
|
||||
responseFormat: "mp3" | "opus" | "pcm" | "wav";
|
||||
extraBody?: Record<string, unknown>;
|
||||
timeoutMs: number;
|
||||
maxBytes?: number;
|
||||
}): Promise<Buffer> {
|
||||
const {
|
||||
text,
|
||||
apiKey,
|
||||
baseUrl,
|
||||
model,
|
||||
voice,
|
||||
speed,
|
||||
instructions,
|
||||
responseFormat,
|
||||
extraBody,
|
||||
timeoutMs,
|
||||
maxBytes = DEFAULT_TTS_MAX_BYTES,
|
||||
} = params;
|
||||
const effectiveInstructions = resolveOpenAITtsInstructions(model, instructions, baseUrl);
|
||||
|
||||
if (!isValidOpenAIModel(model, baseUrl)) {
|
||||
throw new Error(`Invalid model: ${model}`);
|
||||
}
|
||||
if (!isValidOpenAIVoice(voice, baseUrl)) {
|
||||
throw new Error(`Invalid voice: ${voice}`);
|
||||
}
|
||||
|
||||
const requestHeaders = resolveProviderRequestHeaders({
|
||||
provider: "openai",
|
||||
baseUrl,
|
||||
capability: "audio",
|
||||
transport: "http",
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}) ?? {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
const requestBody = JSON.stringify({
|
||||
model,
|
||||
input: text,
|
||||
voice,
|
||||
response_format: responseFormat,
|
||||
...(speed != null && { speed }),
|
||||
...(effectiveInstructions != null && { instructions: effectiveInstructions }),
|
||||
...(extraBody == null ? {} : sanitizeExtraBodyRecord(extraBody)),
|
||||
});
|
||||
const requestUrl = `${baseUrl}/audio/speech`;
|
||||
const debugProxyFetchPatchInstalled = isDebugProxyGlobalFetchPatchInstalled();
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: requestUrl,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: requestHeaders,
|
||||
body: requestBody,
|
||||
},
|
||||
timeoutMs,
|
||||
policy: ssrfPolicyFromHttpBaseUrlAllowedHostname(baseUrl),
|
||||
capture: false,
|
||||
pinDns: debugProxyFetchPatchInstalled ? false : undefined,
|
||||
auditContext: "openai-tts",
|
||||
});
|
||||
try {
|
||||
if (!debugProxyFetchPatchInstalled) {
|
||||
captureHttpExchange({
|
||||
url: requestUrl,
|
||||
method: "POST",
|
||||
requestHeaders,
|
||||
requestBody,
|
||||
response,
|
||||
transport: "http",
|
||||
meta: {
|
||||
provider: "openai",
|
||||
capability: "tts",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await assertOkOrThrowProviderError(response, "OpenAI TTS API error");
|
||||
|
||||
return await readResponseWithLimit(response, maxBytes, {
|
||||
onOverflow: ({ maxBytes: maxBytesLocal }) =>
|
||||
new Error(`OpenAI TTS audio response exceeds ${maxBytesLocal} bytes`),
|
||||
});
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
655
extensions/openai/video-generation-provider.test.ts
Normal file
655
extensions/openai/video-generation-provider.test.ts
Normal file
@@ -0,0 +1,655 @@
|
||||
// Openai tests cover video generation provider plugin behavior.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
getProviderHttpMocks,
|
||||
installProviderHttpMockCleanup,
|
||||
} from "openclaw/plugin-sdk/provider-http-test-mocks";
|
||||
import { expectExplicitVideoGenerationCapabilities } from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const {
|
||||
resolveApiKeyForProviderMock,
|
||||
postJsonRequestMock,
|
||||
postMultipartRequestMock,
|
||||
fetchWithTimeoutMock,
|
||||
fetchWithTimeoutGuardedMock,
|
||||
pollProviderOperationJsonMock,
|
||||
assertOkOrThrowHttpErrorMock,
|
||||
executeProviderOperationWithRetryMock,
|
||||
resolveProviderHttpRequestConfigMock,
|
||||
sanitizeConfiguredModelProviderRequestMock,
|
||||
} = getProviderHttpMocks();
|
||||
|
||||
let buildOpenAIVideoGenerationProvider: typeof import("./video-generation-provider.js").buildOpenAIVideoGenerationProvider;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ buildOpenAIVideoGenerationProvider } = await import("./video-generation-provider.js"));
|
||||
});
|
||||
|
||||
installProviderHttpMockCleanup();
|
||||
|
||||
function postJsonRequest(index = 0): Record<string, unknown> {
|
||||
const request = postJsonRequestMock.mock.calls[index]?.[0] as Record<string, unknown> | undefined;
|
||||
if (!request) {
|
||||
throw new Error(`expected postJsonRequest call ${index}`);
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
function postMultipartRequest(index = 0): Record<string, unknown> {
|
||||
const request = postMultipartRequestMock.mock.calls[index]?.[0] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!request) {
|
||||
throw new Error(`expected postMultipartRequest call ${index}`);
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
function fetchWithTimeoutCall(index: number): [string, RequestInit | undefined, number, unknown] {
|
||||
const call = fetchWithTimeoutMock.mock.calls[index] as
|
||||
| [string, RequestInit | undefined, number, unknown]
|
||||
| undefined;
|
||||
if (!call) {
|
||||
throw new Error(`expected fetchWithTimeout call ${index}`);
|
||||
}
|
||||
return call;
|
||||
}
|
||||
|
||||
function fetchWithTimeoutGuardedCall(
|
||||
index = 0,
|
||||
): [string, RequestInit | undefined, number, unknown, Record<string, unknown> | undefined] {
|
||||
const call = fetchWithTimeoutGuardedMock.mock.calls[index] as
|
||||
| [string, RequestInit | undefined, number, unknown, Record<string, unknown> | undefined]
|
||||
| undefined;
|
||||
if (!call) {
|
||||
throw new Error(`expected fetchWithTimeoutGuarded call ${index}`);
|
||||
}
|
||||
return call;
|
||||
}
|
||||
|
||||
function pollProviderOperationRequest(index = 0): Record<string, unknown> {
|
||||
const request = pollProviderOperationJsonMock.mock.calls[index]?.[0] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!request) {
|
||||
throw new Error(`expected pollProviderOperationJson call ${index}`);
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
function providerHttpConfigRequest(): Record<string, unknown> {
|
||||
const [call] = resolveProviderHttpRequestConfigMock.mock.calls;
|
||||
if (!call) {
|
||||
throw new Error("expected provider HTTP config request");
|
||||
}
|
||||
const [request] = call;
|
||||
if (!request || typeof request !== "object" || Array.isArray(request)) {
|
||||
throw new Error("expected provider HTTP config request");
|
||||
}
|
||||
return request as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function streamedVideoResponse(bytes: string): Response {
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(bytes));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
{ headers: { "content-type": "video/mp4" } },
|
||||
);
|
||||
}
|
||||
|
||||
// Response.json keeps object fixtures on the standard Response body path so the
|
||||
// create read exercises the byte-bounded reader instead of an unbounded res.json().
|
||||
function streamedJsonResponse(payload: unknown): Response {
|
||||
return Response.json(payload);
|
||||
}
|
||||
|
||||
describe("openai video generation provider", () => {
|
||||
it("declares explicit mode capabilities", () => {
|
||||
expectExplicitVideoGenerationCapabilities(buildOpenAIVideoGenerationProvider());
|
||||
});
|
||||
|
||||
it("does not claim size or duration controls for OpenAI video edits", () => {
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
|
||||
expect(provider.capabilities.videoToVideo).toEqual({
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not advertise video generation for OAuth-only OpenAI profiles", () => {
|
||||
const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-openai-video-auth-"));
|
||||
const previousOpenAIKey = process.env.OPENAI_API_KEY;
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
try {
|
||||
fs.writeFileSync(
|
||||
path.join(agentDir, "auth-profiles.json"),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:chatgpt": {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "chatgpt-oauth-token",
|
||||
refresh: "refresh-token",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(buildOpenAIVideoGenerationProvider().isConfigured?.({ agentDir })).toBe(false);
|
||||
} finally {
|
||||
if (previousOpenAIKey === undefined) {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
} else {
|
||||
process.env.OPENAI_API_KEY = previousOpenAIKey;
|
||||
}
|
||||
fs.rmSync(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("requires an OpenAI API key credential for direct video generation", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: "chatgpt-oauth-token",
|
||||
mode: "oauth",
|
||||
} as never);
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await expect(
|
||||
provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "A paper airplane gliding through golden hour light",
|
||||
cfg: {},
|
||||
}),
|
||||
).rejects.toThrow("OpenAI API key missing");
|
||||
|
||||
expect(resolveApiKeyForProviderMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "openai",
|
||||
modelApi: "openai-responses",
|
||||
}),
|
||||
);
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses JSON for text-only Sora requests", async () => {
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: streamedJsonResponse({
|
||||
id: "vid_123",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
id: "vid_123",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
seconds: "4",
|
||||
size: "720x1280",
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/webm" }),
|
||||
arrayBuffer: async () => Buffer.from("webm-bytes"),
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "A paper airplane gliding through golden hour light",
|
||||
cfg: {},
|
||||
durationSeconds: 4,
|
||||
});
|
||||
|
||||
expect(postJsonRequest().url).toBe("https://api.openai.com/v1/videos");
|
||||
const [pollUrl, pollInit, pollTimeout, pollFetch] = fetchWithTimeoutCall(0);
|
||||
expect(pollUrl).toBe("https://api.openai.com/v1/videos/vid_123");
|
||||
expect(pollInit?.method).toBe("GET");
|
||||
expect(pollTimeout).toBe(120000);
|
||||
expect(pollFetch).toBe(fetch);
|
||||
expect(result.videos).toHaveLength(1);
|
||||
expect(result.videos[0]?.mimeType).toBe("video/webm");
|
||||
expect(result.videos[0]?.fileName).toBe("video-1.webm");
|
||||
expect(result.metadata?.videoId).toBe("vid_123");
|
||||
expect(result.metadata?.status).toBe("completed");
|
||||
});
|
||||
|
||||
it("rejects generated video downloads that exceed the configured media cap", async () => {
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: streamedJsonResponse({
|
||||
id: "vid_too_large",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
id: "vid_too_large",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce(streamedVideoResponse("too-large"));
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await expect(
|
||||
provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "short video",
|
||||
cfg: { agents: { defaults: { mediaMaxMb: 0.000001 } } },
|
||||
}),
|
||||
).rejects.toThrow("OpenAI generated video download exceeds 1 bytes");
|
||||
});
|
||||
|
||||
it("uses JSON input_reference.image_url for image-to-video requests", async () => {
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: streamedJsonResponse({
|
||||
id: "vid_456",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
id: "vid_456",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Animate this frame",
|
||||
cfg: {},
|
||||
inputImages: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
|
||||
});
|
||||
|
||||
const createRequest = postJsonRequest();
|
||||
expect(createRequest.url).toBe("https://api.openai.com/v1/videos");
|
||||
expect((createRequest.body as Record<string, unknown>).input_reference).toEqual({
|
||||
image_url: "data:image/png;base64,cG5nLWJ5dGVz",
|
||||
});
|
||||
const [pollUrl, pollInit, pollTimeout, pollFetch] = fetchWithTimeoutCall(0);
|
||||
expect(pollUrl).toBe("https://api.openai.com/v1/videos/vid_456");
|
||||
expect(pollInit?.method).toBe("GET");
|
||||
expect(pollTimeout).toBe(120000);
|
||||
expect(pollFetch).toBe(fetch);
|
||||
});
|
||||
|
||||
it("keeps configured local baseUrl private-network blocked unless explicitly enabled", async () => {
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: streamedJsonResponse({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Render via local relay",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "http://127.0.0.1:44080/v1",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(providerHttpConfigRequest().baseUrl).toBe("http://127.0.0.1:44080/v1");
|
||||
expect(providerHttpConfigRequest().request).toBeUndefined();
|
||||
const createRequest = postJsonRequest();
|
||||
expect(createRequest.url).toBe("http://127.0.0.1:44080/v1/videos");
|
||||
expect(createRequest.allowPrivateNetwork).toBe(false);
|
||||
});
|
||||
|
||||
it("honors configured request allowPrivateNetwork for local video providers", async () => {
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: streamedJsonResponse({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Render via local relay",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "http://127.0.0.1:44080/v1",
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(sanitizeConfiguredModelProviderRequestMock).toHaveBeenCalledWith({
|
||||
allowPrivateNetwork: true,
|
||||
});
|
||||
expect(providerHttpConfigRequest().baseUrl).toBe("http://127.0.0.1:44080/v1");
|
||||
expect(providerHttpConfigRequest().request).toEqual({ allowPrivateNetwork: true });
|
||||
const createRequest = postJsonRequest();
|
||||
expect(createRequest.url).toBe("http://127.0.0.1:44080/v1/videos");
|
||||
expect(createRequest.allowPrivateNetwork).toBe(true);
|
||||
const statusRequest = pollProviderOperationRequest();
|
||||
expect(statusRequest.url).toBe("http://127.0.0.1:44080/v1/videos/vid_local");
|
||||
expect(statusRequest.allowPrivateNetwork).toBe(true);
|
||||
expect(statusRequest.auditContext).toBe("openai-video-status");
|
||||
const [downloadUrl, downloadInit, downloadTimeout, downloadFetch, downloadOptions] =
|
||||
fetchWithTimeoutGuardedCall();
|
||||
expect(downloadUrl).toBe("http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video");
|
||||
expect(downloadInit?.method).toBe("GET");
|
||||
expect(downloadTimeout).toBe(120000);
|
||||
expect(downloadFetch).toBe(fetch);
|
||||
expect(downloadOptions).toEqual({
|
||||
ssrfPolicy: { allowPrivateNetwork: true },
|
||||
auditContext: "openai-video-download",
|
||||
});
|
||||
});
|
||||
|
||||
it("retries guarded local video downloads after transient HTTP errors", async () => {
|
||||
const firstRelease = vi.fn(async () => {});
|
||||
const secondRelease = vi.fn(async () => {});
|
||||
assertOkOrThrowHttpErrorMock
|
||||
.mockImplementationOnce(async () => {})
|
||||
.mockImplementationOnce(async () => {})
|
||||
.mockImplementationOnce(async (_response, label) => {
|
||||
throw new Error(label);
|
||||
})
|
||||
.mockImplementationOnce(async () => {});
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: streamedJsonResponse({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
});
|
||||
fetchWithTimeoutGuardedMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response("busy", { status: 503, statusText: "Service Unavailable" }),
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: firstRelease,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
},
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: secondRelease,
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Render via local relay",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "http://127.0.0.1:44080/v1",
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.videos[0]?.buffer?.toString()).toBe("mp4-bytes");
|
||||
expect(executeProviderOperationWithRetryMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ provider: "openai", stage: "download" }),
|
||||
);
|
||||
expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledTimes(2);
|
||||
expect(firstRelease).toHaveBeenCalledTimes(1);
|
||||
expect(secondRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("releases guarded local video download requests when HTTP errors throw", async () => {
|
||||
const firstRelease = vi.fn(async () => {});
|
||||
const secondRelease = vi.fn(async () => {});
|
||||
assertOkOrThrowHttpErrorMock
|
||||
.mockImplementationOnce(async () => {})
|
||||
.mockImplementationOnce(async () => {})
|
||||
.mockImplementationOnce(async (_response, label) => {
|
||||
throw new Error(label);
|
||||
})
|
||||
.mockImplementationOnce(async (_response, label) => {
|
||||
throw new Error(label);
|
||||
});
|
||||
postJsonRequestMock.mockResolvedValue({
|
||||
response: streamedJsonResponse({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
fetchWithTimeoutMock.mockResolvedValueOnce({
|
||||
json: async () => ({
|
||||
id: "vid_local",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
});
|
||||
fetchWithTimeoutGuardedMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response("busy", { status: 503, statusText: "Service Unavailable" }),
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: firstRelease,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response("busy", { status: 503, statusText: "Service Unavailable" }),
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: secondRelease,
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await expect(
|
||||
provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Render via local relay",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "http://127.0.0.1:44080/v1",
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("OpenAI video download failed");
|
||||
|
||||
expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledTimes(2);
|
||||
expect(firstRelease).toHaveBeenCalledTimes(1);
|
||||
expect(secondRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uses the video edits endpoint for video-to-video uploads", async () => {
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(
|
||||
streamedJsonResponse({
|
||||
id: "vid_789",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
streamedJsonResponse({
|
||||
id: "vid_789",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Remix this clip",
|
||||
cfg: {},
|
||||
inputVideos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
|
||||
});
|
||||
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
const createRequest = postMultipartRequest();
|
||||
expect(createRequest.url).toBe("https://api.openai.com/v1/videos/edits");
|
||||
expect(createRequest.body).toBeInstanceOf(FormData);
|
||||
const form = createRequest.body as FormData;
|
||||
expect(form.get("prompt")).toBe("Remix this clip");
|
||||
expect(form.get("model")).toBe("sora-2");
|
||||
expect(form.get("video")).toBeInstanceOf(File);
|
||||
expect(form.get("input_reference")).toBeNull();
|
||||
expect(createRequest.timeoutMs).toBe(120000);
|
||||
expect(createRequest.fetchFn).toBe(fetch);
|
||||
expect(createRequest.allowPrivateNetwork).toBe(false);
|
||||
});
|
||||
|
||||
it("honors configured request allowPrivateNetwork for multipart video uploads", async () => {
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(
|
||||
streamedJsonResponse({
|
||||
id: "vid_789",
|
||||
model: "sora-2",
|
||||
status: "queued",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
streamedJsonResponse({
|
||||
id: "vid_789",
|
||||
model: "sora-2",
|
||||
status: "completed",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
});
|
||||
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
await provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Remix this clip",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "http://127.0.0.1:44080/v1",
|
||||
request: { allowPrivateNetwork: true },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
inputVideos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
|
||||
});
|
||||
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
const createRequest = postMultipartRequest();
|
||||
expect(createRequest.url).toBe("http://127.0.0.1:44080/v1/videos/edits");
|
||||
expect(createRequest.body).toBeInstanceOf(FormData);
|
||||
expect(createRequest.allowPrivateNetwork).toBe(true);
|
||||
expect(pollProviderOperationRequest().allowPrivateNetwork).toBe(true);
|
||||
expect(fetchWithTimeoutGuardedCall()[4]).toEqual({
|
||||
ssrfPolicy: { allowPrivateNetwork: true },
|
||||
auditContext: "openai-video-download",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects multiple reference assets", async () => {
|
||||
const provider = buildOpenAIVideoGenerationProvider();
|
||||
|
||||
await expect(
|
||||
provider.generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "Animate these",
|
||||
cfg: {},
|
||||
inputImages: [{ buffer: Buffer.from("a"), mimeType: "image/png" }],
|
||||
inputVideos: [{ buffer: Buffer.from("b"), mimeType: "video/mp4" }],
|
||||
}),
|
||||
).rejects.toThrow("OpenAI video generation supports at most one reference image or video.");
|
||||
});
|
||||
});
|
||||
477
extensions/openai/video-generation-provider.ts
Normal file
477
extensions/openai/video-generation-provider.ts
Normal file
@@ -0,0 +1,477 @@
|
||||
// Openai provider module implements model/runtime integration.
|
||||
import { toImageDataUrl } from "openclaw/plugin-sdk/image-generation";
|
||||
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
|
||||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
createProviderOperationDeadline,
|
||||
createProviderOperationTimeoutResolver,
|
||||
executeProviderOperationWithRetry,
|
||||
fetchProviderDownloadResponse,
|
||||
fetchWithTimeoutGuarded,
|
||||
pollProviderOperationJson,
|
||||
postJsonRequest,
|
||||
postMultipartRequest,
|
||||
readProviderJsonResponse,
|
||||
resolveProviderOperationTimeoutMs,
|
||||
resolveProviderHttpRequestConfig,
|
||||
sanitizeConfiguredModelProviderRequest,
|
||||
type ProviderOperationTimeoutMs,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import { resolveConfiguredOpenAIBaseUrl } from "./shared.js";
|
||||
|
||||
const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1";
|
||||
const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2";
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const POLL_INTERVAL_MS = 2_500;
|
||||
const MAX_POLL_ATTEMPTS = 120;
|
||||
const DEFAULT_GENERATED_VIDEO_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const OPENAI_VIDEO_SECONDS = [4, 8, 12] as const;
|
||||
const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"] as const;
|
||||
|
||||
type OpenAIVideoRequestPolicy = {
|
||||
allowPrivateNetwork: boolean;
|
||||
dispatcherPolicy?: Parameters<typeof postJsonRequest>[0]["dispatcherPolicy"];
|
||||
};
|
||||
|
||||
type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed";
|
||||
|
||||
type OpenAIReferenceAsset = {
|
||||
kind: "image" | "video";
|
||||
file: File;
|
||||
buffer: Buffer;
|
||||
mimeType: string;
|
||||
};
|
||||
|
||||
type OpenAIVideoResponse = {
|
||||
id?: string;
|
||||
model?: string;
|
||||
status?: OpenAIVideoStatus;
|
||||
prompt?: string | null;
|
||||
seconds?: string;
|
||||
size?: string;
|
||||
error?: {
|
||||
code?: string;
|
||||
message?: string;
|
||||
} | null;
|
||||
};
|
||||
|
||||
function toBlobBytes(buffer: Buffer): ArrayBuffer {
|
||||
const arrayBuffer = new ArrayBuffer(buffer.byteLength);
|
||||
new Uint8Array(arrayBuffer).set(buffer);
|
||||
return arrayBuffer;
|
||||
}
|
||||
|
||||
function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined {
|
||||
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||
return undefined;
|
||||
}
|
||||
const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds));
|
||||
const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) =>
|
||||
Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best,
|
||||
);
|
||||
return String(nearest) as "4" | "8" | "12";
|
||||
}
|
||||
|
||||
function resolveGeneratedVideoMaxBytes(req: VideoGenerationRequest): number {
|
||||
const configured = req.cfg.agents?.defaults?.mediaMaxMb;
|
||||
if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) {
|
||||
return Math.floor(configured * 1024 * 1024);
|
||||
}
|
||||
return DEFAULT_GENERATED_VIDEO_MAX_BYTES;
|
||||
}
|
||||
|
||||
function resolveSize(params: {
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
resolution?: string;
|
||||
}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined {
|
||||
const explicitSize = normalizeOptionalString(params.size);
|
||||
if (
|
||||
explicitSize &&
|
||||
OPENAI_VIDEO_SIZES.includes(explicitSize as (typeof OPENAI_VIDEO_SIZES)[number])
|
||||
) {
|
||||
return explicitSize as (typeof OPENAI_VIDEO_SIZES)[number];
|
||||
}
|
||||
switch (normalizeOptionalString(params.aspectRatio)) {
|
||||
case "9:16":
|
||||
return "720x1280";
|
||||
case "16:9":
|
||||
return "1280x720";
|
||||
case "4:7":
|
||||
return "1024x1792";
|
||||
case "7:4":
|
||||
return "1792x1024";
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (params.resolution === "1080P") {
|
||||
return "1792x1024";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveReferenceAsset(req: VideoGenerationRequest): OpenAIReferenceAsset | null {
|
||||
const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])];
|
||||
if (allAssets.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (allAssets.length > 1) {
|
||||
throw new Error("OpenAI video generation supports at most one reference image or video.");
|
||||
}
|
||||
const [asset] = allAssets;
|
||||
if (!asset?.buffer) {
|
||||
throw new Error(
|
||||
"OpenAI video generation currently requires local image/video uploads for reference assets.",
|
||||
);
|
||||
}
|
||||
const kind = (req.inputVideos?.length ?? 0) > 0 ? "video" : "image";
|
||||
const mimeType =
|
||||
normalizeOptionalString(asset.mimeType) || (kind === "video" ? "video/mp4" : "image/png");
|
||||
const extension =
|
||||
extensionForMime(mimeType)?.slice(1) ?? (mimeType.startsWith("video/") ? "mp4" : "png");
|
||||
const fileName =
|
||||
normalizeOptionalString(asset.fileName) ||
|
||||
`${kind === "video" ? "reference-video" : "reference-image"}.${extension}`;
|
||||
return {
|
||||
kind,
|
||||
file: new File([toBlobBytes(asset.buffer)], fileName, { type: mimeType }),
|
||||
buffer: asset.buffer,
|
||||
mimeType,
|
||||
};
|
||||
}
|
||||
|
||||
async function pollOpenAIVideo(
|
||||
params: {
|
||||
videoId: string;
|
||||
headers: Headers;
|
||||
timeoutMs?: number;
|
||||
baseUrl: string;
|
||||
fetchFn: typeof fetch;
|
||||
} & OpenAIVideoRequestPolicy,
|
||||
): Promise<OpenAIVideoResponse> {
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: params.timeoutMs,
|
||||
label: `OpenAI video generation task ${params.videoId}`,
|
||||
});
|
||||
return await pollProviderOperationJson<OpenAIVideoResponse>({
|
||||
url: `${params.baseUrl}/videos/${params.videoId}`,
|
||||
headers: params.headers,
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
fetchFn: params.fetchFn,
|
||||
maxAttempts: MAX_POLL_ATTEMPTS,
|
||||
pollIntervalMs: POLL_INTERVAL_MS,
|
||||
requestFailedMessage: "OpenAI video status request failed",
|
||||
timeoutMessage: `OpenAI video generation task ${params.videoId} did not finish in time`,
|
||||
allowPrivateNetwork: params.allowPrivateNetwork,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: "openai-video-status",
|
||||
isComplete: (payload) => payload.status === "completed",
|
||||
getFailureMessage: (payload) =>
|
||||
payload.status === "failed"
|
||||
? normalizeOptionalString(payload.error?.message) || "OpenAI video generation failed"
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function resolveOpenAIVideoDownloadTimeoutMs(timeoutMs: ProviderOperationTimeoutMs | undefined) {
|
||||
const resolved = typeof timeoutMs === "function" ? timeoutMs() : timeoutMs;
|
||||
return typeof resolved === "number" && Number.isFinite(resolved) && resolved > 0
|
||||
? resolved
|
||||
: DEFAULT_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
async function fetchOpenAIVideoDownload(
|
||||
params: {
|
||||
url: string;
|
||||
init: RequestInit;
|
||||
timeoutMs?: ProviderOperationTimeoutMs;
|
||||
fetchFn: typeof fetch;
|
||||
} & OpenAIVideoRequestPolicy,
|
||||
) {
|
||||
if (!params.allowPrivateNetwork && !params.dispatcherPolicy) {
|
||||
const response = await fetchProviderDownloadResponse({
|
||||
url: params.url,
|
||||
init: params.init,
|
||||
timeoutMs: params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
fetchFn: params.fetchFn,
|
||||
provider: "openai",
|
||||
requestFailedMessage: "OpenAI video download failed",
|
||||
});
|
||||
return {
|
||||
response,
|
||||
release: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
return await executeProviderOperationWithRetry({
|
||||
provider: "openai",
|
||||
stage: "download",
|
||||
operation: async () => {
|
||||
const result = await fetchWithTimeoutGuarded(
|
||||
params.url,
|
||||
params.init,
|
||||
resolveOpenAIVideoDownloadTimeoutMs(params.timeoutMs),
|
||||
params.fetchFn,
|
||||
{
|
||||
...(params.allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}),
|
||||
...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}),
|
||||
auditContext: "openai-video-download",
|
||||
},
|
||||
);
|
||||
try {
|
||||
await assertOkOrThrowHttpError(result.response, "OpenAI video download failed");
|
||||
return result;
|
||||
} catch (error) {
|
||||
await result.release();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function downloadOpenAIVideo(
|
||||
params: {
|
||||
videoId: string;
|
||||
headers: Headers;
|
||||
timeoutMs?: ProviderOperationTimeoutMs;
|
||||
baseUrl: string;
|
||||
fetchFn: typeof fetch;
|
||||
maxBytes: number;
|
||||
} & OpenAIVideoRequestPolicy,
|
||||
): Promise<GeneratedVideoAsset> {
|
||||
const url = new URL(`${params.baseUrl}/videos/${params.videoId}/content`);
|
||||
url.searchParams.set("variant", "video");
|
||||
const { response, release } = await fetchOpenAIVideoDownload({
|
||||
url: url.toString(),
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: new Headers({
|
||||
...Object.fromEntries(params.headers.entries()),
|
||||
Accept: "application/binary",
|
||||
}),
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
fetchFn: params.fetchFn,
|
||||
allowPrivateNetwork: params.allowPrivateNetwork,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
});
|
||||
try {
|
||||
const mimeType = normalizeOptionalString(response.headers.get("content-type")) ?? "video/mp4";
|
||||
const buffer = await readResponseWithLimit(response, params.maxBytes, {
|
||||
onOverflow: ({ maxBytes }) =>
|
||||
new Error(`OpenAI generated video download exceeds ${maxBytes} bytes`),
|
||||
});
|
||||
return {
|
||||
buffer,
|
||||
mimeType,
|
||||
fileName: `video-1.${extensionForMime(mimeType)?.slice(1) ?? "mp4"}`,
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
export function buildOpenAIVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
id: "openai",
|
||||
label: "OpenAI",
|
||||
defaultModel: DEFAULT_OPENAI_VIDEO_MODEL,
|
||||
models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro"],
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "openai",
|
||||
agentDir,
|
||||
profileTypes: ["api_key"],
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
maxDurationSeconds: 12,
|
||||
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||
supportsSize: true,
|
||||
sizes: OPENAI_VIDEO_SIZES,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 12,
|
||||
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||
supportsSize: true,
|
||||
sizes: OPENAI_VIDEO_SIZES,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
},
|
||||
},
|
||||
async generateVideo(req) {
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "openai",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
modelApi: "openai-responses",
|
||||
});
|
||||
if (!auth.apiKey || (auth.mode !== undefined && auth.mode !== "api-key")) {
|
||||
throw new Error("OpenAI API key missing");
|
||||
}
|
||||
|
||||
const fetchFn = fetch;
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: req.timeoutMs,
|
||||
label: "OpenAI video generation",
|
||||
});
|
||||
const providerConfig = req.cfg.models?.providers?.openai;
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg),
|
||||
defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL,
|
||||
request: sanitizeConfiguredModelProviderRequest(providerConfig?.request),
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${auth.apiKey}`,
|
||||
},
|
||||
provider: "openai",
|
||||
capability: "video",
|
||||
transport: "http",
|
||||
});
|
||||
|
||||
const model = normalizeOptionalString(req.model) ?? DEFAULT_OPENAI_VIDEO_MODEL;
|
||||
const seconds = resolveDurationSeconds(req.durationSeconds);
|
||||
const size = resolveSize({
|
||||
size: req.size,
|
||||
aspectRatio: req.aspectRatio,
|
||||
resolution: req.resolution,
|
||||
});
|
||||
const referenceAsset = resolveReferenceAsset(req);
|
||||
const requestResult = referenceAsset
|
||||
? referenceAsset.kind === "image"
|
||||
? await (() => {
|
||||
const jsonHeaders = new Headers(headers);
|
||||
jsonHeaders.set("Content-Type", "application/json");
|
||||
return postJsonRequest({
|
||||
url: `${baseUrl}/videos`,
|
||||
headers: jsonHeaders,
|
||||
body: {
|
||||
prompt: req.prompt,
|
||||
model,
|
||||
...(seconds ? { seconds } : {}),
|
||||
...(size ? { size } : {}),
|
||||
input_reference: {
|
||||
image_url: toImageDataUrl(referenceAsset),
|
||||
},
|
||||
},
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
})()
|
||||
: await (() => {
|
||||
const form = new FormData();
|
||||
form.set("prompt", req.prompt);
|
||||
form.set("model", model);
|
||||
form.set("video", referenceAsset.file);
|
||||
const multipartHeaders = new Headers(headers);
|
||||
multipartHeaders.delete("Content-Type");
|
||||
return postMultipartRequest({
|
||||
url: `${baseUrl}/videos/edits`,
|
||||
headers: multipartHeaders,
|
||||
body: form,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
})()
|
||||
: await (() => {
|
||||
const jsonHeaders = new Headers(headers);
|
||||
jsonHeaders.set("Content-Type", "application/json");
|
||||
return postJsonRequest({
|
||||
url: `${baseUrl}/videos`,
|
||||
headers: jsonHeaders,
|
||||
body: {
|
||||
prompt: req.prompt,
|
||||
model,
|
||||
...(seconds ? { seconds } : {}),
|
||||
...(size ? { size } : {}),
|
||||
},
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
})();
|
||||
const { response, release } = requestResult;
|
||||
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "OpenAI video generation failed");
|
||||
const submitted = await readProviderJsonResponse<OpenAIVideoResponse>(
|
||||
response,
|
||||
"OpenAI video generation failed",
|
||||
);
|
||||
const videoId = normalizeOptionalString(submitted.id);
|
||||
if (!videoId) {
|
||||
throw new Error("OpenAI video generation response missing video id");
|
||||
}
|
||||
const completed = await pollOpenAIVideo({
|
||||
videoId,
|
||||
headers,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
baseUrl,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
const video = await downloadOpenAIVideo({
|
||||
videoId,
|
||||
headers,
|
||||
timeoutMs: createProviderOperationTimeoutResolver({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
baseUrl,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
maxBytes: resolveGeneratedVideoMaxBytes(req),
|
||||
});
|
||||
return {
|
||||
videos: [video],
|
||||
model: completed.model ?? submitted.model ?? model,
|
||||
metadata: {
|
||||
videoId,
|
||||
status: completed.status,
|
||||
seconds: completed.seconds ?? submitted.seconds,
|
||||
size: completed.size ?? submitted.size,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user