Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
217
extensions/voyage/embedding-batch.test.ts
Normal file
217
extensions/voyage/embedding-batch.test.ts
Normal file
@@ -0,0 +1,217 @@
|
||||
// Voyage batch tests cover bounded status/error response reads.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { VoyageEmbeddingClient } from "./embedding-provider.js";
|
||||
import { testing } from "./embedding-batch.js";
|
||||
|
||||
const { fetchVoyageBatchStatus, readVoyageBatchError, VOYAGE_BATCH_RESPONSE_MAX_BYTES } = testing;
|
||||
|
||||
function buildClient(): VoyageEmbeddingClient {
|
||||
return {
|
||||
baseUrl: "https://api.voyageai.test/v1",
|
||||
headers: { authorization: "Bearer test" },
|
||||
model: "voyage-3",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build deps whose withRemoteHttpResponse drives the real onResponse against a
|
||||
* caller-provided Response, so the bounded readers run exactly as in production.
|
||||
*/
|
||||
function buildDeps(response: Response): Parameters<typeof fetchVoyageBatchStatus>[0]["deps"] {
|
||||
return {
|
||||
now: () => 0,
|
||||
sleep: async () => {},
|
||||
postJsonWithRetry: (async () => {
|
||||
throw new Error("postJsonWithRetry should not be called in these tests");
|
||||
}) as never,
|
||||
uploadBatchJsonlFile: (async () => {
|
||||
throw new Error("uploadBatchJsonlFile should not be called in these tests");
|
||||
}) as never,
|
||||
withRemoteHttpResponse: (async (params: { onResponse: (res: Response) => Promise<unknown> }) =>
|
||||
await params.onResponse(response)) as never,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A streaming JSON-ish body that proves an oversized response stops being read
|
||||
* before the whole advertised payload is buffered into memory. getReadCount
|
||||
* reports how many chunks were pulled; cancel() flips wasCanceled.
|
||||
*/
|
||||
function streamingResponse(params: { chunkCount: number; chunkSize: number; status?: number }): {
|
||||
response: Response;
|
||||
getReadCount: () => number;
|
||||
wasCanceled: () => boolean;
|
||||
} {
|
||||
let reads = 0;
|
||||
let canceled = false;
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (reads >= params.chunkCount) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
reads += 1;
|
||||
controller.enqueue(encoder.encode("a".repeat(params.chunkSize)));
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
});
|
||||
return {
|
||||
response: new Response(stream, {
|
||||
status: params.status ?? 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
getReadCount: () => reads,
|
||||
wasCanceled: () => canceled,
|
||||
};
|
||||
}
|
||||
|
||||
describe("voyage batch bounded reads", () => {
|
||||
it("uses a 16 MiB cap for batch status/error responses", () => {
|
||||
expect(VOYAGE_BATCH_RESPONSE_MAX_BYTES).toBe(16 * 1024 * 1024);
|
||||
});
|
||||
|
||||
it("parses a well-formed batch status response under the byte cap", async () => {
|
||||
const response = new Response(JSON.stringify({ id: "batch_1", status: "completed" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
|
||||
const status = await fetchVoyageBatchStatus({
|
||||
client: buildClient(),
|
||||
batchId: "batch_1",
|
||||
deps: buildDeps(response),
|
||||
});
|
||||
|
||||
expect(status).toEqual({ id: "batch_1", status: "completed" });
|
||||
});
|
||||
|
||||
it("caps an oversized batch status stream instead of buffering the whole body", async () => {
|
||||
const streamed = streamingResponse({ chunkCount: 64, chunkSize: 1024 });
|
||||
|
||||
await expect(
|
||||
fetchVoyageBatchStatus({
|
||||
client: buildClient(),
|
||||
batchId: "batch_1",
|
||||
deps: buildDeps(streamed.response),
|
||||
maxResponseBytes: 4096,
|
||||
}),
|
||||
).rejects.toThrow(/voyage-batch-status: JSON response exceeds 4096 bytes/);
|
||||
|
||||
// Stream was cancelled mid-flight: fewer chunks read than the full payload.
|
||||
expect(streamed.getReadCount()).toBeLessThan(64);
|
||||
expect(streamed.wasCanceled()).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves the full NDJSON parse chain for an under-cap error file", async () => {
|
||||
// Multi-line NDJSON with a blank line proves the bounded read does not
|
||||
// disturb the original trim/split("\n")/JSON.parse/extractBatchErrorMessage
|
||||
// pipeline: the first useful error message is still extracted byte-for-byte
|
||||
// identically to the pre-change `await res.text()` path.
|
||||
const body = [
|
||||
JSON.stringify({ custom_id: "req-0", response: { status_code: 200 } }),
|
||||
"",
|
||||
JSON.stringify({ custom_id: "req-1", error: { message: "voyage upstream rejected" } }),
|
||||
JSON.stringify({ custom_id: "req-2", error: { message: "second error ignored" } }),
|
||||
"",
|
||||
].join("\n");
|
||||
const response = new Response(body, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/x-ndjson" },
|
||||
});
|
||||
|
||||
const message = await readVoyageBatchError({
|
||||
client: buildClient(),
|
||||
errorFileId: "file_1",
|
||||
deps: buildDeps(response),
|
||||
});
|
||||
|
||||
// extractBatchErrorMessage returns the first line carrying a message, so the
|
||||
// success line is skipped and the second error is not surfaced.
|
||||
expect(message).toBe("voyage upstream rejected");
|
||||
});
|
||||
|
||||
it("returns undefined for an empty error file via the original empty-body branch", async () => {
|
||||
// Whitespace-only body must still hit the `!text.trim()` short-circuit after
|
||||
// decoding the bounded buffer, returning undefined exactly as before.
|
||||
const response = new Response(" \n", {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/x-ndjson" },
|
||||
});
|
||||
|
||||
const message = await readVoyageBatchError({
|
||||
client: buildClient(),
|
||||
errorFileId: "file_1",
|
||||
deps: buildDeps(response),
|
||||
});
|
||||
|
||||
expect(message).toBeUndefined();
|
||||
});
|
||||
|
||||
it("fail-softs an oversized error file into formatUnavailableBatchError by design", async () => {
|
||||
const streamed = streamingResponse({ chunkCount: 64, chunkSize: 1024 });
|
||||
|
||||
// Intended behavior: an over-cap error file must NOT throw out of
|
||||
// readVoyageBatchError. An unbounded error body would otherwise OOM the
|
||||
// worker, so the bounded overflow error is caught and degraded into a
|
||||
// diagnostic string via formatUnavailableBatchError. We accept the lost
|
||||
// detail; the overflow message names the cap so the truncation is visible.
|
||||
const readError = async () =>
|
||||
await readVoyageBatchError({
|
||||
client: buildClient(),
|
||||
errorFileId: "file_1",
|
||||
deps: buildDeps(streamed.response),
|
||||
maxResponseBytes: 4096,
|
||||
});
|
||||
|
||||
await expect(readError()).resolves.toMatch(
|
||||
/error file unavailable: voyage batch error file content exceeds 4096 bytes/,
|
||||
);
|
||||
|
||||
// The bounded reader still cancels the stream mid-flight rather than
|
||||
// buffering the whole advertised payload before failing soft.
|
||||
expect(streamed.getReadCount()).toBeLessThan(64);
|
||||
expect(streamed.wasCanceled()).toBe(true);
|
||||
});
|
||||
|
||||
it("caps an oversized non-OK (error) diagnostic body instead of buffering it whole", async () => {
|
||||
// Regression for the non-OK gap: `assertVoyageResponseOk` previously read the
|
||||
// 4xx/5xx diagnostic body with an unbounded `await res.text()`. A hostile
|
||||
// endpoint can return a 500 with a never-ending body, so that read must be
|
||||
// bounded too. Drive a streaming 500 through the real status path and assert
|
||||
// the bounded overflow error fires and the stream is cancelled mid-flight.
|
||||
const streamed = streamingResponse({ chunkCount: 64, chunkSize: 1024, status: 500 });
|
||||
|
||||
await expect(
|
||||
fetchVoyageBatchStatus({
|
||||
client: buildClient(),
|
||||
batchId: "batch_1",
|
||||
deps: buildDeps(streamed.response),
|
||||
maxResponseBytes: 4096,
|
||||
}),
|
||||
).rejects.toThrow(/voyage batch status failed: 500 \(error body exceeds 4096 bytes\)/);
|
||||
|
||||
// Stream was cancelled mid-flight rather than draining the whole body.
|
||||
expect(streamed.getReadCount()).toBeLessThan(64);
|
||||
expect(streamed.wasCanceled()).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves the diagnostic shape for a small non-OK (error) body", async () => {
|
||||
// Under-cap non-OK body must still surface the original
|
||||
// `${context}: ${status} ${text}` diagnostic byte-for-byte.
|
||||
const response = new Response("voyage upstream is down", {
|
||||
status: 503,
|
||||
headers: { "content-type": "text/plain" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
fetchVoyageBatchStatus({
|
||||
client: buildClient(),
|
||||
batchId: "batch_1",
|
||||
deps: buildDeps(response),
|
||||
}),
|
||||
).rejects.toThrow(/voyage batch status failed: 503 voyage upstream is down/);
|
||||
});
|
||||
});
|
||||
353
extensions/voyage/embedding-batch.ts
Normal file
353
extensions/voyage/embedding-batch.ts
Normal file
@@ -0,0 +1,353 @@
|
||||
// Voyage plugin module implements embedding batch behavior.
|
||||
import { createInterface } from "node:readline";
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
applyEmbeddingBatchOutputLine,
|
||||
buildBatchHeaders,
|
||||
buildEmbeddingBatchGroupOptions,
|
||||
EMBEDDING_BATCH_ENDPOINT,
|
||||
extractBatchErrorMessage,
|
||||
formatUnavailableBatchError,
|
||||
normalizeBatchBaseUrl,
|
||||
postJsonWithRetry,
|
||||
resolveBatchCompletionFromStatus,
|
||||
resolveCompletedBatchResult,
|
||||
runEmbeddingBatchGroups,
|
||||
throwIfBatchTerminalFailure,
|
||||
type EmbeddingBatchExecutionParams,
|
||||
type EmbeddingBatchStatus,
|
||||
type BatchCompletionResult,
|
||||
type ProviderBatchOutputLine,
|
||||
uploadBatchJsonlFile,
|
||||
withRemoteHttpResponse,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http";
|
||||
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
|
||||
import { normalizeStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type { VoyageEmbeddingClient } from "./embedding-provider.js";
|
||||
|
||||
/**
|
||||
* Voyage Batch API Input Line format.
|
||||
* See: https://docs.voyageai.com/docs/batch-inference
|
||||
*/
|
||||
type VoyageBatchRequest = {
|
||||
custom_id: string;
|
||||
body: {
|
||||
input: string | string[];
|
||||
};
|
||||
};
|
||||
|
||||
type VoyageBatchStatus = EmbeddingBatchStatus;
|
||||
type VoyageBatchOutputLine = ProviderBatchOutputLine;
|
||||
|
||||
const VOYAGE_BATCH_ENDPOINT = EMBEDDING_BATCH_ENDPOINT;
|
||||
const VOYAGE_BATCH_COMPLETION_WINDOW = "12h";
|
||||
const VOYAGE_BATCH_MAX_REQUESTS = 50000;
|
||||
// Voyage batch status/error responses are untrusted external bodies. Cap them
|
||||
// the same way other bundled providers do (16 MiB) so a misbehaving or hostile
|
||||
// endpoint cannot stream an unbounded body into memory before we parse it.
|
||||
const VOYAGE_BATCH_RESPONSE_MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
type VoyageBatchDeps = {
|
||||
now: () => number;
|
||||
sleep: (ms: number) => Promise<void>;
|
||||
postJsonWithRetry: typeof postJsonWithRetry;
|
||||
uploadBatchJsonlFile: typeof uploadBatchJsonlFile;
|
||||
withRemoteHttpResponse: typeof withRemoteHttpResponse;
|
||||
};
|
||||
|
||||
function resolveVoyageBatchDeps(overrides: Partial<VoyageBatchDeps> | undefined): VoyageBatchDeps {
|
||||
return {
|
||||
now: overrides?.now ?? Date.now,
|
||||
sleep:
|
||||
overrides?.sleep ??
|
||||
(async (ms: number) =>
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
})),
|
||||
postJsonWithRetry: overrides?.postJsonWithRetry ?? postJsonWithRetry,
|
||||
uploadBatchJsonlFile: overrides?.uploadBatchJsonlFile ?? uploadBatchJsonlFile,
|
||||
withRemoteHttpResponse: overrides?.withRemoteHttpResponse ?? withRemoteHttpResponse,
|
||||
};
|
||||
}
|
||||
|
||||
async function assertVoyageResponseOk(
|
||||
res: Response,
|
||||
context: string,
|
||||
maxBytes: number = VOYAGE_BATCH_RESPONSE_MAX_BYTES,
|
||||
): Promise<void> {
|
||||
if (!res.ok) {
|
||||
// The non-OK diagnostic body is just as untrusted as the success body: a
|
||||
// misbehaving or hostile endpoint can return a 4xx/5xx with an unbounded
|
||||
// body, and the old `await res.text()` buffered it whole before we threw.
|
||||
// Read it through the same bounded reader (16 MiB cap, stream cancelled on
|
||||
// overflow) while preserving the original `${context}: ${status} ${text}`
|
||||
// diagnostic shape for backward compatibility.
|
||||
const bytes = await readResponseWithLimit(res, maxBytes, {
|
||||
onOverflow: ({ maxBytes: maxBytesLocal }) =>
|
||||
new Error(`${context}: ${res.status} (error body exceeds ${maxBytesLocal} bytes)`),
|
||||
});
|
||||
const text = new TextDecoder().decode(bytes);
|
||||
throw new Error(`${context}: ${res.status} ${text}`);
|
||||
}
|
||||
}
|
||||
|
||||
function buildVoyageBatchRequest<T>(params: {
|
||||
client: VoyageEmbeddingClient;
|
||||
path: string;
|
||||
onResponse: (res: Response) => Promise<T>;
|
||||
}) {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.client);
|
||||
return {
|
||||
url: `${baseUrl}/${params.path}`,
|
||||
ssrfPolicy: params.client.ssrfPolicy,
|
||||
init: {
|
||||
headers: buildBatchHeaders(params.client, { json: true }),
|
||||
},
|
||||
onResponse: params.onResponse,
|
||||
};
|
||||
}
|
||||
|
||||
async function submitVoyageBatch(params: {
|
||||
client: VoyageEmbeddingClient;
|
||||
requests: VoyageBatchRequest[];
|
||||
agentId: string;
|
||||
deps: VoyageBatchDeps;
|
||||
}): Promise<VoyageBatchStatus> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.client);
|
||||
const inputFileId = await params.deps.uploadBatchJsonlFile({
|
||||
client: params.client,
|
||||
requests: params.requests,
|
||||
errorPrefix: "voyage batch file upload failed",
|
||||
});
|
||||
|
||||
// 2. Create batch job using Voyage Batches API
|
||||
return await params.deps.postJsonWithRetry<VoyageBatchStatus>({
|
||||
url: `${baseUrl}/batches`,
|
||||
headers: buildBatchHeaders(params.client, { json: true }),
|
||||
ssrfPolicy: params.client.ssrfPolicy,
|
||||
body: {
|
||||
input_file_id: inputFileId,
|
||||
endpoint: VOYAGE_BATCH_ENDPOINT,
|
||||
completion_window: VOYAGE_BATCH_COMPLETION_WINDOW,
|
||||
request_params: {
|
||||
model: params.client.model,
|
||||
input_type: "document",
|
||||
},
|
||||
metadata: {
|
||||
source: "clawdbot-memory",
|
||||
agent: params.agentId,
|
||||
},
|
||||
},
|
||||
errorPrefix: "voyage batch create failed",
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchVoyageBatchStatus(params: {
|
||||
client: VoyageEmbeddingClient;
|
||||
batchId: string;
|
||||
deps: VoyageBatchDeps;
|
||||
maxResponseBytes?: number;
|
||||
}): Promise<VoyageBatchStatus> {
|
||||
const maxBytes = params.maxResponseBytes ?? VOYAGE_BATCH_RESPONSE_MAX_BYTES;
|
||||
return await params.deps.withRemoteHttpResponse(
|
||||
buildVoyageBatchRequest({
|
||||
client: params.client,
|
||||
path: `batches/${params.batchId}`,
|
||||
onResponse: async (res) => {
|
||||
await assertVoyageResponseOk(res, "voyage batch status failed", maxBytes);
|
||||
return await readProviderJsonResponse<VoyageBatchStatus>(res, "voyage-batch-status", {
|
||||
maxBytes,
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function readVoyageBatchError(params: {
|
||||
client: VoyageEmbeddingClient;
|
||||
errorFileId: string;
|
||||
deps: VoyageBatchDeps;
|
||||
maxResponseBytes?: number;
|
||||
}): Promise<string | undefined> {
|
||||
const maxBytes = params.maxResponseBytes ?? VOYAGE_BATCH_RESPONSE_MAX_BYTES;
|
||||
try {
|
||||
return await params.deps.withRemoteHttpResponse(
|
||||
buildVoyageBatchRequest({
|
||||
client: params.client,
|
||||
path: `files/${params.errorFileId}/content`,
|
||||
onResponse: async (res) => {
|
||||
await assertVoyageResponseOk(res, "voyage batch error file content failed", maxBytes);
|
||||
const bytes = await readResponseWithLimit(res, maxBytes, {
|
||||
onOverflow: ({ maxBytes: maxBytesLocal }) =>
|
||||
new Error(`voyage batch error file content exceeds ${maxBytesLocal} bytes`),
|
||||
});
|
||||
const text = new TextDecoder().decode(bytes);
|
||||
if (!text.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
const lines = normalizeStringEntries(text.split("\n")).map(
|
||||
(line) => JSON.parse(line) as VoyageBatchOutputLine,
|
||||
);
|
||||
return extractBatchErrorMessage(lines);
|
||||
},
|
||||
}),
|
||||
);
|
||||
} catch (err) {
|
||||
return formatUnavailableBatchError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForVoyageBatch(params: {
|
||||
client: VoyageEmbeddingClient;
|
||||
batchId: string;
|
||||
wait: boolean;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
debug?: (message: string, data?: Record<string, unknown>) => void;
|
||||
initial?: VoyageBatchStatus;
|
||||
deps: VoyageBatchDeps;
|
||||
}): Promise<BatchCompletionResult> {
|
||||
const start = params.deps.now();
|
||||
let current: VoyageBatchStatus | undefined = params.initial;
|
||||
while (true) {
|
||||
const status =
|
||||
current ??
|
||||
(await fetchVoyageBatchStatus({
|
||||
client: params.client,
|
||||
batchId: params.batchId,
|
||||
deps: params.deps,
|
||||
}));
|
||||
const state = status.status ?? "unknown";
|
||||
if (state === "completed") {
|
||||
return resolveBatchCompletionFromStatus({
|
||||
provider: "voyage",
|
||||
batchId: params.batchId,
|
||||
status,
|
||||
});
|
||||
}
|
||||
await throwIfBatchTerminalFailure({
|
||||
provider: "voyage",
|
||||
status: { ...status, id: params.batchId },
|
||||
readError: async (errorFileId) =>
|
||||
await readVoyageBatchError({
|
||||
client: params.client,
|
||||
errorFileId,
|
||||
deps: params.deps,
|
||||
}),
|
||||
});
|
||||
if (!params.wait) {
|
||||
throw new Error(`voyage batch ${params.batchId} still ${state}; wait disabled`);
|
||||
}
|
||||
if (params.deps.now() - start > params.timeoutMs) {
|
||||
throw new Error(`voyage batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
|
||||
}
|
||||
params.debug?.(`voyage batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`);
|
||||
await params.deps.sleep(params.pollIntervalMs);
|
||||
current = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runVoyageEmbeddingBatches(
|
||||
params: {
|
||||
client: VoyageEmbeddingClient;
|
||||
agentId: string;
|
||||
requests: VoyageBatchRequest[];
|
||||
deps?: Partial<VoyageBatchDeps>;
|
||||
} & EmbeddingBatchExecutionParams,
|
||||
): Promise<Map<string, number[]>> {
|
||||
const deps = resolveVoyageBatchDeps(params.deps);
|
||||
return await runEmbeddingBatchGroups({
|
||||
...buildEmbeddingBatchGroupOptions(params, {
|
||||
maxRequests: VOYAGE_BATCH_MAX_REQUESTS,
|
||||
debugLabel: "memory embeddings: voyage batch submit",
|
||||
}),
|
||||
runGroup: async ({ group, groupIndex, groups, byCustomId, pollIntervalMs, timeoutMs }) => {
|
||||
const batchInfo = await submitVoyageBatch({
|
||||
client: params.client,
|
||||
requests: group,
|
||||
agentId: params.agentId,
|
||||
deps,
|
||||
});
|
||||
if (!batchInfo.id) {
|
||||
throw new Error("voyage batch create failed: missing batch id");
|
||||
}
|
||||
const batchId = batchInfo.id;
|
||||
|
||||
params.debug?.("memory embeddings: voyage batch created", {
|
||||
batchId: batchInfo.id,
|
||||
status: batchInfo.status,
|
||||
group: groupIndex + 1,
|
||||
groups,
|
||||
requests: group.length,
|
||||
});
|
||||
|
||||
const completed = await resolveCompletedBatchResult({
|
||||
provider: "voyage",
|
||||
status: batchInfo,
|
||||
wait: params.wait,
|
||||
waitForBatch: async () =>
|
||||
await waitForVoyageBatch({
|
||||
client: params.client,
|
||||
batchId,
|
||||
wait: params.wait,
|
||||
pollIntervalMs,
|
||||
timeoutMs,
|
||||
debug: params.debug,
|
||||
initial: batchInfo,
|
||||
deps,
|
||||
}),
|
||||
});
|
||||
|
||||
const baseUrl = normalizeBatchBaseUrl(params.client);
|
||||
const errors: string[] = [];
|
||||
const remaining = new Set(group.map((request) => request.custom_id));
|
||||
|
||||
await deps.withRemoteHttpResponse({
|
||||
url: `${baseUrl}/files/${completed.outputFileId}/content`,
|
||||
ssrfPolicy: params.client.ssrfPolicy,
|
||||
init: {
|
||||
headers: buildBatchHeaders(params.client, { json: true }),
|
||||
},
|
||||
onResponse: async (contentRes) => {
|
||||
// Same bounded non-OK diagnostic read as the status/error-file paths:
|
||||
// the failure body is untrusted, so cap it instead of `await text()`.
|
||||
await assertVoyageResponseOk(contentRes, "voyage batch file content failed");
|
||||
|
||||
if (!contentRes.body) {
|
||||
return;
|
||||
}
|
||||
const reader = createInterface({
|
||||
input: Readable.fromWeb(
|
||||
contentRes.body as unknown as import("stream/web").ReadableStream,
|
||||
),
|
||||
terminal: false,
|
||||
});
|
||||
|
||||
for await (const rawLine of reader) {
|
||||
if (!rawLine.trim()) {
|
||||
continue;
|
||||
}
|
||||
const line = JSON.parse(rawLine) as VoyageBatchOutputLine;
|
||||
applyEmbeddingBatchOutputLine({ line, remaining, errors, byCustomId });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`voyage batch ${batchInfo.id} failed: ${errors.join("; ")}`);
|
||||
}
|
||||
if (remaining.size > 0) {
|
||||
throw new Error(
|
||||
`voyage batch ${batchInfo.id} missing ${remaining.size} embedding responses`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
fetchVoyageBatchStatus,
|
||||
readVoyageBatchError,
|
||||
VOYAGE_BATCH_RESPONSE_MAX_BYTES,
|
||||
} as const;
|
||||
91
extensions/voyage/embedding-provider.ts
Normal file
91
extensions/voyage/embedding-provider.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
// Voyage provider module implements model/runtime integration.
|
||||
import {
|
||||
fetchRemoteEmbeddingVectors,
|
||||
normalizeEmbeddingModelWithPrefixes,
|
||||
resolveRemoteEmbeddingBearerClient,
|
||||
type MemoryEmbeddingProvider,
|
||||
type MemoryEmbeddingProviderCreateOptions,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import type { SsrFPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
|
||||
export type VoyageEmbeddingClient = {
|
||||
baseUrl: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
model: string;
|
||||
};
|
||||
|
||||
export const DEFAULT_VOYAGE_EMBEDDING_MODEL = "voyage-4-large";
|
||||
const DEFAULT_VOYAGE_BASE_URL = "https://api.voyageai.com/v1";
|
||||
const VOYAGE_MAX_INPUT_TOKENS: Record<string, number> = {
|
||||
"voyage-3": 32000,
|
||||
"voyage-3-lite": 16000,
|
||||
"voyage-code-3": 32000,
|
||||
};
|
||||
|
||||
function normalizeVoyageModel(model: string): string {
|
||||
return normalizeEmbeddingModelWithPrefixes({
|
||||
model,
|
||||
defaultModel: DEFAULT_VOYAGE_EMBEDDING_MODEL,
|
||||
prefixes: ["voyage/"],
|
||||
});
|
||||
}
|
||||
|
||||
export async function createVoyageEmbeddingProvider(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<{ provider: MemoryEmbeddingProvider; client: VoyageEmbeddingClient }> {
|
||||
const client = await resolveVoyageEmbeddingClient(options);
|
||||
const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
|
||||
|
||||
const embed = async (
|
||||
input: string[],
|
||||
input_type?: "query" | "document",
|
||||
signal?: AbortSignal,
|
||||
): Promise<number[][]> => {
|
||||
if (input.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const body: { model: string; input: string[]; input_type?: "query" | "document" } = {
|
||||
model: client.model,
|
||||
input,
|
||||
};
|
||||
if (input_type) {
|
||||
body.input_type = input_type;
|
||||
}
|
||||
|
||||
return await fetchRemoteEmbeddingVectors({
|
||||
url,
|
||||
headers: client.headers,
|
||||
ssrfPolicy: client.ssrfPolicy,
|
||||
signal,
|
||||
body,
|
||||
errorPrefix: "voyage embeddings failed",
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
provider: {
|
||||
id: "voyage",
|
||||
model: client.model,
|
||||
maxInputTokens: VOYAGE_MAX_INPUT_TOKENS[client.model],
|
||||
embedQuery: async (text, optionsValue) => {
|
||||
const [vec] = await embed([text], "query", optionsValue?.signal);
|
||||
return vec ?? [];
|
||||
},
|
||||
embedBatch: async (texts, optionsLocal) => embed(texts, "document", optionsLocal?.signal),
|
||||
},
|
||||
client,
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveVoyageEmbeddingClient(
|
||||
options: MemoryEmbeddingProviderCreateOptions,
|
||||
): Promise<VoyageEmbeddingClient> {
|
||||
const { baseUrl, headers, ssrfPolicy } = await resolveRemoteEmbeddingBearerClient({
|
||||
provider: "voyage",
|
||||
options,
|
||||
defaultBaseUrl: DEFAULT_VOYAGE_BASE_URL,
|
||||
});
|
||||
const model = normalizeVoyageModel(options.model);
|
||||
return { baseUrl, headers, ssrfPolicy, model };
|
||||
}
|
||||
12
extensions/voyage/index.ts
Normal file
12
extensions/voyage/index.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
// Voyage plugin entrypoint registers its OpenClaw integration.
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { voyageMemoryEmbeddingProviderAdapter } from "./memory-embedding-adapter.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "voyage",
|
||||
name: "Voyage Embeddings",
|
||||
description: "Bundled Voyage memory embedding provider plugin",
|
||||
register(api) {
|
||||
api.registerMemoryEmbeddingProvider(voyageMemoryEmbeddingProviderAdapter);
|
||||
},
|
||||
});
|
||||
57
extensions/voyage/memory-embedding-adapter.ts
Normal file
57
extensions/voyage/memory-embedding-adapter.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
// Voyage plugin module implements memory embedding adapter behavior.
|
||||
import {
|
||||
isMissingEmbeddingApiKeyError,
|
||||
mapBatchEmbeddingsByIndex,
|
||||
sanitizeEmbeddingCacheHeaders,
|
||||
type MemoryEmbeddingProviderAdapter,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { runVoyageEmbeddingBatches } from "./embedding-batch.js";
|
||||
import {
|
||||
createVoyageEmbeddingProvider,
|
||||
DEFAULT_VOYAGE_EMBEDDING_MODEL,
|
||||
} from "./embedding-provider.js";
|
||||
|
||||
export const voyageMemoryEmbeddingProviderAdapter: MemoryEmbeddingProviderAdapter = {
|
||||
id: "voyage",
|
||||
defaultModel: DEFAULT_VOYAGE_EMBEDDING_MODEL,
|
||||
transport: "remote",
|
||||
authProviderId: "voyage",
|
||||
autoSelectPriority: 40,
|
||||
allowExplicitWhenConfiguredAuto: true,
|
||||
shouldContinueAutoSelection: isMissingEmbeddingApiKeyError,
|
||||
create: async (options) => {
|
||||
const { provider, client } = await createVoyageEmbeddingProvider({
|
||||
...options,
|
||||
provider: "voyage",
|
||||
fallback: "none",
|
||||
});
|
||||
return {
|
||||
provider,
|
||||
runtime: {
|
||||
id: "voyage",
|
||||
cacheKeyData: {
|
||||
provider: "voyage",
|
||||
baseUrl: client.baseUrl,
|
||||
model: client.model,
|
||||
headers: sanitizeEmbeddingCacheHeaders(client.headers, ["authorization"]),
|
||||
},
|
||||
batchEmbed: async (batch) => {
|
||||
const byCustomId = await runVoyageEmbeddingBatches({
|
||||
client,
|
||||
agentId: batch.agentId,
|
||||
requests: batch.chunks.map((chunk, index) => ({
|
||||
custom_id: String(index),
|
||||
body: { input: chunk.text },
|
||||
})),
|
||||
wait: batch.wait,
|
||||
concurrency: batch.concurrency,
|
||||
pollIntervalMs: batch.pollIntervalMs,
|
||||
timeoutMs: batch.timeoutMs,
|
||||
debug: batch.debug,
|
||||
});
|
||||
return mapBatchEmbeddingsByIndex(byCustomId, batch.chunks.length);
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
23
extensions/voyage/openclaw.plugin.json
Normal file
23
extensions/voyage/openclaw.plugin.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"id": "voyage",
|
||||
"activation": {
|
||||
"onStartup": false
|
||||
},
|
||||
"enabledByDefault": true,
|
||||
"contracts": {
|
||||
"memoryEmbeddingProviders": ["voyage"]
|
||||
},
|
||||
"setup": {
|
||||
"providers": [
|
||||
{
|
||||
"id": "voyage",
|
||||
"envVars": ["VOYAGE_API_KEY"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {}
|
||||
}
|
||||
}
|
||||
15
extensions/voyage/package.json
Normal file
15
extensions/voyage/package.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "@openclaw/voyage-provider",
|
||||
"version": "2026.6.11",
|
||||
"private": true,
|
||||
"description": "OpenClaw Voyage embedding provider plugin",
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user