Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
244
scripts/bench-model.ts
Normal file
244
scripts/bench-model.ts
Normal file
@@ -0,0 +1,244 @@
|
||||
// Bench Model script supports OpenClaw repository automation.
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { completeSimple, type Model } from "openclaw/plugin-sdk/llm";
|
||||
import { parseStrictIntegerOption } from "./lib/dev-tooling-safety.ts";
|
||||
|
||||
type Usage = {
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
totalTokens?: number;
|
||||
};
|
||||
|
||||
type RunResult = {
|
||||
durationMs: number;
|
||||
usage?: Usage;
|
||||
};
|
||||
|
||||
type CliOptions = {
|
||||
help: boolean;
|
||||
prompt: string;
|
||||
runs: number;
|
||||
};
|
||||
|
||||
const DEFAULT_PROMPT = "Reply with a single word: ok. No punctuation or extra text.";
|
||||
const DEFAULT_RUNS = 10;
|
||||
const BOOLEAN_FLAGS = new Set(["--help", "-h"]);
|
||||
const VALUE_FLAGS = new Set(["--prompt", "--runs"]);
|
||||
|
||||
class CliArgumentError extends Error {
|
||||
override name = "CliArgumentError";
|
||||
}
|
||||
|
||||
function readValue(argv: string[], index: number, flag: string): string {
|
||||
const value = argv[index + 1]?.trim() ?? "";
|
||||
if (!value || value.startsWith("-")) {
|
||||
throw new CliArgumentError(`${flag} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateCliArgs(argv: string[]): void {
|
||||
const seenValueFlags = new Set<string>();
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index] ?? "";
|
||||
if (BOOLEAN_FLAGS.has(arg)) {
|
||||
continue;
|
||||
}
|
||||
if (VALUE_FLAGS.has(arg)) {
|
||||
if (seenValueFlags.has(arg)) {
|
||||
throw new CliArgumentError(`${arg} was provided more than once`);
|
||||
}
|
||||
seenValueFlags.add(arg);
|
||||
readValue(argv, index, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new CliArgumentError(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseArg(argv: string[], flag: string): string | undefined {
|
||||
const index = argv.indexOf(flag);
|
||||
if (index === -1) {
|
||||
return undefined;
|
||||
}
|
||||
return readValue(argv, index, flag);
|
||||
}
|
||||
|
||||
function parseRuns(raw: string | undefined): number {
|
||||
return parseStrictIntegerOption({
|
||||
fallback: DEFAULT_RUNS,
|
||||
label: "--runs",
|
||||
min: 1,
|
||||
raw,
|
||||
});
|
||||
}
|
||||
|
||||
function parseArgs(argv = process.argv.slice(2)): CliOptions {
|
||||
validateCliArgs(argv);
|
||||
return {
|
||||
help: argv.includes("--help") || argv.includes("-h"),
|
||||
prompt: parseArg(argv, "--prompt") ?? DEFAULT_PROMPT,
|
||||
runs: parseRuns(parseArg(argv, "--runs")),
|
||||
};
|
||||
}
|
||||
|
||||
function printUsage(): void {
|
||||
console.log(`OpenClaw model latency benchmark
|
||||
|
||||
Usage:
|
||||
node --import tsx scripts/bench-model.ts [options]
|
||||
|
||||
Options:
|
||||
--runs <n> Runs per model (default: ${DEFAULT_RUNS})
|
||||
--prompt <text> Prompt to send to each model
|
||||
--help, -h Show this text
|
||||
|
||||
Environment:
|
||||
ANTHROPIC_API_KEY
|
||||
MINIMAX_API_KEY
|
||||
MINIMAX_BASE_URL
|
||||
MINIMAX_MODEL
|
||||
`);
|
||||
}
|
||||
|
||||
function median(values: number[]): number {
|
||||
if (values.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const sorted = [...values].toSorted((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
if (sorted.length % 2 === 0) {
|
||||
return Math.round((sorted[mid - 1] + sorted[mid]) / 2);
|
||||
}
|
||||
return sorted[mid];
|
||||
}
|
||||
|
||||
async function runModel(opts: {
|
||||
label: string;
|
||||
model: Model;
|
||||
apiKey: string;
|
||||
runs: number;
|
||||
prompt: string;
|
||||
}): Promise<RunResult[]> {
|
||||
const results: RunResult[] = [];
|
||||
for (let i = 0; i < opts.runs; i += 1) {
|
||||
const started = Date.now();
|
||||
const res = await completeSimple(
|
||||
opts.model,
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: opts.prompt,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
},
|
||||
{ apiKey: opts.apiKey, maxTokens: 64 },
|
||||
);
|
||||
const durationMs = Date.now() - started;
|
||||
results.push({ durationMs, usage: res.usage });
|
||||
console.log(`${opts.label} run ${i + 1}/${opts.runs}: ${durationMs}ms`);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main(argv = process.argv.slice(2)): Promise<void> {
|
||||
const options = parseArgs(argv);
|
||||
if (options.help) {
|
||||
printUsage();
|
||||
return;
|
||||
}
|
||||
|
||||
const anthropicKey = process.env.ANTHROPIC_API_KEY?.trim();
|
||||
const minimaxKey = process.env.MINIMAX_API_KEY?.trim();
|
||||
if (!anthropicKey) {
|
||||
throw new Error("Missing ANTHROPIC_API_KEY in environment.");
|
||||
}
|
||||
if (!minimaxKey) {
|
||||
throw new Error("Missing MINIMAX_API_KEY in environment.");
|
||||
}
|
||||
|
||||
const minimaxBaseUrl = process.env.MINIMAX_BASE_URL?.trim() || "https://api.minimax.io/v1";
|
||||
const minimaxModelId = process.env.MINIMAX_MODEL?.trim() || "MiniMax-M2.1";
|
||||
|
||||
const minimaxModel: Model<"openai-completions"> = {
|
||||
id: minimaxModelId,
|
||||
name: `MiniMax ${minimaxModelId}`,
|
||||
api: "openai-completions",
|
||||
provider: "minimax",
|
||||
baseUrl: minimaxBaseUrl,
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
};
|
||||
const opusModel: Model<"anthropic-messages"> = {
|
||||
id: "claude-opus-4-6",
|
||||
name: "Claude Opus 4.6",
|
||||
api: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 },
|
||||
contextWindow: 200000,
|
||||
maxTokens: 32000,
|
||||
};
|
||||
|
||||
console.log(`Prompt: ${options.prompt}`);
|
||||
console.log(`Runs: ${options.runs}`);
|
||||
console.log("");
|
||||
|
||||
const minimaxResults = await runModel({
|
||||
label: "minimax",
|
||||
model: minimaxModel,
|
||||
apiKey: minimaxKey,
|
||||
runs: options.runs,
|
||||
prompt: options.prompt,
|
||||
});
|
||||
const opusResults = await runModel({
|
||||
label: "opus",
|
||||
model: opusModel,
|
||||
apiKey: anthropicKey,
|
||||
runs: options.runs,
|
||||
prompt: options.prompt,
|
||||
});
|
||||
|
||||
const summarize = (label: string, results: RunResult[]) => {
|
||||
const durations = results.map((r) => r.durationMs);
|
||||
const med = median(durations);
|
||||
const min = Math.min(...durations);
|
||||
const max = Math.max(...durations);
|
||||
return { label, med, min, max };
|
||||
};
|
||||
|
||||
const summary = [summarize("minimax", minimaxResults), summarize("opus", opusResults)];
|
||||
console.log("");
|
||||
console.log("Summary (ms):");
|
||||
for (const row of summary) {
|
||||
console.log(`${row.label.padEnd(7)} median=${row.med} min=${row.min} max=${row.max}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
median,
|
||||
parseArgs,
|
||||
parseRuns,
|
||||
validateCliArgs,
|
||||
};
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
main().catch((err: unknown) => {
|
||||
if (err instanceof CliArgumentError) {
|
||||
console.error(err.message);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
console.error(err instanceof Error ? err.stack : String(err));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user