Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
12
extensions/firecrawl/README.md
Normal file
12
extensions/firecrawl/README.md
Normal file
@@ -0,0 +1,12 @@
|
||||
# OpenClaw Firecrawl Plugin
|
||||
|
||||
Official OpenClaw plugin for Firecrawl.
|
||||
|
||||
Install from OpenClaw:
|
||||
|
||||
```bash
|
||||
openclaw plugins install @openclaw/firecrawl-plugin
|
||||
openclaw gateway restart
|
||||
```
|
||||
|
||||
See <https://docs.openclaw.ai/tools/firecrawl> for setup and configuration.
|
||||
68
extensions/firecrawl/api.ts
Normal file
68
extensions/firecrawl/api.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
// Firecrawl API module exposes the plugin public contract.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { readStringValue } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { runFirecrawlScrape } from "./src/firecrawl-client.js";
|
||||
|
||||
export type FetchFirecrawlContentParams = {
|
||||
url: string;
|
||||
extractMode: "markdown" | "text";
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
onlyMainContent: boolean;
|
||||
maxAgeMs: number;
|
||||
proxy: "auto" | "basic" | "stealth";
|
||||
storeInCache: boolean;
|
||||
timeoutSeconds: number;
|
||||
maxChars?: number;
|
||||
};
|
||||
|
||||
export type FetchFirecrawlContentResult = {
|
||||
text: string;
|
||||
title?: string;
|
||||
finalUrl?: string;
|
||||
status?: number;
|
||||
warning?: string;
|
||||
};
|
||||
|
||||
export async function fetchFirecrawlContent(
|
||||
params: FetchFirecrawlContentParams,
|
||||
): Promise<FetchFirecrawlContentResult> {
|
||||
const cfg: OpenClawConfig = {
|
||||
plugins: {
|
||||
entries: {
|
||||
firecrawl: {
|
||||
enabled: true,
|
||||
config: {
|
||||
webFetch: {
|
||||
apiKey: params.apiKey,
|
||||
baseUrl: params.baseUrl,
|
||||
onlyMainContent: params.onlyMainContent,
|
||||
maxAgeMs: params.maxAgeMs,
|
||||
timeoutSeconds: params.timeoutSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runFirecrawlScrape({
|
||||
cfg,
|
||||
url: params.url,
|
||||
extractMode: params.extractMode,
|
||||
maxChars: params.maxChars,
|
||||
proxy: params.proxy,
|
||||
storeInCache: params.storeInCache,
|
||||
onlyMainContent: params.onlyMainContent,
|
||||
maxAgeMs: params.maxAgeMs,
|
||||
timeoutSeconds: params.timeoutSeconds,
|
||||
});
|
||||
|
||||
return {
|
||||
text: typeof result.text === "string" ? result.text : "",
|
||||
title: readStringValue(result.title),
|
||||
finalUrl: readStringValue(result.finalUrl),
|
||||
status: typeof result.status === "number" ? result.status : undefined,
|
||||
warning: readStringValue(result.warning),
|
||||
};
|
||||
}
|
||||
18
extensions/firecrawl/index.ts
Normal file
18
extensions/firecrawl/index.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
// Firecrawl plugin entrypoint registers its OpenClaw integration.
|
||||
import { definePluginEntry, type AnyAgentTool } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { createFirecrawlWebFetchProvider } from "./src/firecrawl-fetch-provider.js";
|
||||
import { createFirecrawlScrapeTool } from "./src/firecrawl-scrape-tool.js";
|
||||
import { createFirecrawlWebSearchProvider } from "./src/firecrawl-search-provider.js";
|
||||
import { createFirecrawlSearchTool } from "./src/firecrawl-search-tool.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "firecrawl",
|
||||
name: "Firecrawl Plugin",
|
||||
description: "Bundled Firecrawl search and scrape plugin",
|
||||
register(api) {
|
||||
api.registerWebFetchProvider(createFirecrawlWebFetchProvider());
|
||||
api.registerWebSearchProvider(createFirecrawlWebSearchProvider());
|
||||
api.registerTool(createFirecrawlSearchTool(api) as AnyAgentTool);
|
||||
api.registerTool(createFirecrawlScrapeTool(api) as AnyAgentTool);
|
||||
},
|
||||
});
|
||||
21
extensions/firecrawl/npm-shrinkwrap.json
generated
Normal file
21
extensions/firecrawl/npm-shrinkwrap.json
generated
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "@openclaw/firecrawl-plugin",
|
||||
"version": "2026.6.11",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@openclaw/firecrawl-plugin",
|
||||
"version": "2026.6.11",
|
||||
"dependencies": {
|
||||
"typebox": "1.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/typebox": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.3.tgz",
|
||||
"integrity": "sha512-URXGUE31PJDQC+PtRMJeLdF4kmmOdFoVPikPCtV2oOIhUpNpppEdIz7W8bH8cFYPYHdDpaRvqwdegMTmHliudg==",
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
||||
83
extensions/firecrawl/openclaw.plugin.json
Normal file
83
extensions/firecrawl/openclaw.plugin.json
Normal file
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"id": "firecrawl",
|
||||
"activation": {
|
||||
"onStartup": false
|
||||
},
|
||||
"setup": {
|
||||
"providers": [
|
||||
{
|
||||
"id": "firecrawl",
|
||||
"envVars": ["FIRECRAWL_API_KEY"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"uiHints": {
|
||||
"webSearch.apiKey": {
|
||||
"label": "Firecrawl Search API Key",
|
||||
"help": "Firecrawl API key for web search (fallback: FIRECRAWL_API_KEY env var).",
|
||||
"sensitive": true,
|
||||
"placeholder": "fc-..."
|
||||
},
|
||||
"webSearch.baseUrl": {
|
||||
"label": "Firecrawl Search Base URL",
|
||||
"help": "Firecrawl Search base URL override."
|
||||
},
|
||||
"webFetch.apiKey": {
|
||||
"label": "Firecrawl Fetch API Key",
|
||||
"help": "Optional for hosted keyless scraping; add a key for higher limits (fallback: FIRECRAWL_API_KEY env var).",
|
||||
"sensitive": true,
|
||||
"placeholder": "fc-..."
|
||||
},
|
||||
"webFetch.baseUrl": {
|
||||
"label": "Firecrawl Fetch Base URL",
|
||||
"help": "Firecrawl Fetch base URL override."
|
||||
}
|
||||
},
|
||||
"contracts": {
|
||||
"webFetchProviders": ["firecrawl"],
|
||||
"webSearchProviders": ["firecrawl"],
|
||||
"tools": ["firecrawl_search", "firecrawl_scrape"]
|
||||
},
|
||||
"configContracts": {
|
||||
"compatibilityRuntimePaths": ["tools.web.search.apiKey"]
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"webSearch": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"apiKey": {
|
||||
"type": ["string", "object"]
|
||||
},
|
||||
"baseUrl": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"webFetch": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"apiKey": {
|
||||
"type": ["string", "object"]
|
||||
},
|
||||
"baseUrl": {
|
||||
"type": "string"
|
||||
},
|
||||
"onlyMainContent": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"maxAgeMs": {
|
||||
"type": "number"
|
||||
},
|
||||
"timeoutSeconds": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
38
extensions/firecrawl/package.json
Normal file
38
extensions/firecrawl/package.json
Normal file
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"name": "@openclaw/firecrawl-plugin",
|
||||
"version": "2026.6.11",
|
||||
"description": "OpenClaw Firecrawl plugin.",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/openclaw/openclaw"
|
||||
},
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"typebox": "1.3.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
],
|
||||
"install": {
|
||||
"clawhubSpec": "clawhub:@openclaw/firecrawl-plugin",
|
||||
"npmSpec": "@openclaw/firecrawl-plugin",
|
||||
"defaultChoice": "npm",
|
||||
"minHostVersion": ">=2026.6.8"
|
||||
},
|
||||
"compat": {
|
||||
"pluginApi": ">=2026.6.11"
|
||||
},
|
||||
"build": {
|
||||
"openclawVersion": "2026.6.11",
|
||||
"bundledDist": false
|
||||
},
|
||||
"release": {
|
||||
"publishToClawHub": true,
|
||||
"publishToNpm": true
|
||||
}
|
||||
}
|
||||
}
|
||||
239
extensions/firecrawl/src/config.ts
Normal file
239
extensions/firecrawl/src/config.ts
Normal file
@@ -0,0 +1,239 @@
|
||||
// Firecrawl helper module supports config behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { canResolveEnvSecretRefInReadOnlyPath } from "openclaw/plugin-sdk/extension-shared";
|
||||
import { resolvePositiveTimeoutSeconds } from "openclaw/plugin-sdk/provider-web-fetch";
|
||||
import { resolveSecretInputString, normalizeSecretInput } from "openclaw/plugin-sdk/secret-input";
|
||||
|
||||
export const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
|
||||
export const DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS = 30;
|
||||
export const DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS = 60;
|
||||
export const DEFAULT_FIRECRAWL_MAX_AGE_MS = 172_800_000;
|
||||
const FIRECRAWL_API_KEY_ENV_VAR = "FIRECRAWL_API_KEY";
|
||||
|
||||
type WebSearchConfig = NonNullable<OpenClawConfig["tools"]>["web"] extends infer Web
|
||||
? Web extends { search?: infer Search }
|
||||
? Search
|
||||
: undefined
|
||||
: undefined;
|
||||
|
||||
type WebFetchConfig = NonNullable<OpenClawConfig["tools"]>["web"] extends infer Web
|
||||
? Web extends { fetch?: infer Fetch }
|
||||
? Fetch
|
||||
: undefined
|
||||
: undefined;
|
||||
|
||||
type FirecrawlSearchConfig =
|
||||
| {
|
||||
apiKey?: unknown;
|
||||
baseUrl?: string;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
type PluginEntryConfig =
|
||||
| {
|
||||
webSearch?: {
|
||||
apiKey?: unknown;
|
||||
baseUrl?: string;
|
||||
};
|
||||
webFetch?: {
|
||||
apiKey?: unknown;
|
||||
baseUrl?: string;
|
||||
onlyMainContent?: boolean;
|
||||
maxAgeMs?: number;
|
||||
timeoutSeconds?: number;
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
|
||||
type FirecrawlFetchConfig =
|
||||
| {
|
||||
apiKey?: unknown;
|
||||
baseUrl?: string;
|
||||
onlyMainContent?: boolean;
|
||||
maxAgeMs?: number;
|
||||
timeoutSeconds?: number;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
function resolveSearchConfig(cfg?: OpenClawConfig): WebSearchConfig {
|
||||
const search = cfg?.tools?.web?.search;
|
||||
if (!search || typeof search !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return search;
|
||||
}
|
||||
|
||||
function resolveFetchConfig(cfg?: OpenClawConfig): WebFetchConfig {
|
||||
const fetch = cfg?.tools?.web?.fetch;
|
||||
if (!fetch || typeof fetch !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return fetch;
|
||||
}
|
||||
|
||||
export function resolveFirecrawlSearchConfig(cfg?: OpenClawConfig): FirecrawlSearchConfig {
|
||||
const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config as PluginEntryConfig;
|
||||
const pluginWebSearch = pluginConfig?.webSearch;
|
||||
if (pluginWebSearch && typeof pluginWebSearch === "object" && !Array.isArray(pluginWebSearch)) {
|
||||
return pluginWebSearch;
|
||||
}
|
||||
const search = resolveSearchConfig(cfg);
|
||||
if (!search || typeof search !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const firecrawl = "firecrawl" in search ? search.firecrawl : undefined;
|
||||
if (!firecrawl || typeof firecrawl !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return firecrawl as FirecrawlSearchConfig;
|
||||
}
|
||||
|
||||
function resolveFirecrawlFetchConfig(cfg?: OpenClawConfig): FirecrawlFetchConfig {
|
||||
const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config as PluginEntryConfig;
|
||||
const pluginWebFetch = pluginConfig?.webFetch;
|
||||
if (pluginWebFetch && typeof pluginWebFetch === "object" && !Array.isArray(pluginWebFetch)) {
|
||||
return pluginWebFetch;
|
||||
}
|
||||
const fetch = resolveFetchConfig(cfg);
|
||||
if (!fetch || typeof fetch !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const firecrawl = "firecrawl" in fetch ? fetch.firecrawl : undefined;
|
||||
if (!firecrawl || typeof firecrawl !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return firecrawl as FirecrawlFetchConfig;
|
||||
}
|
||||
|
||||
type ConfiguredSecretResolution =
|
||||
| { status: "available"; value: string }
|
||||
| { status: "missing" }
|
||||
| { status: "blocked" };
|
||||
|
||||
function resolveConfiguredSecret(
|
||||
value: unknown,
|
||||
path: string,
|
||||
cfg?: OpenClawConfig,
|
||||
): ConfiguredSecretResolution {
|
||||
const resolved = resolveSecretInputString({
|
||||
value,
|
||||
path,
|
||||
defaults: cfg?.secrets?.defaults,
|
||||
mode: "inspect",
|
||||
});
|
||||
if (resolved.status === "available") {
|
||||
const normalized = normalizeSecretInput(resolved.value);
|
||||
return normalized ? { status: "available", value: normalized } : { status: "missing" };
|
||||
}
|
||||
if (resolved.status === "missing") {
|
||||
return { status: "missing" };
|
||||
}
|
||||
if (resolved.ref.source !== "env") {
|
||||
return { status: "blocked" };
|
||||
}
|
||||
const envVarName = resolved.ref.id.trim();
|
||||
if (envVarName !== FIRECRAWL_API_KEY_ENV_VAR) {
|
||||
return { status: "blocked" };
|
||||
}
|
||||
if (
|
||||
!canResolveEnvSecretRefInReadOnlyPath({
|
||||
cfg,
|
||||
provider: resolved.ref.provider,
|
||||
id: envVarName,
|
||||
})
|
||||
) {
|
||||
return { status: "blocked" };
|
||||
}
|
||||
const envValue = normalizeSecretInput(process.env[envVarName]);
|
||||
return envValue ? { status: "available", value: envValue } : { status: "missing" };
|
||||
}
|
||||
|
||||
export function resolveFirecrawlApiKey(cfg?: OpenClawConfig): string | undefined {
|
||||
const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config as PluginEntryConfig;
|
||||
const search = resolveFirecrawlSearchConfig(cfg);
|
||||
const fetch = resolveFirecrawlFetchConfig(cfg);
|
||||
const configuredCandidates: Array<{ value: unknown; path: string }> = [
|
||||
{
|
||||
value: pluginConfig?.webFetch?.apiKey,
|
||||
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
|
||||
},
|
||||
{
|
||||
value: search?.apiKey,
|
||||
path: "plugins.entries.firecrawl.config.webSearch.apiKey",
|
||||
},
|
||||
{
|
||||
value: search?.apiKey,
|
||||
path: "tools.web.search.firecrawl.apiKey",
|
||||
},
|
||||
{
|
||||
value: fetch?.apiKey,
|
||||
path: "tools.web.fetch.firecrawl.apiKey",
|
||||
},
|
||||
];
|
||||
let blockedConfiguredSecret = false;
|
||||
for (const candidate of configuredCandidates) {
|
||||
const resolved = resolveConfiguredSecret(candidate.value, candidate.path, cfg);
|
||||
if (resolved.status === "available") {
|
||||
return resolved.value;
|
||||
}
|
||||
if (resolved.status === "blocked") {
|
||||
blockedConfiguredSecret = true;
|
||||
}
|
||||
}
|
||||
if (blockedConfiguredSecret) {
|
||||
return undefined;
|
||||
}
|
||||
return normalizeSecretInput(process.env[FIRECRAWL_API_KEY_ENV_VAR]) || undefined;
|
||||
}
|
||||
|
||||
export function resolveFirecrawlBaseUrl(cfg?: OpenClawConfig): string {
|
||||
const search = resolveFirecrawlSearchConfig(cfg);
|
||||
const fetch = resolveFirecrawlFetchConfig(cfg);
|
||||
const configured =
|
||||
(typeof search?.baseUrl === "string" ? search.baseUrl.trim() : "") ||
|
||||
(typeof fetch?.baseUrl === "string" ? fetch.baseUrl.trim() : "") ||
|
||||
normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) ||
|
||||
"";
|
||||
return configured || DEFAULT_FIRECRAWL_BASE_URL;
|
||||
}
|
||||
|
||||
export function resolveFirecrawlOnlyMainContent(cfg?: OpenClawConfig, override?: boolean): boolean {
|
||||
if (typeof override === "boolean") {
|
||||
return override;
|
||||
}
|
||||
const fetch = resolveFirecrawlFetchConfig(cfg);
|
||||
if (typeof fetch?.onlyMainContent === "boolean") {
|
||||
return fetch.onlyMainContent;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function resolveFirecrawlMaxAgeMs(cfg?: OpenClawConfig, override?: number): number {
|
||||
if (typeof override === "number" && Number.isFinite(override) && override >= 0) {
|
||||
return Math.floor(override);
|
||||
}
|
||||
const fetch = resolveFirecrawlFetchConfig(cfg);
|
||||
if (
|
||||
typeof fetch?.maxAgeMs === "number" &&
|
||||
Number.isFinite(fetch.maxAgeMs) &&
|
||||
fetch.maxAgeMs >= 0
|
||||
) {
|
||||
return Math.floor(fetch.maxAgeMs);
|
||||
}
|
||||
return DEFAULT_FIRECRAWL_MAX_AGE_MS;
|
||||
}
|
||||
|
||||
export function resolveFirecrawlScrapeTimeoutSeconds(
|
||||
cfg?: OpenClawConfig,
|
||||
override?: number,
|
||||
): number {
|
||||
const fetch = resolveFirecrawlFetchConfig(cfg);
|
||||
return resolvePositiveTimeoutSeconds(
|
||||
override,
|
||||
resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveFirecrawlSearchTimeoutSeconds(override?: number): number {
|
||||
return resolvePositiveTimeoutSeconds(override, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
|
||||
}
|
||||
721
extensions/firecrawl/src/firecrawl-client.test.ts
Normal file
721
extensions/firecrawl/src/firecrawl-client.test.ts
Normal file
@@ -0,0 +1,721 @@
|
||||
// Firecrawl tests cover firecrawl client behavior — URL safety,
|
||||
// scrape payload parsing, and search-item extraction.
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
let firecrawlClient: typeof import("./firecrawl-client.js").testing;
|
||||
|
||||
beforeAll(async () => {
|
||||
firecrawlClient = (
|
||||
await vi.importActual<typeof import("./firecrawl-client.js")>("./firecrawl-client.js")
|
||||
).testing;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// assertFirecrawlScrapeTargetAllowed
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("assertFirecrawlScrapeTargetAllowed", () => {
|
||||
it("allows valid public HTTPS URLs", () => {
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("https://example.com/page"),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("https://api.firecrawl.dev/v1/scrape"),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects invalid URL strings", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("not a url")).toThrow(
|
||||
"Invalid URL",
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("")).toThrow("Invalid URL");
|
||||
});
|
||||
|
||||
it("rejects non-HTTP(S) protocols", () => {
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("ftp://example.com/file"),
|
||||
).toThrow(/Blocked non-HTTP\(S\) protocol/);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("file:///etc/passwd")).toThrow(
|
||||
/Blocked non-HTTP\(S\) protocol/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("data:text/html,<x>")).toThrow(
|
||||
/Blocked non-HTTP\(S\) protocol/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects private and loopback IP addresses", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://127.0.0.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://10.0.0.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://192.168.1.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://172.16.0.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects blocked hostnames like localhost", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://localhost")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://LOCALHOST")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
});
|
||||
|
||||
it("allows HTTP URLs to public hosts (SSRF check targets the hostname, not the scheme)", () => {
|
||||
// Plain HTTP to a public hostname is not blocked here — the SSRF
|
||||
// layer resolves the hostname to decide if it targets a private network.
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://example.com"),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects IPv6 loopback and private addresses", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://[::1]")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("https://[::1]")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://[fc00::]")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects URL with embedded credentials targeting a blocked host", () => {
|
||||
// Credentials in the URL do not bypass the hostname/IP check.
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://user:pass@127.0.0.1"),
|
||||
).toThrow(/Blocked/);
|
||||
});
|
||||
|
||||
it("rejects bare hostname strings without a scheme as invalid", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("example.com")).toThrow(
|
||||
"Invalid URL",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// resolveSearchItems
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("resolveSearchItems", () => {
|
||||
it("extracts items from a top-level data array (Firecrawl Search API)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com", title: "Example" },
|
||||
{ url: "https://openclaw.ai", title: "OpenClaw" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]).toMatchObject({ url: "https://example.com", title: "Example" });
|
||||
expect(result[1]).toMatchObject({ url: "https://openclaw.ai", title: "OpenClaw" });
|
||||
});
|
||||
|
||||
it("extracts items from a results array", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
results: [{ url: "https://example.org", title: "Org" }],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.org");
|
||||
expect(result[0].title).toBe("Org");
|
||||
});
|
||||
|
||||
it("extracts items from data.results (nested)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: {
|
||||
results: [
|
||||
{ url: "https://example.com/a", title: "A" },
|
||||
{ url: "https://example.com/b", title: "B" },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("extracts items from data.data (doubly nested)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: {
|
||||
data: [{ url: "https://example.com/nested", title: "Nested" }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.com/nested");
|
||||
});
|
||||
|
||||
it("extracts items from data.web array (Firecrawl web search format)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: {
|
||||
web: [{ url: "https://example.com/web", title: "Web Result" }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.com/web");
|
||||
expect(result[0].title).toBe("Web Result");
|
||||
});
|
||||
|
||||
it("extracts items from web.results (top-level)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
web: {
|
||||
results: [{ url: "https://example.com/top-web", title: "Top Web" }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.com/top-web");
|
||||
});
|
||||
|
||||
it("returns an empty array when no search items are present", () => {
|
||||
expect(firecrawlClient.resolveSearchItems({})).toEqual([]);
|
||||
expect(firecrawlClient.resolveSearchItems({ data: "not-an-array" })).toEqual([]);
|
||||
expect(firecrawlClient.resolveSearchItems({ data: [] })).toEqual([]);
|
||||
expect(firecrawlClient.resolveSearchItems({ data: { items: [] } })).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips entries without a resolvable URL", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com/ok", title: "OK" },
|
||||
{ title: "No URL" },
|
||||
{},
|
||||
null,
|
||||
"string entry",
|
||||
42,
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("OK");
|
||||
});
|
||||
|
||||
it("resolves URL from alternate fields: sourceURL, sourceUrl, metadata.sourceURL", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", title: "A" },
|
||||
{ sourceURL: "https://b.com", title: "B" },
|
||||
{ sourceUrl: "https://c.com", title: "C" },
|
||||
{ metadata: { sourceURL: "https://d.com" }, title: "D" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(4);
|
||||
expect(result.map((r) => r.url)).toEqual([
|
||||
"https://a.com",
|
||||
"https://b.com",
|
||||
"https://c.com",
|
||||
"https://d.com",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads description from multiple possible fields", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", description: "explicit desc" },
|
||||
{ url: "https://b.com", snippet: "snippet text" },
|
||||
{ url: "https://c.com", summary: "summary text" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].description).toBe("explicit desc");
|
||||
expect(result[1].description).toBe("snippet text");
|
||||
expect(result[2].description).toBe("summary text");
|
||||
});
|
||||
|
||||
it("reads content from multiple possible fields", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", markdown: "# md" },
|
||||
{ url: "https://b.com", content: "plain content" },
|
||||
{ url: "https://c.com", text: "raw text" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].content).toBe("# md");
|
||||
expect(result[1].content).toBe("plain content");
|
||||
expect(result[2].content).toBe("raw text");
|
||||
});
|
||||
|
||||
it("reads published date from multiple possible fields", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", publishedDate: "2025-01-01" },
|
||||
{ url: "https://b.com", published: "2025-02-02" },
|
||||
{ url: "https://c.com", metadata: { publishedTime: "2025-03-03" } },
|
||||
{ url: "https://d.com", metadata: { publishedDate: "2025-04-04" } },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].published).toBe("2025-01-01");
|
||||
expect(result[1].published).toBe("2025-02-02");
|
||||
expect(result[2].published).toBe("2025-03-03");
|
||||
expect(result[3].published).toBe("2025-04-04");
|
||||
});
|
||||
|
||||
it("resolves siteName by stripping www. prefix from URL hostname", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://www.example.com/page", title: "WWW" },
|
||||
{ url: "https://example.org", title: "No WWW" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].siteName).toBe("example.com");
|
||||
expect(result[1].siteName).toBe("example.org");
|
||||
});
|
||||
|
||||
it("sets description and content to undefined when absent", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [{ url: "https://example.com", title: "Minimal" }],
|
||||
});
|
||||
|
||||
expect(result[0].description).toBeUndefined();
|
||||
expect(result[0].content).toBeUndefined();
|
||||
expect(result[0].published).toBeUndefined();
|
||||
});
|
||||
|
||||
it("falls back from empty url to sourceURL within the same entry", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "", sourceURL: "https://fallback.com", title: "Fallback" },
|
||||
{ sourceURL: "https://only-source.com", title: "Only Source" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].url).toBe("https://fallback.com");
|
||||
expect(result[1].url).toBe("https://only-source.com");
|
||||
});
|
||||
|
||||
it("includes entries with empty title (title defaults to empty string)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com/no-title" },
|
||||
{ url: "https://example.com/with-title", title: "Has Title" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].title).toBe("");
|
||||
expect(result[1].title).toBe("Has Title");
|
||||
});
|
||||
|
||||
it("picks the first candidate array when multiple are present", () => {
|
||||
// The candidates list checks data before results. Both are arrays here,
|
||||
// so data wins and results is ignored.
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [{ url: "https://from-data.com", title: "From Data" }],
|
||||
results: [{ url: "https://from-results.com", title: "From Results" }],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://from-data.com");
|
||||
});
|
||||
|
||||
it("treats non-object metadata as absent (number, string)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com/meta-num", metadata: 42 },
|
||||
{ url: "https://example.com/meta-str", metadata: "oops" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
// Both should still be resolved; metadata fallback should not crash.
|
||||
expect(result[0].url).toBe("https://example.com/meta-num");
|
||||
expect(result[1].url).toBe("https://example.com/meta-str");
|
||||
});
|
||||
|
||||
it("sets siteName to undefined when url is not a valid URL", () => {
|
||||
// resolveSiteName uses new URL() internally and catches errors.
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "not-a-valid-url", title: "Invalid" },
|
||||
{ url: "", title: "Empty URL" }, // will be skipped
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].siteName).toBeUndefined();
|
||||
});
|
||||
|
||||
it("prefers record.title over metadata.title when both are present", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{
|
||||
url: "https://example.com",
|
||||
title: "record title",
|
||||
metadata: { title: "metadata title" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("record title");
|
||||
});
|
||||
|
||||
it("falls back to metadata.title when record.title is absent", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{
|
||||
url: "https://example.com",
|
||||
metadata: { title: "metadata title" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("metadata title");
|
||||
});
|
||||
|
||||
it("falls back to metadata.title when record.title is empty string", () => {
|
||||
// typeof "" === "string" && "" → falsy → falls through to metadata.title
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{
|
||||
url: "https://example.com",
|
||||
title: "",
|
||||
metadata: { title: "metadata title" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("metadata title");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// parseFirecrawlScrapePayload
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("parseFirecrawlScrapePayload", () => {
|
||||
const baseOpts = {
|
||||
url: "https://example.com/page",
|
||||
extractMode: "markdown" as const,
|
||||
maxChars: 50_000,
|
||||
};
|
||||
|
||||
it("parses a standard markdown scrape response", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "# Hello\n\nThis is page content.",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.url).toBe("https://example.com/page");
|
||||
expect(result.extractor).toBe("firecrawl");
|
||||
expect(result.extractMode).toBe("markdown");
|
||||
expect(result.text).toContain("# Hello");
|
||||
expect(result.wrappedLength).toBe((result.text as string).length);
|
||||
expect(result.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back to content field when markdown is absent", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
content: "Fallback content body",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.text).toContain("Fallback content body");
|
||||
});
|
||||
|
||||
it("throws when no content is returned", () => {
|
||||
expect(() =>
|
||||
firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: { data: {} },
|
||||
}),
|
||||
).toThrow(/no content/i);
|
||||
|
||||
expect(() =>
|
||||
firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {},
|
||||
}),
|
||||
).toThrow(/no content/i);
|
||||
});
|
||||
|
||||
it("converts markdown to plain text in text mode", () => {
|
||||
const markdownResult = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 50_000,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "# Heading\n\n**bold** and `code`",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const textResult = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "text",
|
||||
maxChars: 50_000,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "# Heading\n\n**bold** and `code`",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(markdownResult.extractMode).toBe("markdown");
|
||||
expect(textResult.extractMode).toBe("text");
|
||||
// text mode strips markdown syntax: heading markers should be removed
|
||||
expect(textResult.text).not.toContain("# Heading");
|
||||
// The raw lengths differ because text mode strips markdown characters
|
||||
expect(textResult.rawLength as number).toBeLessThan(markdownResult.rawLength as number);
|
||||
});
|
||||
|
||||
it("includes metadata: finalUrl, title, and statusCode", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content with metadata",
|
||||
url: "https://redirected.example.com",
|
||||
statusCode: 200,
|
||||
metadata: {
|
||||
sourceURL: "https://final.example.com/page",
|
||||
title: "Page Title",
|
||||
statusCode: 200,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://final.example.com/page");
|
||||
expect(result.title).toContain("Page Title");
|
||||
expect(result.status).toBe(200);
|
||||
});
|
||||
|
||||
it("falls back to data.url for finalUrl when metadata.sourceURL is absent", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content",
|
||||
url: "https://direct.example.com",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://direct.example.com");
|
||||
});
|
||||
|
||||
it("uses the requested url as finalUrl when no redirect is present", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "no redirect info" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://example.com/page");
|
||||
});
|
||||
|
||||
it("sets title to undefined when metadata title is absent", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "no title" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.title).toBeUndefined();
|
||||
});
|
||||
|
||||
it("sets status to undefined when no statusCode is available", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "no status" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("truncates content when it exceeds maxChars", () => {
|
||||
const longContent = "a".repeat(200);
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 50,
|
||||
payload: {
|
||||
data: { markdown: longContent },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.rawLength as number).toBe(200);
|
||||
});
|
||||
|
||||
it("does not truncate content within maxChars limit", () => {
|
||||
const shortContent = "short content here";
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: shortContent },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.rawLength as number).toBe(shortContent.length);
|
||||
});
|
||||
|
||||
it("handles truncation at exact boundary (not truncated)", () => {
|
||||
const content = "x".repeat(100);
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 100,
|
||||
payload: {
|
||||
data: { markdown: content },
|
||||
},
|
||||
});
|
||||
|
||||
// When raw length equals maxChars, truncateText returns the full text.
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.rawLength as number).toBe(100);
|
||||
});
|
||||
|
||||
it("truncates content one character over maxChars", () => {
|
||||
const content = "x".repeat(101);
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 100,
|
||||
payload: {
|
||||
data: { markdown: content },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.rawLength as number).toBe(101);
|
||||
});
|
||||
|
||||
it("handles maxChars of 0 (truncates everything)", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 0,
|
||||
payload: {
|
||||
data: { markdown: "some content" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.rawLength as number).toBe("some content".length);
|
||||
});
|
||||
|
||||
it("preserves warning string from the response payload", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content with warning" },
|
||||
warning: "Proxy fallback was used for this request",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toContain("Proxy fallback was used");
|
||||
});
|
||||
|
||||
it("omits warning when response has no warning field", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content without warning" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toBeUndefined();
|
||||
});
|
||||
|
||||
it("handles non-string warning gracefully", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content" },
|
||||
warning: 42,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toBeUndefined();
|
||||
});
|
||||
|
||||
it("ignores non-numeric statusCode values", () => {
|
||||
// Firecrawl may return statusCode as a string in some response shapes.
|
||||
// The check is `typeof ... === "number"`, so strings are treated as absent.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content",
|
||||
statusCode: "200",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("treats whitespace-only markdown as valid content", () => {
|
||||
// typeof " " === "string" && " " → truthy → treated as content.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: " " },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.rawLength as number).toBe(3);
|
||||
expect(result.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("silently drops empty-string warning", () => {
|
||||
// typeof "" === "string" && "" → "" is falsy → warning = undefined.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content" },
|
||||
warning: "",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops empty-string metadata.title (treated as absent)", () => {
|
||||
// typeof "" === "string" && "" → falsy → title = undefined.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content",
|
||||
metadata: { title: "" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.title).toBeUndefined();
|
||||
});
|
||||
});
|
||||
619
extensions/firecrawl/src/firecrawl-client.ts
Normal file
619
extensions/firecrawl/src/firecrawl-client.ts
Normal file
@@ -0,0 +1,619 @@
|
||||
// Firecrawl plugin module implements firecrawl client behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
DEFAULT_CACHE_TTL_MINUTES,
|
||||
markdownToText,
|
||||
normalizeCacheKey,
|
||||
readCache,
|
||||
readResponseText,
|
||||
resolveCacheTtlMs,
|
||||
truncateText,
|
||||
withSelfHostedWebToolsEndpoint,
|
||||
withStrictWebToolsEndpoint,
|
||||
writeCache,
|
||||
} from "openclaw/plugin-sdk/provider-web-fetch";
|
||||
import { normalizeSecretInput } from "openclaw/plugin-sdk/secret-input";
|
||||
import { wrapExternalContent, wrapWebContent } from "openclaw/plugin-sdk/security-runtime";
|
||||
import {
|
||||
SsrFBlockedError,
|
||||
isBlockedHostnameOrIp,
|
||||
isPrivateIpAddress,
|
||||
resolvePinnedHostnameWithPolicy,
|
||||
type LookupFn,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
DEFAULT_FIRECRAWL_BASE_URL,
|
||||
resolveFirecrawlApiKey,
|
||||
resolveFirecrawlBaseUrl,
|
||||
resolveFirecrawlMaxAgeMs,
|
||||
resolveFirecrawlOnlyMainContent,
|
||||
resolveFirecrawlScrapeTimeoutSeconds,
|
||||
resolveFirecrawlSearchTimeoutSeconds,
|
||||
} from "./config.js";
|
||||
|
||||
const SEARCH_CACHE = new Map<
|
||||
string,
|
||||
{ value: Record<string, unknown>; expiresAt: number; insertedAt: number }
|
||||
>();
|
||||
const SCRAPE_CACHE = new Map<
|
||||
string,
|
||||
{ value: Record<string, unknown>; expiresAt: number; insertedAt: number }
|
||||
>();
|
||||
const DEFAULT_SEARCH_COUNT = 5;
|
||||
const DEFAULT_SCRAPE_MAX_CHARS = 50_000;
|
||||
const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 64 * 1024 * 1024;
|
||||
const ALLOWED_FIRECRAWL_HOSTS = new Set(["api.firecrawl.dev"]);
|
||||
const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR =
|
||||
"Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
|
||||
const FIRECRAWL_HTTP_PRIVATE_ERROR =
|
||||
"Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
|
||||
|
||||
type FirecrawlEndpointMode = "selfHosted" | "strict";
|
||||
type FirecrawlResolvedEndpoint = {
|
||||
url: string;
|
||||
mode: FirecrawlEndpointMode;
|
||||
};
|
||||
|
||||
type FirecrawlSearchItem = {
|
||||
title: string;
|
||||
url: string;
|
||||
description?: string;
|
||||
content?: string;
|
||||
published?: string;
|
||||
siteName?: string;
|
||||
};
|
||||
|
||||
async function readFirecrawlJsonResponse(
|
||||
response: Response,
|
||||
label: string,
|
||||
opts?: { maxBytes?: number },
|
||||
): Promise<Record<string, unknown>> {
|
||||
return await readProviderJsonResponse<Record<string, unknown>>(response, label, opts);
|
||||
}
|
||||
|
||||
export type FirecrawlSearchParams = {
|
||||
cfg?: OpenClawConfig;
|
||||
query: string;
|
||||
count?: number;
|
||||
timeoutSeconds?: number;
|
||||
sources?: string[];
|
||||
categories?: string[];
|
||||
scrapeResults?: boolean;
|
||||
};
|
||||
|
||||
export type FirecrawlScrapeParams = {
|
||||
cfg?: OpenClawConfig;
|
||||
url: string;
|
||||
extractMode: "markdown" | "text";
|
||||
access?: "credential" | "keyless";
|
||||
maxChars?: number;
|
||||
onlyMainContent?: boolean;
|
||||
maxAgeMs?: number;
|
||||
proxy?: "auto" | "basic" | "stealth";
|
||||
storeInCache?: boolean;
|
||||
timeoutSeconds?: number;
|
||||
};
|
||||
|
||||
export function assertFirecrawlScrapeTargetAllowed(url: string): void {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new SsrFBlockedError("Invalid URL supplied to Firecrawl scrape");
|
||||
}
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new SsrFBlockedError(
|
||||
`Blocked non-HTTP(S) protocol in Firecrawl scrape URL: ${parsed.protocol}`,
|
||||
);
|
||||
}
|
||||
if (isBlockedHostnameOrIp(parsed.hostname)) {
|
||||
throw new SsrFBlockedError(
|
||||
`Blocked hostname or private/internal IP in Firecrawl scrape URL: ${parsed.hostname}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isOfficialFirecrawlEndpoint(url: URL): boolean {
|
||||
return url.protocol === "https:" && ALLOWED_FIRECRAWL_HOSTS.has(url.hostname);
|
||||
}
|
||||
|
||||
async function firecrawlEndpointTargetsPrivateNetwork(
|
||||
url: URL,
|
||||
lookupFn?: LookupFn,
|
||||
): Promise<boolean> {
|
||||
if (isBlockedHostnameOrIp(url.hostname)) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const pinned = await resolvePinnedHostnameWithPolicy(url.hostname, {
|
||||
lookupFn,
|
||||
policy: { allowPrivateNetwork: true },
|
||||
});
|
||||
return pinned.addresses.every((address) => isPrivateIpAddress(address));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function validateFirecrawlBaseUrl(
|
||||
baseUrl: string,
|
||||
lookupFn?: LookupFn,
|
||||
): Promise<FirecrawlEndpointMode> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(baseUrl.trim() || DEFAULT_FIRECRAWL_BASE_URL);
|
||||
} catch {
|
||||
throw new Error("Firecrawl baseUrl must be a valid http:// or https:// URL.");
|
||||
}
|
||||
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
throw new Error("Firecrawl baseUrl must use http:// or https://.");
|
||||
}
|
||||
if (isOfficialFirecrawlEndpoint(url)) {
|
||||
return "strict";
|
||||
}
|
||||
|
||||
const isPrivateTarget = await firecrawlEndpointTargetsPrivateNetwork(url, lookupFn);
|
||||
if (isPrivateTarget) {
|
||||
return "selfHosted";
|
||||
}
|
||||
if (url.protocol === "http:") {
|
||||
throw new Error(FIRECRAWL_HTTP_PRIVATE_ERROR);
|
||||
}
|
||||
throw new Error(`${FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR} Host: ${url.hostname}`);
|
||||
}
|
||||
|
||||
async function resolveEndpoint(
|
||||
baseUrl: string,
|
||||
pathname: "/v2/search" | "/v2/scrape",
|
||||
lookupFn?: LookupFn,
|
||||
): Promise<FirecrawlResolvedEndpoint> {
|
||||
const url = new URL(baseUrl.trim() || DEFAULT_FIRECRAWL_BASE_URL);
|
||||
const mode = await validateFirecrawlBaseUrl(url.toString(), lookupFn);
|
||||
url.username = "";
|
||||
url.password = "";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
url.pathname = pathname;
|
||||
return { url: url.toString(), mode };
|
||||
}
|
||||
|
||||
async function postFirecrawlJson<T>(
|
||||
params: {
|
||||
url: string;
|
||||
mode?: FirecrawlEndpointMode;
|
||||
timeoutSeconds: number;
|
||||
apiKey?: string;
|
||||
body: Record<string, unknown>;
|
||||
errorLabel: string;
|
||||
},
|
||||
parse: (response: Response) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const apiKey = normalizeSecretInput(params.apiKey);
|
||||
const mode = params.mode ?? (await validateFirecrawlBaseUrl(params.url));
|
||||
const withEndpoint =
|
||||
mode === "selfHosted" ? withSelfHostedWebToolsEndpoint : withStrictWebToolsEndpoint;
|
||||
return await withEndpoint(
|
||||
{
|
||||
url: params.url,
|
||||
timeoutSeconds: params.timeoutSeconds,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
// Hosted Firecrawl accepts starter scrape requests without a token.
|
||||
// Send one only when configured so higher-limit accounts still apply.
|
||||
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
||||
},
|
||||
body: JSON.stringify(params.body),
|
||||
},
|
||||
},
|
||||
async ({ response }) => {
|
||||
if (!response.ok) {
|
||||
let detail =
|
||||
typeof response.statusText === "string" && response.statusText.trim()
|
||||
? response.statusText.trim()
|
||||
: "request failed";
|
||||
|
||||
const readJsonPayload = async (): Promise<Record<string, unknown> | null> => {
|
||||
const candidate = response as Response & { clone?: () => Response };
|
||||
const jsonResponse = typeof candidate.clone === "function" ? candidate.clone() : response;
|
||||
try {
|
||||
const body = await readResponseText(jsonResponse, { maxBytes: 64_000 });
|
||||
const payload = JSON.parse(body.text) as unknown;
|
||||
return payload && typeof payload === "object" && !Array.isArray(payload)
|
||||
? (payload as Record<string, unknown>)
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const payload = await readJsonPayload();
|
||||
if (payload) {
|
||||
detail =
|
||||
typeof payload.error === "string"
|
||||
? payload.error
|
||||
: typeof payload.message === "string"
|
||||
? payload.message
|
||||
: detail;
|
||||
} else {
|
||||
const errorBody = await readResponseText(response, { maxBytes: 64_000 });
|
||||
if (errorBody.text) {
|
||||
detail = errorBody.text;
|
||||
}
|
||||
}
|
||||
const safeDetail = wrapWebContent(detail.slice(0, 1_000), "web_fetch");
|
||||
throw new Error(`${params.errorLabel} API error (${response.status}): ${safeDetail}`);
|
||||
}
|
||||
return await parse(response);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function resolveSiteName(urlRaw: string): string | undefined {
|
||||
try {
|
||||
const host = new URL(urlRaw).hostname.replace(/^www\./, "");
|
||||
return host || undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveSearchItems(payload: Record<string, unknown>): FirecrawlSearchItem[] {
|
||||
const candidates = [
|
||||
payload.data,
|
||||
payload.results,
|
||||
(payload.data as { results?: unknown } | undefined)?.results,
|
||||
(payload.data as { data?: unknown } | undefined)?.data,
|
||||
(payload.data as { web?: unknown } | undefined)?.web,
|
||||
(payload.web as { results?: unknown } | undefined)?.results,
|
||||
];
|
||||
const rawItems = candidates.find((candidate) => Array.isArray(candidate));
|
||||
if (!Array.isArray(rawItems)) {
|
||||
return [];
|
||||
}
|
||||
const items: FirecrawlSearchItem[] = [];
|
||||
for (const entry of rawItems) {
|
||||
if (!entry || typeof entry !== "object") {
|
||||
continue;
|
||||
}
|
||||
const record = entry as Record<string, unknown>;
|
||||
const metadata =
|
||||
record.metadata && typeof record.metadata === "object"
|
||||
? (record.metadata as Record<string, unknown>)
|
||||
: undefined;
|
||||
const url =
|
||||
(typeof record.url === "string" && record.url) ||
|
||||
(typeof record.sourceURL === "string" && record.sourceURL) ||
|
||||
(typeof record.sourceUrl === "string" && record.sourceUrl) ||
|
||||
(typeof metadata?.sourceURL === "string" && metadata.sourceURL) ||
|
||||
"";
|
||||
if (!url) {
|
||||
continue;
|
||||
}
|
||||
const title =
|
||||
(typeof record.title === "string" && record.title) ||
|
||||
(typeof metadata?.title === "string" && metadata.title) ||
|
||||
"";
|
||||
const description =
|
||||
(typeof record.description === "string" && record.description) ||
|
||||
(typeof record.snippet === "string" && record.snippet) ||
|
||||
(typeof record.summary === "string" && record.summary) ||
|
||||
undefined;
|
||||
const content =
|
||||
(typeof record.markdown === "string" && record.markdown) ||
|
||||
(typeof record.content === "string" && record.content) ||
|
||||
(typeof record.text === "string" && record.text) ||
|
||||
undefined;
|
||||
const published =
|
||||
(typeof record.publishedDate === "string" && record.publishedDate) ||
|
||||
(typeof record.published === "string" && record.published) ||
|
||||
(typeof metadata?.publishedTime === "string" && metadata.publishedTime) ||
|
||||
(typeof metadata?.publishedDate === "string" && metadata.publishedDate) ||
|
||||
undefined;
|
||||
items.push({
|
||||
title,
|
||||
url,
|
||||
description,
|
||||
content,
|
||||
published,
|
||||
siteName: resolveSiteName(url),
|
||||
});
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
function buildSearchPayload(params: {
|
||||
query: string;
|
||||
provider: "firecrawl";
|
||||
items: FirecrawlSearchItem[];
|
||||
tookMs: number;
|
||||
scrapeResults: boolean;
|
||||
}): Record<string, unknown> {
|
||||
return {
|
||||
query: params.query,
|
||||
provider: params.provider,
|
||||
count: params.items.length,
|
||||
tookMs: params.tookMs,
|
||||
externalContent: {
|
||||
untrusted: true,
|
||||
source: "web_search",
|
||||
provider: params.provider,
|
||||
wrapped: true,
|
||||
},
|
||||
results: params.items.map((entry) => ({
|
||||
title: entry.title ? wrapWebContent(entry.title, "web_search") : "",
|
||||
url: entry.url,
|
||||
description: entry.description ? wrapWebContent(entry.description, "web_search") : "",
|
||||
...(entry.published ? { published: entry.published } : {}),
|
||||
...(entry.siteName ? { siteName: entry.siteName } : {}),
|
||||
...(params.scrapeResults && entry.content
|
||||
? { content: wrapWebContent(entry.content, "web_search") }
|
||||
: {}),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export async function runFirecrawlSearch(
|
||||
params: FirecrawlSearchParams,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const apiKey = resolveFirecrawlApiKey(params.cfg);
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
"web_search (firecrawl) needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webSearch.apiKey.",
|
||||
);
|
||||
}
|
||||
const count =
|
||||
typeof params.count === "number" && Number.isFinite(params.count)
|
||||
? Math.max(1, Math.min(10, Math.floor(params.count)))
|
||||
: DEFAULT_SEARCH_COUNT;
|
||||
const timeoutSeconds = resolveFirecrawlSearchTimeoutSeconds(params.timeoutSeconds);
|
||||
const scrapeResults = params.scrapeResults === true;
|
||||
const sources = Array.isArray(params.sources) ? params.sources.filter(Boolean) : [];
|
||||
const categories = Array.isArray(params.categories) ? params.categories.filter(Boolean) : [];
|
||||
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
||||
const cacheKey = normalizeCacheKey(
|
||||
JSON.stringify({
|
||||
type: "firecrawl-search",
|
||||
q: params.query,
|
||||
count,
|
||||
baseUrl,
|
||||
sources,
|
||||
categories,
|
||||
scrapeResults,
|
||||
}),
|
||||
);
|
||||
const cached = readCache(SEARCH_CACHE, cacheKey);
|
||||
if (cached) {
|
||||
return { ...cached.value, cached: true };
|
||||
}
|
||||
|
||||
const body: Record<string, unknown> = {
|
||||
query: params.query,
|
||||
limit: count,
|
||||
};
|
||||
if (sources.length > 0) {
|
||||
body.sources = sources;
|
||||
}
|
||||
if (categories.length > 0) {
|
||||
body.categories = categories;
|
||||
}
|
||||
if (scrapeResults) {
|
||||
body.scrapeOptions = {
|
||||
formats: ["markdown"],
|
||||
};
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
const endpoint = await resolveEndpoint(baseUrl, "/v2/search");
|
||||
const payload = await postFirecrawlJson(
|
||||
{
|
||||
url: endpoint.url,
|
||||
mode: endpoint.mode,
|
||||
timeoutSeconds,
|
||||
apiKey,
|
||||
body,
|
||||
errorLabel: "Firecrawl Search",
|
||||
},
|
||||
async (response) => {
|
||||
const payloadValue = await readFirecrawlJsonResponse(response, "Firecrawl Search API error");
|
||||
if (payloadValue.success === false) {
|
||||
const error =
|
||||
typeof payloadValue.error === "string"
|
||||
? payloadValue.error
|
||||
: typeof payloadValue.message === "string"
|
||||
? payloadValue.message
|
||||
: "unknown error";
|
||||
throw new Error(`Firecrawl Search API error: ${error}`);
|
||||
}
|
||||
return payloadValue;
|
||||
},
|
||||
);
|
||||
const result = buildSearchPayload({
|
||||
query: params.query,
|
||||
provider: "firecrawl",
|
||||
items: resolveSearchItems(payload),
|
||||
tookMs: Date.now() - start,
|
||||
scrapeResults,
|
||||
});
|
||||
writeCache(
|
||||
SEARCH_CACHE,
|
||||
cacheKey,
|
||||
result,
|
||||
resolveCacheTtlMs(undefined, DEFAULT_CACHE_TTL_MINUTES),
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
function resolveScrapeData(payload: Record<string, unknown>): Record<string, unknown> {
|
||||
const data = payload.data;
|
||||
if (data && typeof data === "object") {
|
||||
return data as Record<string, unknown>;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
export function parseFirecrawlScrapePayload(params: {
|
||||
payload: Record<string, unknown>;
|
||||
url: string;
|
||||
extractMode: "markdown" | "text";
|
||||
maxChars: number;
|
||||
}): Record<string, unknown> {
|
||||
const data = resolveScrapeData(params.payload);
|
||||
const metadata =
|
||||
data.metadata && typeof data.metadata === "object"
|
||||
? (data.metadata as Record<string, unknown>)
|
||||
: undefined;
|
||||
const markdown =
|
||||
(typeof data.markdown === "string" && data.markdown) ||
|
||||
(typeof data.content === "string" && data.content) ||
|
||||
"";
|
||||
if (!markdown) {
|
||||
throw new Error("Firecrawl scrape returned no content.");
|
||||
}
|
||||
const rawText = params.extractMode === "text" ? markdownToText(markdown) : markdown;
|
||||
const truncated = truncateText(rawText, params.maxChars);
|
||||
const wrappedText = wrapExternalContent(truncated.text, {
|
||||
source: "web_fetch",
|
||||
includeWarning: false,
|
||||
});
|
||||
return {
|
||||
url: params.url,
|
||||
finalUrl:
|
||||
(typeof metadata?.sourceURL === "string" && metadata.sourceURL) ||
|
||||
(typeof data.url === "string" && data.url) ||
|
||||
params.url,
|
||||
status:
|
||||
(typeof metadata?.statusCode === "number" && metadata.statusCode) ||
|
||||
(typeof data.statusCode === "number" && data.statusCode) ||
|
||||
undefined,
|
||||
title:
|
||||
typeof metadata?.title === "string" && metadata.title
|
||||
? wrapExternalContent(metadata.title, { source: "web_fetch", includeWarning: false })
|
||||
: undefined,
|
||||
extractor: "firecrawl",
|
||||
extractMode: params.extractMode,
|
||||
externalContent: {
|
||||
untrusted: true,
|
||||
source: "web_fetch",
|
||||
wrapped: true,
|
||||
},
|
||||
truncated: truncated.truncated,
|
||||
rawLength: rawText.length,
|
||||
wrappedLength: wrappedText.length,
|
||||
text: wrappedText,
|
||||
warning:
|
||||
typeof params.payload.warning === "string" && params.payload.warning
|
||||
? wrapExternalContent(params.payload.warning, {
|
||||
source: "web_fetch",
|
||||
includeWarning: false,
|
||||
})
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runFirecrawlScrape(
|
||||
params: FirecrawlScrapeParams,
|
||||
): Promise<Record<string, unknown>> {
|
||||
assertFirecrawlScrapeTargetAllowed(params.url);
|
||||
|
||||
const apiKey = resolveFirecrawlApiKey(params.cfg);
|
||||
// Hosted v2/scrape accepts starter requests without a bearer token.
|
||||
// Only the selected web_fetch provider opts into that access mode.
|
||||
if (!apiKey && params.access !== "keyless") {
|
||||
throw new Error(
|
||||
"firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.",
|
||||
);
|
||||
}
|
||||
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
||||
const timeoutSeconds = resolveFirecrawlScrapeTimeoutSeconds(params.cfg, params.timeoutSeconds);
|
||||
const onlyMainContent = resolveFirecrawlOnlyMainContent(params.cfg, params.onlyMainContent);
|
||||
const maxAgeMs = resolveFirecrawlMaxAgeMs(params.cfg, params.maxAgeMs);
|
||||
const proxy = params.proxy ?? "auto";
|
||||
const storeInCache = params.storeInCache ?? true;
|
||||
const maxChars =
|
||||
typeof params.maxChars === "number" && Number.isFinite(params.maxChars) && params.maxChars > 0
|
||||
? Math.floor(params.maxChars)
|
||||
: DEFAULT_SCRAPE_MAX_CHARS;
|
||||
const cacheKey = normalizeCacheKey(
|
||||
JSON.stringify({
|
||||
type: "firecrawl-scrape",
|
||||
url: params.url,
|
||||
extractMode: params.extractMode,
|
||||
baseUrl,
|
||||
onlyMainContent,
|
||||
maxAgeMs,
|
||||
proxy,
|
||||
storeInCache,
|
||||
maxChars,
|
||||
}),
|
||||
);
|
||||
const cached = readCache(SCRAPE_CACHE, cacheKey);
|
||||
if (cached) {
|
||||
return { ...cached.value, cached: true };
|
||||
}
|
||||
|
||||
const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
|
||||
const payload = await postFirecrawlJson(
|
||||
{
|
||||
url: endpoint.url,
|
||||
mode: endpoint.mode,
|
||||
timeoutSeconds,
|
||||
apiKey,
|
||||
errorLabel: "Firecrawl",
|
||||
body: {
|
||||
url: params.url,
|
||||
formats: ["markdown"],
|
||||
onlyMainContent,
|
||||
timeout: timeoutSeconds * 1000,
|
||||
maxAge: maxAgeMs,
|
||||
proxy,
|
||||
storeInCache,
|
||||
},
|
||||
},
|
||||
async (response) => {
|
||||
const payloadLocal = await readFirecrawlJsonResponse(response, "Firecrawl fetch failed", {
|
||||
// Scrape can legitimately return page bodies before maxChars truncates parsed output.
|
||||
maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES,
|
||||
});
|
||||
if (payloadLocal.success === false) {
|
||||
const detail =
|
||||
typeof payloadLocal.error === "string"
|
||||
? payloadLocal.error
|
||||
: typeof payloadLocal.message === "string"
|
||||
? payloadLocal.message
|
||||
: response.statusText;
|
||||
throw new Error(
|
||||
`Firecrawl fetch failed (${response.status}): ${wrapWebContent(detail, "web_fetch")}`.trim(),
|
||||
);
|
||||
}
|
||||
return payloadLocal;
|
||||
},
|
||||
);
|
||||
const result = parseFirecrawlScrapePayload({
|
||||
payload,
|
||||
url: params.url,
|
||||
extractMode: params.extractMode,
|
||||
maxChars,
|
||||
});
|
||||
writeCache(
|
||||
SCRAPE_CACHE,
|
||||
cacheKey,
|
||||
result,
|
||||
resolveCacheTtlMs(undefined, DEFAULT_CACHE_TTL_MINUTES),
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
assertFirecrawlScrapeTargetAllowed,
|
||||
parseFirecrawlScrapePayload,
|
||||
postFirecrawlJson,
|
||||
readFirecrawlJsonResponse,
|
||||
resolveEndpoint,
|
||||
validateFirecrawlBaseUrl,
|
||||
resolveSearchItems,
|
||||
};
|
||||
export { testing as __testing };
|
||||
71
extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
Normal file
71
extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
// Firecrawl provider module implements model/runtime integration.
|
||||
import type { WebFetchProviderPlugin } from "openclaw/plugin-sdk/provider-web-fetch-contract";
|
||||
|
||||
function ensureRecord(target: Record<string, unknown>, key: string): Record<string, unknown> {
|
||||
const current = target[key];
|
||||
if (current && typeof current === "object" && !Array.isArray(current)) {
|
||||
return current as Record<string, unknown>;
|
||||
}
|
||||
const next: Record<string, unknown> = {};
|
||||
target[key] = next;
|
||||
return next;
|
||||
}
|
||||
|
||||
export const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
|
||||
id: "firecrawl",
|
||||
label: "Firecrawl",
|
||||
hint: "Fetch pages with keyless starter access; add a key for higher limits.",
|
||||
requiresCredential: false,
|
||||
credentialLabel: "Firecrawl API key (optional)",
|
||||
envVars: ["FIRECRAWL_API_KEY"],
|
||||
placeholder: "fc-...",
|
||||
signupUrl: "https://www.firecrawl.dev/",
|
||||
docsUrl: "https://docs.firecrawl.dev",
|
||||
autoDetectOrder: 50,
|
||||
credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey",
|
||||
inactiveSecretPaths: [
|
||||
"plugins.entries.firecrawl.config.webFetch.apiKey",
|
||||
"tools.web.fetch.firecrawl.apiKey",
|
||||
],
|
||||
getCredentialValue: (fetchConfig) => {
|
||||
if (!fetchConfig || typeof fetchConfig !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const legacy = fetchConfig.firecrawl;
|
||||
if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) {
|
||||
return undefined;
|
||||
}
|
||||
if ((legacy as { enabled?: boolean }).enabled === false) {
|
||||
return undefined;
|
||||
}
|
||||
return (legacy as { apiKey?: unknown }).apiKey;
|
||||
},
|
||||
setCredentialValue: (fetchConfigTarget, value) => {
|
||||
const firecrawl = ensureRecord(fetchConfigTarget, "firecrawl");
|
||||
firecrawl.apiKey = value;
|
||||
},
|
||||
getConfiguredCredentialValue: (config) =>
|
||||
(config?.plugins?.entries?.firecrawl?.config as { webFetch?: { apiKey?: unknown } } | undefined)
|
||||
?.webFetch?.apiKey,
|
||||
getConfiguredCredentialFallback: (config) => {
|
||||
const apiKey = (
|
||||
config?.plugins?.entries?.firecrawl?.config as
|
||||
| { webSearch?: { apiKey?: unknown } }
|
||||
| undefined
|
||||
)?.webSearch?.apiKey;
|
||||
return apiKey === undefined
|
||||
? undefined
|
||||
: {
|
||||
path: "plugins.entries.firecrawl.config.webSearch.apiKey",
|
||||
value: apiKey,
|
||||
};
|
||||
},
|
||||
setConfiguredCredentialValue: (configTarget, value) => {
|
||||
const plugins = ensureRecord(configTarget as unknown as Record<string, unknown>, "plugins");
|
||||
const entries = ensureRecord(plugins, "entries");
|
||||
const firecrawlEntry = ensureRecord(entries, "firecrawl");
|
||||
const pluginConfig = ensureRecord(firecrawlEntry, "config");
|
||||
const webFetch = ensureRecord(pluginConfig, "webFetch");
|
||||
webFetch.apiKey = value;
|
||||
},
|
||||
} satisfies Omit<WebFetchProviderPlugin, "applySelectionConfig" | "createTool">;
|
||||
47
extensions/firecrawl/src/firecrawl-fetch-provider.ts
Normal file
47
extensions/firecrawl/src/firecrawl-fetch-provider.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
// Firecrawl provider module implements model/runtime integration.
|
||||
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
||||
import {
|
||||
enablePluginInConfig,
|
||||
type WebFetchProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/provider-web-fetch-contract";
|
||||
import { FIRECRAWL_WEB_FETCH_PROVIDER_SHARED } from "./firecrawl-fetch-provider-shared.js";
|
||||
|
||||
type FirecrawlClientModule = typeof import("./firecrawl-client.js");
|
||||
|
||||
let firecrawlClientModulePromise: Promise<FirecrawlClientModule> | undefined;
|
||||
|
||||
function loadFirecrawlClientModule(): Promise<FirecrawlClientModule> {
|
||||
firecrawlClientModulePromise ??= import("./firecrawl-client.js");
|
||||
return firecrawlClientModulePromise;
|
||||
}
|
||||
|
||||
export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin {
|
||||
return {
|
||||
...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
|
||||
applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config,
|
||||
createTool: ({ config }) => ({
|
||||
description: "Fetch a page using Firecrawl.",
|
||||
parameters: {},
|
||||
execute: async (args) => {
|
||||
const url = typeof args.url === "string" ? args.url : "";
|
||||
const extractMode = args.extractMode === "text" ? "text" : "markdown";
|
||||
const maxChars = readPositiveIntegerParam(args, "maxChars");
|
||||
const proxy =
|
||||
args.proxy === "basic" || args.proxy === "stealth" || args.proxy === "auto"
|
||||
? args.proxy
|
||||
: undefined;
|
||||
const storeInCache = typeof args.storeInCache === "boolean" ? args.storeInCache : undefined;
|
||||
const { runFirecrawlScrape } = await loadFirecrawlClientModule();
|
||||
return await runFirecrawlScrape({
|
||||
cfg: config,
|
||||
url,
|
||||
extractMode,
|
||||
access: "keyless",
|
||||
maxChars,
|
||||
...(proxy ? { proxy } : {}),
|
||||
...(storeInCache !== undefined ? { storeInCache } : {}),
|
||||
});
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
93
extensions/firecrawl/src/firecrawl-scrape-tool.ts
Normal file
93
extensions/firecrawl/src/firecrawl-scrape-tool.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
// Firecrawl plugin module implements firecrawl scrape tool behavior.
|
||||
import { optionalStringEnum } from "openclaw/plugin-sdk/channel-actions";
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-runtime";
|
||||
import {
|
||||
jsonResult,
|
||||
readNonNegativeIntegerParam,
|
||||
readPositiveIntegerParam,
|
||||
readStringParam,
|
||||
} from "openclaw/plugin-sdk/provider-web-search";
|
||||
import { Type } from "typebox";
|
||||
import { runFirecrawlScrape } from "./firecrawl-client.js";
|
||||
|
||||
const FirecrawlScrapeToolSchema = Type.Object(
|
||||
{
|
||||
url: Type.String({ description: "HTTP or HTTPS URL to scrape via Firecrawl." }),
|
||||
extractMode: optionalStringEnum(["markdown", "text"] as const, {
|
||||
description: 'Extraction mode ("markdown" or "text"). Default: markdown.',
|
||||
}),
|
||||
maxChars: Type.Optional(
|
||||
Type.Integer({
|
||||
description: "Maximum characters to return.",
|
||||
minimum: 100,
|
||||
}),
|
||||
),
|
||||
onlyMainContent: Type.Optional(
|
||||
Type.Boolean({
|
||||
description: "Keep only main content when Firecrawl supports it.",
|
||||
}),
|
||||
),
|
||||
maxAgeMs: Type.Optional(
|
||||
Type.Integer({
|
||||
description: "Maximum Firecrawl cache age in milliseconds.",
|
||||
minimum: 0,
|
||||
}),
|
||||
),
|
||||
proxy: optionalStringEnum(["auto", "basic", "stealth"] as const, {
|
||||
description: 'Firecrawl proxy mode ("auto", "basic", or "stealth").',
|
||||
}),
|
||||
storeInCache: Type.Optional(
|
||||
Type.Boolean({
|
||||
description: "Whether Firecrawl should store the scrape in its cache.",
|
||||
}),
|
||||
),
|
||||
timeoutSeconds: Type.Optional(
|
||||
Type.Integer({
|
||||
description: "Timeout in seconds for the Firecrawl scrape request.",
|
||||
minimum: 1,
|
||||
}),
|
||||
),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export function createFirecrawlScrapeTool(api: OpenClawPluginApi) {
|
||||
return {
|
||||
name: "firecrawl_scrape",
|
||||
label: "Firecrawl Scrape",
|
||||
description:
|
||||
"Scrape a page using Firecrawl v2/scrape. Useful for JS-heavy or bot-protected pages where plain web_fetch is weak.",
|
||||
parameters: FirecrawlScrapeToolSchema,
|
||||
execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
|
||||
const url = readStringParam(rawParams, "url", { required: true });
|
||||
const extractMode =
|
||||
readStringParam(rawParams, "extractMode") === "text" ? "text" : "markdown";
|
||||
const maxChars = readPositiveIntegerParam(rawParams, "maxChars");
|
||||
const maxAgeMs = readNonNegativeIntegerParam(rawParams, "maxAgeMs");
|
||||
const timeoutSeconds = readPositiveIntegerParam(rawParams, "timeoutSeconds");
|
||||
const proxyRaw = readStringParam(rawParams, "proxy");
|
||||
const proxy =
|
||||
proxyRaw === "basic" || proxyRaw === "stealth" || proxyRaw === "auto"
|
||||
? proxyRaw
|
||||
: undefined;
|
||||
const onlyMainContent =
|
||||
typeof rawParams.onlyMainContent === "boolean" ? rawParams.onlyMainContent : undefined;
|
||||
const storeInCache =
|
||||
typeof rawParams.storeInCache === "boolean" ? rawParams.storeInCache : undefined;
|
||||
|
||||
return jsonResult(
|
||||
await runFirecrawlScrape({
|
||||
cfg: api.config,
|
||||
url,
|
||||
extractMode,
|
||||
maxChars,
|
||||
onlyMainContent,
|
||||
maxAgeMs,
|
||||
proxy,
|
||||
storeInCache,
|
||||
timeoutSeconds,
|
||||
}),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
43
extensions/firecrawl/src/firecrawl-search-provider.ts
Normal file
43
extensions/firecrawl/src/firecrawl-search-provider.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
||||
// Firecrawl provider module implements model/runtime integration.
|
||||
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
||||
import type { WebSearchProviderPlugin } from "openclaw/plugin-sdk/provider-web-search-contract";
|
||||
import { buildFirecrawlWebSearchProviderBase } from "../web-search-shared.js";
|
||||
|
||||
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client.js"));
|
||||
|
||||
const GenericFirecrawlSearchSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
query: { type: "string", description: "Search query string." },
|
||||
count: {
|
||||
type: "integer",
|
||||
description: "Number of results to return (1-10).",
|
||||
minimum: 1,
|
||||
maximum: 10,
|
||||
},
|
||||
},
|
||||
additionalProperties: false,
|
||||
} satisfies Record<string, unknown>;
|
||||
|
||||
export function createFirecrawlWebSearchProvider(): WebSearchProviderPlugin {
|
||||
return {
|
||||
...buildFirecrawlWebSearchProviderBase(),
|
||||
createTool: (ctx) => ({
|
||||
description:
|
||||
"Search the web using Firecrawl. Returns structured results with snippets from Firecrawl Search. Use firecrawl_search for Firecrawl-specific knobs like sources or categories.",
|
||||
parameters: GenericFirecrawlSearchSchema,
|
||||
execute: async (args) => {
|
||||
const { runFirecrawlSearch } = await loadFirecrawlClientModule();
|
||||
return await runFirecrawlSearch({
|
||||
cfg: ctx.config,
|
||||
query: typeof args.query === "string" ? args.query : "",
|
||||
count: readPositiveIntegerParam(args, "count", {
|
||||
message: "count must be an integer from 1 to 10",
|
||||
max: 10,
|
||||
}),
|
||||
});
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
78
extensions/firecrawl/src/firecrawl-search-tool.ts
Normal file
78
extensions/firecrawl/src/firecrawl-search-tool.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
// Firecrawl plugin module implements firecrawl search tool behavior.
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-runtime";
|
||||
import {
|
||||
jsonResult,
|
||||
readPositiveIntegerParam,
|
||||
readStringArrayParam,
|
||||
readStringParam,
|
||||
} from "openclaw/plugin-sdk/provider-web-search";
|
||||
import { Type } from "typebox";
|
||||
import { runFirecrawlSearch } from "./firecrawl-client.js";
|
||||
|
||||
const FirecrawlSearchToolSchema = Type.Object(
|
||||
{
|
||||
query: Type.String({ description: "Search query string." }),
|
||||
count: Type.Optional(
|
||||
Type.Integer({
|
||||
description: "Number of results to return (1-10).",
|
||||
minimum: 1,
|
||||
maximum: 10,
|
||||
}),
|
||||
),
|
||||
sources: Type.Optional(
|
||||
Type.Array(Type.String(), {
|
||||
description: 'Optional sources list, for example ["web"], ["news"], or ["images"].',
|
||||
}),
|
||||
),
|
||||
categories: Type.Optional(
|
||||
Type.Array(Type.String(), {
|
||||
description: 'Optional Firecrawl categories, for example ["github"] or ["research"].',
|
||||
}),
|
||||
),
|
||||
scrapeResults: Type.Optional(
|
||||
Type.Boolean({
|
||||
description: "Include scraped result content when Firecrawl returns it.",
|
||||
}),
|
||||
),
|
||||
timeoutSeconds: Type.Optional(
|
||||
Type.Integer({
|
||||
description: "Timeout in seconds for the Firecrawl Search request.",
|
||||
minimum: 1,
|
||||
}),
|
||||
),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export function createFirecrawlSearchTool(api: OpenClawPluginApi) {
|
||||
return {
|
||||
name: "firecrawl_search",
|
||||
label: "Firecrawl Search",
|
||||
description:
|
||||
"Search the web using Firecrawl v2/search. Can optionally include scraped content from result pages.",
|
||||
parameters: FirecrawlSearchToolSchema,
|
||||
execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
|
||||
const query = readStringParam(rawParams, "query", { required: true });
|
||||
const count = readPositiveIntegerParam(rawParams, "count", {
|
||||
max: 10,
|
||||
message: "count must be an integer from 1 to 10",
|
||||
});
|
||||
const timeoutSeconds = readPositiveIntegerParam(rawParams, "timeoutSeconds");
|
||||
const sources = readStringArrayParam(rawParams, "sources");
|
||||
const categories = readStringArrayParam(rawParams, "categories");
|
||||
const scrapeResults = rawParams.scrapeResults === true;
|
||||
|
||||
return jsonResult(
|
||||
await runFirecrawlSearch({
|
||||
cfg: api.config,
|
||||
query,
|
||||
count,
|
||||
timeoutSeconds,
|
||||
sources,
|
||||
categories,
|
||||
scrapeResults,
|
||||
}),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
1120
extensions/firecrawl/src/firecrawl-tools.test.ts
Normal file
1120
extensions/firecrawl/src/firecrawl-tools.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
16
extensions/firecrawl/tsconfig.json
Normal file
16
extensions/firecrawl/tsconfig.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"extends": "../tsconfig.package-boundary.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["./*.ts", "./src/**/*.ts"],
|
||||
"exclude": [
|
||||
"./**/*.test.ts",
|
||||
"./dist/**",
|
||||
"./node_modules/**",
|
||||
"./src/test-support/**",
|
||||
"./src/**/*test-helpers.ts",
|
||||
"./src/**/*test-harness.ts",
|
||||
"./src/**/*test-support.ts"
|
||||
]
|
||||
}
|
||||
14
extensions/firecrawl/web-fetch-contract-api.ts
Normal file
14
extensions/firecrawl/web-fetch-contract-api.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
// Firecrawl API module exposes the plugin public contract.
|
||||
import {
|
||||
enablePluginInConfig,
|
||||
type WebFetchProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/provider-web-fetch-contract";
|
||||
import { FIRECRAWL_WEB_FETCH_PROVIDER_SHARED } from "./src/firecrawl-fetch-provider-shared.js";
|
||||
|
||||
export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin {
|
||||
return {
|
||||
...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
|
||||
applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config,
|
||||
createTool: () => null,
|
||||
};
|
||||
}
|
||||
2
extensions/firecrawl/web-fetch-provider.ts
Normal file
2
extensions/firecrawl/web-fetch-provider.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
// Firecrawl provider module implements model/runtime integration.
|
||||
export { createFirecrawlWebFetchProvider } from "./src/firecrawl-fetch-provider.js";
|
||||
10
extensions/firecrawl/web-search-contract-api.ts
Normal file
10
extensions/firecrawl/web-search-contract-api.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
// Firecrawl API module exposes the plugin public contract.
|
||||
import type { WebSearchProviderPlugin } from "openclaw/plugin-sdk/provider-web-search-contract";
|
||||
import { buildFirecrawlWebSearchProviderBase } from "./web-search-shared.js";
|
||||
|
||||
export function createFirecrawlWebSearchProvider(): WebSearchProviderPlugin {
|
||||
return {
|
||||
...buildFirecrawlWebSearchProviderBase(),
|
||||
createTool: () => null,
|
||||
};
|
||||
}
|
||||
2
extensions/firecrawl/web-search-provider.ts
Normal file
2
extensions/firecrawl/web-search-provider.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
// Firecrawl provider module implements model/runtime integration.
|
||||
export { createFirecrawlWebSearchProvider } from "./src/firecrawl-search-provider.js";
|
||||
66
extensions/firecrawl/web-search-shared.ts
Normal file
66
extensions/firecrawl/web-search-shared.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
// Firecrawl plugin module implements web search shared behavior.
|
||||
import {
|
||||
createWebSearchProviderContractFields,
|
||||
enablePluginInConfig,
|
||||
type WebSearchProviderPlugin,
|
||||
} from "openclaw/plugin-sdk/provider-web-search-contract";
|
||||
|
||||
export const FIRECRAWL_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webSearch.apiKey";
|
||||
export const FIRECRAWL_FETCH_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webFetch.apiKey";
|
||||
|
||||
export function getConfiguredFirecrawlFetchCredentialFallback(config?: {
|
||||
plugins?: { entries?: { firecrawl?: { config?: unknown } } };
|
||||
}) {
|
||||
const apiKey = (
|
||||
config?.plugins?.entries?.firecrawl?.config as { webFetch?: { apiKey?: unknown } } | undefined
|
||||
)?.webFetch?.apiKey;
|
||||
return apiKey === undefined
|
||||
? undefined
|
||||
: {
|
||||
path: FIRECRAWL_FETCH_CREDENTIAL_PATH,
|
||||
value: apiKey,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildFirecrawlWebSearchProviderBase(): Omit<WebSearchProviderPlugin, "createTool"> {
|
||||
const contractFields = createWebSearchProviderContractFields({
|
||||
credentialPath: FIRECRAWL_CREDENTIAL_PATH,
|
||||
searchCredential: { type: "scoped", scopeId: "firecrawl" },
|
||||
configuredCredential: { pluginId: "firecrawl" },
|
||||
});
|
||||
|
||||
return {
|
||||
id: "firecrawl",
|
||||
label: "Firecrawl Search",
|
||||
hint: "Structured results with optional result scraping",
|
||||
onboardingScopes: ["text-inference"],
|
||||
credentialLabel: "Firecrawl API key",
|
||||
envVars: ["FIRECRAWL_API_KEY"],
|
||||
placeholder: "fc-...",
|
||||
signupUrl: "https://www.firecrawl.dev/",
|
||||
docsUrl: "https://docs.openclaw.ai/tools/firecrawl",
|
||||
autoDetectOrder: 60,
|
||||
credentialPath: FIRECRAWL_CREDENTIAL_PATH,
|
||||
...contractFields,
|
||||
applySelectionConfig: (config) => {
|
||||
const enabled = enablePluginInConfig(config, "firecrawl");
|
||||
if (!enabled.enabled || enabled.config.tools?.web?.fetch?.provider) {
|
||||
return enabled.config;
|
||||
}
|
||||
return {
|
||||
...enabled.config,
|
||||
tools: {
|
||||
...enabled.config.tools,
|
||||
web: {
|
||||
...enabled.config.tools?.web,
|
||||
fetch: {
|
||||
...enabled.config.tools?.web?.fetch,
|
||||
provider: "firecrawl",
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
getConfiguredCredentialFallback: getConfiguredFirecrawlFetchCredentialFallback,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user