Vendor OpenClaw source as Adolf fork baseline

Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
2026-07-05 09:36:54 +00:00
parent 3216769225
commit bedb527145
21108 changed files with 6010766 additions and 0 deletions
--- a/extensions/web-readability/index.ts
+++ b/extensions/web-readability/index.ts
@@ -0,0 +1,12 @@
+// Web Readability plugin entrypoint registers its OpenClaw integration.
+import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
+
+export default definePluginEntry({
+  id: "web-readability",
+  name: "Web Readability Extraction",
+  description: "Extract readable article content from local HTML web fetch responses.",
+  register() {
+    // Runtime is exposed through web-content-extractor.ts so hot web-fetch paths can
+    // load only the narrow extractor artifact instead of the full plugin entrypoint.
+  },
+});
--- a/extensions/web-readability/openclaw.plugin.json
+++ b/extensions/web-readability/openclaw.plugin.json
@@ -0,0 +1,17 @@
+{
+  "id": "web-readability",
+  "activation": {
+    "onStartup": false
+  },
+  "enabledByDefault": true,
+  "name": "Web Readability Extraction",
+  "description": "Extract readable article content from local HTML web fetch responses.",
+  "contracts": {
+    "webContentExtractors": ["readability"]
+  },
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {}
+  }
+}
--- a/extensions/web-readability/package.json
+++ b/extensions/web-readability/package.json
@@ -0,0 +1,19 @@
+{
+  "name": "@openclaw/web-readability-plugin",
+  "version": "2026.6.11",
+  "private": true,
+  "description": "OpenClaw local Readability web extraction plugin",
+  "type": "module",
+  "dependencies": {
+    "@mozilla/readability": "0.6.0",
+    "linkedom": "0.18.12"
+  },
+  "devDependencies": {
+    "@openclaw/plugin-sdk": "workspace:*"
+  },
+  "openclaw": {
+    "extensions": [
+      "./index.ts"
+    ]
+  }
+}
--- a/extensions/web-readability/web-content-extractor.test.ts
+++ b/extensions/web-readability/web-content-extractor.test.ts
@@ -0,0 +1,75 @@
+// Web Readability tests cover web content extractor plugin behavior.
+import { describe, expect, it } from "vitest";
+import { createReadabilityWebContentExtractor } from "./web-content-extractor.js";
+
+const SAMPLE_HTML = `<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>Example Article</title>
+  </head>
+  <body>
+    <nav>
+      <ul>
+        <li><a href="/home">Home</a></li>
+        <li><a href="/about">About</a></li>
+      </ul>
+    </nav>
+    <main>
+      <article>
+        <h1>Example Article</h1>
+        <p>Main content starts here with enough words to satisfy readability.</p>
+        <p>Second paragraph for a bit more signal.</p>
+      </article>
+    </main>
+    <footer>Footer text</footer>
+  </body>
+</html>`;
+
+type ReadabilityResult = Awaited<
+  ReturnType<ReturnType<typeof createReadabilityWebContentExtractor>["extract"]>
+>;
+
+function requireReadabilityResult(result: ReadabilityResult): NonNullable<ReadabilityResult> {
+  if (!result) {
+    throw new Error("expected readability extraction result");
+  }
+  return result;
+}
+
+describe("web readability extractor", () => {
+  it("extracts readable text", async () => {
+    const extractor = createReadabilityWebContentExtractor();
+    const result = await extractor.extract({
+      html: SAMPLE_HTML,
+      url: "https://example.com/article",
+      extractMode: "text",
+    });
+    const extracted = requireReadabilityResult(result);
+    expect(extracted.text).toContain("Main content starts here");
+    expect(extracted.title).toBe("Example Article");
+  });
+
+  it("extracts readable markdown", async () => {
+    const extractor = createReadabilityWebContentExtractor();
+    const result = await extractor.extract({
+      html: SAMPLE_HTML,
+      url: "https://example.com/article",
+      extractMode: "markdown",
+    });
+    const extracted = requireReadabilityResult(result);
+    expect(extracted.text).toContain("Main content starts here");
+    expect(extracted.title).toBe("Example Article");
+  });
+
+  it("does not count void tags toward the nesting limit", async () => {
+    const extractor = createReadabilityWebContentExtractor();
+    const html = SAMPLE_HTML.replace("<article>", `<article>${"<BR>".repeat(3100)}`);
+    const result = await extractor.extract({
+      html,
+      url: "https://example.com/article",
+      extractMode: "markdown",
+    });
+    expect(requireReadabilityResult(result).text).toContain("Main content starts here");
+  });
+});
--- a/extensions/web-readability/web-content-extractor.ts
+++ b/extensions/web-readability/web-content-extractor.ts
@@ -0,0 +1,209 @@
+// Web Readability plugin module implements web content extractor behavior.
+import type {
+  WebContentExtractionRequest,
+  WebContentExtractionResult,
+  WebContentExtractorPlugin,
+} from "openclaw/plugin-sdk/web-content-extractor";
+import {
+  htmlToMarkdown,
+  normalizeWhitespace,
+  sanitizeHtml,
+  stripInvisibleUnicode,
+} from "openclaw/plugin-sdk/web-content-extractor";
+
+const READABILITY_MAX_HTML_CHARS = 1_000_000;
+const READABILITY_MAX_ESTIMATED_NESTING_DEPTH = 3_000;
+const HTML_VOID_TAGS = new Set([
+  "area",
+  "base",
+  "br",
+  "col",
+  "embed",
+  "hr",
+  "img",
+  "input",
+  "link",
+  "meta",
+  "param",
+  "source",
+  "track",
+  "wbr",
+]);
+
+type ParsedHtml = {
+  document: Document;
+};
+
+type ParseHtml = (html: string) => ParsedHtml;
+
+type ReadabilityResult = {
+  content?: string;
+  textContent?: string | null;
+  title?: string | null;
+};
+
+type ReadabilityInstance = {
+  parse(): ReadabilityResult | null;
+};
+
+type ReadabilityConstructor = new (
+  document: Document,
+  options: { charThreshold: number },
+) => ReadabilityInstance;
+
+type ReadabilityModule = {
+  Readability: ReadabilityConstructor;
+};
+
+type LinkedomModule = {
+  parseHTML: ParseHtml;
+};
+
+const READABILITY_MODULE = "@mozilla/readability";
+const LINKEDOM_MODULE = "linkedom";
+
+let readabilityDepsPromise:
+  | Promise<{
+      Readability: ReadabilityConstructor;
+      parseHTML: ParseHtml;
+    }>
+  | undefined;
+
+async function loadReadabilityDeps(): Promise<{
+  Readability: ReadabilityConstructor;
+  parseHTML: ParseHtml;
+}> {
+  if (!readabilityDepsPromise) {
+    readabilityDepsPromise = Promise.all([
+      import(READABILITY_MODULE) as Promise<ReadabilityModule>,
+      import(LINKEDOM_MODULE) as Promise<LinkedomModule>,
+    ]).then(([readability, linkedom]) => ({
+      Readability: readability.Readability,
+      parseHTML: linkedom.parseHTML,
+    }));
+  }
+  try {
+    return await readabilityDepsPromise;
+  } catch (error) {
+    readabilityDepsPromise = undefined;
+    throw error;
+  }
+}
+
+function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boolean {
+  let depth = 0;
+  const len = html.length;
+  for (let i = 0; i < len; i++) {
+    if (html.charCodeAt(i) !== 60) {
+      continue;
+    }
+    const next = html.charCodeAt(i + 1);
+    if (next === 33 || next === 63) {
+      continue;
+    }
+
+    let j = i + 1;
+    let closing = false;
+    if (html.charCodeAt(j) === 47) {
+      closing = true;
+      j += 1;
+    }
+
+    while (j < len && html.charCodeAt(j) <= 32) {
+      j += 1;
+    }
+
+    const nameStart = j;
+    while (j < len) {
+      const c = html.charCodeAt(j);
+      const isNameChar =
+        (c >= 65 && c <= 90) ||
+        (c >= 97 && c <= 122) ||
+        (c >= 48 && c <= 57) ||
+        c === 58 ||
+        c === 45;
+      if (!isNameChar) {
+        break;
+      }
+      j += 1;
+    }
+
+    if (j === nameStart) {
+      continue;
+    }
+
+    if (closing) {
+      if (depth > 0) {
+        depth -= 1;
+      }
+      continue;
+    }
+    const tagName = html.slice(nameStart, j).toLowerCase();
+    if (HTML_VOID_TAGS.has(tagName)) {
+      continue;
+    }
+
+    let selfClosing = false;
+    for (let k = j; k < len && k < j + 200; k++) {
+      const c = html.charCodeAt(k);
+      if (c === 62) {
+        selfClosing = html.charCodeAt(k - 1) === 47;
+        break;
+      }
+    }
+    if (selfClosing) {
+      continue;
+    }
+
+    depth += 1;
+    if (depth > maxDepth) {
+      return true;
+    }
+  }
+  return false;
+}
+
+async function extractWithReadability(
+  request: WebContentExtractionRequest,
+): Promise<WebContentExtractionResult | null> {
+  const cleanHtml = await sanitizeHtml(request.html);
+  if (
+    cleanHtml.length > READABILITY_MAX_HTML_CHARS ||
+    exceedsEstimatedHtmlNestingDepth(cleanHtml, READABILITY_MAX_ESTIMATED_NESTING_DEPTH)
+  ) {
+    return null;
+  }
+  try {
+    const { Readability, parseHTML } = await loadReadabilityDeps();
+    const { document } = parseHTML(cleanHtml);
+    try {
+      (document as { baseURI?: string }).baseURI = request.url;
+    } catch {
+      // Best-effort base URI for relative links.
+    }
+    const reader = new Readability(document, { charThreshold: 0 });
+    const parsed = reader.parse();
+    if (!parsed?.content) {
+      return null;
+    }
+    const title = parsed.title || undefined;
+    if (request.extractMode === "text") {
+      const text = stripInvisibleUnicode(normalizeWhitespace(parsed.textContent ?? ""));
+      return text ? { text, title } : null;
+    }
+    const rendered = htmlToMarkdown(parsed.content);
+    const text = stripInvisibleUnicode(rendered.text);
+    return text ? { text, title: title ?? rendered.title } : null;
+  } catch {
+    return null;
+  }
+}
+
+export function createReadabilityWebContentExtractor(): WebContentExtractorPlugin {
+  return {
+    id: "readability",
+    label: "Readability",
+    autoDetectOrder: 10,
+    extract: extractWithReadability,
+  };
+}