Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
29
extensions/memory-core/api.ts
Normal file
29
extensions/memory-core/api.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
// Memory Core API module exposes the plugin public contract.
|
||||
export type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
export type {
|
||||
MemoryEmbeddingProbeResult,
|
||||
MemoryProviderStatus,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
export {
|
||||
dedupeDreamDiaryEntries,
|
||||
removeBackfillDiaryEntries,
|
||||
writeBackfillDiaryEntries,
|
||||
} from "./src/dreaming-narrative.js";
|
||||
export { previewGroundedRemMarkdown } from "./src/rem-evidence.js";
|
||||
export { filterRecallEntriesWithinLookback } from "./src/dreaming-phases.js";
|
||||
export { previewRemHarness } from "./src/rem-harness.js";
|
||||
export type { PreviewRemHarnessOptions, PreviewRemHarnessResult } from "./src/rem-harness.js";
|
||||
export { configureMemoryCoreDreamingState } from "./src/dreaming-state.js";
|
||||
export {
|
||||
buildDreamingShadowTrialReport,
|
||||
defaultDreamingShadowTrialReportPath,
|
||||
resolveDreamingShadowTrialRecommendation,
|
||||
writeDreamingShadowTrialReport,
|
||||
} from "./src/dreaming-shadow-trial.js";
|
||||
export type {
|
||||
DreamingShadowTrialInput,
|
||||
DreamingShadowTrialRecommendation,
|
||||
DreamingShadowTrialReport,
|
||||
DreamingShadowTrialVerdict,
|
||||
} from "./src/dreaming-shadow-trial.js";
|
||||
25
extensions/memory-core/cli-metadata.ts
Normal file
25
extensions/memory-core/cli-metadata.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
// Memory Core plugin module implements cli metadata behavior.
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/core";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "memory-core",
|
||||
name: "Memory (Core)",
|
||||
description: "File-backed memory search tools and CLI",
|
||||
register(api) {
|
||||
api.registerCli(
|
||||
async ({ program }) => {
|
||||
const { registerMemoryCli } = await import("./cli.js");
|
||||
registerMemoryCli(program);
|
||||
},
|
||||
{
|
||||
descriptors: [
|
||||
{
|
||||
name: "memory",
|
||||
description: "Search, inspect, and reindex memory files",
|
||||
hasSubcommands: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
2
extensions/memory-core/cli.ts
Normal file
2
extensions/memory-core/cli.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
// Memory Core plugin module implements cli behavior.
|
||||
export { registerMemoryCli } from "./src/cli.js";
|
||||
1471
extensions/memory-core/doctor-contract-api.test.ts
Normal file
1471
extensions/memory-core/doctor-contract-api.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
1248
extensions/memory-core/doctor-contract-api.ts
Normal file
1248
extensions/memory-core/doctor-contract-api.ts
Normal file
File diff suppressed because it is too large
Load Diff
262
extensions/memory-core/index.test.ts
Normal file
262
extensions/memory-core/index.test.ts
Normal file
@@ -0,0 +1,262 @@
|
||||
// Memory Core tests cover index plugin behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { OpenClawPluginCommandDefinition } from "openclaw/plugin-sdk/core";
|
||||
import type { MemoryPluginRuntime } from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildMemoryFlushPlan,
|
||||
DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES,
|
||||
DEFAULT_MEMORY_FLUSH_PROMPT,
|
||||
DEFAULT_MEMORY_FLUSH_SOFT_TOKENS,
|
||||
} from "./src/flush-plan.js";
|
||||
import { buildPromptSection } from "./src/prompt-section.js";
|
||||
|
||||
const closeMemorySearchManagerMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock("./src/runtime-provider.js", () => ({
|
||||
memoryRuntime: {
|
||||
closeAllMemorySearchManagers: vi.fn(async () => {}),
|
||||
closeMemorySearchManager: closeMemorySearchManagerMock,
|
||||
getMemorySearchManager: vi.fn(async () => null),
|
||||
},
|
||||
}));
|
||||
|
||||
import plugin from "./index.js";
|
||||
|
||||
function registerMemoryCoreRuntime(): MemoryPluginRuntime {
|
||||
let runtime: MemoryPluginRuntime | undefined;
|
||||
plugin.register(
|
||||
createTestPluginApi({
|
||||
registerMemoryCapability(capability) {
|
||||
runtime = capability.runtime;
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (!runtime) {
|
||||
throw new Error("expected memory-core to register a memory runtime");
|
||||
}
|
||||
return runtime;
|
||||
}
|
||||
|
||||
describe("buildPromptSection", () => {
|
||||
it("returns empty when no memory tools are available", () => {
|
||||
expect(buildPromptSection({ availableTools: new Set() })).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("describes the two-step flow when both memory tools are available", () => {
|
||||
const result = buildPromptSection({
|
||||
availableTools: new Set(["memory_search", "memory_get"]),
|
||||
});
|
||||
expect(result[0]).toBe("## Memory Recall");
|
||||
expect(result[1]).toContain("run memory_search");
|
||||
expect(result[1]).toContain("then use memory_get");
|
||||
expect(result[1]).toContain("indexed session transcripts");
|
||||
expect(result).toContain(
|
||||
"Citations: include Source: <path#line> when it helps the user verify memory snippets.",
|
||||
);
|
||||
expect(result.at(-1)).toBe("");
|
||||
});
|
||||
|
||||
it("limits the guidance to memory_search when only search is available", () => {
|
||||
const result = buildPromptSection({ availableTools: new Set(["memory_search"]) });
|
||||
expect(result[0]).toBe("## Memory Recall");
|
||||
expect(result[1]).toContain("run memory_search");
|
||||
expect(result[1]).toContain("indexed session transcripts");
|
||||
expect(result[1]).not.toContain("then use memory_get");
|
||||
});
|
||||
|
||||
it("limits the guidance to memory_get when only get is available", () => {
|
||||
const result = buildPromptSection({ availableTools: new Set(["memory_get"]) });
|
||||
expect(result[0]).toBe("## Memory Recall");
|
||||
expect(result[1]).toContain("run memory_get");
|
||||
expect(result[1]).not.toContain("run memory_search");
|
||||
});
|
||||
|
||||
it("includes citations-off instruction when citationsMode is off", () => {
|
||||
const result = buildPromptSection({
|
||||
availableTools: new Set(["memory_search"]),
|
||||
citationsMode: "off",
|
||||
});
|
||||
expect(result).toContain(
|
||||
"Citations are disabled: do not mention file paths or line numbers in replies unless the user explicitly asks.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory-core plugin runtime registration", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("registers the dreaming runtime slash command", () => {
|
||||
let command: OpenClawPluginCommandDefinition | undefined;
|
||||
plugin.register(
|
||||
createTestPluginApi({
|
||||
registerCommand(definition) {
|
||||
command = definition;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(command?.name).toBe("dreaming");
|
||||
expect(command?.acceptsArgs).toBe(true);
|
||||
expect(command?.exposeSenderIsOwner).toBe(true);
|
||||
expect(command?.description).toContain("Enable or disable");
|
||||
});
|
||||
|
||||
it("wires scoped memory search cleanup through the lazy runtime", async () => {
|
||||
const runtime = registerMemoryCoreRuntime();
|
||||
const cfg = {} as OpenClawConfig;
|
||||
|
||||
await runtime.closeMemorySearchManager?.({ cfg, agentId: "main" });
|
||||
|
||||
expect(closeMemorySearchManagerMock).toHaveBeenCalledWith({ cfg, agentId: "main" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildMemoryFlushPlan", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: {
|
||||
userTimezone: "America/New_York",
|
||||
timeFormat: "12",
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
it("replaces YYYY-MM-DD using user timezone and appends current time", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
...cfg,
|
||||
agents: {
|
||||
...cfg.agents,
|
||||
defaults: {
|
||||
...cfg.agents?.defaults,
|
||||
compaction: {
|
||||
memoryFlush: {
|
||||
prompt: "Store durable notes in memory/YYYY-MM-DD.md",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
nowMs: Date.UTC(2026, 1, 16, 15, 0, 0),
|
||||
});
|
||||
|
||||
expect(plan?.prompt).toContain("memory/2026-02-16.md");
|
||||
expect(plan?.prompt).toContain(
|
||||
"Current time: Monday, February 16th, 2026 - 10:00 AM (America/New_York)",
|
||||
);
|
||||
expect(plan?.prompt).toContain("Reference UTC: 2026-02-16 15:00 UTC");
|
||||
expect(plan?.relativePath).toBe("memory/2026-02-16.md");
|
||||
});
|
||||
|
||||
it("does not append a duplicate current time line", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
...cfg,
|
||||
agents: {
|
||||
...cfg.agents,
|
||||
defaults: {
|
||||
...cfg.agents?.defaults,
|
||||
compaction: {
|
||||
memoryFlush: {
|
||||
prompt: "Store notes.\nCurrent time: already present",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
nowMs: Date.UTC(2026, 1, 16, 15, 0, 0),
|
||||
});
|
||||
|
||||
expect(plan?.prompt).toContain("Current time: already present");
|
||||
expect((plan?.prompt.match(/Current time:/g) ?? []).length).toBe(1);
|
||||
});
|
||||
|
||||
it("defaults to safe prompts and gating values", () => {
|
||||
const plan = buildMemoryFlushPlan();
|
||||
expect(plan?.softThresholdTokens).toBe(DEFAULT_MEMORY_FLUSH_SOFT_TOKENS);
|
||||
expect(plan?.forceFlushTranscriptBytes).toBe(DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES);
|
||||
expect(plan?.prompt).toContain("memory/");
|
||||
expect(plan?.prompt).toContain("MEMORY.md");
|
||||
expect(plan?.systemPrompt).toContain("MEMORY.md");
|
||||
});
|
||||
|
||||
it("respects disable flag", () => {
|
||||
expect(
|
||||
buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: { compaction: { memoryFlush: { enabled: false } } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("carries configured memory flush model override", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
compaction: {
|
||||
memoryFlush: {
|
||||
model: "ollama/qwen3:8b",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(plan?.model).toBe("ollama/qwen3:8b");
|
||||
});
|
||||
|
||||
it("falls back to defaults when numeric values are invalid", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
compaction: {
|
||||
reserveTokensFloor: Number.NaN,
|
||||
memoryFlush: {
|
||||
softThresholdTokens: -100,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(plan?.softThresholdTokens).toBe(DEFAULT_MEMORY_FLUSH_SOFT_TOKENS);
|
||||
expect(plan?.forceFlushTranscriptBytes).toBe(DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES);
|
||||
expect(plan?.reserveTokensFloor).toBe(20_000);
|
||||
});
|
||||
|
||||
it("parses forceFlushTranscriptBytes from byte-size strings", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
compaction: {
|
||||
memoryFlush: {
|
||||
forceFlushTranscriptBytes: "3mb",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(plan?.forceFlushTranscriptBytes).toBe(3 * 1024 * 1024);
|
||||
});
|
||||
|
||||
it("keeps overwrite guards in the default prompt", () => {
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toMatch(/APPEND/i);
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("do not overwrite");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("timestamped variant");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("YYYY-MM-DD.md");
|
||||
});
|
||||
});
|
||||
228
extensions/memory-core/index.ts
Normal file
228
extensions/memory-core/index.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
||||
// Memory Core plugin entrypoint registers its OpenClaw integration.
|
||||
import {
|
||||
jsonResult,
|
||||
resolveMemorySearchConfig,
|
||||
resolveSessionAgentIds,
|
||||
type MemoryPluginRuntime,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
import { resolveMemoryBackendConfig } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
import {
|
||||
definePluginEntry,
|
||||
type AnyAgentTool,
|
||||
type OpenClawPluginToolContext,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime";
|
||||
import type { TSchema } from "typebox";
|
||||
import { configureMemoryCoreDreamingState } from "./src/dreaming-state.js";
|
||||
import { registerShortTermPromotionDreaming } from "./src/dreaming.js";
|
||||
import { buildMemoryFlushPlan } from "./src/flush-plan.js";
|
||||
import { buildPromptSection } from "./src/prompt-section.js";
|
||||
|
||||
type MemoryToolsModule = typeof import("./src/tools.js");
|
||||
|
||||
type MemoryToolOptions = {
|
||||
config?: OpenClawConfig;
|
||||
getConfig?: () => OpenClawConfig | undefined;
|
||||
agentId?: string;
|
||||
agentSessionKey?: string;
|
||||
sandboxed?: boolean;
|
||||
oneShotCliRun?: boolean;
|
||||
};
|
||||
|
||||
const loadMemoryToolsModule = createLazyRuntimeModule(() => import("./src/tools.js"));
|
||||
|
||||
const loadRuntimeProviderModule = createLazyRuntimeModule(
|
||||
() => import("./src/runtime-provider.js"),
|
||||
);
|
||||
|
||||
function getToolConfig(options: MemoryToolOptions): OpenClawConfig | undefined {
|
||||
return options.getConfig?.() ?? options.config;
|
||||
}
|
||||
|
||||
function hasMemoryToolContext(options: MemoryToolOptions): boolean {
|
||||
const cfg = getToolConfig(options);
|
||||
if (!cfg) {
|
||||
return false;
|
||||
}
|
||||
const { sessionAgentId: agentId } = resolveSessionAgentIds({
|
||||
sessionKey: options.agentSessionKey,
|
||||
config: cfg,
|
||||
agentId: options.agentId,
|
||||
});
|
||||
return Boolean(resolveMemorySearchConfig(cfg, agentId));
|
||||
}
|
||||
|
||||
const MemorySearchSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
query: { type: "string" },
|
||||
maxResults: { type: "integer", minimum: 1 },
|
||||
minScore: { type: "number" },
|
||||
corpus: { type: "string", enum: ["memory", "wiki", "all", "sessions"] },
|
||||
},
|
||||
required: ["query"],
|
||||
additionalProperties: false,
|
||||
} as const satisfies TSchema;
|
||||
|
||||
const MemoryGetSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
path: { type: "string" },
|
||||
from: { type: "integer", minimum: 1 },
|
||||
lines: { type: "integer", minimum: 1 },
|
||||
corpus: { type: "string", enum: ["memory", "wiki", "all"] },
|
||||
},
|
||||
required: ["path"],
|
||||
additionalProperties: false,
|
||||
} as const satisfies TSchema;
|
||||
|
||||
function createLazyMemoryTool(params: {
|
||||
options: MemoryToolOptions;
|
||||
label: string;
|
||||
name: "memory_search" | "memory_get";
|
||||
description: string;
|
||||
parameters: typeof MemorySearchSchema | typeof MemoryGetSchema;
|
||||
load: (module: MemoryToolsModule, options: MemoryToolOptions) => AnyAgentTool | null;
|
||||
}): AnyAgentTool | null {
|
||||
if (!hasMemoryToolContext(params.options)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let toolPromise: Promise<AnyAgentTool | null> | undefined;
|
||||
const loadTool = async () => {
|
||||
toolPromise ??= loadMemoryToolsModule().then((module) => params.load(module, params.options));
|
||||
return await toolPromise;
|
||||
};
|
||||
|
||||
return {
|
||||
label: params.label,
|
||||
name: params.name,
|
||||
description: params.description,
|
||||
parameters: params.parameters,
|
||||
execute: async (toolCallId, toolParams, signal, onUpdate) => {
|
||||
const tool = await loadTool();
|
||||
if (!tool) {
|
||||
return jsonResult({
|
||||
disabled: true,
|
||||
unavailable: true,
|
||||
error: "memory search unavailable",
|
||||
});
|
||||
}
|
||||
return await tool.execute(toolCallId, toolParams, signal, onUpdate);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createLazyMemorySearchTool(options: MemoryToolOptions): AnyAgentTool | null {
|
||||
return createLazyMemoryTool({
|
||||
options,
|
||||
label: "Memory Search",
|
||||
name: "memory_search",
|
||||
description:
|
||||
"Mandatory recall step: semantically search MEMORY.md + memory/*.md (and optional session transcripts) before answering questions about prior work, decisions, dates, people, preferences, or todos. Optional `corpus=wiki` or `corpus=all` also searches registered compiled-wiki supplements. `corpus=memory` restricts hits to indexed memory files (excludes session transcript chunks from ranking). `corpus=sessions` restricts hits to indexed session transcripts (same visibility rules as session history tools). If response has disabled=true, memory retrieval is unavailable and should be surfaced to the user.",
|
||||
parameters: MemorySearchSchema,
|
||||
load: (module, loadOptions) => module.createMemorySearchTool(loadOptions),
|
||||
});
|
||||
}
|
||||
|
||||
function createLazyMemoryGetTool(options: MemoryToolOptions): AnyAgentTool | null {
|
||||
return createLazyMemoryTool({
|
||||
options,
|
||||
label: "Memory Get",
|
||||
name: "memory_get",
|
||||
description:
|
||||
"Safe exact excerpt read from MEMORY.md or memory/*.md. Defaults to a bounded excerpt when lines are omitted, includes truncation/continuation info when more content exists, and `corpus=wiki` reads from registered compiled-wiki supplements.",
|
||||
parameters: MemoryGetSchema,
|
||||
load: (module, loadOptions) => module.createMemoryGetTool(loadOptions),
|
||||
});
|
||||
}
|
||||
|
||||
function resolveMemoryToolOptions(ctx: OpenClawPluginToolContext): MemoryToolOptions {
|
||||
const getConfig = () => ctx.getRuntimeConfig?.() ?? ctx.runtimeConfig ?? ctx.config;
|
||||
return {
|
||||
config: getConfig(),
|
||||
getConfig,
|
||||
agentId: ctx.agentId,
|
||||
agentSessionKey: ctx.sessionKey,
|
||||
sandboxed: ctx.sandboxed,
|
||||
oneShotCliRun: ctx.oneShotCliRun,
|
||||
};
|
||||
}
|
||||
|
||||
const memoryRuntime: MemoryPluginRuntime = {
|
||||
async getMemorySearchManager(params) {
|
||||
const { memoryRuntime: runtime } = await loadRuntimeProviderModule();
|
||||
return await runtime.getMemorySearchManager(params);
|
||||
},
|
||||
resolveMemoryBackendConfig(params) {
|
||||
return resolveMemoryBackendConfig(params);
|
||||
},
|
||||
async closeAllMemorySearchManagers() {
|
||||
const { memoryRuntime: runtime } = await loadRuntimeProviderModule();
|
||||
await runtime.closeAllMemorySearchManagers?.();
|
||||
},
|
||||
async closeMemorySearchManager(params) {
|
||||
const { memoryRuntime: runtime } = await loadRuntimeProviderModule();
|
||||
await runtime.closeMemorySearchManager?.(params);
|
||||
},
|
||||
};
|
||||
export default definePluginEntry({
|
||||
id: "memory-core",
|
||||
name: "Memory (Core)",
|
||||
description: "File-backed memory search tools and CLI",
|
||||
kind: "memory",
|
||||
register(api) {
|
||||
configureMemoryCoreDreamingState(<T>(options: OpenKeyedStoreOptions) =>
|
||||
api.runtime.state.openKeyedStore<T>(options),
|
||||
);
|
||||
registerShortTermPromotionDreaming(api);
|
||||
api.registerMemoryCapability({
|
||||
promptBuilder: buildPromptSection,
|
||||
flushPlanResolver: buildMemoryFlushPlan,
|
||||
runtime: memoryRuntime,
|
||||
publicArtifacts: {
|
||||
async listArtifacts(params) {
|
||||
const { listMemoryCorePublicArtifacts } = await import("./src/public-artifacts.js");
|
||||
return await listMemoryCorePublicArtifacts(params);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
api.registerTool((ctx) => createLazyMemorySearchTool(resolveMemoryToolOptions(ctx)), {
|
||||
names: ["memory_search"],
|
||||
});
|
||||
|
||||
api.registerTool((ctx) => createLazyMemoryGetTool(resolveMemoryToolOptions(ctx)), {
|
||||
names: ["memory_get"],
|
||||
});
|
||||
|
||||
api.registerCommand({
|
||||
name: "dreaming",
|
||||
description: "Enable or disable memory dreaming.",
|
||||
acceptsArgs: true,
|
||||
exposeSenderIsOwner: true,
|
||||
handler: async (ctx) => {
|
||||
const { handleDreamingCommand } = await import("./src/dreaming-command.js");
|
||||
return await handleDreamingCommand(api, ctx);
|
||||
},
|
||||
});
|
||||
|
||||
api.registerCli(
|
||||
async ({ program }) => {
|
||||
const { registerMemoryCli } = await import("./cli.js");
|
||||
registerMemoryCli(program);
|
||||
},
|
||||
{
|
||||
descriptors: [
|
||||
{
|
||||
name: "memory",
|
||||
description: "Search, inspect, and reindex memory files",
|
||||
hasSubcommands: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
6
extensions/memory-core/manager-runtime.ts
Normal file
6
extensions/memory-core/manager-runtime.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
// Memory Core plugin module implements manager runtime behavior.
|
||||
export {
|
||||
closeAllMemoryIndexManagers,
|
||||
closeMemoryIndexManagersForAgent,
|
||||
MemoryIndexManager,
|
||||
} from "./src/memory/manager-runtime.js";
|
||||
205
extensions/memory-core/openclaw.plugin.json
Normal file
205
extensions/memory-core/openclaw.plugin.json
Normal file
@@ -0,0 +1,205 @@
|
||||
{
|
||||
"id": "memory-core",
|
||||
"activation": {
|
||||
"onStartup": false
|
||||
},
|
||||
"kind": "memory",
|
||||
"contracts": {
|
||||
"tools": ["memory_get", "memory_search"]
|
||||
},
|
||||
"toolMetadata": {
|
||||
"memory_get": {
|
||||
"replaySafe": true
|
||||
}
|
||||
},
|
||||
"commandAliases": [
|
||||
{
|
||||
"name": "dreaming",
|
||||
"kind": "runtime-slash",
|
||||
"cliCommand": "memory"
|
||||
}
|
||||
],
|
||||
"uiHints": {
|
||||
"dreaming.frequency": {
|
||||
"label": "Dreaming Frequency",
|
||||
"placeholder": "0 3 * * *",
|
||||
"help": "Optional cron cadence for the full dreaming sweep (light, REM, then deep)."
|
||||
},
|
||||
"dreaming.model": {
|
||||
"label": "Dreaming Model",
|
||||
"placeholder": "anthropic/claude-sonnet-4-6",
|
||||
"help": "Optional provider/model override for Dream Diary narrative subagent runs. Requires plugins.entries.memory-core.subagent.allowModelOverride."
|
||||
}
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"dreaming": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"frequency": {
|
||||
"type": "string"
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"timezone": {
|
||||
"type": "string"
|
||||
},
|
||||
"verboseLogging": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"storage": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["inline", "separate", "both"]
|
||||
},
|
||||
"separateReports": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"execution": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"defaults": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"phases": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"light": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"lookbackDays": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"dedupeSimilarity": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"execution": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"deep": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"minScore": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"minRecallCount": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"minUniqueQueries": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"recencyHalfLifeDays": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"maxAgeDays": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"maxPromotedSnippetTokens": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"description": "Maximum estimated token count for each short-term recall snippet promoted into MEMORY.md. Provenance metadata remains attached to the entry."
|
||||
},
|
||||
"execution": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"rem": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"lookbackDays": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"minPatternStrength": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"execution": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
29
extensions/memory-core/package.json
Normal file
29
extensions/memory-core/package.json
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "@openclaw/memory-core",
|
||||
"version": "2026.6.11",
|
||||
"private": true,
|
||||
"description": "OpenClaw core memory search plugin",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"chokidar": "5.0.0",
|
||||
"json5": "2.2.3",
|
||||
"typebox": "1.3.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*",
|
||||
"openclaw": "workspace:*"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"openclaw": ">=2026.6.11"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"openclaw": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
36
extensions/memory-core/runtime-api.ts
Normal file
36
extensions/memory-core/runtime-api.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
// Memory Core API module exposes the plugin public contract.
|
||||
export { getMemorySearchManager, MemoryIndexManager } from "./src/memory/index.js";
|
||||
export { memoryRuntime } from "./src/runtime-provider.js";
|
||||
export {
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
getBuiltinMemoryEmbeddingProviderDoctorMetadata,
|
||||
listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata,
|
||||
} from "./src/memory/provider-adapters.js";
|
||||
export { createEmbeddingProvider } from "./src/memory/embeddings.js";
|
||||
export {
|
||||
resolveMemoryCacheSummary,
|
||||
resolveMemoryFtsState,
|
||||
resolveMemoryVectorState,
|
||||
type Tone,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
export { checkQmdBinaryAvailability } from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
export { hasConfiguredMemorySecretInput } from "openclaw/plugin-sdk/memory-core-host-secret";
|
||||
export { auditDreamingArtifacts, repairDreamingArtifacts } from "./src/dreaming-repair.js";
|
||||
export { configureMemoryCoreDreamingState } from "./src/dreaming-state.js";
|
||||
export {
|
||||
auditShortTermPromotionArtifacts,
|
||||
loadShortTermPromotionDreamingStats,
|
||||
removeGroundedShortTermCandidates,
|
||||
repairShortTermPromotionArtifacts,
|
||||
} from "./src/short-term-promotion.js";
|
||||
export type { BuiltinMemoryEmbeddingProviderDoctorMetadata } from "./src/memory/provider-adapters.js";
|
||||
export type {
|
||||
DreamingArtifactsAuditSummary,
|
||||
RepairDreamingArtifactsResult,
|
||||
} from "./src/dreaming-repair.js";
|
||||
export type {
|
||||
RepairShortTermPromotionArtifactsResult,
|
||||
ShortTermDreamingStats,
|
||||
ShortTermDreamingStatsEntry,
|
||||
ShortTermAuditSummary,
|
||||
} from "./src/short-term-promotion.js";
|
||||
27
extensions/memory-core/src/cli.host.runtime.ts
Normal file
27
extensions/memory-core/src/cli.host.runtime.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
// Memory Core plugin module implements cli.host behavior.
|
||||
export {
|
||||
colorize,
|
||||
defaultRuntime,
|
||||
formatErrorMessage,
|
||||
isRich,
|
||||
resolveCommandSecretRefsViaGateway,
|
||||
setVerbose,
|
||||
shortenHomeInString,
|
||||
shortenHomePath,
|
||||
theme,
|
||||
withManager,
|
||||
withProgress,
|
||||
withProgressTotals,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-cli";
|
||||
export {
|
||||
getRuntimeConfig,
|
||||
resolveDefaultAgentId,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
resolveStateDir,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
export {
|
||||
listMemoryFiles,
|
||||
normalizeExtraMemoryPaths,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
export { getMemorySearchManager } from "./memory/index.js";
|
||||
2019
extensions/memory-core/src/cli.runtime.ts
Normal file
2019
extensions/memory-core/src/cli.runtime.ts
Normal file
File diff suppressed because it is too large
Load Diff
2396
extensions/memory-core/src/cli.test.ts
Normal file
2396
extensions/memory-core/src/cli.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
268
extensions/memory-core/src/cli.ts
Normal file
268
extensions/memory-core/src/cli.ts
Normal file
@@ -0,0 +1,268 @@
|
||||
// Memory Core plugin module implements cli behavior.
|
||||
import type { Command } from "commander";
|
||||
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
||||
import {
|
||||
formatDocsLink,
|
||||
formatHelpExamples,
|
||||
theme,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-cli";
|
||||
import {
|
||||
parseStrictNonNegativeInteger,
|
||||
parseStrictPositiveInteger,
|
||||
} from "openclaw/plugin-sdk/number-runtime";
|
||||
import type {
|
||||
MemoryCommandOptions,
|
||||
MemoryPromoteCommandOptions,
|
||||
MemoryPromoteExplainOptions,
|
||||
MemoryRemBackfillOptions,
|
||||
MemoryRemHarnessOptions,
|
||||
MemorySearchCommandOptions,
|
||||
} from "./cli.types.js";
|
||||
import {
|
||||
DEFAULT_PROMOTION_MIN_RECALL_COUNT,
|
||||
DEFAULT_PROMOTION_MIN_SCORE,
|
||||
DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES,
|
||||
} from "./short-term-promotion.js";
|
||||
|
||||
const loadMemoryCliRuntime = createLazyRuntimeModule(() => import("./cli.runtime.js"));
|
||||
|
||||
const DECIMAL_NUMBER_RE = /^[+-]?(?:\d+(?:\.\d+)?|\.\d+)$/;
|
||||
|
||||
export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryStatus(opts);
|
||||
}
|
||||
|
||||
async function runMemoryIndex(opts: MemoryCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryIndex(opts);
|
||||
}
|
||||
|
||||
async function runMemorySearch(queryArg: string | undefined, opts: MemorySearchCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemorySearch(queryArg, opts);
|
||||
}
|
||||
|
||||
async function runMemoryPromote(opts: MemoryPromoteCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryPromote(opts);
|
||||
}
|
||||
|
||||
async function runMemoryPromoteExplain(
|
||||
selectorArg: string | undefined,
|
||||
opts: MemoryPromoteExplainOptions,
|
||||
) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryPromoteExplain(selectorArg, opts);
|
||||
}
|
||||
|
||||
async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryRemHarness(opts);
|
||||
}
|
||||
|
||||
async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryRemBackfill(opts);
|
||||
}
|
||||
|
||||
function invalidCliArgument(message: string): Error & { code: string; exitCode: number } {
|
||||
const error = new Error(message) as Error & { code: string; exitCode: number };
|
||||
error.name = "InvalidArgumentError";
|
||||
// Commander recognizes parser failures by code; keep the import type-only for bundled plugin deps.
|
||||
error.code = "commander.invalidArgument";
|
||||
error.exitCode = 1;
|
||||
return error;
|
||||
}
|
||||
|
||||
function parseMemoryCliNumberOption(value: string, flag: string): number {
|
||||
const trimmed = value.trim();
|
||||
const parsed = DECIMAL_NUMBER_RE.test(trimmed) ? Number(trimmed) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) {
|
||||
throw invalidCliArgument(`${flag} must be a finite number.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseMemoryCliPositiveIntegerOption(value: string, flag: string): number {
|
||||
const parsed = parseStrictPositiveInteger(value);
|
||||
if (parsed === undefined) {
|
||||
throw invalidCliArgument(`${flag} must be a positive integer.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseMemoryCliNonNegativeIntegerOption(value: string, flag: string): number {
|
||||
const parsed = parseStrictNonNegativeInteger(value);
|
||||
if (parsed === undefined) {
|
||||
throw invalidCliArgument(`${flag} must be a non-negative integer.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function registerMemoryCli(program: Command) {
|
||||
const memory = program
|
||||
.command("memory")
|
||||
.description("Search, inspect, and reindex memory files")
|
||||
.addHelpText(
|
||||
"after",
|
||||
() =>
|
||||
`\n${theme.heading("Examples:")}\n${formatHelpExamples([
|
||||
["openclaw memory status", "Show index and provider status."],
|
||||
[
|
||||
"openclaw memory status --fix",
|
||||
"Repair stale recall locks and normalize promotion metadata.",
|
||||
],
|
||||
["openclaw memory status --deep", "Probe embedding provider readiness."],
|
||||
["openclaw memory index --force", "Force a full reindex."],
|
||||
['openclaw memory search "meeting notes"', "Quick search using positional query."],
|
||||
[
|
||||
'openclaw memory search --query "deployment" --max-results 20',
|
||||
"Limit results for focused troubleshooting.",
|
||||
],
|
||||
[
|
||||
`openclaw memory promote --limit 10 --min-score ${DEFAULT_PROMOTION_MIN_SCORE}`,
|
||||
"Review weighted short-term candidates for long-term memory.",
|
||||
],
|
||||
[
|
||||
"openclaw memory promote --apply",
|
||||
"Append top-ranked short-term candidates into MEMORY.md.",
|
||||
],
|
||||
[
|
||||
'openclaw memory promote-explain "router vlan"',
|
||||
"Explain why a specific candidate would or would not promote.",
|
||||
],
|
||||
[
|
||||
"openclaw memory rem-harness --json",
|
||||
"Preview REM reflections, candidate truths, and deep promotion output.",
|
||||
],
|
||||
[
|
||||
"openclaw memory rem-backfill --path ./memory",
|
||||
"Write grounded historical REM entries into DREAMS.md for UI review.",
|
||||
],
|
||||
[
|
||||
"openclaw memory rem-backfill --path ./memory --stage-short-term",
|
||||
"Also seed durable grounded candidates into the live short-term promotion store.",
|
||||
],
|
||||
["openclaw memory status --json", "Output machine-readable JSON (good for scripts)."],
|
||||
])}\n\n${theme.muted("Docs:")} ${formatDocsLink("/cli/memory", "docs.openclaw.ai/cli/memory")}\n`,
|
||||
);
|
||||
|
||||
memory
|
||||
.command("status")
|
||||
.description("Show memory search index status")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--json", "Print JSON")
|
||||
.option("--deep", "Probe embedding provider availability")
|
||||
.option("--index", "Reindex if dirty (implies --deep)")
|
||||
.option("--fix", "Repair stale recall locks and normalize promotion metadata")
|
||||
.option("--verbose", "Verbose logging", false)
|
||||
.action(async (opts: MemoryCommandOptions & { force?: boolean }) => {
|
||||
await runMemoryStatus(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("index")
|
||||
.description("Reindex memory files")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--force", "Force full reindex", false)
|
||||
.option("--verbose", "Verbose logging", false)
|
||||
.action(async (opts: MemoryCommandOptions) => {
|
||||
await runMemoryIndex(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("search")
|
||||
.description("Search memory files")
|
||||
.argument("[query]", "Search query")
|
||||
.option("--query <text>", "Search query (alternative to positional argument)")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--max-results <n>", "Max results", (value: string) =>
|
||||
parseMemoryCliPositiveIntegerOption(value, "--max-results"),
|
||||
)
|
||||
.option("--min-score <n>", "Minimum score", (value: string) =>
|
||||
parseMemoryCliNumberOption(value, "--min-score"),
|
||||
)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (queryArg: string | undefined, opts: MemorySearchCommandOptions) => {
|
||||
await runMemorySearch(queryArg, opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("promote")
|
||||
.description("Rank short-term recalls and optionally append top entries to MEMORY.md")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--limit <n>", "Max candidates", (value: string) =>
|
||||
parseMemoryCliPositiveIntegerOption(value, "--limit"),
|
||||
)
|
||||
.option(
|
||||
"--min-score <n>",
|
||||
`Minimum weighted score (default: ${DEFAULT_PROMOTION_MIN_SCORE})`,
|
||||
(value: string) => parseMemoryCliNumberOption(value, "--min-score"),
|
||||
)
|
||||
.option(
|
||||
"--min-recall-count <n>",
|
||||
`Minimum recall count (default: ${DEFAULT_PROMOTION_MIN_RECALL_COUNT})`,
|
||||
(value: string) => parseMemoryCliNonNegativeIntegerOption(value, "--min-recall-count"),
|
||||
)
|
||||
.option(
|
||||
"--min-unique-queries <n>",
|
||||
`Minimum distinct query count (default: ${DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES})`,
|
||||
(value: string) => parseMemoryCliNonNegativeIntegerOption(value, "--min-unique-queries"),
|
||||
)
|
||||
.option("--apply", "Append selected candidates to MEMORY.md", false)
|
||||
.option("--include-promoted", "Include already promoted candidates", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryPromoteCommandOptions) => {
|
||||
await runMemoryPromote(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("promote-explain")
|
||||
.description("Explain a specific promotion candidate and its score breakdown")
|
||||
.argument("<selector>", "Candidate key, path fragment, or snippet fragment")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--include-promoted", "Include already promoted candidates", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (selectorArg: string | undefined, opts: MemoryPromoteExplainOptions) => {
|
||||
await runMemoryPromoteExplain(selectorArg, opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("rem-harness")
|
||||
.description("Preview REM reflections, candidate truths, and deep promotions without writing")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--path <file-or-dir>", "Seed the harness from historical daily memory file(s)")
|
||||
.option("--grounded", "Also render a grounded day-level REM preview")
|
||||
.option("--include-promoted", "Include already promoted deep candidates", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryRemHarnessOptions) => {
|
||||
await runMemoryRemHarness(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("rem-backfill")
|
||||
.description("Write grounded historical REM summaries into DREAMS.md for UI review")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--path <file-or-dir>", "Historical daily memory file(s) or directory")
|
||||
.option("--rollback", "Remove previously written grounded REM backfill entries", false)
|
||||
.option(
|
||||
"--stage-short-term",
|
||||
"Also seed grounded durable candidates into the short-term promotion store",
|
||||
false,
|
||||
)
|
||||
.option(
|
||||
"--rollback-short-term",
|
||||
"Remove previously seeded grounded short-term candidates",
|
||||
false,
|
||||
)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryRemBackfillOptions) => {
|
||||
await runMemoryRemBackfill(opts);
|
||||
});
|
||||
|
||||
memory.action(() => {
|
||||
memory.outputHelp();
|
||||
process.exitCode = 0;
|
||||
});
|
||||
}
|
||||
42
extensions/memory-core/src/cli.types.ts
Normal file
42
extensions/memory-core/src/cli.types.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
// Memory Core type declarations define plugin contracts.
|
||||
export type MemoryCommandOptions = {
|
||||
agent?: string;
|
||||
json?: boolean;
|
||||
deep?: boolean;
|
||||
index?: boolean;
|
||||
force?: boolean;
|
||||
fix?: boolean;
|
||||
verbose?: boolean;
|
||||
};
|
||||
|
||||
export type MemorySearchCommandOptions = MemoryCommandOptions & {
|
||||
query?: string;
|
||||
maxResults?: number;
|
||||
minScore?: number;
|
||||
};
|
||||
|
||||
export type MemoryPromoteCommandOptions = MemoryCommandOptions & {
|
||||
limit?: number;
|
||||
minScore?: number;
|
||||
minRecallCount?: number;
|
||||
minUniqueQueries?: number;
|
||||
apply?: boolean;
|
||||
includePromoted?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryPromoteExplainOptions = MemoryCommandOptions & {
|
||||
includePromoted?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryRemHarnessOptions = MemoryCommandOptions & {
|
||||
includePromoted?: boolean;
|
||||
path?: string;
|
||||
grounded?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryRemBackfillOptions = MemoryCommandOptions & {
|
||||
path?: string;
|
||||
rollback?: boolean;
|
||||
stageShortTerm?: boolean;
|
||||
rollbackShortTerm?: boolean;
|
||||
};
|
||||
114
extensions/memory-core/src/concept-vocabulary.test.ts
Normal file
114
extensions/memory-core/src/concept-vocabulary.test.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
// Memory Core tests cover concept vocabulary plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
classifyConceptTagScript,
|
||||
deriveConceptTags,
|
||||
summarizeConceptTagScriptCoverage,
|
||||
} from "./concept-vocabulary.js";
|
||||
|
||||
describe("concept vocabulary", () => {
|
||||
it("extracts Unicode-aware concept tags for common European languages", () => {
|
||||
const tags = deriveConceptTags({
|
||||
path: "memory/2026-04-04.md",
|
||||
snippet:
|
||||
"Configuración de gateway, configuration du routeur, Sicherung und Überwachung Glacier.",
|
||||
});
|
||||
|
||||
expect(tags).toStrictEqual([
|
||||
"gateway",
|
||||
"glacier",
|
||||
"routeur",
|
||||
"sicherung",
|
||||
"überwachung",
|
||||
"configuración",
|
||||
"configuration",
|
||||
]);
|
||||
expect(tags).not.toContain("de");
|
||||
expect(tags).not.toContain("du");
|
||||
expect(tags).not.toContain("und");
|
||||
expect(tags).not.toContain("2026-04-04.md");
|
||||
});
|
||||
|
||||
it("preserves short protected-glossary terms past the latin minimum-length gate", () => {
|
||||
const tags = deriveConceptTags({
|
||||
path: "memory/2026-04-04.md",
|
||||
snippet: "Store the session in kv and back up to s3 nightly.",
|
||||
});
|
||||
|
||||
// "kv" and "s3" are 2-char latin glossary entries that the generic min-length-3 gate would drop.
|
||||
expect(tags).toContain("kv");
|
||||
expect(tags).toContain("s3");
|
||||
});
|
||||
|
||||
it("does not surface short glossary terms that only appear inside longer words", () => {
|
||||
const tags = deriveConceptTags({
|
||||
path: "memory/2026-04-04.md",
|
||||
snippet: "Played the mkv recording and tuned the css3 layout.",
|
||||
});
|
||||
|
||||
// "kv"/"s3" are substrings of "mkv"/"css3"; whole-word matching must not emit them as tags.
|
||||
expect(tags).not.toContain("kv");
|
||||
expect(tags).not.toContain("s3");
|
||||
expect(tags).toContain("mkv");
|
||||
expect(tags).toContain("css3");
|
||||
});
|
||||
|
||||
it("extracts protected and segmented CJK concept tags", () => {
|
||||
const tags = deriveConceptTags({
|
||||
path: "memory/2026-04-04.md",
|
||||
snippet:
|
||||
"障害対応ルーター設定とバックアップ確認。路由器备份与网关同步。라우터 백업 페일오버 점검.",
|
||||
});
|
||||
|
||||
expect(tags).toStrictEqual([
|
||||
"バックアップ",
|
||||
"ルーター",
|
||||
"障害対応",
|
||||
"路由器",
|
||||
"备份",
|
||||
"网关",
|
||||
"라우터",
|
||||
"백업",
|
||||
]);
|
||||
expect(tags).not.toContain("ルー");
|
||||
expect(tags).not.toContain("ター");
|
||||
});
|
||||
|
||||
it("classifies concept tags by script family", () => {
|
||||
expect(classifyConceptTagScript("routeur")).toBe("latin");
|
||||
expect(classifyConceptTagScript("路由器")).toBe("cjk");
|
||||
expect(classifyConceptTagScript("qmd路由器")).toBe("mixed");
|
||||
});
|
||||
|
||||
it("drops chat scaffolding stop words from derived concept tags", () => {
|
||||
const tags = deriveConceptTags({
|
||||
path: "memory/.dreams/session-corpus/2026-04-16.txt",
|
||||
snippet:
|
||||
"Assistant: the system should remind you about the Ollama provider setup in your workspace.",
|
||||
});
|
||||
|
||||
expect(tags).toContain("ollama");
|
||||
expect(tags).toContain("provider");
|
||||
expect(tags).not.toContain("assistant");
|
||||
expect(tags).not.toContain("system");
|
||||
expect(tags).not.toContain("the");
|
||||
expect(tags).not.toContain("you");
|
||||
expect(tags).not.toContain("your");
|
||||
});
|
||||
|
||||
it("summarizes entry coverage across latin, cjk, and mixed tags", () => {
|
||||
expect(
|
||||
summarizeConceptTagScriptCoverage([
|
||||
["routeur", "sauvegarde"],
|
||||
["路由器", "备份"],
|
||||
["qmd", "路由器"],
|
||||
["сервер"],
|
||||
]),
|
||||
).toEqual({
|
||||
latinEntryCount: 1,
|
||||
cjkEntryCount: 1,
|
||||
mixedEntryCount: 1,
|
||||
otherEntryCount: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
509
extensions/memory-core/src/concept-vocabulary.ts
Normal file
509
extensions/memory-core/src/concept-vocabulary.ts
Normal file
@@ -0,0 +1,509 @@
|
||||
// Memory Core plugin module implements concept vocabulary behavior.
|
||||
import path from "node:path";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
|
||||
export const MAX_CONCEPT_TAGS = 8;
|
||||
|
||||
type ConceptTagScriptFamily = "latin" | "cjk" | "mixed" | "other";
|
||||
|
||||
export type ConceptTagScriptCoverage = {
|
||||
latinEntryCount: number;
|
||||
cjkEntryCount: number;
|
||||
mixedEntryCount: number;
|
||||
otherEntryCount: number;
|
||||
};
|
||||
|
||||
const LANGUAGE_STOP_WORDS = {
|
||||
shared: [
|
||||
"about",
|
||||
"after",
|
||||
"agent",
|
||||
"again",
|
||||
"also",
|
||||
"assistant",
|
||||
"because",
|
||||
"before",
|
||||
"being",
|
||||
"between",
|
||||
"build",
|
||||
"called",
|
||||
"could",
|
||||
"daily",
|
||||
"default",
|
||||
"deploy",
|
||||
"during",
|
||||
"every",
|
||||
"file",
|
||||
"files",
|
||||
"from",
|
||||
"have",
|
||||
"into",
|
||||
"just",
|
||||
"line",
|
||||
"lines",
|
||||
"long",
|
||||
"main",
|
||||
"make",
|
||||
"memory",
|
||||
"month",
|
||||
"more",
|
||||
"most",
|
||||
"move",
|
||||
"much",
|
||||
"next",
|
||||
"note",
|
||||
"notes",
|
||||
"over",
|
||||
"part",
|
||||
"past",
|
||||
"port",
|
||||
"same",
|
||||
"score",
|
||||
"search",
|
||||
"session",
|
||||
"sessions",
|
||||
"short",
|
||||
"should",
|
||||
"since",
|
||||
"some",
|
||||
"subagent",
|
||||
"system",
|
||||
"than",
|
||||
"that",
|
||||
"their",
|
||||
"there",
|
||||
"these",
|
||||
"they",
|
||||
"this",
|
||||
"through",
|
||||
"today",
|
||||
"user",
|
||||
"using",
|
||||
"with",
|
||||
"work",
|
||||
"workspace",
|
||||
"year",
|
||||
],
|
||||
english: ["and", "are", "for", "into", "its", "our", "the", "then", "were", "you", "your"],
|
||||
spanish: [
|
||||
"al",
|
||||
"con",
|
||||
"como",
|
||||
"de",
|
||||
"del",
|
||||
"el",
|
||||
"en",
|
||||
"es",
|
||||
"la",
|
||||
"las",
|
||||
"los",
|
||||
"para",
|
||||
"por",
|
||||
"que",
|
||||
"se",
|
||||
"sin",
|
||||
"su",
|
||||
"sus",
|
||||
"una",
|
||||
"uno",
|
||||
"unos",
|
||||
"unas",
|
||||
"y",
|
||||
],
|
||||
french: [
|
||||
"au",
|
||||
"aux",
|
||||
"avec",
|
||||
"dans",
|
||||
"de",
|
||||
"des",
|
||||
"du",
|
||||
"en",
|
||||
"est",
|
||||
"et",
|
||||
"la",
|
||||
"le",
|
||||
"les",
|
||||
"ou",
|
||||
"pour",
|
||||
"que",
|
||||
"qui",
|
||||
"sans",
|
||||
"ses",
|
||||
"son",
|
||||
"sur",
|
||||
"une",
|
||||
"un",
|
||||
],
|
||||
german: [
|
||||
"auf",
|
||||
"aus",
|
||||
"bei",
|
||||
"das",
|
||||
"dem",
|
||||
"den",
|
||||
"der",
|
||||
"des",
|
||||
"die",
|
||||
"ein",
|
||||
"eine",
|
||||
"einem",
|
||||
"einen",
|
||||
"einer",
|
||||
"für",
|
||||
"im",
|
||||
"in",
|
||||
"mit",
|
||||
"nach",
|
||||
"oder",
|
||||
"ohne",
|
||||
"über",
|
||||
"und",
|
||||
"von",
|
||||
"zu",
|
||||
"zum",
|
||||
"zur",
|
||||
],
|
||||
cjk: [
|
||||
"が",
|
||||
"から",
|
||||
"する",
|
||||
"して",
|
||||
"した",
|
||||
"で",
|
||||
"と",
|
||||
"に",
|
||||
"の",
|
||||
"は",
|
||||
"へ",
|
||||
"まで",
|
||||
"も",
|
||||
"や",
|
||||
"を",
|
||||
"与",
|
||||
"为",
|
||||
"了",
|
||||
"及",
|
||||
"和",
|
||||
"在",
|
||||
"将",
|
||||
"或",
|
||||
"把",
|
||||
"是",
|
||||
"用",
|
||||
"的",
|
||||
"과",
|
||||
"는",
|
||||
"도",
|
||||
"로",
|
||||
"를",
|
||||
"에",
|
||||
"에서",
|
||||
"와",
|
||||
"은",
|
||||
"으로",
|
||||
"을",
|
||||
"이",
|
||||
"하다",
|
||||
"한",
|
||||
"할",
|
||||
"해",
|
||||
"했다",
|
||||
],
|
||||
pathNoise: [
|
||||
"cjs",
|
||||
"cpp",
|
||||
"cts",
|
||||
"jsx",
|
||||
"json",
|
||||
"md",
|
||||
"mjs",
|
||||
"mts",
|
||||
"text",
|
||||
"toml",
|
||||
"ts",
|
||||
"tsx",
|
||||
"txt",
|
||||
"yaml",
|
||||
"yml",
|
||||
],
|
||||
} as const;
|
||||
|
||||
const CONCEPT_STOP_WORDS = new Set(
|
||||
Object.values(LANGUAGE_STOP_WORDS)
|
||||
.flat()
|
||||
.map((word) => normalizeLowercaseStringOrEmpty(word)),
|
||||
);
|
||||
|
||||
const PROTECTED_GLOSSARY = [
|
||||
"backup",
|
||||
"backups",
|
||||
"embedding",
|
||||
"embeddings",
|
||||
"failover",
|
||||
"gateway",
|
||||
"glacier",
|
||||
"gpt",
|
||||
"kv",
|
||||
"network",
|
||||
"openai",
|
||||
"qmd",
|
||||
"router",
|
||||
"s3",
|
||||
"vlan",
|
||||
"sauvegarde",
|
||||
"routeur",
|
||||
"passerelle",
|
||||
"konfiguration",
|
||||
"sicherung",
|
||||
"überwachung",
|
||||
"configuración",
|
||||
"respaldo",
|
||||
"enrutador",
|
||||
"puerta-de-enlace",
|
||||
"バックアップ",
|
||||
"フェイルオーバー",
|
||||
"ルーター",
|
||||
"ネットワーク",
|
||||
"ゲートウェイ",
|
||||
"障害対応",
|
||||
"路由器",
|
||||
"备份",
|
||||
"故障转移",
|
||||
"网络",
|
||||
"网关",
|
||||
"라우터",
|
||||
"백업",
|
||||
"페일오버",
|
||||
"네트워크",
|
||||
"게이트웨이",
|
||||
"장애대응",
|
||||
].map((word) => normalizeLowercaseStringOrEmpty(word.normalize("NFKC")));
|
||||
|
||||
const COMPOUND_TOKEN_RE = /[\p{L}\p{N}]+(?:[._/-][\p{L}\p{N}]+)+/gu;
|
||||
const LETTER_OR_NUMBER_RE = /[\p{L}\p{N}]/u;
|
||||
const LATIN_RE = /\p{Script=Latin}/u;
|
||||
const HAN_RE = /\p{Script=Han}/u;
|
||||
const HIRAGANA_RE = /\p{Script=Hiragana}/u;
|
||||
const KATAKANA_RE = /\p{Script=Katakana}/u;
|
||||
const HANGUL_RE = /\p{Script=Hangul}/u;
|
||||
|
||||
const DEFAULT_WORD_SEGMENTER =
|
||||
typeof Intl.Segmenter === "function" ? new Intl.Segmenter("und", { granularity: "word" }) : null;
|
||||
|
||||
function containsLetterOrNumber(value: string): boolean {
|
||||
return LETTER_OR_NUMBER_RE.test(value);
|
||||
}
|
||||
|
||||
export function classifyConceptTagScript(tag: string): ConceptTagScriptFamily {
|
||||
const normalized = tag.normalize("NFKC");
|
||||
const hasLatin = LATIN_RE.test(normalized);
|
||||
const hasCjk =
|
||||
HAN_RE.test(normalized) ||
|
||||
HIRAGANA_RE.test(normalized) ||
|
||||
KATAKANA_RE.test(normalized) ||
|
||||
HANGUL_RE.test(normalized);
|
||||
if (hasLatin && hasCjk) {
|
||||
return "mixed";
|
||||
}
|
||||
if (hasCjk) {
|
||||
return "cjk";
|
||||
}
|
||||
if (hasLatin) {
|
||||
return "latin";
|
||||
}
|
||||
return "other";
|
||||
}
|
||||
|
||||
function minimumTokenLengthForScript(script: ConceptTagScriptFamily): number {
|
||||
if (script === "cjk") {
|
||||
return 2;
|
||||
}
|
||||
return 3;
|
||||
}
|
||||
|
||||
function isKanaOnlyToken(value: string): boolean {
|
||||
return (
|
||||
!HAN_RE.test(value) &&
|
||||
!HANGUL_RE.test(value) &&
|
||||
(HIRAGANA_RE.test(value) || KATAKANA_RE.test(value))
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeConceptToken(rawToken: string, fromGlossary = false): string | null {
|
||||
const normalized = normalizeLowercaseStringOrEmpty(
|
||||
rawToken
|
||||
.normalize("NFKC")
|
||||
.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")
|
||||
.replaceAll("_", "-"),
|
||||
);
|
||||
if (!normalized || !containsLetterOrNumber(normalized) || normalized.length > 32) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
/^\d+$/.test(normalized) ||
|
||||
/^\d{4}-\d{2}-\d{2}$/u.test(normalized) ||
|
||||
/^\d{4}-\d{2}-\d{2}\.[\p{L}\p{N}]+$/u.test(normalized)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const script = classifyConceptTagScript(normalized);
|
||||
// Glossary entries are an explicit allowlist of short technical terms (e.g. "kv", "s3"); they
|
||||
// bypass the per-script minimum length that would otherwise discard them.
|
||||
if (!fromGlossary && normalized.length < minimumTokenLengthForScript(script)) {
|
||||
return null;
|
||||
}
|
||||
if (isKanaOnlyToken(normalized) && normalized.length < 3) {
|
||||
return null;
|
||||
}
|
||||
if (CONCEPT_STOP_WORDS.has(normalized)) {
|
||||
return null;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
// Only entries shorter than their script's minimum token length rely on the glossary bypass, and
|
||||
// only those need whole-word matching so they don't fire inside longer words ("kv" in "mkv"). Longer
|
||||
// entries keep substring containment (the shipped behavior, e.g. "backup" tagging inside "backups").
|
||||
// Precomputed so derive() does not reclassify on every call.
|
||||
const GLOSSARY_ENTRIES = PROTECTED_GLOSSARY.map((entry) => ({
|
||||
entry,
|
||||
wholeWord: entry.length < minimumTokenLengthForScript(classifyConceptTagScript(entry)),
|
||||
}));
|
||||
|
||||
function isAlphanumericAt(source: string, index: number): boolean {
|
||||
const ch = source[index];
|
||||
return ch !== undefined && LETTER_OR_NUMBER_RE.test(ch);
|
||||
}
|
||||
|
||||
// True when `entry` occurs as a delimiter-bounded token, not inside a longer word. Keeps short
|
||||
// glossary entries like "kv"/"s3" from firing inside "mkv"/"css3" once they bypass the length gate.
|
||||
function includesStandaloneTerm(source: string, entry: string): boolean {
|
||||
let from = source.indexOf(entry);
|
||||
while (from !== -1) {
|
||||
if (!isAlphanumericAt(source, from - 1) && !isAlphanumericAt(source, from + entry.length)) {
|
||||
return true;
|
||||
}
|
||||
from = source.indexOf(entry, from + 1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function collectGlossaryMatches(source: string): string[] {
|
||||
const normalizedSource = normalizeLowercaseStringOrEmpty(source.normalize("NFKC"));
|
||||
const matches: string[] = [];
|
||||
for (const { entry, wholeWord } of GLOSSARY_ENTRIES) {
|
||||
const present = wholeWord
|
||||
? includesStandaloneTerm(normalizedSource, entry)
|
||||
: normalizedSource.includes(entry);
|
||||
if (present) {
|
||||
matches.push(entry);
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
function collectCompoundTokens(source: string): string[] {
|
||||
return source.match(COMPOUND_TOKEN_RE) ?? [];
|
||||
}
|
||||
|
||||
function collectSegmentTokens(source: string): string[] {
|
||||
if (DEFAULT_WORD_SEGMENTER) {
|
||||
return Array.from(DEFAULT_WORD_SEGMENTER.segment(source), (part) =>
|
||||
part.isWordLike ? part.segment : "",
|
||||
).filter(Boolean);
|
||||
}
|
||||
return source.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
||||
}
|
||||
|
||||
function pushNormalizedTag(
|
||||
tags: string[],
|
||||
rawToken: string,
|
||||
limit: number,
|
||||
fromGlossary = false,
|
||||
): void {
|
||||
const normalized = normalizeConceptToken(rawToken, fromGlossary);
|
||||
if (!normalized || tags.includes(normalized)) {
|
||||
return;
|
||||
}
|
||||
tags.push(normalized);
|
||||
if (tags.length > limit) {
|
||||
tags.splice(limit);
|
||||
}
|
||||
}
|
||||
|
||||
export function deriveConceptTags(params: {
|
||||
path: string;
|
||||
snippet: string;
|
||||
limit?: number;
|
||||
}): string[] {
|
||||
const source = `${path.basename(params.path)} ${params.snippet}`;
|
||||
const limit = Number.isFinite(params.limit)
|
||||
? Math.max(0, Math.floor(params.limit as number))
|
||||
: MAX_CONCEPT_TAGS;
|
||||
if (limit === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const tags: string[] = [];
|
||||
const tokenSources: Array<{ tokens: string[]; fromGlossary: boolean }> = [
|
||||
{ tokens: collectGlossaryMatches(source), fromGlossary: true },
|
||||
{ tokens: collectCompoundTokens(source), fromGlossary: false },
|
||||
{ tokens: collectSegmentTokens(source), fromGlossary: false },
|
||||
];
|
||||
for (const { tokens, fromGlossary } of tokenSources) {
|
||||
for (const rawToken of tokens) {
|
||||
pushNormalizedTag(tags, rawToken, limit, fromGlossary);
|
||||
if (tags.length >= limit) {
|
||||
return tags;
|
||||
}
|
||||
}
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
|
||||
export function summarizeConceptTagScriptCoverage(
|
||||
conceptTagsByEntry: string[][],
|
||||
): ConceptTagScriptCoverage {
|
||||
const coverage: ConceptTagScriptCoverage = {
|
||||
latinEntryCount: 0,
|
||||
cjkEntryCount: 0,
|
||||
mixedEntryCount: 0,
|
||||
otherEntryCount: 0,
|
||||
};
|
||||
|
||||
for (const conceptTags of conceptTagsByEntry) {
|
||||
let hasLatin = false;
|
||||
let hasCjk = false;
|
||||
let hasOther = false;
|
||||
for (const tag of conceptTags) {
|
||||
const family = classifyConceptTagScript(tag);
|
||||
if (family === "mixed") {
|
||||
hasLatin = true;
|
||||
hasCjk = true;
|
||||
continue;
|
||||
}
|
||||
if (family === "latin") {
|
||||
hasLatin = true;
|
||||
continue;
|
||||
}
|
||||
if (family === "cjk") {
|
||||
hasCjk = true;
|
||||
continue;
|
||||
}
|
||||
hasOther = true;
|
||||
}
|
||||
|
||||
if (hasLatin && hasCjk) {
|
||||
coverage.mixedEntryCount += 1;
|
||||
} else if (hasCjk) {
|
||||
coverage.cjkEntryCount += 1;
|
||||
} else if (hasLatin) {
|
||||
coverage.latinEntryCount += 1;
|
||||
} else if (hasOther) {
|
||||
coverage.otherEntryCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return coverage;
|
||||
}
|
||||
56
extensions/memory-core/src/config.test.ts
Normal file
56
extensions/memory-core/src/config.test.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
// Memory Core tests cover config plugin behavior.
|
||||
import fs from "node:fs";
|
||||
import {
|
||||
type JsonSchemaObject,
|
||||
validateJsonSchemaValue,
|
||||
} from "openclaw/plugin-sdk/json-schema-runtime";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const manifest = JSON.parse(
|
||||
fs.readFileSync(new URL("../openclaw.plugin.json", import.meta.url), "utf-8"),
|
||||
) as { configSchema: JsonSchemaObject };
|
||||
|
||||
describe("memory-core manifest config schema", () => {
|
||||
it("accepts dreaming phase thresholds used by QA and runtime", () => {
|
||||
const result = validateJsonSchemaValue({
|
||||
schema: manifest.configSchema,
|
||||
cacheKey: "memory-core.manifest.dreaming-phase-thresholds",
|
||||
value: {
|
||||
dreaming: {
|
||||
enabled: true,
|
||||
timezone: "Europe/London",
|
||||
verboseLogging: true,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
phases: {
|
||||
light: {
|
||||
enabled: true,
|
||||
lookbackDays: 2,
|
||||
limit: 20,
|
||||
dedupeSimilarity: 0.9,
|
||||
},
|
||||
deep: {
|
||||
enabled: true,
|
||||
limit: 10,
|
||||
minScore: 0,
|
||||
minRecallCount: 3,
|
||||
minUniqueQueries: 3,
|
||||
recencyHalfLifeDays: 14,
|
||||
maxAgeDays: 30,
|
||||
},
|
||||
rem: {
|
||||
enabled: true,
|
||||
lookbackDays: 7,
|
||||
limit: 10,
|
||||
minPatternStrength: 0.75,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
});
|
||||
});
|
||||
218
extensions/memory-core/src/dreaming-command.test.ts
Normal file
218
extensions/memory-core/src/dreaming-command.test.ts
Normal file
@@ -0,0 +1,218 @@
|
||||
// Memory Core tests cover dreaming command plugin behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { PluginCommandContext } from "openclaw/plugin-sdk/core";
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { handleDreamingCommand } from "./dreaming-command.js";
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function resolveStoredDreaming(config: OpenClawConfig): Record<string, unknown> {
|
||||
const entry = asRecord(config.plugins?.entries?.["memory-core"]);
|
||||
const pluginConfig = asRecord(entry?.config);
|
||||
return asRecord(pluginConfig?.dreaming) ?? {};
|
||||
}
|
||||
|
||||
function createHarness(initialConfig: OpenClawConfig = {}) {
|
||||
let runtimeConfig: OpenClawConfig = initialConfig;
|
||||
|
||||
const runtime = {
|
||||
config: {
|
||||
current: vi.fn(() => runtimeConfig),
|
||||
loadConfig: vi.fn(() => runtimeConfig),
|
||||
mutateConfigFile: vi.fn(async ({ mutate }: { mutate: (draft: OpenClawConfig) => void }) => {
|
||||
const draft = structuredClone(runtimeConfig);
|
||||
mutate(draft);
|
||||
runtimeConfig = draft;
|
||||
return {
|
||||
path: "/tmp/openclaw.json",
|
||||
previousHash: null,
|
||||
persistedHash: null,
|
||||
snapshot: {},
|
||||
nextConfig: runtimeConfig,
|
||||
afterWrite: { mode: "auto" },
|
||||
followUp: { mode: "auto", requiresRestart: false },
|
||||
result: undefined,
|
||||
};
|
||||
}),
|
||||
replaceConfigFile: vi.fn(async ({ nextConfig }: { nextConfig: OpenClawConfig }) => {
|
||||
runtimeConfig = nextConfig;
|
||||
}),
|
||||
writeConfigFile: vi.fn(async (nextConfig: OpenClawConfig) => {
|
||||
runtimeConfig = nextConfig;
|
||||
}),
|
||||
},
|
||||
} as unknown as OpenClawPluginApi["runtime"];
|
||||
|
||||
const api = {
|
||||
runtime,
|
||||
} as unknown as OpenClawPluginApi;
|
||||
|
||||
return {
|
||||
api,
|
||||
runtime,
|
||||
getRuntimeConfig: () => runtimeConfig,
|
||||
};
|
||||
}
|
||||
|
||||
function createCommandContext(
|
||||
args?: string,
|
||||
overrides?: Partial<Pick<PluginCommandContext, "gatewayClientScopes" | "senderIsOwner">>,
|
||||
): PluginCommandContext {
|
||||
return {
|
||||
channel: "webchat",
|
||||
isAuthorizedSender: true,
|
||||
commandBody: args ? `/dreaming ${args}` : "/dreaming",
|
||||
args,
|
||||
config: {},
|
||||
gatewayClientScopes: overrides?.gatewayClientScopes,
|
||||
senderIsOwner: overrides?.senderIsOwner,
|
||||
requestConversationBinding: async () => ({ status: "error", message: "unsupported" }),
|
||||
detachConversationBinding: async () => ({ removed: false }),
|
||||
getCurrentConversationBinding: async () => null,
|
||||
};
|
||||
}
|
||||
|
||||
async function runDreamingCommand(
|
||||
harness: ReturnType<typeof createHarness>,
|
||||
args?: string,
|
||||
overrides?: Partial<Pick<PluginCommandContext, "gatewayClientScopes" | "senderIsOwner">>,
|
||||
) {
|
||||
return await handleDreamingCommand(harness.api, createCommandContext(args, overrides));
|
||||
}
|
||||
|
||||
describe("memory-core /dreaming command", () => {
|
||||
it("shows phase explanations when invoked without args", async () => {
|
||||
const harness = createHarness();
|
||||
const result = await runDreamingCommand(harness);
|
||||
|
||||
expect(result.text).toContain("Usage: /dreaming status");
|
||||
expect(result.text).toContain("Dreaming status:");
|
||||
expect(result.text).toContain("- implementation detail: each sweep runs light -> REM -> deep.");
|
||||
expect(result.text).toContain(
|
||||
"- deep is the only stage that writes durable entries to MEMORY.md.",
|
||||
);
|
||||
});
|
||||
|
||||
it("blocks non-owner external channel callers from persisting dreaming config", async () => {
|
||||
const harness = createHarness();
|
||||
|
||||
const result = await runDreamingCommand(harness, "off");
|
||||
|
||||
expect(result.text).toContain(
|
||||
"requires owner status for channel callers or operator.admin for gateway clients",
|
||||
);
|
||||
expect(harness.runtime.config.mutateConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows owner external channel callers to persist global enablement", async () => {
|
||||
const harness = createHarness({
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
phases: {
|
||||
deep: {
|
||||
minScore: 0.9,
|
||||
},
|
||||
},
|
||||
frequency: "0 */6 * * *",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await runDreamingCommand(harness, "off", {
|
||||
senderIsOwner: true,
|
||||
});
|
||||
|
||||
expect(harness.runtime.config.mutateConfigFile).toHaveBeenCalledTimes(1);
|
||||
const storedDreaming = resolveStoredDreaming(harness.getRuntimeConfig());
|
||||
expect(storedDreaming.enabled).toBe(false);
|
||||
expect(storedDreaming.frequency).toBe("0 */6 * * *");
|
||||
expect(result.text).toContain("Dreaming disabled.");
|
||||
});
|
||||
|
||||
it("blocks unscoped gateway callers from persisting dreaming config", async () => {
|
||||
const harness = createHarness();
|
||||
|
||||
const result = await runDreamingCommand(harness, "off", {
|
||||
gatewayClientScopes: [],
|
||||
});
|
||||
|
||||
expect(result.text).toContain(
|
||||
"requires owner status for channel callers or operator.admin for gateway clients",
|
||||
);
|
||||
expect(harness.runtime.config.mutateConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("blocks write-scoped gateway callers from persisting dreaming config", async () => {
|
||||
const harness = createHarness();
|
||||
|
||||
const result = await runDreamingCommand(harness, "off", {
|
||||
gatewayClientScopes: ["operator.write"],
|
||||
});
|
||||
|
||||
expect(result.text).toContain(
|
||||
"requires owner status for channel callers or operator.admin for gateway clients",
|
||||
);
|
||||
expect(harness.runtime.config.mutateConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows admin-scoped gateway callers to persist dreaming config", async () => {
|
||||
const harness = createHarness();
|
||||
|
||||
const result = await runDreamingCommand(harness, "on", {
|
||||
gatewayClientScopes: ["operator.admin"],
|
||||
});
|
||||
|
||||
expect(harness.runtime.config.mutateConfigFile).toHaveBeenCalledTimes(1);
|
||||
expect(resolveStoredDreaming(harness.getRuntimeConfig()).enabled).toBe(true);
|
||||
expect(result.text).toContain("Dreaming enabled.");
|
||||
});
|
||||
|
||||
it("returns status without mutating config", async () => {
|
||||
const harness = createHarness({
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
frequency: "15 */8 * * *",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
userTimezone: "America/Los_Angeles",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await runDreamingCommand(harness, "status");
|
||||
|
||||
expect(result.text).toContain("Dreaming status:");
|
||||
expect(result.text).toContain("- enabled: off (America/Los_Angeles)");
|
||||
expect(result.text).toContain("- sweep cadence: 15 */8 * * *");
|
||||
expect(result.text).toContain("- promotion policy: score>=0.8, recalls>=3, uniqueQueries>=3");
|
||||
expect(harness.runtime.config.mutateConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shows usage for invalid args and does not mutate config", async () => {
|
||||
const harness = createHarness();
|
||||
const result = await runDreamingCommand(harness, "unknown-mode");
|
||||
|
||||
expect(result.text).toContain("Usage: /dreaming status");
|
||||
expect(harness.runtime.config.mutateConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
135
extensions/memory-core/src/dreaming-command.ts
Normal file
135
extensions/memory-core/src/dreaming-command.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
// Memory Core plugin module implements dreaming command behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { resolveMemoryDreamingConfig } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import type { OpenClawPluginApi, PluginCommandContext } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { asRecord } from "./dreaming-shared.js";
|
||||
import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js";
|
||||
|
||||
function resolveMemoryCorePluginConfig(cfg: OpenClawConfig): Record<string, unknown> {
|
||||
const entry = asRecord(cfg.plugins?.entries?.["memory-core"]);
|
||||
return asRecord(entry?.config) ?? {};
|
||||
}
|
||||
|
||||
function updateDreamingEnabledInConfig(cfg: OpenClawConfig, enabled: boolean): OpenClawConfig {
|
||||
const entries = { ...cfg.plugins?.entries };
|
||||
const existingEntry = asRecord(entries["memory-core"]) ?? {};
|
||||
const existingConfig = asRecord(existingEntry.config) ?? {};
|
||||
const existingSleep = asRecord(existingConfig.dreaming) ?? {};
|
||||
entries["memory-core"] = {
|
||||
...existingEntry,
|
||||
config: {
|
||||
...existingConfig,
|
||||
dreaming: {
|
||||
...existingSleep,
|
||||
enabled,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...cfg,
|
||||
plugins: {
|
||||
...cfg.plugins,
|
||||
entries,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatEnabled(value: boolean): string {
|
||||
return value ? "on" : "off";
|
||||
}
|
||||
|
||||
function formatPhaseGuide(): string {
|
||||
return [
|
||||
"- implementation detail: each sweep runs light -> REM -> deep.",
|
||||
"- deep is the only stage that writes durable entries to MEMORY.md.",
|
||||
"- DREAMS.md is for human-readable dreaming summaries and diary entries.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function formatStatus(cfg: OpenClawConfig): string {
|
||||
const pluginConfig = resolveMemoryCorePluginConfig(cfg);
|
||||
const dreaming = resolveMemoryDreamingConfig({
|
||||
pluginConfig,
|
||||
cfg,
|
||||
});
|
||||
const deep = resolveShortTermPromotionDreamingConfig({ pluginConfig, cfg });
|
||||
const timezone = dreaming.timezone ? ` (${dreaming.timezone})` : "";
|
||||
|
||||
return [
|
||||
"Dreaming status:",
|
||||
`- enabled: ${formatEnabled(dreaming.enabled)}${timezone}`,
|
||||
`- sweep cadence: ${dreaming.frequency}`,
|
||||
`- promotion policy: score>=${deep.minScore}, recalls>=${deep.minRecallCount}, uniqueQueries>=${deep.minUniqueQueries}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function formatUsage(includeStatus: string): string {
|
||||
return [
|
||||
"Usage: /dreaming status",
|
||||
"Usage: /dreaming on|off",
|
||||
"",
|
||||
includeStatus,
|
||||
"",
|
||||
"Phases:",
|
||||
formatPhaseGuide(),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function lacksAdminOrOwnerForDreamingMutation(params: {
|
||||
gatewayClientScopes?: readonly string[];
|
||||
senderIsOwner?: boolean;
|
||||
}): boolean {
|
||||
if (Array.isArray(params.gatewayClientScopes)) {
|
||||
return !params.gatewayClientScopes.includes("operator.admin");
|
||||
}
|
||||
return params.senderIsOwner !== true;
|
||||
}
|
||||
|
||||
export async function handleDreamingCommand(api: OpenClawPluginApi, ctx: PluginCommandContext) {
|
||||
const args = ctx.args?.trim() ?? "";
|
||||
const [firstToken = ""] = args
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map((token) => normalizeLowercaseStringOrEmpty(token));
|
||||
const currentConfig = api.runtime.config.current() as OpenClawConfig;
|
||||
|
||||
if (!firstToken || firstToken === "help" || firstToken === "options" || firstToken === "phases") {
|
||||
return { text: formatUsage(formatStatus(currentConfig)) };
|
||||
}
|
||||
|
||||
if (firstToken === "status") {
|
||||
return { text: formatStatus(currentConfig) };
|
||||
}
|
||||
|
||||
if (firstToken === "on" || firstToken === "off") {
|
||||
if (
|
||||
lacksAdminOrOwnerForDreamingMutation({
|
||||
gatewayClientScopes: ctx.gatewayClientScopes,
|
||||
senderIsOwner: ctx.senderIsOwner,
|
||||
})
|
||||
) {
|
||||
return {
|
||||
text: "⚠️ /dreaming on|off requires owner status for channel callers or operator.admin for gateway clients.",
|
||||
};
|
||||
}
|
||||
const enabled = firstToken === "on";
|
||||
const committed = await api.runtime.config.mutateConfigFile({
|
||||
afterWrite: { mode: "auto" },
|
||||
mutate: (draft) => {
|
||||
const nextConfig = updateDreamingEnabledInConfig(draft, enabled);
|
||||
Object.assign(draft, nextConfig);
|
||||
},
|
||||
});
|
||||
return {
|
||||
text: [
|
||||
`Dreaming ${enabled ? "enabled" : "disabled"}.`,
|
||||
"",
|
||||
formatStatus(committed.nextConfig),
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
return { text: formatUsage(formatStatus(currentConfig)) };
|
||||
}
|
||||
150
extensions/memory-core/src/dreaming-dreams-file.ts
Normal file
150
extensions/memory-core/src/dreaming-dreams-file.ts
Normal file
@@ -0,0 +1,150 @@
|
||||
// Memory Core helpers for safe managed DREAMS.md updates.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { createAsyncLock } from "openclaw/plugin-sdk/async-lock-runtime";
|
||||
import { extractErrorCode } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { resolveGlobalMap } from "openclaw/plugin-sdk/global-singleton";
|
||||
import { replaceManagedMarkdownBlock } from "openclaw/plugin-sdk/memory-host-markdown";
|
||||
import { readRegularFile, replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime";
|
||||
|
||||
const DREAMS_FILENAMES = ["DREAMS.md", "dreams.md"] as const;
|
||||
const DEEP_START_MARKER = "<!-- openclaw:dreaming:deep:start -->";
|
||||
const DEEP_END_MARKER = "<!-- openclaw:dreaming:deep:end -->";
|
||||
const DREAMS_FILE_LOCKS_KEY = Symbol.for("openclaw.memoryCore.dreamingNarrative.fileLocks");
|
||||
|
||||
type DreamsFileLockEntry = {
|
||||
withLock: ReturnType<typeof createAsyncLock>;
|
||||
refs: number;
|
||||
};
|
||||
|
||||
const dreamsFileLocks = resolveGlobalMap<string, DreamsFileLockEntry>(DREAMS_FILE_LOCKS_KEY);
|
||||
|
||||
export async function resolveDreamsPath(workspaceDir: string): Promise<string> {
|
||||
for (const name of DREAMS_FILENAMES) {
|
||||
const target = path.join(workspaceDir, name);
|
||||
try {
|
||||
await fs.access(target);
|
||||
return target;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return path.join(workspaceDir, DREAMS_FILENAMES[0]);
|
||||
}
|
||||
|
||||
function isEmptyDreamsReadError(err: unknown): boolean {
|
||||
const code = extractErrorCode(err);
|
||||
if (
|
||||
code === "ENOENT" ||
|
||||
code === "ENOTDIR" ||
|
||||
code === "not-found" ||
|
||||
code === "not-file" ||
|
||||
code === "path-alias" ||
|
||||
code === "path-mismatch" ||
|
||||
code === "symlink"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return err instanceof Error && err.message === "path must be a regular file";
|
||||
}
|
||||
|
||||
export async function readDreamsFile(dreamsPath: string): Promise<string> {
|
||||
try {
|
||||
return (await readRegularFile({ filePath: dreamsPath })).buffer.toString("utf-8");
|
||||
} catch (err) {
|
||||
if (isEmptyDreamsReadError(err)) {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function assertSafeDreamsPath(dreamsPath: string): Promise<void> {
|
||||
const stat = await fs.lstat(dreamsPath).catch((err: unknown) => {
|
||||
if (extractErrorCode(err) === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!stat) {
|
||||
return;
|
||||
}
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new Error("Refusing to write symlinked DREAMS.md");
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
throw new Error("Refusing to write non-file DREAMS.md");
|
||||
}
|
||||
}
|
||||
|
||||
async function writeDreamsFileAtomic(dreamsPath: string, content: string): Promise<void> {
|
||||
await assertSafeDreamsPath(dreamsPath);
|
||||
await replaceFileAtomic({
|
||||
filePath: dreamsPath,
|
||||
content,
|
||||
mode: 0o600,
|
||||
preserveExistingMode: true,
|
||||
tempPrefix: `${path.basename(dreamsPath)}.dreams`,
|
||||
throwOnCleanupError: true,
|
||||
});
|
||||
}
|
||||
|
||||
export async function updateDreamsFile<T>(params: {
|
||||
workspaceDir: string;
|
||||
updater: (
|
||||
existing: string,
|
||||
dreamsPath: string,
|
||||
) =>
|
||||
| Promise<{ content: string; result: T; shouldWrite?: boolean }>
|
||||
| {
|
||||
content: string;
|
||||
result: T;
|
||||
shouldWrite?: boolean;
|
||||
};
|
||||
}): Promise<T> {
|
||||
const dreamsPath = await resolveDreamsPath(params.workspaceDir);
|
||||
await fs.mkdir(path.dirname(dreamsPath), { recursive: true });
|
||||
let lockEntry = dreamsFileLocks.get(dreamsPath);
|
||||
if (!lockEntry) {
|
||||
lockEntry = { withLock: createAsyncLock(), refs: 0 };
|
||||
dreamsFileLocks.set(dreamsPath, lockEntry);
|
||||
}
|
||||
lockEntry.refs += 1;
|
||||
try {
|
||||
return await lockEntry.withLock(async () => {
|
||||
const existing = await readDreamsFile(dreamsPath);
|
||||
const { content, result, shouldWrite = true } = await params.updater(existing, dreamsPath);
|
||||
if (shouldWrite) {
|
||||
await writeDreamsFileAtomic(dreamsPath, content.endsWith("\n") ? content : `${content}\n`);
|
||||
}
|
||||
return result;
|
||||
});
|
||||
} finally {
|
||||
lockEntry.refs -= 1;
|
||||
if (lockEntry.refs <= 0 && dreamsFileLocks.get(dreamsPath) === lockEntry) {
|
||||
dreamsFileLocks.delete(dreamsPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateDeepDreamsFile(params: {
|
||||
workspaceDir: string;
|
||||
bodyLines: string[];
|
||||
}): Promise<string> {
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No durable changes.";
|
||||
return await updateDreamsFile({
|
||||
workspaceDir: params.workspaceDir,
|
||||
updater: (existing, dreamsPath) => ({
|
||||
content: replaceManagedMarkdownBlock({
|
||||
original: existing,
|
||||
heading: "## Deep Sleep",
|
||||
startMarker: DEEP_START_MARKER,
|
||||
endMarker: DEEP_END_MARKER,
|
||||
body,
|
||||
}),
|
||||
result: dreamsPath,
|
||||
}),
|
||||
});
|
||||
}
|
||||
34
extensions/memory-core/src/dreaming-events.ts
Normal file
34
extensions/memory-core/src/dreaming-events.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
// Memory Core plugin module implements structured dreaming event helpers.
|
||||
import type { MemoryDreamingPhaseName } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events";
|
||||
import { formatErrorMessage } from "./dreaming-shared.js";
|
||||
import { resolveMemoryCoreNowMs, resolveMemoryCoreTimestamp } from "./time.js";
|
||||
|
||||
type Logger = {
|
||||
warn: (message: string) => void;
|
||||
};
|
||||
|
||||
export async function appendFailedDreamingEvent(params: {
|
||||
workspaceDir: string;
|
||||
phase: MemoryDreamingPhaseName;
|
||||
error: string;
|
||||
storageMode: "inline" | "separate" | "both";
|
||||
nowMs?: number;
|
||||
logger: Logger;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
await appendMemoryHostEvent(params.workspaceDir, {
|
||||
type: "memory.dream.completed",
|
||||
timestamp: resolveMemoryCoreTimestamp(resolveMemoryCoreNowMs(params.nowMs)),
|
||||
phase: params.phase,
|
||||
outcome: "failed",
|
||||
error: params.error,
|
||||
lineCount: 0,
|
||||
storageMode: params.storageMode,
|
||||
});
|
||||
} catch (err) {
|
||||
params.logger.warn(
|
||||
`memory-core: failed to write ${params.phase} dreaming outcome event for workspace ${params.workspaceDir}: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
276
extensions/memory-core/src/dreaming-markdown.test.ts
Normal file
276
extensions/memory-core/src/dreaming-markdown.test.ts
Normal file
@@ -0,0 +1,276 @@
|
||||
// Memory Core tests cover dreaming markdown plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { writeDailyDreamingPhaseBlock, writeDeepDreamingReport } from "./dreaming-markdown.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempWorkspace } = createMemoryCoreTestHarness();
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function expectPathMissing(targetPath: string): Promise<void> {
|
||||
const error = await fs.access(targetPath).then(
|
||||
() => undefined,
|
||||
(accessError: unknown) => accessError,
|
||||
);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as NodeJS.ErrnoException).code).toBe("ENOENT");
|
||||
}
|
||||
|
||||
function requireInlinePath(result: { inlinePath?: string }): string {
|
||||
if (!result.inlinePath) {
|
||||
throw new Error("Expected inline dreaming markdown path");
|
||||
}
|
||||
return result.inlinePath;
|
||||
}
|
||||
|
||||
function requireReportPath(reportPath: string | undefined): string {
|
||||
if (!reportPath) {
|
||||
throw new Error("Expected deep dreaming report path");
|
||||
}
|
||||
return reportPath;
|
||||
}
|
||||
|
||||
describe("dreaming markdown storage", () => {
|
||||
const nowMs = Date.parse("2026-04-05T10:00:00Z");
|
||||
const timezone = "UTC";
|
||||
|
||||
it("writes inline light dreaming output into the daily memory file", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
const result = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: remember the API key is fake"],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
const inlinePath = requireInlinePath(result);
|
||||
expect(inlinePath).toBe(path.join(workspaceDir, "memory", "2026-04-05.md"));
|
||||
const content = await fs.readFile(inlinePath, "utf-8");
|
||||
expect(content).toContain("## Light Sleep");
|
||||
expect(content).toContain("- Candidate: remember the API key is fake");
|
||||
});
|
||||
|
||||
it("falls back when the injected timestamp is outside Date range", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(Date.UTC(2026, 4, 30, 12, 0, 0));
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
const result = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: bounded fallback"],
|
||||
nowMs: 8_640_000_000_000_001,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(requireInlinePath(result)).toBe(path.join(workspaceDir, "memory", "2026-05-30.md"));
|
||||
});
|
||||
|
||||
it("keeps multiple inline phases in the shared daily memory file", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: first block"],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
bodyLines: ["- Theme: `focus` kept surfacing."],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
const dreamsPath = path.join(workspaceDir, "memory", "2026-04-05.md");
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content).toContain("## Light Sleep");
|
||||
expect(content).toContain("## REM Sleep");
|
||||
expect(content).toContain("- Candidate: first block");
|
||||
expect(content).toContain("- Theme: `focus` kept surfacing.");
|
||||
});
|
||||
|
||||
it("keeps daily phase output separate from lowercase dreams.md diaries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
const lowercasePath = path.join(workspaceDir, "dreams.md");
|
||||
await fs.writeFile(lowercasePath, "# Scratch\n\n", "utf-8");
|
||||
|
||||
const result = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
bodyLines: ["- Theme: `glacier` kept surfacing."],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
const inlinePath = requireInlinePath(result);
|
||||
expect(inlinePath).toBe(path.join(workspaceDir, "memory", "2026-04-05.md"));
|
||||
const content = await fs.readFile(inlinePath, "utf-8");
|
||||
expect(content).toContain("## REM Sleep");
|
||||
expect(content).toContain("- Theme: `glacier` kept surfacing.");
|
||||
await expect(fs.readFile(lowercasePath, "utf-8")).resolves.toBe("# Scratch\n\n");
|
||||
});
|
||||
|
||||
it("still writes deep reports to the per-phase report directory", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
const reportPath = await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: ["- Promoted: durable preference"],
|
||||
storage: {
|
||||
mode: "separate",
|
||||
separateReports: false,
|
||||
},
|
||||
nowMs: Date.parse("2026-04-05T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
|
||||
const requiredReportPath = requireReportPath(reportPath);
|
||||
expect(requiredReportPath).toBe(
|
||||
path.join(workspaceDir, "memory", "dreaming", "deep", "2026-04-05.md"),
|
||||
);
|
||||
const content = await fs.readFile(requiredReportPath, "utf-8");
|
||||
expect(content).toContain("# Deep Sleep");
|
||||
expect(content).toContain("- Promoted: durable preference");
|
||||
|
||||
const dreamsContent = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(dreamsContent).toContain("## Deep Sleep");
|
||||
expect(dreamsContent).toContain("<!-- openclaw:dreaming:deep:start -->");
|
||||
expect(dreamsContent).toContain("- Promoted: durable preference");
|
||||
});
|
||||
|
||||
it("writes the deep summary to DREAMS.md without a separate report in inline mode", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
const reportPath = await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: ["- Ranked 3 candidate(s) for durable promotion."],
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
nowMs: Date.parse("2026-04-05T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
|
||||
expect(reportPath).toBeUndefined();
|
||||
await expectPathMissing(path.join(workspaceDir, "memory", "dreaming", "deep", "2026-04-05.md"));
|
||||
const dreamsContent = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(dreamsContent).toContain("## Deep Sleep");
|
||||
expect(dreamsContent).toContain("- Ranked 3 candidate(s) for durable promotion.");
|
||||
});
|
||||
|
||||
it("replaces the managed deep summary while preserving the diary block", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(
|
||||
dreamsPath,
|
||||
[
|
||||
"# Dream Diary",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:start -->",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"*April 4, 2026, 3:00 AM*",
|
||||
"",
|
||||
"The old diary entry stays.",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:end -->",
|
||||
"",
|
||||
"## Deep Sleep",
|
||||
"<!-- openclaw:dreaming:deep:start -->",
|
||||
"- Old summary.",
|
||||
"<!-- openclaw:dreaming:deep:end -->",
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: ["- New summary."],
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
nowMs,
|
||||
timezone,
|
||||
});
|
||||
|
||||
const dreamsContent = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(dreamsContent).toContain("The old diary entry stays.");
|
||||
expect(dreamsContent).toContain("- New summary.");
|
||||
expect(dreamsContent).not.toContain("- Old summary.");
|
||||
});
|
||||
|
||||
it("reuses existing lowercase dreams.md for deep summaries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
const lowercasePath = path.join(workspaceDir, "dreams.md");
|
||||
await fs.writeFile(lowercasePath, "# Existing dreams\n", "utf-8");
|
||||
|
||||
await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: ["- Lowercase target."],
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
nowMs,
|
||||
timezone,
|
||||
});
|
||||
|
||||
const dreamsContent = await fs.readFile(lowercasePath, "utf-8");
|
||||
expect(dreamsContent).toContain("# Existing dreams");
|
||||
expect(dreamsContent).toContain("- Lowercase target.");
|
||||
});
|
||||
|
||||
it("refuses to overwrite a symlinked DREAMS.md for deep summaries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
const targetPath = path.join(workspaceDir, "outside.txt");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(targetPath, "outside\n", "utf-8");
|
||||
await fs.symlink(targetPath, dreamsPath);
|
||||
|
||||
await expect(
|
||||
writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: ["- Do not escape workspace."],
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
nowMs,
|
||||
timezone,
|
||||
}),
|
||||
).rejects.toThrow("Refusing to write symlinked DREAMS.md");
|
||||
await expect(fs.readFile(targetPath, "utf-8")).resolves.toBe("outside\n");
|
||||
});
|
||||
});
|
||||
158
extensions/memory-core/src/dreaming-markdown.ts
Normal file
158
extensions/memory-core/src/dreaming-markdown.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
// Memory Core plugin module implements dreaming markdown behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
formatMemoryDreamingDay,
|
||||
type MemoryDreamingPhaseName,
|
||||
type MemoryDreamingStorageConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events";
|
||||
import {
|
||||
replaceManagedMarkdownBlock,
|
||||
withTrailingNewline,
|
||||
} from "openclaw/plugin-sdk/memory-host-markdown";
|
||||
import { updateDeepDreamsFile } from "./dreaming-dreams-file.js";
|
||||
import { resolveMemoryCoreNowMs, resolveMemoryCoreTimestamp } from "./time.js";
|
||||
|
||||
const DAILY_PHASE_HEADINGS: Record<Exclude<MemoryDreamingPhaseName, "deep">, string> = {
|
||||
light: "## Light Sleep",
|
||||
rem: "## REM Sleep",
|
||||
};
|
||||
|
||||
const DAILY_PHASE_LABELS: Record<Exclude<MemoryDreamingPhaseName, "deep">, string> = {
|
||||
light: "light",
|
||||
rem: "rem",
|
||||
};
|
||||
|
||||
function resolvePhaseMarkers(phase: Exclude<MemoryDreamingPhaseName, "deep">): {
|
||||
start: string;
|
||||
end: string;
|
||||
} {
|
||||
const label = DAILY_PHASE_LABELS[phase];
|
||||
return {
|
||||
start: `<!-- openclaw:dreaming:${label}:start -->`,
|
||||
end: `<!-- openclaw:dreaming:${label}:end -->`,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDailyMemoryPath(workspaceDir: string, epochMs: number, timezone?: string): string {
|
||||
const isoDay = formatMemoryDreamingDay(epochMs, timezone);
|
||||
return path.join(workspaceDir, "memory", `${isoDay}.md`);
|
||||
}
|
||||
|
||||
function resolveSeparateReportPath(
|
||||
workspaceDir: string,
|
||||
phase: MemoryDreamingPhaseName,
|
||||
epochMs: number,
|
||||
timezone?: string,
|
||||
): string {
|
||||
const isoDay = formatMemoryDreamingDay(epochMs, timezone);
|
||||
return path.join(workspaceDir, "memory", "dreaming", phase, `${isoDay}.md`);
|
||||
}
|
||||
|
||||
function shouldWriteInline(storage: MemoryDreamingStorageConfig): boolean {
|
||||
return storage.mode === "inline" || storage.mode === "both";
|
||||
}
|
||||
|
||||
function shouldWriteSeparate(storage: MemoryDreamingStorageConfig): boolean {
|
||||
return storage.mode === "separate" || storage.mode === "both" || storage.separateReports;
|
||||
}
|
||||
|
||||
export async function writeDailyDreamingPhaseBlock(params: {
|
||||
workspaceDir: string;
|
||||
phase: Exclude<MemoryDreamingPhaseName, "deep">;
|
||||
bodyLines: string[];
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
storage: MemoryDreamingStorageConfig;
|
||||
}): Promise<{ inlinePath?: string; reportPath?: string }> {
|
||||
const nowMs = resolveMemoryCoreNowMs(params.nowMs);
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No notable updates.";
|
||||
let inlinePath: string | undefined;
|
||||
let reportPath: string | undefined;
|
||||
|
||||
if (shouldWriteInline(params.storage)) {
|
||||
inlinePath = resolveDailyMemoryPath(params.workspaceDir, nowMs, params.timezone);
|
||||
await fs.mkdir(path.dirname(inlinePath), { recursive: true });
|
||||
const original = await fs.readFile(inlinePath, "utf-8").catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
const markers = resolvePhaseMarkers(params.phase);
|
||||
const updated = replaceManagedMarkdownBlock({
|
||||
original,
|
||||
heading: DAILY_PHASE_HEADINGS[params.phase],
|
||||
startMarker: markers.start,
|
||||
endMarker: markers.end,
|
||||
body,
|
||||
});
|
||||
await fs.writeFile(inlinePath, withTrailingNewline(updated), "utf-8");
|
||||
}
|
||||
|
||||
if (shouldWriteSeparate(params.storage)) {
|
||||
reportPath = resolveSeparateReportPath(
|
||||
params.workspaceDir,
|
||||
params.phase,
|
||||
nowMs,
|
||||
params.timezone,
|
||||
);
|
||||
await fs.mkdir(path.dirname(reportPath), { recursive: true });
|
||||
const report = [
|
||||
`# ${params.phase === "light" ? "Light Sleep" : "REM Sleep"}`,
|
||||
"",
|
||||
body,
|
||||
"",
|
||||
].join("\n");
|
||||
await fs.writeFile(reportPath, report, "utf-8");
|
||||
}
|
||||
|
||||
await appendMemoryHostEvent(params.workspaceDir, {
|
||||
type: "memory.dream.completed",
|
||||
timestamp: resolveMemoryCoreTimestamp(nowMs),
|
||||
phase: params.phase,
|
||||
outcome: "completed",
|
||||
...(inlinePath ? { inlinePath } : {}),
|
||||
...(reportPath ? { reportPath } : {}),
|
||||
lineCount: params.bodyLines.length,
|
||||
storageMode: params.storage.mode,
|
||||
});
|
||||
|
||||
return {
|
||||
...(inlinePath ? { inlinePath } : {}),
|
||||
...(reportPath ? { reportPath } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeDeepDreamingReport(params: {
|
||||
workspaceDir: string;
|
||||
bodyLines: string[];
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
storage: MemoryDreamingStorageConfig;
|
||||
}): Promise<string | undefined> {
|
||||
const nowMs = resolveMemoryCoreNowMs(params.nowMs);
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No durable changes.";
|
||||
const inlinePath = await updateDeepDreamsFile({
|
||||
workspaceDir: params.workspaceDir,
|
||||
bodyLines: params.bodyLines,
|
||||
});
|
||||
let reportPath: string | undefined;
|
||||
if (shouldWriteSeparate(params.storage)) {
|
||||
reportPath = resolveSeparateReportPath(params.workspaceDir, "deep", nowMs, params.timezone);
|
||||
await fs.mkdir(path.dirname(reportPath), { recursive: true });
|
||||
await fs.writeFile(reportPath, `# Deep Sleep\n\n${body}\n`, "utf-8");
|
||||
}
|
||||
await appendMemoryHostEvent(params.workspaceDir, {
|
||||
type: "memory.dream.completed",
|
||||
timestamp: resolveMemoryCoreTimestamp(nowMs),
|
||||
phase: "deep",
|
||||
outcome: "completed",
|
||||
inlinePath,
|
||||
...(reportPath ? { reportPath } : {}),
|
||||
lineCount: params.bodyLines.length,
|
||||
storageMode: params.storage.mode,
|
||||
});
|
||||
return reportPath;
|
||||
}
|
||||
1557
extensions/memory-core/src/dreaming-narrative.test.ts
Normal file
1557
extensions/memory-core/src/dreaming-narrative.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
1039
extensions/memory-core/src/dreaming-narrative.ts
Normal file
1039
extensions/memory-core/src/dreaming-narrative.ts
Normal file
File diff suppressed because it is too large
Load Diff
3360
extensions/memory-core/src/dreaming-phases.test.ts
Normal file
3360
extensions/memory-core/src/dreaming-phases.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
2032
extensions/memory-core/src/dreaming-phases.ts
Normal file
2032
extensions/memory-core/src/dreaming-phases.ts
Normal file
File diff suppressed because it is too large
Load Diff
256
extensions/memory-core/src/dreaming-repair.test.ts
Normal file
256
extensions/memory-core/src/dreaming-repair.test.ts
Normal file
@@ -0,0 +1,256 @@
|
||||
// Memory Core tests cover dreaming repair plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { auditDreamingArtifacts, repairDreamingArtifacts } from "./dreaming-repair.js";
|
||||
import {
|
||||
configureMemoryCoreDreamingStateForTests,
|
||||
DREAMING_DAILY_INGESTION_NAMESPACE,
|
||||
DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
|
||||
DREAMING_SESSION_INGESTION_SEEN_NAMESPACE,
|
||||
readMemoryCoreWorkspaceEntries,
|
||||
resetMemoryCoreDreamingStateForTests,
|
||||
writeMemoryCoreWorkspaceEntries,
|
||||
} from "./dreaming-state.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
await configureMemoryCoreDreamingStateForTests();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
resetMemoryCoreDreamingStateForTests();
|
||||
});
|
||||
|
||||
async function createWorkspace(): Promise<string> {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "dreaming-repair-test-"));
|
||||
tempDirs.push(workspaceDir);
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", ".dreams"), { recursive: true });
|
||||
return workspaceDir;
|
||||
}
|
||||
|
||||
function requireArchiveDir(archiveDir: string | undefined): string {
|
||||
if (!archiveDir) {
|
||||
throw new Error("Expected dreaming repair to create an archive directory");
|
||||
}
|
||||
return archiveDir;
|
||||
}
|
||||
|
||||
async function expectPathMissing(targetPath: string): Promise<void> {
|
||||
let error: unknown;
|
||||
try {
|
||||
await fs.access(targetPath);
|
||||
} catch (caught) {
|
||||
error = caught;
|
||||
}
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as NodeJS.ErrnoException).code).toBe("ENOENT");
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
while (tempDirs.length > 0) {
|
||||
const dir = tempDirs.pop();
|
||||
if (dir) {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("dreaming artifact repair", () => {
|
||||
it("detects self-ingested dreaming corpus lines", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
await fs
|
||||
.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-corpus", "2026-04-11.txt"),
|
||||
[
|
||||
"[main/dreaming-main.jsonl#L4] regular session text",
|
||||
"[main/dreaming-narrative-light.jsonl#L1] Write a dream diary entry from these memory fragments:",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
)
|
||||
.catch(async () => {
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", ".dreams", "session-corpus"), {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-corpus", "2026-04-11.txt"),
|
||||
[
|
||||
"[main/dreaming-main.jsonl#L4] regular session text",
|
||||
"[main/dreaming-narrative-light.jsonl#L1] Write a dream diary entry from these memory fragments:",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
});
|
||||
|
||||
const audit = await auditDreamingArtifacts({ workspaceDir });
|
||||
|
||||
expect(audit.sessionCorpusFileCount).toBe(1);
|
||||
expect(audit.suspiciousSessionCorpusFileCount).toBe(1);
|
||||
expect(audit.suspiciousSessionCorpusLineCount).toBe(1);
|
||||
expect(audit.issues).toStrictEqual([
|
||||
{
|
||||
severity: "warn",
|
||||
code: "dreaming-session-corpus-self-ingested",
|
||||
message:
|
||||
"Dreaming session corpus appears to contain self-ingested narrative content (1 suspicious line).",
|
||||
fixable: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not flag ordinary transcript text that merely mentions dreaming-narrative", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", ".dreams", "session-corpus"), {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-corpus", "2026-04-11.txt"),
|
||||
[
|
||||
"[main/chat.jsonl#L4] regular session text",
|
||||
"[main/chat.jsonl#L5] We should inspect the dreaming-narrative session behavior tomorrow.",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const audit = await auditDreamingArtifacts({ workspaceDir });
|
||||
|
||||
expect(audit.suspiciousSessionCorpusFileCount).toBe(0);
|
||||
expect(audit.suspiciousSessionCorpusLineCount).toBe(0);
|
||||
expect(audit.issues).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("rejects relative workspace paths during audit and repair", async () => {
|
||||
await expect(auditDreamingArtifacts({ workspaceDir: "relative/workspace" })).rejects.toThrow(
|
||||
"workspaceDir must be an absolute path",
|
||||
);
|
||||
await expect(repairDreamingArtifacts({ workspaceDir: "relative/workspace" })).rejects.toThrow(
|
||||
"workspaceDir must be an absolute path",
|
||||
);
|
||||
});
|
||||
|
||||
it("archives derived dreaming artifacts without touching the diary by default", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
const sessionCorpusDir = path.join(workspaceDir, "memory", ".dreams", "session-corpus");
|
||||
await fs.mkdir(sessionCorpusDir, { recursive: true });
|
||||
await fs.writeFile(path.join(sessionCorpusDir, "2026-04-11.txt"), "corpus\n", "utf-8");
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-ingestion.json"),
|
||||
JSON.stringify({ version: 3, files: {}, seenMessages: {} }, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(dreamsPath, "# Dream Diary\n", "utf-8");
|
||||
|
||||
const repair = await repairDreamingArtifacts({
|
||||
workspaceDir,
|
||||
now: new Date("2026-04-11T21:30:00.000Z"),
|
||||
});
|
||||
|
||||
expect(repair.changed).toBe(true);
|
||||
expect(repair.archivedSessionCorpus).toBe(true);
|
||||
expect(repair.archivedSessionIngestion).toBe(true);
|
||||
expect(repair.archivedDreamsDiary).toBe(false);
|
||||
const archiveDir = requireArchiveDir(repair.archiveDir);
|
||||
expect(archiveDir).toBe(
|
||||
path.join(workspaceDir, ".openclaw-repair", "dreaming", "2026-04-11T21-30-00-000Z"),
|
||||
);
|
||||
await expectPathMissing(sessionCorpusDir);
|
||||
await expectPathMissing(path.join(workspaceDir, "memory", ".dreams", "session-ingestion.json"));
|
||||
await expect(fs.readFile(dreamsPath, "utf-8")).resolves.toContain("# Dream Diary");
|
||||
const archivedEntries = await fs.readdir(archiveDir);
|
||||
expect(archivedEntries.filter((entry) => entry.startsWith("session-corpus."))).not.toEqual([]);
|
||||
expect(
|
||||
archivedEntries.filter((entry) => entry.startsWith("session-ingestion.json.")),
|
||||
).not.toEqual([]);
|
||||
});
|
||||
|
||||
it("clears sqlite session ingestion state when archiving session corpus", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
const sessionCorpusDir = path.join(workspaceDir, "memory", ".dreams", "session-corpus");
|
||||
await fs.mkdir(sessionCorpusDir, { recursive: true });
|
||||
await fs.writeFile(path.join(sessionCorpusDir, "2026-04-11.txt"), "corpus\n", "utf-8");
|
||||
await Promise.all([
|
||||
writeMemoryCoreWorkspaceEntries({
|
||||
namespace: DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
|
||||
workspaceDir,
|
||||
entries: [
|
||||
{
|
||||
key: "main/session.jsonl",
|
||||
value: {
|
||||
lastSize: 120,
|
||||
lastMtimeMs: 1_000,
|
||||
lastContentHash: "hash",
|
||||
cursorLine: 42,
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
writeMemoryCoreWorkspaceEntries({
|
||||
namespace: DREAMING_SESSION_INGESTION_SEEN_NAMESPACE,
|
||||
workspaceDir,
|
||||
entries: [
|
||||
{
|
||||
key: "main:0",
|
||||
value: { scope: "main", index: 0, hashes: ["message-hash"] },
|
||||
},
|
||||
],
|
||||
}),
|
||||
]);
|
||||
|
||||
const repair = await repairDreamingArtifacts({ workspaceDir });
|
||||
|
||||
expect(repair.archivedSessionCorpus).toBe(true);
|
||||
await expect(
|
||||
readMemoryCoreWorkspaceEntries({
|
||||
namespace: DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
|
||||
workspaceDir,
|
||||
}),
|
||||
).resolves.toEqual([]);
|
||||
await expect(
|
||||
readMemoryCoreWorkspaceEntries({
|
||||
namespace: DREAMING_SESSION_INGESTION_SEEN_NAMESPACE,
|
||||
workspaceDir,
|
||||
}),
|
||||
).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("reports ingestion state present from SQLite when legacy JSON is absent", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
// Write SQLite ingestion entries but NO legacy session-ingestion.json
|
||||
await writeMemoryCoreWorkspaceEntries({
|
||||
namespace: DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
|
||||
workspaceDir,
|
||||
entries: [
|
||||
{
|
||||
key: "main/session.jsonl",
|
||||
value: { lastSize: 120, lastMtimeMs: 1_000, lastContentHash: "hash", cursorLine: 42 },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const audit = await auditDreamingArtifacts({ workspaceDir });
|
||||
|
||||
expect(audit.sessionIngestionExists).toBe(true);
|
||||
});
|
||||
|
||||
it("reports ingestion state present from SQLite daily namespace", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
// Only daily ingestion namespace has rows
|
||||
await writeMemoryCoreWorkspaceEntries({
|
||||
namespace: DREAMING_DAILY_INGESTION_NAMESPACE,
|
||||
workspaceDir,
|
||||
entries: [
|
||||
{
|
||||
key: "2026-06-10",
|
||||
value: { ingestedAt: Date.now() },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const audit = await auditDreamingArtifacts({ workspaceDir });
|
||||
|
||||
expect(audit.sessionIngestionExists).toBe(true);
|
||||
});
|
||||
});
|
||||
337
extensions/memory-core/src/dreaming-repair.ts
Normal file
337
extensions/memory-core/src/dreaming-repair.ts
Normal file
@@ -0,0 +1,337 @@
|
||||
// Memory Core plugin module implements dreaming repair behavior.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { extractErrorCode } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
clearMemoryCoreWorkspaceNamespace,
|
||||
DREAMING_DAILY_INGESTION_NAMESPACE,
|
||||
DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
|
||||
DREAMING_SESSION_INGESTION_SEEN_NAMESPACE,
|
||||
readMemoryCoreWorkspaceEntries,
|
||||
} from "./dreaming-state.js";
|
||||
|
||||
type DreamingArtifactsAuditIssue = {
|
||||
severity: "warn" | "error";
|
||||
code:
|
||||
| "dreaming-session-corpus-unreadable"
|
||||
| "dreaming-session-corpus-self-ingested"
|
||||
| "dreaming-session-ingestion-unreadable"
|
||||
| "dreaming-diary-unreadable";
|
||||
message: string;
|
||||
fixable: boolean;
|
||||
};
|
||||
|
||||
export type DreamingArtifactsAuditSummary = {
|
||||
dreamsPath?: string;
|
||||
sessionCorpusDir: string;
|
||||
sessionCorpusFileCount: number;
|
||||
suspiciousSessionCorpusFileCount: number;
|
||||
suspiciousSessionCorpusLineCount: number;
|
||||
sessionIngestionPath: string;
|
||||
sessionIngestionExists: boolean;
|
||||
issues: DreamingArtifactsAuditIssue[];
|
||||
};
|
||||
|
||||
export type RepairDreamingArtifactsResult = {
|
||||
changed: boolean;
|
||||
archiveDir?: string;
|
||||
archivedDreamsDiary: boolean;
|
||||
archivedSessionCorpus: boolean;
|
||||
archivedSessionIngestion: boolean;
|
||||
archivedPaths: string[];
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
const DREAMS_FILENAMES = ["DREAMS.md", "dreams.md"] as const;
|
||||
const SESSION_CORPUS_RELATIVE_DIR = path.join("memory", ".dreams", "session-corpus");
|
||||
const SESSION_INGESTION_RELATIVE_PATH = path.join("memory", ".dreams", "session-ingestion.json");
|
||||
const REPAIR_ARCHIVE_RELATIVE_DIR = path.join(".openclaw-repair", "dreaming");
|
||||
const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-";
|
||||
const DREAMING_NARRATIVE_PROMPT_PREFIX = "Write a dream diary entry from these memory fragments";
|
||||
|
||||
function requireAbsoluteWorkspaceDir(rawWorkspaceDir: string): string {
|
||||
const trimmed = rawWorkspaceDir.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("workspaceDir is required");
|
||||
}
|
||||
if (!path.isAbsolute(trimmed)) {
|
||||
throw new Error("workspaceDir must be an absolute path");
|
||||
}
|
||||
return path.resolve(trimmed);
|
||||
}
|
||||
|
||||
async function resolveExistingDreamsPath(workspaceDir: string): Promise<string | undefined> {
|
||||
for (const fileName of DREAMS_FILENAMES) {
|
||||
const candidate = path.join(workspaceDir, fileName);
|
||||
try {
|
||||
await fs.access(candidate);
|
||||
return candidate;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function listSessionCorpusFiles(sessionCorpusDir: string): Promise<string[]> {
|
||||
const entries = await fs.readdir(sessionCorpusDir, { withFileTypes: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".txt"))
|
||||
.map((entry) => path.join(sessionCorpusDir, entry.name))
|
||||
.toSorted();
|
||||
}
|
||||
|
||||
function isSuspiciousSessionCorpusLine(line: string): boolean {
|
||||
return (
|
||||
line.includes(DREAMING_NARRATIVE_PROMPT_PREFIX) &&
|
||||
(line.includes(DREAMING_NARRATIVE_RUN_PREFIX) || line.includes("dreaming-narrative-"))
|
||||
);
|
||||
}
|
||||
|
||||
function buildArchiveTimestamp(now: Date): string {
|
||||
return now.toISOString().replace(/[:.]/g, "-");
|
||||
}
|
||||
|
||||
async function ensureArchivablePath(targetPath: string): Promise<"file" | "dir" | null> {
|
||||
const stat = await fs.lstat(targetPath).catch((err: unknown) => {
|
||||
if (extractErrorCode(err) === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!stat) {
|
||||
return null;
|
||||
}
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new Error(`Refusing to archive symlinked path: ${targetPath}`);
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
return "dir";
|
||||
}
|
||||
if (stat.isFile()) {
|
||||
return "file";
|
||||
}
|
||||
throw new Error(`Refusing to archive non-file artifact: ${targetPath}`);
|
||||
}
|
||||
|
||||
async function moveToArchive(params: {
|
||||
targetPath: string;
|
||||
archiveDir: string;
|
||||
}): Promise<string | null> {
|
||||
const kind = await ensureArchivablePath(params.targetPath);
|
||||
if (!kind) {
|
||||
return null;
|
||||
}
|
||||
await fs.mkdir(params.archiveDir, { recursive: true });
|
||||
const baseName = path.basename(params.targetPath);
|
||||
const destination = path.join(params.archiveDir, `${baseName}.${randomUUID()}`);
|
||||
await fs.rename(params.targetPath, destination);
|
||||
return destination;
|
||||
}
|
||||
|
||||
async function clearSessionIngestionState(workspaceDir: string): Promise<void> {
|
||||
await Promise.all([
|
||||
clearMemoryCoreWorkspaceNamespace({
|
||||
namespace: DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
|
||||
workspaceDir,
|
||||
}),
|
||||
clearMemoryCoreWorkspaceNamespace({
|
||||
namespace: DREAMING_SESSION_INGESTION_SEEN_NAMESPACE,
|
||||
workspaceDir,
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
export async function auditDreamingArtifacts(params: {
|
||||
workspaceDir: string;
|
||||
}): Promise<DreamingArtifactsAuditSummary> {
|
||||
const workspaceDir = requireAbsoluteWorkspaceDir(params.workspaceDir);
|
||||
const dreamsPath = await resolveExistingDreamsPath(workspaceDir);
|
||||
const sessionCorpusDir = path.join(workspaceDir, SESSION_CORPUS_RELATIVE_DIR);
|
||||
const sessionIngestionPath = path.join(workspaceDir, SESSION_INGESTION_RELATIVE_PATH);
|
||||
const issues: DreamingArtifactsAuditIssue[] = [];
|
||||
let sessionCorpusFileCount = 0;
|
||||
let suspiciousSessionCorpusFileCount = 0;
|
||||
let suspiciousSessionCorpusLineCount = 0;
|
||||
let sessionIngestionExists = false;
|
||||
|
||||
if (dreamsPath) {
|
||||
try {
|
||||
await fs.access(dreamsPath);
|
||||
} catch (err) {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
code: "dreaming-diary-unreadable",
|
||||
message: `Dream diary could not be inspected: ${(err as NodeJS.ErrnoException).code ?? "error"}.`,
|
||||
fixable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const corpusFiles = await listSessionCorpusFiles(sessionCorpusDir);
|
||||
sessionCorpusFileCount = corpusFiles.length;
|
||||
for (const corpusFile of corpusFiles) {
|
||||
const content = await fs.readFile(corpusFile, "utf-8");
|
||||
const suspiciousLines = content
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0 && isSuspiciousSessionCorpusLine(line));
|
||||
if (suspiciousLines.length > 0) {
|
||||
suspiciousSessionCorpusFileCount += 1;
|
||||
suspiciousSessionCorpusLineCount += suspiciousLines.length;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
code: "dreaming-session-corpus-unreadable",
|
||||
message: `Dreaming session corpus could not be inspected: ${(err as NodeJS.ErrnoException).code ?? "error"}.`,
|
||||
fixable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.access(sessionIngestionPath);
|
||||
sessionIngestionExists = true;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
code: "dreaming-session-ingestion-unreadable",
|
||||
message: `Dreaming session-ingestion state could not be inspected: ${(err as NodeJS.ErrnoException).code ?? "error"}.`,
|
||||
fixable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to SQLite plugin state when the legacy JSON file was archived by migration.
|
||||
if (!sessionIngestionExists) {
|
||||
try {
|
||||
const ingestionNamespaces = [
|
||||
DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
|
||||
DREAMING_SESSION_INGESTION_SEEN_NAMESPACE,
|
||||
DREAMING_DAILY_INGESTION_NAMESPACE,
|
||||
] as const;
|
||||
for (const namespace of ingestionNamespaces) {
|
||||
const entries = await readMemoryCoreWorkspaceEntries({
|
||||
namespace,
|
||||
workspaceDir,
|
||||
});
|
||||
if (entries.length > 0) {
|
||||
sessionIngestionExists = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// SQLite plugin state unavailable — keep filesystem-only result.
|
||||
}
|
||||
}
|
||||
|
||||
if (suspiciousSessionCorpusLineCount > 0) {
|
||||
issues.push({
|
||||
severity: "warn",
|
||||
code: "dreaming-session-corpus-self-ingested",
|
||||
message: `Dreaming session corpus appears to contain self-ingested narrative content (${suspiciousSessionCorpusLineCount} suspicious line${suspiciousSessionCorpusLineCount === 1 ? "" : "s"}).`,
|
||||
fixable: true,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...(dreamsPath ? { dreamsPath } : {}),
|
||||
sessionCorpusDir,
|
||||
sessionCorpusFileCount,
|
||||
suspiciousSessionCorpusFileCount,
|
||||
suspiciousSessionCorpusLineCount,
|
||||
sessionIngestionPath,
|
||||
sessionIngestionExists,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
export async function repairDreamingArtifacts(params: {
|
||||
workspaceDir: string;
|
||||
archiveDiary?: boolean;
|
||||
now?: Date;
|
||||
}): Promise<RepairDreamingArtifactsResult> {
|
||||
const workspaceDir = requireAbsoluteWorkspaceDir(params.workspaceDir);
|
||||
const warnings: string[] = [];
|
||||
const archivedPaths: string[] = [];
|
||||
let archiveDir: string | undefined;
|
||||
let archivedDreamsDiary = false;
|
||||
let archivedSessionCorpus = false;
|
||||
let archivedSessionIngestion = false;
|
||||
|
||||
const ensureArchiveDir = () => {
|
||||
archiveDir ??= path.join(
|
||||
workspaceDir,
|
||||
REPAIR_ARCHIVE_RELATIVE_DIR,
|
||||
buildArchiveTimestamp(params.now ?? new Date()),
|
||||
);
|
||||
return archiveDir;
|
||||
};
|
||||
|
||||
const archivePathIfPresent = async (targetPath: string): Promise<string | null> => {
|
||||
try {
|
||||
return await moveToArchive({ targetPath, archiveDir: ensureArchiveDir() });
|
||||
} catch (err) {
|
||||
warnings.push(err instanceof Error ? err.message : String(err));
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const sessionCorpusDestination = await archivePathIfPresent(
|
||||
path.join(workspaceDir, SESSION_CORPUS_RELATIVE_DIR),
|
||||
);
|
||||
if (sessionCorpusDestination) {
|
||||
archivedSessionCorpus = true;
|
||||
archivedPaths.push(sessionCorpusDestination);
|
||||
}
|
||||
|
||||
const sessionIngestionDestination = await archivePathIfPresent(
|
||||
path.join(workspaceDir, SESSION_INGESTION_RELATIVE_PATH),
|
||||
);
|
||||
if (sessionIngestionDestination) {
|
||||
archivedSessionIngestion = true;
|
||||
archivedPaths.push(sessionIngestionDestination);
|
||||
}
|
||||
|
||||
if (sessionCorpusDestination || sessionIngestionDestination) {
|
||||
try {
|
||||
await clearSessionIngestionState(workspaceDir);
|
||||
} catch (err) {
|
||||
warnings.push(
|
||||
`Failed clearing dreaming session-ingestion SQLite state: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (params.archiveDiary) {
|
||||
const dreamsPath = await resolveExistingDreamsPath(workspaceDir);
|
||||
if (dreamsPath) {
|
||||
const dreamsDestination = await archivePathIfPresent(dreamsPath);
|
||||
if (dreamsDestination) {
|
||||
archivedDreamsDiary = true;
|
||||
archivedPaths.push(dreamsDestination);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const changed = archivedDreamsDiary || archivedSessionCorpus || archivedSessionIngestion;
|
||||
return {
|
||||
changed,
|
||||
...(archiveDir ? { archiveDir } : {}),
|
||||
archivedDreamsDiary,
|
||||
archivedSessionCorpus,
|
||||
archivedSessionIngestion,
|
||||
archivedPaths,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
174
extensions/memory-core/src/dreaming-shadow-trial.test.ts
Normal file
174
extensions/memory-core/src/dreaming-shadow-trial.test.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
// Memory Core tests cover dreaming shadow trial plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildDreamingShadowTrialReport,
|
||||
defaultDreamingShadowTrialReportPath,
|
||||
resolveDreamingShadowTrialRecommendation,
|
||||
writeDreamingShadowTrialReport,
|
||||
} from "./dreaming-shadow-trial.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempWorkspace } = createMemoryCoreTestHarness();
|
||||
|
||||
const baseInput = {
|
||||
candidate: "The user prefers release notes with exact verification commands.",
|
||||
trialPrompt: "Prepare a release readiness note.",
|
||||
baselineOutcome: "Mentions tests passed without the exact command.",
|
||||
candidateOutcome: "Includes the exact verification command and remaining risk.",
|
||||
reason: "The candidate improves the release reply without exposing private data.",
|
||||
riskFlags: ["no secret exposure", "no outdated preference conflict"],
|
||||
evidenceRefs: ["memory/2026-05-18.md#L30-L49"],
|
||||
};
|
||||
|
||||
describe("dreaming shadow trial runner", () => {
|
||||
it("maps verdicts to report-only recommendations", () => {
|
||||
expect(resolveDreamingShadowTrialRecommendation("helpful")).toBe("promote");
|
||||
expect(resolveDreamingShadowTrialRecommendation("neutral")).toBe("defer");
|
||||
expect(resolveDreamingShadowTrialRecommendation("harmful")).toBe("reject");
|
||||
});
|
||||
|
||||
it("builds the stable shadow-trial report contract", () => {
|
||||
const report = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "helpful",
|
||||
nowMs: Date.parse("2026-05-18T18:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(report.recommendation).toBe("promote");
|
||||
expect(report.promotionAction).toBe("report-only");
|
||||
expect(report.markdown).toContain("candidate: The user prefers release notes");
|
||||
expect(report.markdown).toContain("baseline outcome: Mentions tests passed");
|
||||
expect(report.markdown).toContain("candidate outcome: Includes the exact verification command");
|
||||
expect(report.markdown).toContain("verdict: helpful");
|
||||
expect(report.markdown).toContain("recommendation: promote");
|
||||
expect(report.markdown).toContain("risk flags:");
|
||||
expect(report.markdown).toContain("- no secret exposure");
|
||||
expect(report.markdown).toContain("evidence refs:");
|
||||
expect(report.markdown).toContain("promotion action: report-only");
|
||||
expect(report.markdown).not.toContain("promoted to MEMORY.md");
|
||||
});
|
||||
|
||||
it("writes only the shadow-trial report and leaves MEMORY.md unchanged", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-shadow-trial-");
|
||||
const memoryPath = path.join(workspaceDir, "MEMORY.md");
|
||||
await fs.writeFile(memoryPath, "# Memory\n\nExisting durable memory.\n", "utf-8");
|
||||
|
||||
const report = await writeDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "neutral",
|
||||
workspaceDir,
|
||||
nowMs: Date.parse("2026-05-18T18:00:00.000Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
|
||||
expect(report.recommendation).toBe("defer");
|
||||
expect(path.dirname(report.reportPath!)).toBe(
|
||||
path.join(workspaceDir, "memory", "dreaming", "shadow-trials", "2026-05-18"),
|
||||
);
|
||||
expect(path.basename(report.reportPath!)).toMatch(/^[a-f0-9]{12}\.md$/);
|
||||
await expect(fs.readFile(memoryPath, "utf-8")).resolves.toBe(
|
||||
"# Memory\n\nExisting durable memory.\n",
|
||||
);
|
||||
expect(report.reportPath).toBeTruthy();
|
||||
await expect(fs.readFile(report.reportPath!, "utf-8")).resolves.toContain(
|
||||
"promotion action: report-only",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses the configured dreaming timezone for the default report day", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-shadow-trial-timezone-");
|
||||
|
||||
const report = await writeDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "helpful",
|
||||
workspaceDir,
|
||||
nowMs: Date.parse("2026-05-18T21:30:00.000Z"),
|
||||
timezone: "Asia/Riyadh",
|
||||
});
|
||||
|
||||
expect(path.dirname(report.reportPath!)).toBe(
|
||||
path.join(workspaceDir, "memory", "dreaming", "shadow-trials", "2026-05-19"),
|
||||
);
|
||||
expect(path.basename(report.reportPath!)).toMatch(/^[a-f0-9]{12}\.md$/);
|
||||
await expect(fs.readFile(report.reportPath!, "utf-8")).resolves.toContain(
|
||||
"recommendation: promote",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps distinct same-day trials in separate default report files", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-shadow-trial-collisions-");
|
||||
const nowMs = Date.parse("2026-05-18T18:00:00.000Z");
|
||||
|
||||
const first = await writeDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "helpful",
|
||||
workspaceDir,
|
||||
nowMs,
|
||||
});
|
||||
const second = await writeDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
candidate: "The user prefers terse release notes with exact verification commands.",
|
||||
verdict: "helpful",
|
||||
workspaceDir,
|
||||
nowMs,
|
||||
});
|
||||
|
||||
expect(first.reportPath).not.toBe(second.reportPath);
|
||||
expect(path.dirname(first.reportPath!)).toBe(path.dirname(second.reportPath!));
|
||||
await expect(fs.readFile(first.reportPath!, "utf-8")).resolves.toContain(
|
||||
"candidate: The user prefers release notes",
|
||||
);
|
||||
await expect(fs.readFile(second.reportPath!, "utf-8")).resolves.toContain(
|
||||
"candidate: The user prefers terse release notes",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps risky candidates reject-only without promoting durable memory", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-shadow-trial-risk-");
|
||||
const reportPath = defaultDreamingShadowTrialReportPath({
|
||||
...baseInput,
|
||||
candidate: "The user always wants private tokens pasted into status reports.",
|
||||
candidateOutcome: "Includes a private token in the release reply.",
|
||||
verdict: "harmful",
|
||||
reason: "The candidate creates secret exposure risk.",
|
||||
riskFlags: ["secret exposure"],
|
||||
workspaceDir,
|
||||
nowMs: Date.parse("2026-05-19T01:00:00.000Z"),
|
||||
});
|
||||
|
||||
const report = await writeDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
candidate: "The user always wants private tokens pasted into status reports.",
|
||||
candidateOutcome: "Includes a private token in the release reply.",
|
||||
verdict: "harmful",
|
||||
reason: "The candidate creates secret exposure risk.",
|
||||
riskFlags: ["secret exposure"],
|
||||
workspaceDir,
|
||||
reportPath,
|
||||
});
|
||||
|
||||
expect(report.recommendation).toBe("reject");
|
||||
expect(report.markdown).toContain("verdict: harmful");
|
||||
expect(report.markdown).toContain("recommendation: reject");
|
||||
expect(report.markdown).toContain("promotion action: report-only");
|
||||
await expect(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8")).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps missing evidence as empty machine data while rendering markdown placeholders", () => {
|
||||
const report = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "neutral",
|
||||
riskFlags: [],
|
||||
evidenceRefs: [],
|
||||
});
|
||||
|
||||
expect(report.riskFlags).toEqual([]);
|
||||
expect(report.evidenceRefs).toEqual([]);
|
||||
expect(report.markdown).toContain("- none recorded");
|
||||
expect(report.markdown).toContain("- none supplied");
|
||||
});
|
||||
});
|
||||
242
extensions/memory-core/src/dreaming-shadow-trial.ts
Normal file
242
extensions/memory-core/src/dreaming-shadow-trial.ts
Normal file
@@ -0,0 +1,242 @@
|
||||
// Memory Core plugin module implements dreaming shadow trial behavior.
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { formatMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
|
||||
export type DreamingShadowTrialVerdict = "helpful" | "neutral" | "harmful";
|
||||
export type DreamingShadowTrialRecommendation = "promote" | "defer" | "reject";
|
||||
|
||||
export type DreamingShadowTrialInput = {
|
||||
candidate: string;
|
||||
trialPrompt: string;
|
||||
baselineOutcome: string;
|
||||
candidateOutcome: string;
|
||||
verdict: DreamingShadowTrialVerdict;
|
||||
reason: string;
|
||||
riskFlags?: string[];
|
||||
evidenceRefs?: string[];
|
||||
workspaceDir?: string;
|
||||
reportPath?: string;
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
};
|
||||
|
||||
export type DreamingShadowTrialReport = {
|
||||
candidate: string;
|
||||
trialPrompt: string;
|
||||
baselineOutcome: string;
|
||||
candidateOutcome: string;
|
||||
verdict: DreamingShadowTrialVerdict;
|
||||
recommendation: DreamingShadowTrialRecommendation;
|
||||
reason: string;
|
||||
riskFlags: string[];
|
||||
evidenceRefs: string[];
|
||||
promotionAction: "report-only";
|
||||
reportPath?: string;
|
||||
markdown: string;
|
||||
};
|
||||
|
||||
function normalizeRequiredText(value: string, label: string): string {
|
||||
const normalized = value.trim().replace(/\s+/g, " ");
|
||||
if (!normalized) {
|
||||
throw new Error(`dreaming shadow trial requires ${label}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeList(values: string[] | undefined, fallback: string): string[] {
|
||||
const normalized = (values ?? []).map((value) => value.trim()).filter(Boolean);
|
||||
return normalized.length > 0 ? normalized : [fallback];
|
||||
}
|
||||
|
||||
function normalizeDataList(values: string[] | undefined): string[] {
|
||||
return (values ?? []).map((value) => value.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
export function resolveDreamingShadowTrialRecommendation(
|
||||
verdict: DreamingShadowTrialVerdict,
|
||||
): DreamingShadowTrialRecommendation {
|
||||
if (verdict === "helpful") {
|
||||
return "promote";
|
||||
}
|
||||
if (verdict === "harmful") {
|
||||
return "reject";
|
||||
}
|
||||
return "defer";
|
||||
}
|
||||
|
||||
function formatList(values: string[]): string {
|
||||
return values.map((value) => `- ${value}`).join("\n");
|
||||
}
|
||||
|
||||
function resolveReportContentHash(params: {
|
||||
candidate: string;
|
||||
trialPrompt: string;
|
||||
baselineOutcome: string;
|
||||
candidateOutcome: string;
|
||||
verdict: DreamingShadowTrialVerdict;
|
||||
reason: string;
|
||||
riskFlags: string[];
|
||||
evidenceRefs: string[];
|
||||
}): string {
|
||||
const seed = JSON.stringify([
|
||||
params.candidate,
|
||||
params.trialPrompt,
|
||||
params.baselineOutcome,
|
||||
params.candidateOutcome,
|
||||
params.verdict,
|
||||
params.reason,
|
||||
params.riskFlags,
|
||||
params.evidenceRefs,
|
||||
]);
|
||||
return crypto.createHash("sha256").update(seed).digest("hex").slice(0, 12);
|
||||
}
|
||||
|
||||
export function defaultDreamingShadowTrialReportPath(params: {
|
||||
workspaceDir: string;
|
||||
candidate: string;
|
||||
trialPrompt: string;
|
||||
baselineOutcome: string;
|
||||
candidateOutcome: string;
|
||||
verdict: DreamingShadowTrialVerdict;
|
||||
reason: string;
|
||||
riskFlags?: string[];
|
||||
evidenceRefs?: string[];
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
}): string {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const day = formatMemoryDreamingDay(nowMs, params.timezone);
|
||||
const contentHash = resolveReportContentHash({
|
||||
candidate: normalizeRequiredText(params.candidate, "candidate"),
|
||||
trialPrompt: normalizeRequiredText(params.trialPrompt, "trialPrompt"),
|
||||
baselineOutcome: normalizeRequiredText(params.baselineOutcome, "baselineOutcome"),
|
||||
candidateOutcome: normalizeRequiredText(params.candidateOutcome, "candidateOutcome"),
|
||||
verdict: params.verdict,
|
||||
reason: normalizeRequiredText(params.reason, "reason"),
|
||||
riskFlags: normalizeDataList(params.riskFlags),
|
||||
evidenceRefs: normalizeDataList(params.evidenceRefs),
|
||||
});
|
||||
return path.join(
|
||||
params.workspaceDir,
|
||||
"memory",
|
||||
"dreaming",
|
||||
"shadow-trials",
|
||||
day,
|
||||
`${contentHash}.md`,
|
||||
);
|
||||
}
|
||||
|
||||
function resolveReportPath(params: {
|
||||
workspaceDir?: string;
|
||||
candidate: string;
|
||||
trialPrompt: string;
|
||||
baselineOutcome: string;
|
||||
candidateOutcome: string;
|
||||
verdict: DreamingShadowTrialVerdict;
|
||||
reason: string;
|
||||
riskFlags: string[];
|
||||
evidenceRefs: string[];
|
||||
reportPath?: string;
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
}): string | undefined {
|
||||
if (params.reportPath) {
|
||||
if (path.isAbsolute(params.reportPath)) {
|
||||
return params.reportPath;
|
||||
}
|
||||
if (!params.workspaceDir) {
|
||||
throw new Error("dreaming shadow trial relative reportPath requires workspaceDir");
|
||||
}
|
||||
return path.join(params.workspaceDir, params.reportPath);
|
||||
}
|
||||
if (!params.workspaceDir) {
|
||||
return undefined;
|
||||
}
|
||||
return defaultDreamingShadowTrialReportPath({
|
||||
workspaceDir: params.workspaceDir,
|
||||
candidate: params.candidate,
|
||||
trialPrompt: params.trialPrompt,
|
||||
baselineOutcome: params.baselineOutcome,
|
||||
candidateOutcome: params.candidateOutcome,
|
||||
verdict: params.verdict,
|
||||
reason: params.reason,
|
||||
riskFlags: params.riskFlags,
|
||||
evidenceRefs: params.evidenceRefs,
|
||||
nowMs: params.nowMs,
|
||||
timezone: params.timezone,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDreamingShadowTrialReport(
|
||||
input: DreamingShadowTrialInput,
|
||||
): DreamingShadowTrialReport {
|
||||
const candidate = normalizeRequiredText(input.candidate, "candidate");
|
||||
const trialPrompt = normalizeRequiredText(input.trialPrompt, "trialPrompt");
|
||||
const baselineOutcome = normalizeRequiredText(input.baselineOutcome, "baselineOutcome");
|
||||
const candidateOutcome = normalizeRequiredText(input.candidateOutcome, "candidateOutcome");
|
||||
const reason = normalizeRequiredText(input.reason, "reason");
|
||||
const riskFlags = normalizeDataList(input.riskFlags);
|
||||
const evidenceRefs = normalizeDataList(input.evidenceRefs);
|
||||
const recommendation = resolveDreamingShadowTrialRecommendation(input.verdict);
|
||||
const reportPath = resolveReportPath({
|
||||
workspaceDir: input.workspaceDir,
|
||||
candidate,
|
||||
trialPrompt,
|
||||
baselineOutcome,
|
||||
candidateOutcome,
|
||||
verdict: input.verdict,
|
||||
reason,
|
||||
riskFlags,
|
||||
evidenceRefs,
|
||||
reportPath: input.reportPath,
|
||||
nowMs: input.nowMs,
|
||||
timezone: input.timezone,
|
||||
});
|
||||
|
||||
const markdown = [
|
||||
"# Dreaming Shadow Trial Report",
|
||||
"",
|
||||
`candidate: ${candidate}`,
|
||||
`trial prompt: ${trialPrompt}`,
|
||||
`baseline outcome: ${baselineOutcome}`,
|
||||
`candidate outcome: ${candidateOutcome}`,
|
||||
`verdict: ${input.verdict}`,
|
||||
`recommendation: ${recommendation}`,
|
||||
`reason: ${reason}`,
|
||||
"risk flags:",
|
||||
formatList(normalizeList(riskFlags, "none recorded")),
|
||||
"evidence refs:",
|
||||
formatList(normalizeList(evidenceRefs, "none supplied")),
|
||||
"promotion action: report-only",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
return {
|
||||
candidate,
|
||||
trialPrompt,
|
||||
baselineOutcome,
|
||||
candidateOutcome,
|
||||
verdict: input.verdict,
|
||||
recommendation,
|
||||
reason,
|
||||
riskFlags,
|
||||
evidenceRefs,
|
||||
promotionAction: "report-only",
|
||||
...(reportPath ? { reportPath } : {}),
|
||||
markdown,
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeDreamingShadowTrialReport(
|
||||
input: DreamingShadowTrialInput & { workspaceDir: string },
|
||||
): Promise<DreamingShadowTrialReport> {
|
||||
const report = buildDreamingShadowTrialReport(input);
|
||||
if (!report.reportPath) {
|
||||
throw new Error("dreaming shadow trial report path could not be resolved");
|
||||
}
|
||||
await fs.mkdir(path.dirname(report.reportPath), { recursive: true });
|
||||
await fs.writeFile(report.reportPath, report.markdown, "utf-8");
|
||||
return report;
|
||||
}
|
||||
41
extensions/memory-core/src/dreaming-shared.test.ts
Normal file
41
extensions/memory-core/src/dreaming-shared.test.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
// Memory Core tests cover dreaming shared plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { includesSystemEventToken } from "./dreaming-shared.js";
|
||||
|
||||
const TOKEN = "__openclaw_memory_core_short_term_promotion_dream__";
|
||||
|
||||
describe("includesSystemEventToken", () => {
|
||||
it("matches the bare token", () => {
|
||||
expect(includesSystemEventToken(TOKEN, TOKEN)).toBe(true);
|
||||
});
|
||||
|
||||
it("matches a token wrapped by an isolated-cron `[cron:<id>]` prefix", () => {
|
||||
expect(includesSystemEventToken(`[cron:abc-123] ${TOKEN}`, TOKEN)).toBe(true);
|
||||
});
|
||||
|
||||
it("matches the token on its own line within multiline content", () => {
|
||||
expect(includesSystemEventToken(`leading text\n${TOKEN}\ntrailing`, TOKEN)).toBe(true);
|
||||
});
|
||||
|
||||
it("does NOT match a user message that merely embeds the token mid-sentence", () => {
|
||||
expect(
|
||||
includesSystemEventToken(`please tell me about ${TOKEN} when you have time`, TOKEN),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does NOT match a user message with the token in a code-fence-style block", () => {
|
||||
expect(
|
||||
includesSystemEventToken(`here is a snippet:\n\`${TOKEN}\`\nwhat does that do?`, TOKEN),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does NOT match an arbitrary wrapper the runtime does not produce", () => {
|
||||
expect(includesSystemEventToken(`[somewrap] ${TOKEN}`, TOKEN)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for empty inputs", () => {
|
||||
expect(includesSystemEventToken("", TOKEN)).toBe(false);
|
||||
expect(includesSystemEventToken(TOKEN, "")).toBe(false);
|
||||
expect(includesSystemEventToken(" ", TOKEN)).toBe(false);
|
||||
});
|
||||
});
|
||||
33
extensions/memory-core/src/dreaming-shared.ts
Normal file
33
extensions/memory-core/src/dreaming-shared.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
// Memory Core plugin module implements dreaming shared behavior.
|
||||
export { asNullableRecord as asRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
export { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
export function normalizeTrimmedString(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
export function includesSystemEventToken(cleanedBody: string, eventText: string): boolean {
|
||||
const normalizedBody = normalizeTrimmedString(cleanedBody);
|
||||
const normalizedEventText = normalizeTrimmedString(eventText);
|
||||
if (!normalizedBody || !normalizedEventText) {
|
||||
return false;
|
||||
}
|
||||
if (normalizedBody === normalizedEventText) {
|
||||
return true;
|
||||
}
|
||||
return normalizedBody.split(/\r?\n/).some((line) => {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === normalizedEventText) {
|
||||
return true;
|
||||
}
|
||||
// Isolated cron turns wrap the payload with a `[cron:<id>] ...` prefix; strip
|
||||
// that one known wrapper before matching so the dream sentinel still triggers
|
||||
// without falling back to a broad substring match (which would let any user
|
||||
// message embedding the token surface as a dream cron firing).
|
||||
return trimmed.replace(/^\[cron:[^\]]+\]\s*/, "") === normalizedEventText;
|
||||
});
|
||||
}
|
||||
180
extensions/memory-core/src/dreaming-state.ts
Normal file
180
extensions/memory-core/src/dreaming-state.ts
Normal file
@@ -0,0 +1,180 @@
|
||||
// Memory Core dreaming state lives in SQLite-backed plugin state.
|
||||
import { createHash } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
OpenKeyedStoreOptions,
|
||||
PluginStateKeyedStore,
|
||||
} from "openclaw/plugin-sdk/plugin-state-runtime";
|
||||
|
||||
export const MEMORY_CORE_PLUGIN_ID = "memory-core";
|
||||
export const DREAMING_DAILY_INGESTION_NAMESPACE = "dreaming-daily-ingestion";
|
||||
export const DREAMING_SESSION_INGESTION_FILES_NAMESPACE = "dreaming-session-ingestion-files";
|
||||
export const DREAMING_SESSION_INGESTION_SEEN_NAMESPACE = "dreaming-session-ingestion-seen";
|
||||
export const SHORT_TERM_RECALL_NAMESPACE = "short-term-recall";
|
||||
export const SHORT_TERM_PHASE_SIGNAL_NAMESPACE = "short-term-phase-signals";
|
||||
export const SHORT_TERM_META_NAMESPACE = "short-term-meta";
|
||||
export const SHORT_TERM_LOCK_NAMESPACE = "short-term-locks";
|
||||
|
||||
export const DREAMING_WORKSPACE_STATE_MAX_ENTRIES = 50_000;
|
||||
export const SHORT_TERM_LOCK_MAX_ENTRIES = 4_096;
|
||||
export const SESSION_SEEN_HASHES_PER_CHUNK = 512;
|
||||
|
||||
export type MemoryCoreOpenKeyedStore = <T>(
|
||||
options: OpenKeyedStoreOptions,
|
||||
) => PluginStateKeyedStore<T>;
|
||||
|
||||
type WorkspaceValue<T> = {
|
||||
version: 1;
|
||||
workspaceKey: string;
|
||||
workspaceDir: string;
|
||||
key: string;
|
||||
value: T;
|
||||
};
|
||||
|
||||
export type MemoryCoreWorkspaceEntry<T> = { key: string; value: T };
|
||||
|
||||
type MemoryCoreWorkspaceParams = {
|
||||
namespace: string;
|
||||
workspaceDir: string;
|
||||
};
|
||||
|
||||
type WriteMemoryCoreWorkspaceEntriesParams<T> = MemoryCoreWorkspaceParams & {
|
||||
entries: Array<MemoryCoreWorkspaceEntry<T>>;
|
||||
};
|
||||
|
||||
type WriteMemoryCoreWorkspaceEntryParams<T> = MemoryCoreWorkspaceParams &
|
||||
MemoryCoreWorkspaceEntry<T>;
|
||||
|
||||
let configuredOpenKeyedStore: MemoryCoreOpenKeyedStore | undefined;
|
||||
|
||||
export function configureMemoryCoreDreamingState(openKeyedStore: MemoryCoreOpenKeyedStore): void {
|
||||
configuredOpenKeyedStore = openKeyedStore;
|
||||
}
|
||||
|
||||
export async function configureMemoryCoreDreamingStateForTests(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<void> {
|
||||
const { createPluginStateKeyedStoreForTests } =
|
||||
await import("openclaw/plugin-sdk/plugin-state-test-runtime");
|
||||
const testEnv = { ...env };
|
||||
configureMemoryCoreDreamingState(<T>(options: OpenKeyedStoreOptions) =>
|
||||
createPluginStateKeyedStoreForTests<T>(MEMORY_CORE_PLUGIN_ID, { ...options, env: testEnv }),
|
||||
);
|
||||
}
|
||||
|
||||
export function resetMemoryCoreDreamingStateForTests(): void {
|
||||
configuredOpenKeyedStore = undefined;
|
||||
}
|
||||
|
||||
export function openMemoryCoreStateStore<T>(
|
||||
options: OpenKeyedStoreOptions,
|
||||
): PluginStateKeyedStore<T> {
|
||||
if (!configuredOpenKeyedStore) {
|
||||
throw new Error("memory-core dreaming SQLite state store is not configured");
|
||||
}
|
||||
return configuredOpenKeyedStore<T>(options);
|
||||
}
|
||||
|
||||
export function normalizeMemoryCoreWorkspaceKey(workspaceDir: string): string {
|
||||
const resolved = path.resolve(workspaceDir).replace(/\\/g, "/");
|
||||
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
||||
}
|
||||
|
||||
export function memoryCoreWorkspaceStateKey(workspaceDir: string): string {
|
||||
return createHash("sha256").update(normalizeMemoryCoreWorkspaceKey(workspaceDir)).digest("hex");
|
||||
}
|
||||
|
||||
export function memoryCoreWorkspaceEntryKey(workspaceDir: string, logicalKey: string): string {
|
||||
const workspaceKey = memoryCoreWorkspaceStateKey(workspaceDir);
|
||||
const itemKey = createHash("sha256").update(logicalKey).digest("hex");
|
||||
return `${workspaceKey}:${itemKey}`;
|
||||
}
|
||||
|
||||
export function memoryCoreStateReference(namespace: string, workspaceDir: string): string {
|
||||
return `plugin-state:${MEMORY_CORE_PLUGIN_ID}/${namespace}/${memoryCoreWorkspaceStateKey(workspaceDir)}`;
|
||||
}
|
||||
|
||||
function openWorkspaceStore<T>(namespace: string): PluginStateKeyedStore<WorkspaceValue<T>> {
|
||||
return openMemoryCoreStateStore<WorkspaceValue<T>>({
|
||||
namespace,
|
||||
maxEntries: DREAMING_WORKSPACE_STATE_MAX_ENTRIES,
|
||||
});
|
||||
}
|
||||
|
||||
// Caller owns typed decoding for values read from plugin state.
|
||||
export function readMemoryCoreWorkspaceEntries<T>(
|
||||
params: MemoryCoreWorkspaceParams,
|
||||
): Promise<Array<MemoryCoreWorkspaceEntry<T>>>;
|
||||
export async function readMemoryCoreWorkspaceEntries(
|
||||
params: MemoryCoreWorkspaceParams,
|
||||
): Promise<Array<MemoryCoreWorkspaceEntry<unknown>>> {
|
||||
const workspaceKey = memoryCoreWorkspaceStateKey(params.workspaceDir);
|
||||
const prefix = `${workspaceKey}:`;
|
||||
const entries = await openWorkspaceStore<unknown>(params.namespace).entries();
|
||||
return entries
|
||||
.filter((entry) => entry.key.startsWith(prefix) && entry.value.workspaceKey === workspaceKey)
|
||||
.map((entry) => ({ key: entry.value.key, value: entry.value.value }));
|
||||
}
|
||||
|
||||
// Caller owns typed encoding for values written to plugin state.
|
||||
export function writeMemoryCoreWorkspaceEntries<T>(
|
||||
params: WriteMemoryCoreWorkspaceEntriesParams<T>,
|
||||
): Promise<void>;
|
||||
export async function writeMemoryCoreWorkspaceEntries(
|
||||
params: WriteMemoryCoreWorkspaceEntriesParams<unknown>,
|
||||
): Promise<void> {
|
||||
const store = openWorkspaceStore<unknown>(params.namespace);
|
||||
const workspaceKey = memoryCoreWorkspaceStateKey(params.workspaceDir);
|
||||
const prefix = `${workspaceKey}:`;
|
||||
const replacementKeys = new Set<string>();
|
||||
for (const entry of params.entries) {
|
||||
const stateKey = memoryCoreWorkspaceEntryKey(params.workspaceDir, entry.key);
|
||||
replacementKeys.add(stateKey);
|
||||
await store.register(stateKey, {
|
||||
version: 1,
|
||||
workspaceKey,
|
||||
workspaceDir: path.resolve(params.workspaceDir),
|
||||
key: entry.key,
|
||||
value: entry.value,
|
||||
});
|
||||
}
|
||||
for (const entry of await store.entries()) {
|
||||
if (entry.key.startsWith(prefix) && !replacementKeys.has(entry.key)) {
|
||||
await store.delete(entry.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Caller owns typed encoding for values written to plugin state.
|
||||
export function writeMemoryCoreWorkspaceEntry<T>(
|
||||
params: WriteMemoryCoreWorkspaceEntryParams<T>,
|
||||
): Promise<void>;
|
||||
export async function writeMemoryCoreWorkspaceEntry(
|
||||
params: WriteMemoryCoreWorkspaceEntryParams<unknown>,
|
||||
): Promise<void> {
|
||||
const workspaceKey = memoryCoreWorkspaceStateKey(params.workspaceDir);
|
||||
await openWorkspaceStore<unknown>(params.namespace).register(
|
||||
memoryCoreWorkspaceEntryKey(params.workspaceDir, params.key),
|
||||
{
|
||||
version: 1,
|
||||
workspaceKey,
|
||||
workspaceDir: path.resolve(params.workspaceDir),
|
||||
key: params.key,
|
||||
value: params.value,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function clearMemoryCoreWorkspaceNamespace(params: {
|
||||
namespace: string;
|
||||
workspaceDir: string;
|
||||
}): Promise<void> {
|
||||
const store = openWorkspaceStore(params.namespace);
|
||||
const workspaceKey = memoryCoreWorkspaceStateKey(params.workspaceDir);
|
||||
const prefix = `${workspaceKey}:`;
|
||||
for (const entry of await store.entries()) {
|
||||
if (entry.key.startsWith(prefix)) {
|
||||
await store.delete(entry.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
2883
extensions/memory-core/src/dreaming.test.ts
Normal file
2883
extensions/memory-core/src/dreaming.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
1009
extensions/memory-core/src/dreaming.ts
Normal file
1009
extensions/memory-core/src/dreaming.ts
Normal file
File diff suppressed because it is too large
Load Diff
19
extensions/memory-core/src/flush-plan.test.ts
Normal file
19
extensions/memory-core/src/flush-plan.test.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
// Memory Core tests cover flush plan plugin behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildMemoryFlushPlan } from "./flush-plan.js";
|
||||
|
||||
describe("buildMemoryFlushPlan", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("falls back when the injected timestamp is outside Date range", () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(Date.UTC(2026, 4, 30, 12, 0, 0));
|
||||
|
||||
const plan = buildMemoryFlushPlan({
|
||||
nowMs: 8_640_000_000_000_001,
|
||||
});
|
||||
|
||||
expect(plan?.relativePath).toBe("memory/2026-05-30.md");
|
||||
});
|
||||
});
|
||||
142
extensions/memory-core/src/flush-plan.ts
Normal file
142
extensions/memory-core/src/flush-plan.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
// Memory Core plugin module implements flush plan behavior.
|
||||
import {
|
||||
DEFAULT_AGENT_COMPACTION_RESERVE_TOKENS_FLOOR,
|
||||
parseNonNegativeByteSize,
|
||||
resolveCronStyleNow,
|
||||
SILENT_REPLY_TOKEN,
|
||||
type MemoryFlushPlan,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
import { resolveMemoryCoreNowMs } from "./time.js";
|
||||
|
||||
export const DEFAULT_MEMORY_FLUSH_SOFT_TOKENS = 4000;
|
||||
export const DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
const MEMORY_FLUSH_TARGET_HINT =
|
||||
"Store durable memories only in memory/YYYY-MM-DD.md (create memory/ if needed).";
|
||||
const MEMORY_FLUSH_APPEND_ONLY_HINT =
|
||||
"If memory/YYYY-MM-DD.md already exists, APPEND new content only and do not overwrite existing entries.";
|
||||
const MEMORY_FLUSH_READ_ONLY_HINT =
|
||||
"Treat workspace bootstrap/reference files such as MEMORY.md, DREAMS.md, SOUL.md, TOOLS.md, and AGENTS.md as read-only during this flush; never overwrite, replace, or edit them.";
|
||||
const MEMORY_FLUSH_REQUIRED_HINTS = [
|
||||
MEMORY_FLUSH_TARGET_HINT,
|
||||
MEMORY_FLUSH_APPEND_ONLY_HINT,
|
||||
MEMORY_FLUSH_READ_ONLY_HINT,
|
||||
];
|
||||
|
||||
export const DEFAULT_MEMORY_FLUSH_PROMPT = [
|
||||
"Pre-compaction memory flush.",
|
||||
MEMORY_FLUSH_TARGET_HINT,
|
||||
MEMORY_FLUSH_READ_ONLY_HINT,
|
||||
MEMORY_FLUSH_APPEND_ONLY_HINT,
|
||||
"Do NOT create timestamped variant files (e.g., YYYY-MM-DD-HHMM.md); always use the canonical YYYY-MM-DD.md filename.",
|
||||
`If nothing to store, reply with ${SILENT_REPLY_TOKEN}.`,
|
||||
].join(" ");
|
||||
|
||||
const DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT = [
|
||||
"Pre-compaction memory flush turn.",
|
||||
"The session is near auto-compaction; capture durable memories to disk.",
|
||||
MEMORY_FLUSH_TARGET_HINT,
|
||||
MEMORY_FLUSH_READ_ONLY_HINT,
|
||||
MEMORY_FLUSH_APPEND_ONLY_HINT,
|
||||
`You may reply, but usually ${SILENT_REPLY_TOKEN} is correct.`,
|
||||
].join(" ");
|
||||
|
||||
function formatDateStampInTimezone(nowMs: number, timezone: string): string {
|
||||
const parts = new Intl.DateTimeFormat("en-US", {
|
||||
timeZone: timezone,
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
}).formatToParts(new Date(nowMs));
|
||||
const year = parts.find((part) => part.type === "year")?.value;
|
||||
const month = parts.find((part) => part.type === "month")?.value;
|
||||
const day = parts.find((part) => part.type === "day")?.value;
|
||||
if (year && month && day) {
|
||||
return `${year}-${month}-${day}`;
|
||||
}
|
||||
return new Date(resolveMemoryCoreNowMs(nowMs)).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function normalizeNonNegativeInt(value: unknown): number | null {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
return null;
|
||||
}
|
||||
const int = Math.floor(value);
|
||||
return int >= 0 ? int : null;
|
||||
}
|
||||
|
||||
function ensureNoReplyHint(text: string): string {
|
||||
if (text.includes(SILENT_REPLY_TOKEN)) {
|
||||
return text;
|
||||
}
|
||||
return `${text}\n\nIf no user-visible reply is needed, start with ${SILENT_REPLY_TOKEN}.`;
|
||||
}
|
||||
|
||||
function ensureMemoryFlushSafetyHints(text: string): string {
|
||||
let next = text.trim();
|
||||
for (const hint of MEMORY_FLUSH_REQUIRED_HINTS) {
|
||||
if (!next.includes(hint)) {
|
||||
next = next ? `${next}\n\n${hint}` : hint;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function appendCurrentTimeLine(text: string, timeLine: string): string {
|
||||
const trimmed = text.trimEnd();
|
||||
if (!trimmed) {
|
||||
return timeLine;
|
||||
}
|
||||
if (trimmed.includes("Current time:")) {
|
||||
return trimmed;
|
||||
}
|
||||
return `${trimmed}\n${timeLine}`;
|
||||
}
|
||||
|
||||
export function buildMemoryFlushPlan(
|
||||
params: {
|
||||
cfg?: OpenClawConfig;
|
||||
nowMs?: number;
|
||||
} = {},
|
||||
): MemoryFlushPlan | null {
|
||||
const resolved = params;
|
||||
const nowMs = resolveMemoryCoreNowMs(resolved.nowMs);
|
||||
const cfg = resolved.cfg;
|
||||
const defaults = cfg?.agents?.defaults?.compaction?.memoryFlush;
|
||||
if (defaults?.enabled === false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const softThresholdTokens =
|
||||
normalizeNonNegativeInt(defaults?.softThresholdTokens) ?? DEFAULT_MEMORY_FLUSH_SOFT_TOKENS;
|
||||
const forceFlushTranscriptBytes =
|
||||
parseNonNegativeByteSize(defaults?.forceFlushTranscriptBytes) ??
|
||||
DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES;
|
||||
const reserveTokensFloor =
|
||||
normalizeNonNegativeInt(cfg?.agents?.defaults?.compaction?.reserveTokensFloor) ??
|
||||
DEFAULT_AGENT_COMPACTION_RESERVE_TOKENS_FLOOR;
|
||||
|
||||
const { timeLine, userTimezone } = resolveCronStyleNow(cfg ?? {}, nowMs);
|
||||
const dateStamp = formatDateStampInTimezone(nowMs, userTimezone);
|
||||
const relativePath = `memory/${dateStamp}.md`;
|
||||
|
||||
const promptBase = ensureNoReplyHint(
|
||||
ensureMemoryFlushSafetyHints(defaults?.prompt?.trim() || DEFAULT_MEMORY_FLUSH_PROMPT),
|
||||
);
|
||||
const systemPrompt = ensureNoReplyHint(
|
||||
ensureMemoryFlushSafetyHints(
|
||||
defaults?.systemPrompt?.trim() || DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT,
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
softThresholdTokens,
|
||||
forceFlushTranscriptBytes,
|
||||
reserveTokensFloor,
|
||||
model: defaults?.model?.trim() || undefined,
|
||||
prompt: appendCurrentTimeLine(promptBase.replaceAll("YYYY-MM-DD", dateStamp), timeLine),
|
||||
systemPrompt: systemPrompt.replaceAll("YYYY-MM-DD", dateStamp),
|
||||
relativePath,
|
||||
};
|
||||
}
|
||||
179
extensions/memory-core/src/memory-budget.test.ts
Normal file
179
extensions/memory-core/src/memory-budget.test.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
// Memory Core tests cover memory budget plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { compactMemoryForBudget, DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js";
|
||||
|
||||
function promotionSection(date: string, sizeChars: number): string {
|
||||
const heading = `## Promoted From Short-Term Memory (${date})\n`;
|
||||
const padding = "x".repeat(Math.max(0, sizeChars - heading.length));
|
||||
return `${heading}${padding}`;
|
||||
}
|
||||
|
||||
describe("compactMemoryForBudget — bounded MEMORY.md compaction (regression for #73691)", () => {
|
||||
it("returns existing memory unchanged when total fits the budget", () => {
|
||||
const existing = "# Long-Term Memory\n\nSome content.\n";
|
||||
const newSection = "\n## Promoted From Short-Term Memory (2026-04-29)\n- entry\n";
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 1_000,
|
||||
});
|
||||
expect(result.compacted).toBe(existing);
|
||||
expect(result.droppedDates).toEqual([]);
|
||||
});
|
||||
|
||||
it("drops the oldest promotion section first when over budget", () => {
|
||||
const oldest = promotionSection("2026-04-10", 500);
|
||||
const newer = promotionSection("2026-04-20", 500);
|
||||
const existing = `${oldest}\n${newer}`;
|
||||
const newSection = `\n${promotionSection("2026-04-29", 500)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 1_200,
|
||||
});
|
||||
expect(result.droppedDates).toEqual(["2026-04-10"]);
|
||||
expect(result.compacted).not.toContain("(2026-04-10)");
|
||||
expect(result.compacted).toContain("(2026-04-20)");
|
||||
});
|
||||
|
||||
it("drops sections in ascending date order regardless of file order", () => {
|
||||
// File has sections in non-chronological order; algorithm must drop oldest by date.
|
||||
const newer = promotionSection("2026-04-25", 400);
|
||||
const oldest = promotionSection("2026-04-10", 400);
|
||||
const middle = promotionSection("2026-04-18", 400);
|
||||
const existing = `${newer}\n${oldest}\n${middle}`;
|
||||
const newSection = `\n${promotionSection("2026-04-29", 400)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 1_300,
|
||||
});
|
||||
// Drop oldest first; if still over budget, drop next oldest.
|
||||
expect(result.droppedDates[0]).toBe("2026-04-10");
|
||||
expect(result.compacted).not.toContain("(2026-04-10)");
|
||||
});
|
||||
|
||||
it("preserves user-authored content (non-promotion sections)", () => {
|
||||
const userSection = "## My Notes\n\nImportant user content I do not want dropped.\n";
|
||||
const oldest = promotionSection("2026-04-10", 800);
|
||||
const existing = `# Long-Term Memory\n\n${userSection}\n${oldest}`;
|
||||
const newSection = `\n${promotionSection("2026-04-29", 600)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 800,
|
||||
});
|
||||
expect(result.droppedDates).toContain("2026-04-10");
|
||||
expect(result.compacted).toContain("## My Notes");
|
||||
expect(result.compacted).toContain("Important user content");
|
||||
expect(result.compacted).toContain("# Long-Term Memory");
|
||||
});
|
||||
|
||||
it("drops every promotion section when budget cannot be satisfied otherwise", () => {
|
||||
const existing = [
|
||||
promotionSection("2026-04-10", 600),
|
||||
promotionSection("2026-04-15", 600),
|
||||
promotionSection("2026-04-20", 600),
|
||||
].join("\n");
|
||||
const newSection = `\n${promotionSection("2026-04-29", 600)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 700,
|
||||
});
|
||||
expect(result.droppedDates).toEqual(["2026-04-10", "2026-04-15", "2026-04-20"]);
|
||||
expect(result.compacted).not.toContain("Promoted From Short-Term Memory");
|
||||
});
|
||||
|
||||
it("returns existing unchanged when the file has no promotion sections (cannot compact)", () => {
|
||||
const existing = "# Long-Term Memory\n\nLots of user content here.\n".repeat(50);
|
||||
const newSection = `\n${promotionSection("2026-04-29", 200)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 500,
|
||||
});
|
||||
expect(result.compacted).toBe(existing);
|
||||
expect(result.droppedDates).toEqual([]);
|
||||
});
|
||||
|
||||
it("treats budgetChars <= 0 as 'no budget' and returns existing unchanged", () => {
|
||||
const existing = promotionSection("2026-04-10", 500);
|
||||
const newSection = `\n${promotionSection("2026-04-29", 500)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 0,
|
||||
});
|
||||
expect(result.compacted).toBe(existing);
|
||||
expect(result.droppedDates).toEqual([]);
|
||||
});
|
||||
|
||||
it("handles empty existing memory cleanly", () => {
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: "",
|
||||
newSection: promotionSection("2026-04-29", 500),
|
||||
budgetChars: 100,
|
||||
});
|
||||
expect(result.compacted).toBe("");
|
||||
expect(result.droppedDates).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves a non-promotion ## heading sandwiched between promotion sections", () => {
|
||||
const existing =
|
||||
`${promotionSection("2026-04-10", 400)}\n` +
|
||||
"## My Reflections\nMy own notes.\n\n" +
|
||||
promotionSection("2026-04-20", 400);
|
||||
const newSection = `\n${promotionSection("2026-04-29", 400)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 900,
|
||||
});
|
||||
expect(result.droppedDates).toContain("2026-04-10");
|
||||
expect(result.compacted).toContain("## My Reflections");
|
||||
expect(result.compacted).toContain("My own notes.");
|
||||
});
|
||||
|
||||
it("does not prepend a spurious leading newline when input starts with a ## heading", () => {
|
||||
// Regression for greptile P2 #1: parseMemoryBlocks's flush guard previously
|
||||
// pushed an empty preserved block when content started directly with `##`,
|
||||
// making compacted output start with an extra `\n`.
|
||||
const existing = `${promotionSection("2026-04-10", 200)}\n${promotionSection("2026-04-20", 200)}`;
|
||||
const newSection = `\n${promotionSection("2026-04-29", 200)}`;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: 500,
|
||||
});
|
||||
expect(result.compacted.startsWith("\n")).toBe(false);
|
||||
expect(result.compacted.startsWith("## Promoted From Short-Term Memory")).toBe(true);
|
||||
});
|
||||
|
||||
it("respects writer overhead reserve so on-disk size stays inside the budget", () => {
|
||||
// Regression for greptile P2 #2: budget check previously ignored the
|
||||
// header (~20 chars) and trailing newline (1 char) the caller adds.
|
||||
const existing = `${promotionSection("2026-04-10", 1_000)}\n${promotionSection("2026-04-20", 1_000)}`;
|
||||
const newSection = `\n${promotionSection("2026-04-29", 1_000)}`;
|
||||
const budget = 2_000;
|
||||
const result = compactMemoryForBudget({
|
||||
existingMemory: existing,
|
||||
newSection,
|
||||
budgetChars: budget,
|
||||
});
|
||||
const headerOverhead = 20; // "# Long-Term Memory\n\n"
|
||||
const trailingNewline = 1;
|
||||
expect(
|
||||
result.compacted.length + newSection.length + headerOverhead + trailingNewline,
|
||||
).toBeLessThanOrEqual(budget);
|
||||
});
|
||||
|
||||
it("exposes a sane default budget below the bootstrap injection cap", () => {
|
||||
// Bootstrap injection is capped at 12_000 chars per file (see
|
||||
// src/agents/embedded-agent-helpers/bootstrap.ts). The MEMORY.md budget
|
||||
// must stay strictly below that to leave room for headers and so
|
||||
// promoted content keeps reaching new sessions.
|
||||
expect(DEFAULT_MEMORY_FILE_MAX_CHARS).toBeLessThan(12_000);
|
||||
expect(DEFAULT_MEMORY_FILE_MAX_CHARS).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
164
extensions/memory-core/src/memory-budget.ts
Normal file
164
extensions/memory-core/src/memory-budget.ts
Normal file
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* Bounded MEMORY.md compaction for dreaming/promotion writes.
|
||||
*
|
||||
* Background: the dreaming pipeline appends promoted entries to MEMORY.md
|
||||
* via short-term-promotion.applyShortTermPromotions. Without a size budget,
|
||||
* MEMORY.md grows unboundedly across deep-phase sweeps and eventually
|
||||
* exceeds bootstrap's per-file injection cap, breaking session bootstrap.
|
||||
* See issue #73691.
|
||||
*
|
||||
* Strategy: drop the OLDEST auto-promoted sections (date-ordered) until
|
||||
* the file plus the new section fit within the budget. User-authored
|
||||
* content (anything that is not a `## Promoted From Short-Term Memory
|
||||
* (DATE)` section) is preserved unconditionally — only dreaming-owned
|
||||
* sections are eligible for compaction.
|
||||
*/
|
||||
|
||||
const PROMOTION_SECTION_HEADING_RE = /^## Promoted From Short-Term Memory \(([^)]+)\)\s*$/;
|
||||
|
||||
/**
|
||||
* Default budget for MEMORY.md content on disk, in characters. Chosen to
|
||||
* stay safely below the bootstrap injection cap (~12KB per file at the
|
||||
* time of writing) so promoted memory keeps reaching new sessions instead
|
||||
* of being silently dropped by bootstrap truncation.
|
||||
*/
|
||||
export const DEFAULT_MEMORY_FILE_MAX_CHARS = 10_000;
|
||||
|
||||
/**
|
||||
* Reserve for writer-side overhead that the helper does not see directly:
|
||||
* the `# Long-Term Memory\n\n` header re-emitted when compaction empties
|
||||
* out (20 chars) and `withTrailingNewline`'s trailing `\n` (1 char). See
|
||||
* the actual write expression in `applyShortTermPromotions`. Subtracting
|
||||
* this from `budgetChars` keeps the on-disk file inside the caller's
|
||||
* stated budget instead of exceeding it by up to ~21 chars in edge cases.
|
||||
*/
|
||||
const WRITE_OVERHEAD_RESERVE = 21;
|
||||
|
||||
type MemoryBlock =
|
||||
| { kind: "preserved"; text: string }
|
||||
| { kind: "promotion"; date: string; text: string };
|
||||
|
||||
function parseMemoryBlocks(content: string): MemoryBlock[] {
|
||||
if (content.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const lines = content.split(/\r?\n/);
|
||||
const blocks: MemoryBlock[] = [];
|
||||
let currentLines: string[] = [];
|
||||
let currentKind: "preserved" | "promotion" = "preserved";
|
||||
let currentDate: string | undefined;
|
||||
|
||||
const flush = () => {
|
||||
if (currentLines.length === 0) {
|
||||
return;
|
||||
}
|
||||
const text = currentLines.join("\n");
|
||||
if (currentKind === "promotion" && currentDate) {
|
||||
blocks.push({ kind: "promotion", date: currentDate, text });
|
||||
} else {
|
||||
blocks.push({ kind: "preserved", text });
|
||||
}
|
||||
currentLines = [];
|
||||
currentKind = "preserved";
|
||||
currentDate = undefined;
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith("## ")) {
|
||||
flush();
|
||||
const match = PROMOTION_SECTION_HEADING_RE.exec(line);
|
||||
if (match) {
|
||||
currentKind = "promotion";
|
||||
currentDate = match[1];
|
||||
} else {
|
||||
currentKind = "preserved";
|
||||
}
|
||||
currentLines = [line];
|
||||
} else {
|
||||
currentLines.push(line);
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function joinBlocks(blocks: MemoryBlock[]): string {
|
||||
return blocks.map((block) => block.text).join("\n");
|
||||
}
|
||||
|
||||
export type CompactMemoryParams = {
|
||||
existingMemory: string;
|
||||
newSection: string;
|
||||
budgetChars: number;
|
||||
};
|
||||
|
||||
export type CompactMemoryResult = {
|
||||
compacted: string;
|
||||
droppedDates: string[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Drop oldest auto-promotion sections from `existingMemory` until
|
||||
* `existingMemory + newSection` fits within `budgetChars`. Returns the
|
||||
* (possibly trimmed) existing memory and the dates of dropped sections.
|
||||
*
|
||||
* Guarantees:
|
||||
* - Non-promotion content (user-authored markdown, the file header, any
|
||||
* `##` heading not matching the promotion pattern) is preserved.
|
||||
* - Promotion sections are dropped in ascending date order (oldest first).
|
||||
* - If `existingMemory + newSection` already fits the budget, the existing
|
||||
* memory is returned unchanged.
|
||||
* - If the budget cannot be satisfied even by dropping every promotion
|
||||
* section, the function drops them all and returns; the caller writes
|
||||
* the new section anyway. This is the "log and continue" failure mode —
|
||||
* refusing the new write would silently swallow the freshest material.
|
||||
*/
|
||||
export function compactMemoryForBudget(params: CompactMemoryParams): CompactMemoryResult {
|
||||
const { existingMemory, newSection, budgetChars } = params;
|
||||
if (budgetChars <= 0) {
|
||||
return { compacted: existingMemory, droppedDates: [] };
|
||||
}
|
||||
|
||||
// Reserve writer-side header + trailing-newline overhead so the on-disk
|
||||
// file actually fits the caller's stated budget.
|
||||
const effectiveBudget = Math.max(0, budgetChars - WRITE_OVERHEAD_RESERVE);
|
||||
|
||||
if (existingMemory.length + newSection.length <= effectiveBudget) {
|
||||
return { compacted: existingMemory, droppedDates: [] };
|
||||
}
|
||||
|
||||
const blocks = parseMemoryBlocks(existingMemory);
|
||||
const promotionEntries = blocks
|
||||
.map((block, index) =>
|
||||
block.kind === "promotion" ? { index, date: block.date, length: block.text.length } : null,
|
||||
)
|
||||
.filter((entry): entry is { index: number; date: string; length: number } => entry !== null)
|
||||
.toSorted((a, b) => a.date.localeCompare(b.date));
|
||||
|
||||
if (promotionEntries.length === 0) {
|
||||
return { compacted: existingMemory, droppedDates: [] };
|
||||
}
|
||||
|
||||
const droppedIndices = new Set<number>();
|
||||
const droppedDates: string[] = [];
|
||||
let projectedExistingSize = existingMemory.length;
|
||||
// Block boundaries cost one newline each in joinBlocks; subtract a
|
||||
// newline along with the block text so the projection stays honest.
|
||||
const blockSeparatorCost = blocks.length > 1 ? 1 : 0;
|
||||
|
||||
for (const entry of promotionEntries) {
|
||||
if (projectedExistingSize + newSection.length <= effectiveBudget) {
|
||||
break;
|
||||
}
|
||||
droppedIndices.add(entry.index);
|
||||
droppedDates.push(entry.date);
|
||||
projectedExistingSize = Math.max(0, projectedExistingSize - entry.length - blockSeparatorCost);
|
||||
}
|
||||
|
||||
if (droppedIndices.size === 0) {
|
||||
return { compacted: existingMemory, droppedDates: [] };
|
||||
}
|
||||
|
||||
const remaining = blocks.filter((_, index) => !droppedIndices.has(index));
|
||||
return { compacted: joinBlocks(remaining), droppedDates };
|
||||
}
|
||||
221
extensions/memory-core/src/memory-events.test.ts
Normal file
221
extensions/memory-core/src/memory-events.test.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
// Memory Core tests cover memory events plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
readMemoryHostEventRecords,
|
||||
readMemoryHostEvents,
|
||||
resolveMemoryHostEventLogPath,
|
||||
} from "openclaw/plugin-sdk/memory-host-events";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js";
|
||||
import {
|
||||
applyShortTermPromotions,
|
||||
rankShortTermPromotionCandidates,
|
||||
recordShortTermRecalls,
|
||||
} from "./short-term-promotion.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempWorkspace } = createMemoryCoreTestHarness();
|
||||
|
||||
describe("memory host event journal integration", () => {
|
||||
it("records recall and promotion events from short-term promotion flows", async () => {
|
||||
const workspaceDir = await createTempWorkspace("memory-core-events-");
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", "2026-04-05.md"),
|
||||
"# Daily\n\nalpha\nbeta\ngamma\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "alpha memory",
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-05.md",
|
||||
startLine: 3,
|
||||
endLine: 4,
|
||||
score: 0.92,
|
||||
snippet: "alpha beta",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 0, 0),
|
||||
});
|
||||
|
||||
const candidates = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 5, 0),
|
||||
});
|
||||
const applied = await applyShortTermPromotions({
|
||||
workspaceDir,
|
||||
candidates,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 10, 0),
|
||||
});
|
||||
|
||||
expect(applied.applied).toBe(1);
|
||||
|
||||
const events = await readMemoryHostEvents({ workspaceDir });
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"memory.recall.recorded",
|
||||
"memory.promotion.applied",
|
||||
]);
|
||||
const recallEvent = events[0];
|
||||
if (recallEvent?.type !== "memory.recall.recorded") {
|
||||
throw new Error("expected recall event");
|
||||
}
|
||||
expect(recallEvent.resultCount).toBe(1);
|
||||
expect(recallEvent.query).toBe("alpha memory");
|
||||
const promotionEvent = events[1];
|
||||
if (promotionEvent?.type !== "memory.promotion.applied") {
|
||||
throw new Error("expected promotion event");
|
||||
}
|
||||
expect(promotionEvent.applied).toBe(1);
|
||||
});
|
||||
|
||||
it("records skipped recall events for durable memory hits excluded from short-term promotion", async () => {
|
||||
const workspaceDir = await createTempWorkspace("memory-core-skipped-recall-events-");
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", "decisoes"), { recursive: true });
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", "idiomas"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "MEMORY.md"),
|
||||
"# Memory\n\nAlpha durable note.\n",
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", "decisoes", "2026-06.md"),
|
||||
"# Decisoes\n\nAlpha monthly decision.\n",
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", "idiomas", "PLANO.md"),
|
||||
"# Plano\n\nAlpha language plan.\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "alpha durable memory",
|
||||
results: [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 3,
|
||||
endLine: 3,
|
||||
score: 0.91,
|
||||
snippet: "Alpha durable note.",
|
||||
source: "memory",
|
||||
},
|
||||
{
|
||||
path: "memory/decisoes/2026-06.md",
|
||||
startLine: 3,
|
||||
endLine: 3,
|
||||
score: 0.88,
|
||||
snippet: "Alpha monthly decision.",
|
||||
source: "memory",
|
||||
},
|
||||
{
|
||||
path: "memory/idiomas/PLANO.md",
|
||||
startLine: 3,
|
||||
endLine: 3,
|
||||
score: 0.83,
|
||||
snippet: "Alpha language plan.",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
nowMs: Date.UTC(2026, 5, 13, 9, 0, 0),
|
||||
});
|
||||
|
||||
const candidates = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.UTC(2026, 5, 13, 9, 5, 0),
|
||||
});
|
||||
const events = await readMemoryHostEventRecords({ workspaceDir });
|
||||
|
||||
expect(candidates).toEqual([]);
|
||||
expect(events.map((event) => event.type)).toEqual(["memory.recall.skipped"]);
|
||||
const skippedEvent = events[0];
|
||||
if (skippedEvent?.type !== "memory.recall.skipped") {
|
||||
throw new Error("expected skipped recall event");
|
||||
}
|
||||
expect(skippedEvent.query).toBe("alpha durable memory");
|
||||
expect(skippedEvent.reason).toBe("non-short-term-memory-path");
|
||||
expect(skippedEvent.eligibleResultCount).toBe(0);
|
||||
expect(skippedEvent.skippedResultCount).toBe(3);
|
||||
expect(skippedEvent.results.map((result) => result.path)).toEqual([
|
||||
"MEMORY.md",
|
||||
"memory/decisoes/2026-06.md",
|
||||
"memory/idiomas/PLANO.md",
|
||||
]);
|
||||
expect(
|
||||
skippedEvent.results.every((result) => result.reason === "non-short-term-memory-path"),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("records dreaming completion events when phase artifacts are written", async () => {
|
||||
const workspaceDir = await createTempWorkspace("memory-core-dream-events-");
|
||||
|
||||
const written = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- staged note", "- second note"],
|
||||
nowMs: Date.UTC(2026, 3, 5, 13, 0, 0),
|
||||
storage: { mode: "both", separateReports: true },
|
||||
});
|
||||
|
||||
const events = await readMemoryHostEvents({ workspaceDir });
|
||||
|
||||
expect(written.inlinePath).toBe(path.join(workspaceDir, "memory", "2026-04-05.md"));
|
||||
expect(written.reportPath).toBe(
|
||||
path.join(workspaceDir, "memory", "dreaming", "light", "2026-04-05.md"),
|
||||
);
|
||||
await expect(fs.readFile(written.inlinePath ?? "", "utf8")).resolves.toContain("- staged note");
|
||||
await expect(fs.readFile(written.reportPath ?? "", "utf8")).resolves.toContain("- second note");
|
||||
expect(events).toHaveLength(1);
|
||||
const dreamEvent = events[0];
|
||||
if (dreamEvent?.type !== "memory.dream.completed") {
|
||||
throw new Error("expected dream completion event");
|
||||
}
|
||||
expect(dreamEvent.phase).toBe("light");
|
||||
expect(dreamEvent.outcome).toBe("completed");
|
||||
expect(dreamEvent.lineCount).toBe(2);
|
||||
expect(dreamEvent.storageMode).toBe("both");
|
||||
});
|
||||
|
||||
it("keeps legacy dreaming completion events without outcome readable", async () => {
|
||||
const workspaceDir = await createTempWorkspace("memory-core-legacy-dream-events-");
|
||||
const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir);
|
||||
await fs.mkdir(path.dirname(eventLogPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
eventLogPath,
|
||||
`${JSON.stringify({
|
||||
type: "memory.dream.completed",
|
||||
timestamp: "2026-04-05T13:00:00.000Z",
|
||||
phase: "deep",
|
||||
inlinePath: path.join(workspaceDir, "DREAMS.md"),
|
||||
lineCount: 2,
|
||||
storageMode: "inline",
|
||||
})}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const events = await readMemoryHostEvents({ workspaceDir });
|
||||
|
||||
expect(events).toHaveLength(1);
|
||||
const dreamEvent = events[0];
|
||||
if (dreamEvent?.type !== "memory.dream.completed") {
|
||||
throw new Error("expected dream completion event");
|
||||
}
|
||||
expect(dreamEvent.outcome).toBeUndefined();
|
||||
expect(dreamEvent.phase).toBe("deep");
|
||||
});
|
||||
});
|
||||
162
extensions/memory-core/src/memory-tool-manager.test-mocks.ts
Normal file
162
extensions/memory-core/src/memory-tool-manager.test-mocks.ts
Normal file
@@ -0,0 +1,162 @@
|
||||
// Memory Core plugin module implements memory tool manager mock behavior.
|
||||
import type { MemorySearchRuntimeDebug } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
import { vi } from "vitest";
|
||||
|
||||
type SearchImpl = (opts?: {
|
||||
maxResults?: number;
|
||||
minScore?: number;
|
||||
sessionKey?: string;
|
||||
qmdSearchModeOverride?: "query" | "search" | "vsearch";
|
||||
onDebug?: (debug: MemorySearchRuntimeDebug) => void;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<unknown[]>;
|
||||
export type MemoryReadParams = { relPath: string; from?: number; lines?: number };
|
||||
type MemoryReadResult = {
|
||||
text: string;
|
||||
path: string;
|
||||
truncated?: boolean;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
nextFrom?: number;
|
||||
};
|
||||
type MemoryBackend = "builtin" | "qmd";
|
||||
|
||||
let backend: MemoryBackend = "builtin";
|
||||
let workspaceDir = "/workspace";
|
||||
let customStatus: Record<string, unknown> | undefined;
|
||||
let searchImpl: SearchImpl = async () => [];
|
||||
let getManagerImpl:
|
||||
| ((params: { cfg?: unknown; agentId?: string; purpose?: string }) => Promise<{
|
||||
manager?: unknown;
|
||||
error?: string;
|
||||
}>)
|
||||
| undefined;
|
||||
let readFileImpl: (params: MemoryReadParams) => Promise<MemoryReadResult> = async (params) => ({
|
||||
text: "",
|
||||
path: params.relPath,
|
||||
from: params.from ?? 1,
|
||||
lines: params.lines ?? 120,
|
||||
});
|
||||
|
||||
const stubManager = {
|
||||
search: vi.fn(async (_query: string, opts?: Parameters<SearchImpl>[0]) => await searchImpl(opts)),
|
||||
readFile: vi.fn(async (params: MemoryReadParams) => await readFileImpl(params)),
|
||||
status: () => ({
|
||||
backend,
|
||||
files: 1,
|
||||
chunks: 1,
|
||||
dirty: false,
|
||||
workspaceDir,
|
||||
dbPath: "/workspace/.memory/index.sqlite",
|
||||
provider: "builtin",
|
||||
model: "builtin",
|
||||
requestedProvider: "builtin",
|
||||
sources: ["memory" as const],
|
||||
sourceCounts: [{ source: "memory" as const, files: 1, chunks: 1 }],
|
||||
custom: customStatus,
|
||||
}),
|
||||
sync: vi.fn(),
|
||||
probeVectorAvailability: vi.fn(async () => true),
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
const getMemorySearchManagerMock = vi.fn(
|
||||
async (params: { cfg?: unknown; agentId?: string; purpose?: string }) =>
|
||||
getManagerImpl ? await getManagerImpl(params) : { manager: stubManager },
|
||||
);
|
||||
const readAgentMemoryFileMock = vi.fn(
|
||||
async (params: MemoryReadParams) => await readFileImpl(params),
|
||||
);
|
||||
|
||||
vi.mock("./tools.runtime.js", () => ({
|
||||
resolveMemoryBackendConfig: ({
|
||||
cfg,
|
||||
}: {
|
||||
cfg?: { memory?: { backend?: string; qmd?: unknown } };
|
||||
}) => ({
|
||||
backend,
|
||||
qmd: cfg?.memory?.qmd,
|
||||
}),
|
||||
getMemorySearchManager: getMemorySearchManagerMock,
|
||||
readAgentMemoryFile: readAgentMemoryFileMock,
|
||||
}));
|
||||
|
||||
export function setMemoryBackend(next: MemoryBackend): void {
|
||||
backend = next;
|
||||
}
|
||||
|
||||
export function setMemoryWorkspaceDir(next: string): void {
|
||||
workspaceDir = next;
|
||||
}
|
||||
|
||||
export function setMemoryCustomStatus(next: Record<string, unknown> | undefined): void {
|
||||
customStatus = next;
|
||||
}
|
||||
|
||||
export function setMemorySearchImpl(next: SearchImpl): void {
|
||||
searchImpl = next;
|
||||
}
|
||||
|
||||
export function setMemorySearchManagerImpl(
|
||||
next: (params: { cfg?: unknown; agentId?: string; purpose?: string }) => Promise<{
|
||||
manager?: unknown;
|
||||
error?: string;
|
||||
}>,
|
||||
): void {
|
||||
getManagerImpl = next;
|
||||
}
|
||||
|
||||
export function setMemoryReadFileImpl(
|
||||
next: (params: MemoryReadParams) => Promise<MemoryReadResult>,
|
||||
): void {
|
||||
readFileImpl = next;
|
||||
}
|
||||
|
||||
export function resetMemoryToolMockState(overrides?: {
|
||||
backend?: MemoryBackend;
|
||||
searchImpl?: SearchImpl;
|
||||
readFileImpl?: (params: MemoryReadParams) => Promise<MemoryReadResult>;
|
||||
}): void {
|
||||
backend = overrides?.backend ?? "builtin";
|
||||
workspaceDir = "/workspace";
|
||||
customStatus = undefined;
|
||||
getManagerImpl = undefined;
|
||||
searchImpl = overrides?.searchImpl ?? (async () => []);
|
||||
readFileImpl =
|
||||
overrides?.readFileImpl ??
|
||||
(async (params: MemoryReadParams) => ({
|
||||
text: "",
|
||||
path: params.relPath,
|
||||
from: params.from ?? 1,
|
||||
lines: params.lines ?? 120,
|
||||
}));
|
||||
vi.clearAllMocks();
|
||||
}
|
||||
|
||||
export function getMemorySearchManagerMockCalls(): number {
|
||||
return getMemorySearchManagerMock.mock.calls.length;
|
||||
}
|
||||
|
||||
export function getMemorySyncMockCalls(): number {
|
||||
return stubManager.sync.mock.calls.length;
|
||||
}
|
||||
|
||||
export function getMemoryCloseMockCalls(): number {
|
||||
return stubManager.close.mock.calls.length;
|
||||
}
|
||||
|
||||
export function getMemorySearchManagerMockConfigs(): unknown[] {
|
||||
return getMemorySearchManagerMock.mock.calls.map(([params]) => params.cfg);
|
||||
}
|
||||
|
||||
export function getMemorySearchManagerMockParams(): Array<{
|
||||
cfg?: unknown;
|
||||
agentId?: string;
|
||||
purpose?: string;
|
||||
}> {
|
||||
return getMemorySearchManagerMock.mock.calls.map(([params]) => params);
|
||||
}
|
||||
|
||||
export function getReadAgentMemoryFileMockCalls(): number {
|
||||
return readAgentMemoryFileMock.mock.calls.length;
|
||||
}
|
||||
32
extensions/memory-core/src/memory/embedding.test-mocks.ts
Normal file
32
extensions/memory-core/src/memory/embedding.test-mocks.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
// Memory Core plugin module implements embedding mocks behavior.
|
||||
import { vi } from "vitest";
|
||||
import "./test-runtime-mocks.js";
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
embedBatch: vi.fn(async (texts: string[]) => texts.map(() => [0, 1, 0])),
|
||||
embedQuery: vi.fn(async () => [0, 1, 0]),
|
||||
}));
|
||||
|
||||
export function resetEmbeddingMocks(): void {
|
||||
hoisted.embedBatch.mockReset();
|
||||
hoisted.embedQuery.mockReset();
|
||||
hoisted.embedBatch.mockImplementation(async (texts: string[]) => texts.map(() => [0, 1, 0]));
|
||||
hoisted.embedQuery.mockImplementation(async () => [0, 1, 0]);
|
||||
}
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
resolveEmbeddingProviderAdapterId: (providerId: string) => providerId,
|
||||
resolveEmbeddingProviderAdapterTransport: (providerId: string) =>
|
||||
providerId === "local" ? "local" : "remote",
|
||||
resolveEmbeddingProviderIndexIdentity: () => undefined,
|
||||
createEmbeddingProvider: async () => ({
|
||||
requestedProvider: "openai",
|
||||
provider: {
|
||||
id: "mock",
|
||||
model: "mock-embed",
|
||||
maxInputTokens: 8192,
|
||||
embedQuery: hoisted.embedQuery,
|
||||
embedBatch: hoisted.embedBatch,
|
||||
},
|
||||
}),
|
||||
}));
|
||||
271
extensions/memory-core/src/memory/embeddings.test.ts
Normal file
271
extensions/memory-core/src/memory/embeddings.test.ts
Normal file
@@ -0,0 +1,271 @@
|
||||
// Memory Core tests cover embeddings plugin behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { EmbeddingProviderAdapter } from "openclaw/plugin-sdk/embedding-providers";
|
||||
import type { MemoryEmbeddingProviderAdapter } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createEmbeddingProvider, resolveEmbeddingProviderFallbackModel } from "./embeddings.js";
|
||||
|
||||
const mockEmbeddingRegistry = vi.hoisted(() => ({
|
||||
genericAdapters: [] as EmbeddingProviderAdapter[],
|
||||
adapters: [] as MemoryEmbeddingProviderAdapter[],
|
||||
genericLookupConfigs: [] as Array<OpenClawConfig | undefined>,
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/embedding-providers", () => ({
|
||||
getEmbeddingProvider: (id: string, config?: OpenClawConfig) => {
|
||||
mockEmbeddingRegistry.genericLookupConfigs.push(config);
|
||||
return mockEmbeddingRegistry.genericAdapters.find((adapter) => adapter.id === id);
|
||||
},
|
||||
listEmbeddingProviders: () => [...mockEmbeddingRegistry.genericAdapters],
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/memory-core-host-engine-embeddings", () => ({
|
||||
DEFAULT_LOCAL_MODEL: "nomic-embed-text",
|
||||
createLocalEmbeddingProvider: async () => {
|
||||
throw new Error("local embedding provider is not used by these tests");
|
||||
},
|
||||
getMemoryEmbeddingProvider: (id: string) =>
|
||||
mockEmbeddingRegistry.adapters.find((adapter) => adapter.id === id),
|
||||
listMemoryEmbeddingProviders: () => [...mockEmbeddingRegistry.adapters],
|
||||
listRegisteredMemoryEmbeddingProviderAdapters: () => [...mockEmbeddingRegistry.adapters],
|
||||
listRegisteredMemoryEmbeddingProviders: () =>
|
||||
mockEmbeddingRegistry.adapters.map((adapter) => ({ adapter })),
|
||||
}));
|
||||
|
||||
const missingBedrockCredentialsError = new Error(
|
||||
'No API key found for provider "bedrock". AWS credentials are not available.',
|
||||
);
|
||||
|
||||
function createOptions(provider: string) {
|
||||
return {
|
||||
config: {
|
||||
plugins: {
|
||||
deny: [
|
||||
"amazon-bedrock",
|
||||
"github-copilot",
|
||||
"google",
|
||||
"lmstudio",
|
||||
"memory-core",
|
||||
"mistral",
|
||||
"ollama",
|
||||
"openai",
|
||||
"voyage",
|
||||
],
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
provider,
|
||||
fallback: "none",
|
||||
model: "",
|
||||
};
|
||||
}
|
||||
|
||||
function createMissingCredentialsAdapter(
|
||||
overrides: Partial<MemoryEmbeddingProviderAdapter> = {},
|
||||
): MemoryEmbeddingProviderAdapter {
|
||||
return {
|
||||
id: "bedrock",
|
||||
transport: "remote",
|
||||
autoSelectPriority: 60,
|
||||
formatSetupError: (err) => (err instanceof Error ? err.message : String(err)),
|
||||
shouldContinueAutoSelection: (err) =>
|
||||
err instanceof Error && err.message.includes("No API key found for provider"),
|
||||
create: async () => {
|
||||
throw missingBedrockCredentialsError;
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function clearMemoryEmbeddingProviders(): void {
|
||||
mockEmbeddingRegistry.genericAdapters = [];
|
||||
mockEmbeddingRegistry.adapters = [];
|
||||
mockEmbeddingRegistry.genericLookupConfigs = [];
|
||||
}
|
||||
|
||||
function registerGenericEmbeddingProvider(adapter: EmbeddingProviderAdapter): void {
|
||||
mockEmbeddingRegistry.genericAdapters = mockEmbeddingRegistry.genericAdapters.filter(
|
||||
(candidate) => candidate.id !== adapter.id,
|
||||
);
|
||||
mockEmbeddingRegistry.genericAdapters.push(adapter);
|
||||
}
|
||||
|
||||
function registerMemoryEmbeddingProvider(adapter: MemoryEmbeddingProviderAdapter): void {
|
||||
mockEmbeddingRegistry.adapters = mockEmbeddingRegistry.adapters.filter(
|
||||
(candidate) => candidate.id !== adapter.id,
|
||||
);
|
||||
mockEmbeddingRegistry.adapters.push(adapter);
|
||||
}
|
||||
|
||||
describe("createEmbeddingProvider", () => {
|
||||
beforeEach(() => {
|
||||
clearMemoryEmbeddingProviders();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearMemoryEmbeddingProviders();
|
||||
});
|
||||
|
||||
it("normalizes legacy auto mode to OpenAI", async () => {
|
||||
registerMemoryEmbeddingProvider(createMissingCredentialsAdapter({ id: "bedrock" }));
|
||||
registerMemoryEmbeddingProvider({
|
||||
id: "openai",
|
||||
transport: "remote",
|
||||
autoSelectPriority: 20,
|
||||
create: async () => ({
|
||||
provider: {
|
||||
id: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
embedQuery: async () => [1],
|
||||
embedBatch: async (texts) => texts.map(() => [1]),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await createEmbeddingProvider(createOptions("auto"));
|
||||
|
||||
expect(result.provider?.id).toBe("openai");
|
||||
expect(result.requestedProvider).toBe("openai");
|
||||
});
|
||||
|
||||
it("still throws missing credentials for an explicit provider request", async () => {
|
||||
registerMemoryEmbeddingProvider(createMissingCredentialsAdapter());
|
||||
|
||||
await expect(createEmbeddingProvider(createOptions("bedrock"))).rejects.toThrow(
|
||||
missingBedrockCredentialsError.message,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not run priority-based auto-selection after a skippable setup failure", async () => {
|
||||
registerMemoryEmbeddingProvider(createMissingCredentialsAdapter({ autoSelectPriority: 10 }));
|
||||
registerMemoryEmbeddingProvider({
|
||||
id: "openai",
|
||||
transport: "remote",
|
||||
autoSelectPriority: 20,
|
||||
create: async () => ({
|
||||
provider: {
|
||||
id: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
embedQuery: async () => [1],
|
||||
embedBatch: async (texts) => texts.map(() => [1]),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await createEmbeddingProvider(createOptions("auto"));
|
||||
|
||||
expect(result.provider?.id).toBe("openai");
|
||||
expect(result.requestedProvider).toBe("openai");
|
||||
});
|
||||
|
||||
it("uses a generic embedding provider when no memory-specific provider exists", async () => {
|
||||
registerGenericEmbeddingProvider({
|
||||
id: "openai-compatible",
|
||||
create: async () => ({
|
||||
provider: {
|
||||
id: "generic",
|
||||
model: "generic-model",
|
||||
embed: async (_input, options) => (options?.inputType === "query" ? [1] : [2]),
|
||||
embedBatch: async (inputs, options) =>
|
||||
inputs.map(() => (options?.inputType === "document" ? [3] : [4])),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const options = createOptions("openai-compatible");
|
||||
const result = await createEmbeddingProvider(options);
|
||||
|
||||
expect(result.provider?.id).toBe("generic");
|
||||
expect(mockEmbeddingRegistry.genericLookupConfigs).toEqual([options.config]);
|
||||
await expect(result.provider?.embedQuery("hello")).resolves.toEqual([1]);
|
||||
await expect(result.provider?.embedBatch(["doc"])).resolves.toEqual([[3]]);
|
||||
});
|
||||
|
||||
it("keeps memory-specific providers authoritative during dual registration", async () => {
|
||||
registerGenericEmbeddingProvider({
|
||||
id: "openai-compatible",
|
||||
create: async () => ({
|
||||
provider: {
|
||||
id: "generic",
|
||||
model: "generic-model",
|
||||
embed: async (_input, options) => (options?.inputType === "query" ? [1] : [2]),
|
||||
embedBatch: async (inputs, options) =>
|
||||
inputs.map(() => (options?.inputType === "document" ? [3] : [4])),
|
||||
},
|
||||
}),
|
||||
});
|
||||
registerMemoryEmbeddingProvider({
|
||||
id: "openai-compatible",
|
||||
create: async () => ({
|
||||
provider: {
|
||||
id: "legacy",
|
||||
model: "legacy-model",
|
||||
embedQuery: async () => [0],
|
||||
embedBatch: async (texts) => texts.map(() => [0]),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await createEmbeddingProvider(createOptions("openai-compatible"));
|
||||
|
||||
expect(result.provider?.id).toBe("legacy");
|
||||
await expect(result.provider?.embedQuery("hello")).resolves.toEqual([0]);
|
||||
});
|
||||
|
||||
it("reports the llama.cpp plugin install command when local is unregistered", async () => {
|
||||
await expect(createEmbeddingProvider(createOptions("local"))).rejects.toThrow(
|
||||
"openclaw plugins install @openclaw/llama-cpp-provider",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not auto-select generic providers by priority policy", async () => {
|
||||
registerMemoryEmbeddingProvider({
|
||||
id: "openai-compatible",
|
||||
transport: "remote",
|
||||
autoSelectPriority: 20,
|
||||
create: async () => ({
|
||||
provider: {
|
||||
id: "legacy",
|
||||
model: "legacy-model",
|
||||
embedQuery: async () => [1],
|
||||
embedBatch: async (texts) => texts.map(() => [1]),
|
||||
},
|
||||
}),
|
||||
});
|
||||
registerGenericEmbeddingProvider({
|
||||
id: "openai-compatible",
|
||||
create: async () => ({
|
||||
provider: {
|
||||
id: "generic",
|
||||
model: "generic-model",
|
||||
embed: async () => [2],
|
||||
embedBatch: async (inputs) => inputs.map(() => [2]),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(createEmbeddingProvider(createOptions("auto"))).rejects.toThrow(
|
||||
"Unknown memory embedding provider: openai",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses config-scoped lookup for generic fallback model resolution", () => {
|
||||
registerGenericEmbeddingProvider({
|
||||
id: "openai-compatible",
|
||||
defaultModel: "generic-default",
|
||||
create: async () => ({
|
||||
provider: null,
|
||||
}),
|
||||
});
|
||||
const options = createOptions("openai-compatible");
|
||||
|
||||
const model = resolveEmbeddingProviderFallbackModel(
|
||||
"openai-compatible",
|
||||
"source-model",
|
||||
options.config,
|
||||
);
|
||||
|
||||
expect(model).toBe("generic-default");
|
||||
expect(mockEmbeddingRegistry.genericLookupConfigs).toEqual([options.config]);
|
||||
});
|
||||
});
|
||||
279
extensions/memory-core/src/memory/embeddings.ts
Normal file
279
extensions/memory-core/src/memory/embeddings.ts
Normal file
@@ -0,0 +1,279 @@
|
||||
// Memory Core plugin module implements embeddings behavior.
|
||||
import {
|
||||
getEmbeddingProvider,
|
||||
type EmbeddingProviderAdapter,
|
||||
type EmbeddingProvider as GenericEmbeddingProvider,
|
||||
type EmbeddingProviderRuntime as GenericEmbeddingProviderRuntime,
|
||||
} from "openclaw/plugin-sdk/embedding-providers";
|
||||
import {
|
||||
getMemoryEmbeddingProvider as getLegacyMemoryEmbeddingProvider,
|
||||
type MemoryEmbeddingProvider,
|
||||
type MemoryEmbeddingProviderAdapter,
|
||||
type MemoryEmbeddingProviderCreateOptions,
|
||||
type MemoryEmbeddingProviderRuntime,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { formatErrorMessage } from "../dreaming-shared.js";
|
||||
|
||||
export type EmbeddingProvider = MemoryEmbeddingProvider;
|
||||
export type EmbeddingProviderId = string;
|
||||
export type EmbeddingProviderRequest = string;
|
||||
type EmbeddingProviderFallback = string;
|
||||
export type EmbeddingProviderRuntime = MemoryEmbeddingProviderRuntime;
|
||||
|
||||
export type EmbeddingProviderResult = {
|
||||
provider: EmbeddingProvider | null;
|
||||
requestedProvider: EmbeddingProviderRequest;
|
||||
fallbackFrom?: string;
|
||||
fallbackReason?: string;
|
||||
providerUnavailableReason?: string;
|
||||
runtime?: EmbeddingProviderRuntime;
|
||||
};
|
||||
|
||||
type CreateEmbeddingProviderOptions = MemoryEmbeddingProviderCreateOptions & {
|
||||
provider: EmbeddingProviderRequest;
|
||||
fallback: EmbeddingProviderFallback;
|
||||
};
|
||||
|
||||
const DEFAULT_MEMORY_EMBEDDING_PROVIDER = "openai";
|
||||
const LOCAL_LLAMA_CPP_PROVIDER_ID = "local";
|
||||
|
||||
function createMissingLlamaCppProviderError(): Error {
|
||||
return new Error(
|
||||
[
|
||||
"Unknown memory embedding provider: local.",
|
||||
"Local GGUF embeddings are provided by the official llama.cpp provider plugin.",
|
||||
"Install it with: openclaw plugins install @openclaw/llama-cpp-provider",
|
||||
"Then restart OpenClaw and retry: openclaw memory status --deep",
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
function adaptGenericEmbeddingProvider(
|
||||
provider: GenericEmbeddingProvider,
|
||||
): MemoryEmbeddingProvider {
|
||||
return {
|
||||
id: provider.id,
|
||||
model: provider.model,
|
||||
...(typeof provider.maxInputTokens === "number"
|
||||
? { maxInputTokens: provider.maxInputTokens }
|
||||
: {}),
|
||||
embedQuery: async (text, options) =>
|
||||
await provider.embed(text, {
|
||||
...options,
|
||||
inputType: "query",
|
||||
}),
|
||||
embedBatch: async (texts, options) =>
|
||||
await provider.embedBatch(texts, {
|
||||
...options,
|
||||
inputType: "document",
|
||||
}),
|
||||
embedBatchInputs: async (inputs, options) =>
|
||||
await provider.embedBatch(inputs, {
|
||||
...options,
|
||||
inputType: "document",
|
||||
}),
|
||||
...(provider.close ? { close: provider.close } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function adaptGenericRuntime(
|
||||
runtime: GenericEmbeddingProviderRuntime | undefined,
|
||||
): MemoryEmbeddingProviderRuntime | undefined {
|
||||
if (!runtime) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
id: runtime.id,
|
||||
...(runtime.cacheKeyData ? { cacheKeyData: runtime.cacheKeyData } : {}),
|
||||
...(runtime.indexIdentityAliases?.length
|
||||
? { indexIdentityAliases: runtime.indexIdentityAliases }
|
||||
: {}),
|
||||
...(typeof runtime.inlineQueryTimeoutMs === "number"
|
||||
? { inlineQueryTimeoutMs: runtime.inlineQueryTimeoutMs }
|
||||
: {}),
|
||||
...(typeof runtime.inlineBatchTimeoutMs === "number"
|
||||
? { inlineBatchTimeoutMs: runtime.inlineBatchTimeoutMs }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function adaptGenericEmbeddingAdapter(
|
||||
adapter: EmbeddingProviderAdapter,
|
||||
): MemoryEmbeddingProviderAdapter {
|
||||
const resolveIndexIdentity = adapter.resolveIndexIdentity;
|
||||
return {
|
||||
id: adapter.id,
|
||||
...(adapter.defaultModel ? { defaultModel: adapter.defaultModel } : {}),
|
||||
...(adapter.transport ? { transport: adapter.transport } : {}),
|
||||
...(adapter.authProviderId ? { authProviderId: adapter.authProviderId } : {}),
|
||||
...(adapter.formatSetupError ? { formatSetupError: adapter.formatSetupError } : {}),
|
||||
...(resolveIndexIdentity
|
||||
? {
|
||||
resolveIndexIdentity: (options: MemoryEmbeddingProviderCreateOptions) =>
|
||||
resolveIndexIdentity({
|
||||
...options,
|
||||
...(typeof options.outputDimensionality === "number"
|
||||
? { dimensions: options.outputDimensionality }
|
||||
: {}),
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
create: async (options) => {
|
||||
const result = await adapter.create({
|
||||
...options,
|
||||
...(typeof options.outputDimensionality === "number"
|
||||
? { dimensions: options.outputDimensionality }
|
||||
: {}),
|
||||
});
|
||||
return {
|
||||
provider: result.provider ? adaptGenericEmbeddingProvider(result.provider) : null,
|
||||
runtime: adaptGenericRuntime(result.runtime),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatProviderError(adapter: MemoryEmbeddingProviderAdapter, err: unknown): string {
|
||||
return adapter.formatSetupError?.(err) ?? formatErrorMessage(err);
|
||||
}
|
||||
|
||||
function getAdapter(
|
||||
id: string,
|
||||
config?: MemoryEmbeddingProviderCreateOptions["config"],
|
||||
): MemoryEmbeddingProviderAdapter {
|
||||
const adapter = getLegacyMemoryEmbeddingProvider(id, config);
|
||||
if (adapter) {
|
||||
return adapter;
|
||||
}
|
||||
const genericAdapter = getEmbeddingProvider(id, config);
|
||||
if (genericAdapter) {
|
||||
return adaptGenericEmbeddingAdapter(genericAdapter);
|
||||
}
|
||||
if (id === LOCAL_LLAMA_CPP_PROVIDER_ID) {
|
||||
throw createMissingLlamaCppProviderError();
|
||||
}
|
||||
throw new Error(`Unknown memory embedding provider: ${id}`);
|
||||
}
|
||||
|
||||
function resolveProviderModel(
|
||||
adapter: MemoryEmbeddingProviderAdapter,
|
||||
requestedModel: string,
|
||||
): string {
|
||||
const trimmed = requestedModel.trim();
|
||||
if (trimmed) {
|
||||
return trimmed;
|
||||
}
|
||||
return adapter.defaultModel ?? "";
|
||||
}
|
||||
|
||||
export function resolveEmbeddingProviderFallbackModel(
|
||||
providerId: string,
|
||||
fallbackSourceModel: string,
|
||||
config?: MemoryEmbeddingProviderCreateOptions["config"],
|
||||
): string {
|
||||
const adapter =
|
||||
getLegacyMemoryEmbeddingProvider(providerId, config) ??
|
||||
getEmbeddingProvider(providerId, config);
|
||||
return adapter?.defaultModel ?? fallbackSourceModel;
|
||||
}
|
||||
|
||||
export function resolveEmbeddingProviderAdapterId(
|
||||
providerId: string,
|
||||
config?: MemoryEmbeddingProviderCreateOptions["config"],
|
||||
): string | undefined {
|
||||
try {
|
||||
return getAdapter(providerId, config).id;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveEmbeddingProviderAdapterTransport(
|
||||
providerId: string,
|
||||
config?: MemoryEmbeddingProviderCreateOptions["config"],
|
||||
): MemoryEmbeddingProviderAdapter["transport"] {
|
||||
try {
|
||||
return getAdapter(providerId, config).transport;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveEmbeddingProviderIndexIdentity(options: CreateEmbeddingProviderOptions) {
|
||||
const provider =
|
||||
options.provider === "auto" ? DEFAULT_MEMORY_EMBEDDING_PROVIDER : options.provider;
|
||||
try {
|
||||
const adapter = getAdapter(provider, options.config);
|
||||
const model = resolveProviderModel(adapter, options.model);
|
||||
const identity = adapter.resolveIndexIdentity?.({
|
||||
...options,
|
||||
provider,
|
||||
model,
|
||||
});
|
||||
return identity
|
||||
? {
|
||||
provider: { id: adapter.id, model: identity.model },
|
||||
cacheKeyData: identity.cacheKeyData,
|
||||
aliases: identity.aliases,
|
||||
}
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function createWithAdapter(
|
||||
adapter: MemoryEmbeddingProviderAdapter,
|
||||
options: CreateEmbeddingProviderOptions,
|
||||
): Promise<EmbeddingProviderResult> {
|
||||
const result = await adapter.create({
|
||||
...options,
|
||||
model: resolveProviderModel(adapter, options.model),
|
||||
});
|
||||
return {
|
||||
provider: result.provider,
|
||||
requestedProvider: options.provider,
|
||||
runtime: result.runtime,
|
||||
};
|
||||
}
|
||||
|
||||
export async function createEmbeddingProvider(
|
||||
options: CreateEmbeddingProviderOptions,
|
||||
): Promise<EmbeddingProviderResult> {
|
||||
const provider =
|
||||
options.provider === "auto" ? DEFAULT_MEMORY_EMBEDDING_PROVIDER : options.provider;
|
||||
const primaryAdapter = getAdapter(provider, options.config);
|
||||
try {
|
||||
return await createWithAdapter(primaryAdapter, {
|
||||
...options,
|
||||
provider,
|
||||
});
|
||||
} catch (primaryErr) {
|
||||
const reason = formatProviderError(primaryAdapter, primaryErr);
|
||||
if (options.fallback && options.fallback !== "none" && options.fallback !== provider) {
|
||||
const fallbackAdapter = getAdapter(options.fallback, options.config);
|
||||
try {
|
||||
const fallbackResult = await createWithAdapter(fallbackAdapter, {
|
||||
...options,
|
||||
provider: options.fallback,
|
||||
});
|
||||
return {
|
||||
...fallbackResult,
|
||||
requestedProvider: provider,
|
||||
fallbackFrom: provider,
|
||||
fallbackReason: reason,
|
||||
};
|
||||
} catch (fallbackErr) {
|
||||
const fallbackReason = formatProviderError(fallbackAdapter, fallbackErr);
|
||||
const wrapped = new Error(
|
||||
`${reason}\n\nFallback to ${options.fallback} failed: ${fallbackReason}`,
|
||||
) as Error & { cause?: unknown };
|
||||
wrapped.cause = primaryErr;
|
||||
throw wrapped;
|
||||
}
|
||||
}
|
||||
const wrapped = new Error(reason) as Error & { cause?: unknown };
|
||||
wrapped.cause = primaryErr;
|
||||
throw wrapped;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
// Memory Core tests cover generic embedding provider.bridge plugin behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type {
|
||||
EmbeddingInput,
|
||||
EmbeddingProviderCallOptions,
|
||||
} from "openclaw/plugin-sdk/embedding-providers";
|
||||
import {
|
||||
createPluginRegistryFixture,
|
||||
registerVirtualTestPlugin,
|
||||
} from "openclaw/plugin-sdk/plugin-test-contracts";
|
||||
import {
|
||||
clearEmbeddingProviders,
|
||||
getRegisteredEmbeddingProvider,
|
||||
listRegisteredEmbeddingProviders,
|
||||
type RegisteredEmbeddingProvider,
|
||||
restoreRegisteredEmbeddingProviders,
|
||||
clearMemoryEmbeddingProviders,
|
||||
listRegisteredMemoryEmbeddingProviders,
|
||||
type RegisteredMemoryEmbeddingProvider,
|
||||
restoreRegisteredMemoryEmbeddingProviders,
|
||||
} from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createEmbeddingProvider, resolveEmbeddingProviderIndexIdentity } from "./embeddings.js";
|
||||
|
||||
type CapturedCall = {
|
||||
kind: "embed" | "embedBatch";
|
||||
input: EmbeddingInput | EmbeddingInput[];
|
||||
options: EmbeddingProviderCallOptions | undefined;
|
||||
};
|
||||
|
||||
let embeddingProvidersSnapshot: RegisteredEmbeddingProvider[];
|
||||
let memoryEmbeddingProvidersSnapshot: RegisteredMemoryEmbeddingProvider[];
|
||||
|
||||
function createOptions(config: OpenClawConfig) {
|
||||
return {
|
||||
config,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
provider: "virtual-generic",
|
||||
fallback: "none",
|
||||
model: "virtual-model",
|
||||
outputDimensionality: 7,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
embeddingProvidersSnapshot = listRegisteredEmbeddingProviders();
|
||||
memoryEmbeddingProvidersSnapshot = listRegisteredMemoryEmbeddingProviders();
|
||||
clearEmbeddingProviders();
|
||||
clearMemoryEmbeddingProviders();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
restoreRegisteredEmbeddingProviders(embeddingProvidersSnapshot);
|
||||
restoreRegisteredMemoryEmbeddingProviders(memoryEmbeddingProvidersSnapshot);
|
||||
});
|
||||
|
||||
describe("memory-core generic embedding provider bridge", () => {
|
||||
it("adapts a contract-declared generic embedding plugin into explicit memory requests", async () => {
|
||||
const calls: CapturedCall[] = [];
|
||||
const { config, registry } = createPluginRegistryFixture({
|
||||
plugins: {
|
||||
enabled: false,
|
||||
},
|
||||
} as OpenClawConfig);
|
||||
|
||||
registerVirtualTestPlugin({
|
||||
registry,
|
||||
config,
|
||||
id: "virtual-generic-plugin",
|
||||
name: "Virtual Generic Embeddings",
|
||||
contracts: {
|
||||
embeddingProviders: ["virtual-generic"],
|
||||
},
|
||||
register(api) {
|
||||
api.registerEmbeddingProvider({
|
||||
id: "virtual-generic",
|
||||
transport: "remote",
|
||||
defaultModel: "virtual-default",
|
||||
resolveIndexIdentity: (options) => ({
|
||||
model: options.model,
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: options.model,
|
||||
dimensions: options.dimensions,
|
||||
},
|
||||
aliases: [
|
||||
{
|
||||
model: "virtual-model-legacy",
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: "virtual-model-legacy",
|
||||
dimensions: options.dimensions,
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
create: async (options) => {
|
||||
expect(options.model).toBe("virtual-model");
|
||||
expect(options.dimensions).toBe(7);
|
||||
expect(options.config).toBe(config);
|
||||
return {
|
||||
provider: {
|
||||
id: "virtual-generic",
|
||||
model: options.model,
|
||||
dimensions: options.dimensions,
|
||||
maxInputTokens: 2048,
|
||||
embed: async (input, callOptions) => {
|
||||
calls.push({ kind: "embed", input, options: callOptions });
|
||||
return callOptions?.inputType === "query" ? [1, 2, 3] : [0];
|
||||
},
|
||||
embedBatch: async (inputs, callOptions) => {
|
||||
calls.push({ kind: "embedBatch", input: inputs, options: callOptions });
|
||||
return inputs.map((_input, index) =>
|
||||
callOptions?.inputType === "document" ? [index, 7] : [0],
|
||||
);
|
||||
},
|
||||
},
|
||||
runtime: {
|
||||
id: "virtual-generic",
|
||||
inlineQueryTimeoutMs: 1234,
|
||||
inlineBatchTimeoutMs: 5678,
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: options.model,
|
||||
dimensions: options.dimensions,
|
||||
},
|
||||
indexIdentityAliases: [
|
||||
{
|
||||
model: "virtual-model-legacy",
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: "virtual-model-legacy",
|
||||
dimensions: options.dimensions,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
expect(getRegisteredEmbeddingProvider("virtual-generic")?.ownerPluginId).toBe(
|
||||
"virtual-generic-plugin",
|
||||
);
|
||||
expect(registry.registry.embeddingProviders.map((entry) => entry.provider.id)).toEqual([
|
||||
"virtual-generic",
|
||||
]);
|
||||
expect(listRegisteredMemoryEmbeddingProviders()).toEqual([]);
|
||||
|
||||
expect(resolveEmbeddingProviderIndexIdentity(createOptions(config))).toEqual({
|
||||
provider: { id: "virtual-generic", model: "virtual-model" },
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: "virtual-model",
|
||||
dimensions: 7,
|
||||
},
|
||||
aliases: [
|
||||
{
|
||||
model: "virtual-model-legacy",
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: "virtual-model-legacy",
|
||||
dimensions: 7,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const result = await createEmbeddingProvider(createOptions(config));
|
||||
|
||||
expect(result.requestedProvider).toBe("virtual-generic");
|
||||
expect(result.provider).toMatchObject({
|
||||
id: "virtual-generic",
|
||||
model: "virtual-model",
|
||||
maxInputTokens: 2048,
|
||||
});
|
||||
expect(result.runtime).toEqual({
|
||||
id: "virtual-generic",
|
||||
inlineQueryTimeoutMs: 1234,
|
||||
inlineBatchTimeoutMs: 5678,
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: "virtual-model",
|
||||
dimensions: 7,
|
||||
},
|
||||
indexIdentityAliases: [
|
||||
{
|
||||
model: "virtual-model-legacy",
|
||||
cacheKeyData: {
|
||||
provider: "virtual-generic",
|
||||
model: "virtual-model-legacy",
|
||||
dimensions: 7,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(result.provider?.embedQuery("query")).resolves.toEqual([1, 2, 3]);
|
||||
await expect(result.provider?.embedBatch(["doc-a", "doc-b"])).resolves.toEqual([
|
||||
[0, 7],
|
||||
[1, 7],
|
||||
]);
|
||||
await expect(
|
||||
result.provider?.embedBatchInputs?.([
|
||||
{
|
||||
text: "structured doc",
|
||||
parts: [{ type: "text", text: "structured doc" }],
|
||||
},
|
||||
]),
|
||||
).resolves.toEqual([[0, 7]]);
|
||||
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
kind: "embed",
|
||||
input: "query",
|
||||
options: { inputType: "query" },
|
||||
},
|
||||
{
|
||||
kind: "embedBatch",
|
||||
input: ["doc-a", "doc-b"],
|
||||
options: { inputType: "document" },
|
||||
},
|
||||
{
|
||||
kind: "embedBatch",
|
||||
input: [
|
||||
{
|
||||
text: "structured doc",
|
||||
parts: [{ type: "text", text: "structured doc" }],
|
||||
},
|
||||
],
|
||||
options: { inputType: "document" },
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,250 @@
|
||||
// Memory Core tests cover generic embedding provider.integration plugin behavior.
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import {
|
||||
clearEmbeddingProviders,
|
||||
clearMemoryEmbeddingProviders,
|
||||
getActivePluginRegistry,
|
||||
listRegisteredEmbeddingProviders,
|
||||
listRegisteredMemoryEmbeddingProviders,
|
||||
restoreRegisteredEmbeddingProviders,
|
||||
restoreRegisteredMemoryEmbeddingProviders,
|
||||
} from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createEmbeddingProvider } from "./embeddings.js";
|
||||
|
||||
type CapturedRequest = {
|
||||
method: string | undefined;
|
||||
url: string | undefined;
|
||||
headers: IncomingMessage["headers"];
|
||||
body: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type TestServer = {
|
||||
baseUrl: string;
|
||||
requests: CapturedRequest[];
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
const servers: TestServer[] = [];
|
||||
let registeredEmbeddingProvidersSnapshot: ReturnType<typeof listRegisteredEmbeddingProviders>;
|
||||
let registeredMemoryEmbeddingProvidersSnapshot: ReturnType<
|
||||
typeof listRegisteredMemoryEmbeddingProviders
|
||||
>;
|
||||
let restoreActiveMemoryEmbeddingProviders: (() => void) | undefined;
|
||||
|
||||
async function readJsonBody(req: IncomingMessage): Promise<Record<string, unknown>> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function startEmbeddingServer(): Promise<TestServer> {
|
||||
const requests: CapturedRequest[] = [];
|
||||
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
||||
void (async () => {
|
||||
try {
|
||||
const body = await readJsonBody(req);
|
||||
requests.push({
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
headers: req.headers,
|
||||
body,
|
||||
});
|
||||
const input = body.input;
|
||||
const texts = Array.isArray(input) ? input : [input];
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
object: "list",
|
||||
data: texts.map((text, index) => ({
|
||||
object: "embedding",
|
||||
embedding: [String(text).length, index + 0.5, 3],
|
||||
index,
|
||||
})),
|
||||
model: body.model,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
res.writeHead(500, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
const address = server.address() as AddressInfo;
|
||||
const testServer = {
|
||||
baseUrl: `http://127.0.0.1:${address.port}/v1`,
|
||||
requests,
|
||||
close: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
}),
|
||||
};
|
||||
servers.push(testServer);
|
||||
return testServer;
|
||||
}
|
||||
|
||||
function createMemoryEmbeddingOptions(overrides?: {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
baseUrl?: string;
|
||||
}) {
|
||||
return {
|
||||
config: {
|
||||
plugins: {
|
||||
enabled: false,
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
provider: overrides?.provider ?? "openai-compatible",
|
||||
fallback: "none",
|
||||
model: overrides?.model ?? "text-embedding-bge-m3",
|
||||
inputType: "default",
|
||||
queryInputType: "query",
|
||||
documentInputType: "document",
|
||||
remote: {
|
||||
baseUrl: overrides?.baseUrl,
|
||||
apiKey: "fixture-token",
|
||||
headers: {
|
||||
Authorization: "Bearer ignored",
|
||||
"x-api-key": "hidden",
|
||||
"x-deployment": "tenant-a",
|
||||
},
|
||||
},
|
||||
outputDimensionality: 3,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
registeredEmbeddingProvidersSnapshot = listRegisteredEmbeddingProviders();
|
||||
registeredMemoryEmbeddingProvidersSnapshot = listRegisteredMemoryEmbeddingProviders();
|
||||
clearEmbeddingProviders();
|
||||
clearMemoryEmbeddingProviders();
|
||||
|
||||
const activeRegistry = getActivePluginRegistry();
|
||||
if (activeRegistry) {
|
||||
const memoryEmbeddingProviders = activeRegistry.memoryEmbeddingProviders;
|
||||
activeRegistry.memoryEmbeddingProviders = [];
|
||||
restoreActiveMemoryEmbeddingProviders = () => {
|
||||
activeRegistry.memoryEmbeddingProviders = memoryEmbeddingProviders;
|
||||
};
|
||||
} else {
|
||||
restoreActiveMemoryEmbeddingProviders = undefined;
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const pendingServers = servers.splice(0);
|
||||
await Promise.all(pendingServers.map((server) => server.close()));
|
||||
restoreRegisteredEmbeddingProviders(registeredEmbeddingProvidersSnapshot);
|
||||
restoreRegisteredMemoryEmbeddingProviders(registeredMemoryEmbeddingProvidersSnapshot);
|
||||
restoreActiveMemoryEmbeddingProviders?.();
|
||||
});
|
||||
|
||||
describe("memory-core generic embedding provider bridge", () => {
|
||||
it("uses the core OpenAI-compatible provider through the generic registry and memory bridge", async () => {
|
||||
const server = await startEmbeddingServer();
|
||||
|
||||
expect(listRegisteredMemoryEmbeddingProviders()).toEqual([]);
|
||||
expect(listRegisteredEmbeddingProviders()).toMatchObject([
|
||||
{
|
||||
ownerPluginId: "core",
|
||||
adapter: { id: "openai-compatible" },
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await createEmbeddingProvider(
|
||||
createMemoryEmbeddingOptions({ baseUrl: ` ${server.baseUrl}/ ` }),
|
||||
);
|
||||
|
||||
expect(result.provider?.id).toBe("openai-compatible");
|
||||
expect(result.provider?.model).toBe("text-embedding-bge-m3");
|
||||
expect(result.runtime).toMatchObject({
|
||||
id: "openai-compatible",
|
||||
inlineBatchTimeoutMs: 600_000,
|
||||
cacheKeyData: {
|
||||
provider: "openai-compatible",
|
||||
baseUrl: server.baseUrl,
|
||||
model: "text-embedding-bge-m3",
|
||||
dimensions: 3,
|
||||
inputType: "default",
|
||||
queryInputType: "query",
|
||||
documentInputType: "document",
|
||||
headers: {
|
||||
accept: "application/json",
|
||||
"content-type": "application/json",
|
||||
"x-deployment": "tenant-a",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(server.requests).toHaveLength(0);
|
||||
|
||||
await expect(result.provider?.embedQuery("hello")).resolves.toEqual([5, 0.5, 3]);
|
||||
await expect(result.provider?.embedBatch(["a", "abcd"])).resolves.toEqual([
|
||||
[1, 0.5, 3],
|
||||
[4, 1.5, 3],
|
||||
]);
|
||||
await expect(
|
||||
result.provider?.embedBatchInputs?.([
|
||||
{
|
||||
text: "structured doc",
|
||||
parts: [{ type: "text", text: "structured doc" }],
|
||||
},
|
||||
]),
|
||||
).resolves.toEqual([[14, 0.5, 3]]);
|
||||
|
||||
expect(server.requests).toHaveLength(3);
|
||||
expect(server.requests[0]).toMatchObject({
|
||||
method: "POST",
|
||||
url: "/v1/embeddings",
|
||||
body: {
|
||||
model: "text-embedding-bge-m3",
|
||||
input: ["hello"],
|
||||
dimensions: 3,
|
||||
input_type: "query",
|
||||
},
|
||||
});
|
||||
expect(server.requests[0]?.body).not.toHaveProperty("encoding_format");
|
||||
expect(server.requests[0]?.headers.authorization).toBe("Bearer fixture-token");
|
||||
expect(server.requests[0]?.headers["x-api-key"]).toBe("hidden");
|
||||
expect(server.requests[0]?.headers["x-deployment"]).toBe("tenant-a");
|
||||
expect(server.requests[1]?.body).toEqual({
|
||||
model: "text-embedding-bge-m3",
|
||||
input: ["a", "abcd"],
|
||||
dimensions: 3,
|
||||
input_type: "document",
|
||||
});
|
||||
expect(server.requests[2]?.body).toEqual({
|
||||
model: "text-embedding-bge-m3",
|
||||
input: ["structured doc"],
|
||||
dimensions: 3,
|
||||
input_type: "document",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not make generic embedding providers memory auto-selection candidates", async () => {
|
||||
const server = await startEmbeddingServer();
|
||||
|
||||
await expect(
|
||||
createEmbeddingProvider(
|
||||
createMemoryEmbeddingOptions({
|
||||
provider: "auto",
|
||||
baseUrl: server.baseUrl,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("Unknown memory embedding provider: openai");
|
||||
expect(server.requests).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
105
extensions/memory-core/src/memory/hybrid.test.ts
Normal file
105
extensions/memory-core/src/memory/hybrid.test.ts
Normal file
@@ -0,0 +1,105 @@
|
||||
// Memory Core tests cover hybrid plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
|
||||
|
||||
describe("memory hybrid helpers", () => {
|
||||
it("buildFtsQuery tokenizes and AND-joins", () => {
|
||||
expect(buildFtsQuery("hello world")).toBe('"hello" AND "world"');
|
||||
expect(buildFtsQuery("FOO_bar baz-1")).toBe('"FOO_bar" AND "baz" AND "1"');
|
||||
expect(buildFtsQuery("金银价格")).toBe('"金银价格"');
|
||||
expect(buildFtsQuery("価格 2026年")).toBe('"価格" AND "2026年"');
|
||||
expect(buildFtsQuery(" ")).toBeNull();
|
||||
});
|
||||
|
||||
it("bm25RankToScore is monotonic and clamped", () => {
|
||||
expect(bm25RankToScore(0)).toBeCloseTo(1);
|
||||
expect(bm25RankToScore(1)).toBeCloseTo(0.5);
|
||||
expect(bm25RankToScore(10)).toBeLessThan(bm25RankToScore(1));
|
||||
expect(bm25RankToScore(-100)).toBeCloseTo(1, 1);
|
||||
});
|
||||
|
||||
it("bm25RankToScore preserves FTS5 BM25 relevance ordering", () => {
|
||||
const strongest = bm25RankToScore(-4.2);
|
||||
const middle = bm25RankToScore(-2.1);
|
||||
const weakest = bm25RankToScore(-0.5);
|
||||
|
||||
expect(strongest).toBeGreaterThan(middle);
|
||||
expect(middle).toBeGreaterThan(weakest);
|
||||
expect(strongest).not.toBe(middle);
|
||||
expect(middle).not.toBe(weakest);
|
||||
});
|
||||
|
||||
it("mergeHybridResults unions by id and combines weighted scores", async () => {
|
||||
const merged = await mergeHybridResults({
|
||||
vectorWeight: 0.7,
|
||||
textWeight: 0.3,
|
||||
vector: [
|
||||
{
|
||||
id: "a",
|
||||
path: "memory/a.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
source: "memory",
|
||||
snippet: "vec-a",
|
||||
vectorScore: 0.9,
|
||||
},
|
||||
],
|
||||
keyword: [
|
||||
{
|
||||
id: "b",
|
||||
path: "memory/b.md",
|
||||
startLine: 3,
|
||||
endLine: 4,
|
||||
source: "memory",
|
||||
snippet: "kw-b",
|
||||
textScore: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(merged).toHaveLength(2);
|
||||
const a = merged.find((r) => r.path === "memory/a.md");
|
||||
const b = merged.find((r) => r.path === "memory/b.md");
|
||||
expect(a?.score).toBeCloseTo(0.7 * 0.9);
|
||||
expect(a?.vectorScore).toBeCloseTo(0.9);
|
||||
expect(a?.textScore).toBe(0);
|
||||
expect(b?.score).toBeCloseTo(0.3 * 1);
|
||||
expect(b?.vectorScore).toBe(0);
|
||||
expect(b?.textScore).toBeCloseTo(1);
|
||||
});
|
||||
|
||||
it("mergeHybridResults prefers keyword snippet when ids overlap", async () => {
|
||||
const merged = await mergeHybridResults({
|
||||
vectorWeight: 0.5,
|
||||
textWeight: 0.5,
|
||||
vector: [
|
||||
{
|
||||
id: "a",
|
||||
path: "memory/a.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
source: "memory",
|
||||
snippet: "vec-a",
|
||||
vectorScore: 0.2,
|
||||
},
|
||||
],
|
||||
keyword: [
|
||||
{
|
||||
id: "a",
|
||||
path: "memory/a.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
source: "memory",
|
||||
snippet: "kw-a",
|
||||
textScore: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(merged).toHaveLength(1);
|
||||
expect(merged[0]?.snippet).toBe("kw-a");
|
||||
expect(merged[0]?.score).toBeCloseTo(0.5 * 0.2 + 0.5 * 1);
|
||||
expect(merged[0]?.vectorScore).toBeCloseTo(0.2);
|
||||
expect(merged[0]?.textScore).toBeCloseTo(1);
|
||||
});
|
||||
});
|
||||
156
extensions/memory-core/src/memory/hybrid.ts
Normal file
156
extensions/memory-core/src/memory/hybrid.ts
Normal file
@@ -0,0 +1,156 @@
|
||||
// Memory Core plugin module implements hybrid behavior.
|
||||
import { normalizeStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { applyMMRToHybridResults, type MMRConfig, DEFAULT_MMR_CONFIG } from "./mmr.js";
|
||||
import {
|
||||
applyTemporalDecayToHybridResults,
|
||||
type TemporalDecayConfig,
|
||||
DEFAULT_TEMPORAL_DECAY_CONFIG,
|
||||
} from "./temporal-decay.js";
|
||||
|
||||
type HybridSource = string;
|
||||
|
||||
type HybridVectorResult = {
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
source: HybridSource;
|
||||
snippet: string;
|
||||
vectorScore: number;
|
||||
};
|
||||
|
||||
type HybridKeywordResult = {
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
source: HybridSource;
|
||||
snippet: string;
|
||||
textScore: number;
|
||||
};
|
||||
|
||||
export function buildFtsQuery(raw: string): string | null {
|
||||
const tokens = normalizeStringEntries(raw.match(/[\p{L}\p{N}_]+/gu) ?? []);
|
||||
if (tokens.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
|
||||
return quoted.join(" AND ");
|
||||
}
|
||||
|
||||
export function bm25RankToScore(rank: number): number {
|
||||
if (!Number.isFinite(rank)) {
|
||||
return 1 / (1 + 999);
|
||||
}
|
||||
if (rank < 0) {
|
||||
const relevance = -rank;
|
||||
return relevance / (1 + relevance);
|
||||
}
|
||||
return 1 / (1 + rank);
|
||||
}
|
||||
|
||||
export async function mergeHybridResults(params: {
|
||||
vector: HybridVectorResult[];
|
||||
keyword: HybridKeywordResult[];
|
||||
vectorWeight: number;
|
||||
textWeight: number;
|
||||
workspaceDir?: string;
|
||||
/** MMR configuration for diversity-aware re-ranking */
|
||||
mmr?: Partial<MMRConfig>;
|
||||
/** Temporal decay configuration for recency-aware scoring */
|
||||
temporalDecay?: Partial<TemporalDecayConfig>;
|
||||
/** Test hook for deterministic time-dependent behavior */
|
||||
nowMs?: number;
|
||||
}): Promise<
|
||||
Array<{
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
vectorScore: number;
|
||||
textScore: number;
|
||||
snippet: string;
|
||||
source: HybridSource;
|
||||
}>
|
||||
> {
|
||||
const byId = new Map<
|
||||
string,
|
||||
{
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
source: HybridSource;
|
||||
snippet: string;
|
||||
vectorScore: number;
|
||||
textScore: number;
|
||||
}
|
||||
>();
|
||||
|
||||
for (const r of params.vector) {
|
||||
byId.set(r.id, {
|
||||
id: r.id,
|
||||
path: r.path,
|
||||
startLine: r.startLine,
|
||||
endLine: r.endLine,
|
||||
source: r.source,
|
||||
snippet: r.snippet,
|
||||
vectorScore: r.vectorScore,
|
||||
textScore: 0,
|
||||
});
|
||||
}
|
||||
|
||||
for (const r of params.keyword) {
|
||||
const existing = byId.get(r.id);
|
||||
if (existing) {
|
||||
existing.textScore = r.textScore;
|
||||
if (r.snippet && r.snippet.length > 0) {
|
||||
existing.snippet = r.snippet;
|
||||
}
|
||||
} else {
|
||||
byId.set(r.id, {
|
||||
id: r.id,
|
||||
path: r.path,
|
||||
startLine: r.startLine,
|
||||
endLine: r.endLine,
|
||||
source: r.source,
|
||||
snippet: r.snippet,
|
||||
vectorScore: 0,
|
||||
textScore: r.textScore,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const merged = Array.from(byId.values()).map((entry) => {
|
||||
const score = params.vectorWeight * entry.vectorScore + params.textWeight * entry.textScore;
|
||||
return {
|
||||
path: entry.path,
|
||||
startLine: entry.startLine,
|
||||
endLine: entry.endLine,
|
||||
score,
|
||||
vectorScore: entry.vectorScore,
|
||||
textScore: entry.textScore,
|
||||
snippet: entry.snippet,
|
||||
source: entry.source,
|
||||
};
|
||||
});
|
||||
|
||||
// Keep component scores as raw retrieval diagnostics; temporal decay and MMR
|
||||
// only adjust or reorder the combined ranking score.
|
||||
const temporalDecayConfig = { ...DEFAULT_TEMPORAL_DECAY_CONFIG, ...params.temporalDecay };
|
||||
const decayed = await applyTemporalDecayToHybridResults({
|
||||
results: merged,
|
||||
temporalDecay: temporalDecayConfig,
|
||||
workspaceDir: params.workspaceDir,
|
||||
nowMs: params.nowMs,
|
||||
});
|
||||
const sorted = decayed.toSorted((a, b) => b.score - a.score);
|
||||
|
||||
// Apply MMR re-ranking if enabled
|
||||
const mmrConfig = { ...DEFAULT_MMR_CONFIG, ...params.mmr };
|
||||
if (mmrConfig.enabled) {
|
||||
return applyMMRToHybridResults(sorted, mmrConfig);
|
||||
}
|
||||
|
||||
return sorted;
|
||||
}
|
||||
2315
extensions/memory-core/src/memory/index.test.ts
Normal file
2315
extensions/memory-core/src/memory/index.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
14
extensions/memory-core/src/memory/index.ts
Normal file
14
extensions/memory-core/src/memory/index.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
// Memory Core plugin entrypoint registers its OpenClaw integration.
|
||||
export { MemoryIndexManager } from "./manager.js";
|
||||
export type {
|
||||
MemoryEmbeddingProbeResult,
|
||||
MemorySearchManager,
|
||||
MemorySearchResult,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
export {
|
||||
closeAllMemorySearchManagers,
|
||||
closeMemorySearchManager,
|
||||
getMemorySearchManager,
|
||||
type MemorySearchManagerPurpose,
|
||||
type MemorySearchManagerResult,
|
||||
} from "./search-manager.js";
|
||||
33
extensions/memory-core/src/memory/manager-async-state.ts
Normal file
33
extensions/memory-core/src/memory/manager-async-state.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
// Memory Core plugin module implements manager async state behavior.
|
||||
export async function startAsyncSearchSync(params: {
|
||||
enabled: boolean;
|
||||
dirty: boolean;
|
||||
sessionsDirty: boolean;
|
||||
sync: (params: { reason: string }) => Promise<void>;
|
||||
onError: (err: unknown) => void;
|
||||
}): Promise<void> {
|
||||
if (!params.enabled || (!params.dirty && !params.sessionsDirty)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await params.sync({ reason: "search" });
|
||||
} catch (err: unknown) {
|
||||
params.onError(err);
|
||||
}
|
||||
}
|
||||
|
||||
export async function awaitPendingManagerWork(params: {
|
||||
pendingSync?: Promise<void> | null;
|
||||
pendingProviderInit?: Promise<void> | null;
|
||||
}): Promise<void> {
|
||||
if (params.pendingSync) {
|
||||
try {
|
||||
await params.pendingSync;
|
||||
} catch {}
|
||||
}
|
||||
if (params.pendingProviderInit) {
|
||||
try {
|
||||
await params.pendingProviderInit;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Memory Core tests cover manager batch state plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
MEMORY_BATCH_FAILURE_LIMIT,
|
||||
recordMemoryBatchFailure,
|
||||
resetMemoryBatchFailureState,
|
||||
} from "./manager-batch-state.js";
|
||||
|
||||
describe("memory batch state", () => {
|
||||
it("resets failures after recovery", () => {
|
||||
expect(
|
||||
resetMemoryBatchFailureState({
|
||||
enabled: true,
|
||||
count: 1,
|
||||
lastError: "batch failed",
|
||||
lastProvider: "openai",
|
||||
}),
|
||||
).toEqual({
|
||||
enabled: true,
|
||||
count: 0,
|
||||
lastError: undefined,
|
||||
lastProvider: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("disables batching after repeated failures", () => {
|
||||
const once = recordMemoryBatchFailure(
|
||||
{ enabled: true, count: 0 },
|
||||
{ provider: "openai", message: "batch failed", attempts: 1 },
|
||||
);
|
||||
expect(once).toEqual({
|
||||
enabled: true,
|
||||
count: 1,
|
||||
lastError: "batch failed",
|
||||
lastProvider: "openai",
|
||||
});
|
||||
|
||||
const twice = recordMemoryBatchFailure(once, {
|
||||
provider: "openai",
|
||||
message: "batch failed again",
|
||||
attempts: 1,
|
||||
});
|
||||
expect(twice).toEqual({
|
||||
enabled: false,
|
||||
count: MEMORY_BATCH_FAILURE_LIMIT,
|
||||
lastError: "batch failed again",
|
||||
lastProvider: "openai",
|
||||
});
|
||||
});
|
||||
|
||||
it("force-disables batching immediately", () => {
|
||||
expect(
|
||||
recordMemoryBatchFailure(
|
||||
{ enabled: true, count: 0 },
|
||||
{ provider: "gemini", message: "not available", forceDisable: true },
|
||||
),
|
||||
).toEqual({
|
||||
enabled: false,
|
||||
count: MEMORY_BATCH_FAILURE_LIMIT,
|
||||
lastError: "not available",
|
||||
lastProvider: "gemini",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves disabled state unchanged", () => {
|
||||
expect(
|
||||
recordMemoryBatchFailure(
|
||||
{ enabled: false, count: MEMORY_BATCH_FAILURE_LIMIT, lastError: "x", lastProvider: "y" },
|
||||
{ provider: "openai", message: "ignored" },
|
||||
),
|
||||
).toEqual({
|
||||
enabled: false,
|
||||
count: MEMORY_BATCH_FAILURE_LIMIT,
|
||||
lastError: "x",
|
||||
lastProvider: "y",
|
||||
});
|
||||
});
|
||||
});
|
||||
45
extensions/memory-core/src/memory/manager-batch-state.ts
Normal file
45
extensions/memory-core/src/memory/manager-batch-state.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
// Memory Core plugin module implements manager batch state behavior.
|
||||
export const MEMORY_BATCH_FAILURE_LIMIT = 2;
|
||||
|
||||
type MemoryBatchFailureState = {
|
||||
enabled: boolean;
|
||||
count: number;
|
||||
lastError?: string;
|
||||
lastProvider?: string;
|
||||
};
|
||||
|
||||
export function resetMemoryBatchFailureState(
|
||||
state: MemoryBatchFailureState,
|
||||
): MemoryBatchFailureState {
|
||||
return {
|
||||
...state,
|
||||
count: 0,
|
||||
lastError: undefined,
|
||||
lastProvider: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function recordMemoryBatchFailure(
|
||||
state: MemoryBatchFailureState,
|
||||
params: {
|
||||
provider: string;
|
||||
message: string;
|
||||
attempts?: number;
|
||||
forceDisable?: boolean;
|
||||
},
|
||||
): MemoryBatchFailureState {
|
||||
if (!state.enabled) {
|
||||
return state;
|
||||
}
|
||||
const increment = params.forceDisable
|
||||
? MEMORY_BATCH_FAILURE_LIMIT
|
||||
: Math.max(1, params.attempts ?? 1);
|
||||
const count = state.count + increment;
|
||||
const enabled = !(params.forceDisable || count >= MEMORY_BATCH_FAILURE_LIMIT);
|
||||
return {
|
||||
enabled,
|
||||
count,
|
||||
lastError: params.message,
|
||||
lastProvider: params.provider,
|
||||
};
|
||||
}
|
||||
172
extensions/memory-core/src/memory/manager-cache.test.ts
Normal file
172
extensions/memory-core/src/memory/manager-cache.test.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
// Memory Core tests cover manager cache plugin behavior.
|
||||
import {
|
||||
closeManagedCacheEntries,
|
||||
getOrCreateManagedCacheEntry,
|
||||
resolveSingletonManagedCache,
|
||||
type ManagedCache,
|
||||
} from "./manager-cache.js";
|
||||
|
||||
function createDeferred<T = void>(): {
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T | PromiseLike<T>) => void;
|
||||
reject: (reason?: unknown) => void;
|
||||
} {
|
||||
let resolve: ((value: T | PromiseLike<T>) => void) | undefined;
|
||||
let reject: ((reason?: unknown) => void) | undefined;
|
||||
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
if (!resolve || !reject) {
|
||||
throw new Error("Expected deferred callbacks to be initialized");
|
||||
}
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
type TestEntry = {
|
||||
id: string;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
function createTestCache(): ManagedCache<TestEntry> {
|
||||
return resolveSingletonManagedCache<TestEntry>(Symbol("openclaw.manager-cache.test"));
|
||||
}
|
||||
|
||||
function createEntry(id: string): TestEntry {
|
||||
return {
|
||||
id,
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
describe("manager cache", () => {
|
||||
const cachesForCleanup: ManagedCache<TestEntry>[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
cachesForCleanup.splice(0).map((cache) =>
|
||||
closeManagedCacheEntries({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
}),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it("repairs an invalid singleton cache shape", async () => {
|
||||
const cacheKey = Symbol("openclaw.manager-cache.corrupt-test");
|
||||
(globalThis as Record<PropertyKey, unknown>)[cacheKey] = {};
|
||||
|
||||
const cache = resolveSingletonManagedCache<TestEntry>(cacheKey);
|
||||
cachesForCleanup.push(cache);
|
||||
const entry = await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
create: async () => createEntry("repaired"),
|
||||
});
|
||||
|
||||
expect(entry.id).toBe("repaired");
|
||||
expect(cache.cache).toBeInstanceOf(Map);
|
||||
expect(cache.pending).toBeInstanceOf(Map);
|
||||
delete (globalThis as Record<PropertyKey, unknown>)[cacheKey];
|
||||
});
|
||||
|
||||
it("deduplicates concurrent creation for the same cache key", async () => {
|
||||
const cache = createTestCache();
|
||||
cachesForCleanup.push(cache);
|
||||
let createCalls = 0;
|
||||
|
||||
const results = await Promise.all(
|
||||
Array.from(
|
||||
{ length: 12 },
|
||||
async () =>
|
||||
await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
create: async () => {
|
||||
createCalls += 1;
|
||||
await Promise.resolve();
|
||||
return createEntry("shared");
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
expect(results).toHaveLength(12);
|
||||
expect(new Set(results).size).toBe(1);
|
||||
expect(createCalls).toBe(1);
|
||||
});
|
||||
|
||||
it("waits for pending creation before global teardown closes cached entries", async () => {
|
||||
const cache = createTestCache();
|
||||
const first = createEntry("first");
|
||||
const second = createEntry("second");
|
||||
cachesForCleanup.push(cache);
|
||||
const gate = createDeferred();
|
||||
|
||||
const pendingFirst = getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
create: async () => {
|
||||
await gate.promise;
|
||||
return first;
|
||||
},
|
||||
});
|
||||
|
||||
const teardown = closeManagedCacheEntries({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
});
|
||||
gate.resolve();
|
||||
|
||||
await teardown;
|
||||
expect(first.close).toHaveBeenCalledTimes(1);
|
||||
|
||||
const resolvedFirst = await pendingFirst;
|
||||
const resolvedSecond = await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
create: async () => second,
|
||||
});
|
||||
|
||||
expect(resolvedFirst).toBe(first);
|
||||
expect(resolvedSecond).toBe(second);
|
||||
expect(resolvedSecond).not.toBe(resolvedFirst);
|
||||
});
|
||||
|
||||
it("bypasses identity caching for status-only callers", async () => {
|
||||
const cache = createTestCache();
|
||||
cachesForCleanup.push(cache);
|
||||
let createCalls = 0;
|
||||
|
||||
const first = await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
bypassCache: true,
|
||||
create: async () => {
|
||||
createCalls += 1;
|
||||
return createEntry(`status-${createCalls}`);
|
||||
},
|
||||
});
|
||||
const second = await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
bypassCache: true,
|
||||
create: async () => {
|
||||
createCalls += 1;
|
||||
return createEntry(`status-${createCalls}`);
|
||||
},
|
||||
});
|
||||
|
||||
expect(first).not.toBe(second);
|
||||
expect(createCalls).toBe(2);
|
||||
expect(cache.cache.size).toBe(0);
|
||||
});
|
||||
});
|
||||
92
extensions/memory-core/src/memory/manager-cache.ts
Normal file
92
extensions/memory-core/src/memory/manager-cache.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
// Memory Core plugin module implements manager cache behavior.
|
||||
import { resolveGlobalSingleton } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
|
||||
type Closable = {
|
||||
close?: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
export type ManagedCache<T> = {
|
||||
cache: Map<string, T>;
|
||||
pending: Map<string, Promise<T>>;
|
||||
};
|
||||
|
||||
export function resolveSingletonManagedCache<T>(cacheKey: symbol): ManagedCache<T> {
|
||||
const resolved = resolveGlobalSingleton<unknown>(cacheKey, () => ({
|
||||
cache: new Map<string, T>(),
|
||||
pending: new Map<string, Promise<T>>(),
|
||||
}));
|
||||
if (
|
||||
typeof resolved === "object" &&
|
||||
resolved !== null &&
|
||||
(resolved as Partial<ManagedCache<T>>).cache instanceof Map &&
|
||||
(resolved as Partial<ManagedCache<T>>).pending instanceof Map
|
||||
) {
|
||||
return resolved as ManagedCache<T>;
|
||||
}
|
||||
const repaired: ManagedCache<T> = {
|
||||
cache: new Map<string, T>(),
|
||||
pending: new Map<string, Promise<T>>(),
|
||||
};
|
||||
(globalThis as Record<PropertyKey, unknown>)[cacheKey] = repaired;
|
||||
return repaired;
|
||||
}
|
||||
|
||||
export async function getOrCreateManagedCacheEntry<T>(params: {
|
||||
cache: Map<string, T>;
|
||||
pending: Map<string, Promise<T>>;
|
||||
key: string;
|
||||
bypassCache?: boolean;
|
||||
create: () => Promise<T> | T;
|
||||
}): Promise<T> {
|
||||
if (params.bypassCache) {
|
||||
return await params.create();
|
||||
}
|
||||
const existing = params.cache.get(params.key);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const pending = params.pending.get(params.key);
|
||||
if (pending) {
|
||||
return pending;
|
||||
}
|
||||
const createPromise = (async () => {
|
||||
const refreshed = params.cache.get(params.key);
|
||||
if (refreshed) {
|
||||
return refreshed;
|
||||
}
|
||||
const entry = await params.create();
|
||||
params.cache.set(params.key, entry);
|
||||
return entry;
|
||||
})();
|
||||
params.pending.set(params.key, createPromise);
|
||||
try {
|
||||
return await createPromise;
|
||||
} finally {
|
||||
if (params.pending.get(params.key) === createPromise) {
|
||||
params.pending.delete(params.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function closeManagedCacheEntries<T extends Closable>(params: {
|
||||
cache: Map<string, T>;
|
||||
pending: Map<string, Promise<T>>;
|
||||
onCloseError?: (err: unknown) => void;
|
||||
}): Promise<void> {
|
||||
const pending = Array.from(params.pending.values());
|
||||
if (pending.length > 0) {
|
||||
await Promise.allSettled(pending);
|
||||
}
|
||||
const entries = Array.from(params.cache.values());
|
||||
params.cache.clear();
|
||||
for (const entry of entries) {
|
||||
if (typeof entry.close !== "function") {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await entry.close();
|
||||
} catch (err) {
|
||||
params.onCloseError?.(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
237
extensions/memory-core/src/memory/manager-db.test.ts
Normal file
237
extensions/memory-core/src/memory/manager-db.test.ts
Normal file
@@ -0,0 +1,237 @@
|
||||
// Memory Core tests cover shared agent database publication and shadow cleanup.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
ensureMemoryIndexSchema,
|
||||
loadSqliteVecExtension,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
cleanupAgedMemoryReindexTempFiles,
|
||||
publishMemoryDatabaseTables,
|
||||
readMemoryDatabaseRevision,
|
||||
} from "./manager-db.js";
|
||||
import { acquireMemoryReindexLock } from "./manager-reindex-lock.js";
|
||||
|
||||
function ensureTestMemorySchema(db: DatabaseSync, cacheEnabled = true): void {
|
||||
ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled,
|
||||
ftsEnabled: false,
|
||||
});
|
||||
}
|
||||
|
||||
async function expectPathMissing(targetPath: string): Promise<void> {
|
||||
await expect(fs.access(targetPath)).rejects.toThrow("ENOENT");
|
||||
}
|
||||
|
||||
describe("memory manager database publication", () => {
|
||||
let fixtureRoot = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-memory-db-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("removes a stale vector table when the shadow index has no vectors", async () => {
|
||||
const targetPath = path.join(fixtureRoot, "target.sqlite");
|
||||
const sourcePath = path.join(fixtureRoot, "source.sqlite");
|
||||
const targetDb = new DatabaseSync(targetPath);
|
||||
const sourceDb = new DatabaseSync(sourcePath);
|
||||
try {
|
||||
ensureTestMemorySchema(targetDb);
|
||||
ensureTestMemorySchema(sourceDb);
|
||||
targetDb.exec("CREATE TABLE memory_index_chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)");
|
||||
targetDb
|
||||
.prepare("INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)")
|
||||
.run("stale", "[]");
|
||||
sourceDb.close();
|
||||
|
||||
await publishMemoryDatabaseTables({
|
||||
targetDb,
|
||||
sourcePath,
|
||||
metaKey: "memory_index_meta",
|
||||
expectedRevision: readMemoryDatabaseRevision(targetDb),
|
||||
});
|
||||
|
||||
expect(
|
||||
targetDb
|
||||
.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'memory_index_chunks_vec'",
|
||||
)
|
||||
.get(),
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
try {
|
||||
sourceDb.close();
|
||||
} catch {}
|
||||
targetDb.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("loads sqlite-vec on the target before publishing a shadow vector table", async () => {
|
||||
const targetPath = path.join(fixtureRoot, "target.sqlite");
|
||||
const sourcePath = path.join(fixtureRoot, "source.sqlite");
|
||||
const targetDb = new DatabaseSync(targetPath, { allowExtension: true });
|
||||
const sourceDb = new DatabaseSync(sourcePath, { allowExtension: true });
|
||||
try {
|
||||
ensureTestMemorySchema(targetDb);
|
||||
ensureTestMemorySchema(sourceDb);
|
||||
const sourceVector = await loadSqliteVecExtension({ db: sourceDb });
|
||||
if (!sourceVector.ok) {
|
||||
return;
|
||||
}
|
||||
sourceDb.exec(`
|
||||
CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0(
|
||||
id TEXT PRIMARY KEY,
|
||||
embedding FLOAT[3]
|
||||
)
|
||||
`);
|
||||
sourceDb
|
||||
.prepare("INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)")
|
||||
.run("vector", JSON.stringify([0, 1, 0]));
|
||||
sourceDb.close();
|
||||
|
||||
await publishMemoryDatabaseTables({
|
||||
targetDb,
|
||||
sourcePath,
|
||||
metaKey: "memory_index_meta",
|
||||
expectedRevision: readMemoryDatabaseRevision(targetDb),
|
||||
vectorExtensionPath: sourceVector.extensionPath,
|
||||
});
|
||||
|
||||
expect(targetDb.prepare("SELECT id FROM memory_index_chunks_vec").all()).toEqual([
|
||||
{ id: "vector" },
|
||||
]);
|
||||
} finally {
|
||||
try {
|
||||
sourceDb.close();
|
||||
} catch {}
|
||||
targetDb.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a stale shadow publish after a concurrent live memory update", async () => {
|
||||
const targetPath = path.join(fixtureRoot, "target.sqlite");
|
||||
const sourcePath = path.join(fixtureRoot, "source.sqlite");
|
||||
const targetDb = new DatabaseSync(targetPath);
|
||||
const sourceDb = new DatabaseSync(sourcePath);
|
||||
let concurrentDb: DatabaseSync | undefined;
|
||||
try {
|
||||
ensureTestMemorySchema(targetDb);
|
||||
ensureTestMemorySchema(sourceDb);
|
||||
targetDb
|
||||
.prepare(
|
||||
"INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)",
|
||||
)
|
||||
.run("memory.md", "memory", "published", 1, 1);
|
||||
sourceDb
|
||||
.prepare(
|
||||
"INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)",
|
||||
)
|
||||
.run("memory.md", "memory", "shadow", 1, 1);
|
||||
const expectedRevision = readMemoryDatabaseRevision(targetDb);
|
||||
sourceDb.close();
|
||||
|
||||
concurrentDb = new DatabaseSync(targetPath);
|
||||
concurrentDb
|
||||
.prepare("UPDATE memory_index_sources SET hash = ? WHERE path = ? AND source = ?")
|
||||
.run("newer", "memory.md", "memory");
|
||||
concurrentDb.close();
|
||||
concurrentDb = undefined;
|
||||
|
||||
await expect(
|
||||
publishMemoryDatabaseTables({
|
||||
targetDb,
|
||||
sourcePath,
|
||||
metaKey: "memory_index_meta",
|
||||
expectedRevision,
|
||||
}),
|
||||
).rejects.toThrow(/changed while full reindex was building/);
|
||||
expect(
|
||||
targetDb
|
||||
.prepare("SELECT hash FROM memory_index_sources WHERE path = ? AND source = ?")
|
||||
.get("memory.md", "memory"),
|
||||
).toEqual({ hash: "newer" });
|
||||
} finally {
|
||||
try {
|
||||
concurrentDb?.close();
|
||||
} catch {}
|
||||
try {
|
||||
sourceDb.close();
|
||||
} catch {}
|
||||
targetDb.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves the live embedding cache when the shadow index has caching disabled", async () => {
|
||||
const targetPath = path.join(fixtureRoot, "target.sqlite");
|
||||
const sourcePath = path.join(fixtureRoot, "source.sqlite");
|
||||
const targetDb = new DatabaseSync(targetPath);
|
||||
const sourceDb = new DatabaseSync(sourcePath);
|
||||
try {
|
||||
ensureTestMemorySchema(targetDb);
|
||||
ensureTestMemorySchema(sourceDb, false);
|
||||
targetDb
|
||||
.prepare(
|
||||
`INSERT INTO memory_embedding_cache (
|
||||
provider, model, provider_key, hash, embedding, dims, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run("test", "model", "key", "hash", "[]", 0, 1);
|
||||
sourceDb.close();
|
||||
|
||||
await publishMemoryDatabaseTables({
|
||||
targetDb,
|
||||
sourcePath,
|
||||
metaKey: "memory_index_meta",
|
||||
expectedRevision: readMemoryDatabaseRevision(targetDb),
|
||||
});
|
||||
|
||||
expect(targetDb.prepare("SELECT hash FROM memory_embedding_cache").all()).toEqual([
|
||||
{ hash: "hash" },
|
||||
]);
|
||||
} finally {
|
||||
try {
|
||||
sourceDb.close();
|
||||
} catch {}
|
||||
targetDb.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("removes aged orphan shadows but preserves young and locked shadows", async () => {
|
||||
const databasePath = path.join(fixtureRoot, "agent.sqlite");
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.close();
|
||||
const oldShadow = `${databasePath}.memory-reindex-11111111-2222-3333-4444-555555555555`;
|
||||
const youngShadow = `${databasePath}.memory-reindex-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee`;
|
||||
const lockedShadow = `${databasePath}.memory-reindex-99999999-aaaa-bbbb-cccc-dddddddddddd`;
|
||||
const old = new Date(Date.now() - 48 * 60 * 60_000);
|
||||
|
||||
for (const suffix of ["", "-wal", "-journal"]) {
|
||||
await fs.writeFile(`${oldShadow}${suffix}`, "orphan");
|
||||
await fs.utimes(`${oldShadow}${suffix}`, old, old);
|
||||
}
|
||||
await fs.writeFile(youngShadow, "active");
|
||||
await fs.writeFile(lockedShadow, "locked");
|
||||
await fs.utimes(lockedShadow, old, old);
|
||||
|
||||
const lock = acquireMemoryReindexLock(databasePath);
|
||||
cleanupAgedMemoryReindexTempFiles(databasePath);
|
||||
await expect(fs.access(lockedShadow)).resolves.toBeUndefined();
|
||||
lock.release();
|
||||
|
||||
cleanupAgedMemoryReindexTempFiles(databasePath);
|
||||
|
||||
await expectPathMissing(oldShadow);
|
||||
await expectPathMissing(`${oldShadow}-wal`);
|
||||
await expectPathMissing(`${oldShadow}-journal`);
|
||||
await expectPathMissing(lockedShadow);
|
||||
await expect(fs.access(youngShadow)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
317
extensions/memory-core/src/memory/manager-db.ts
Normal file
317
extensions/memory-core/src/memory/manager-db.ts
Normal file
@@ -0,0 +1,317 @@
|
||||
// Memory Core plugin module implements manager db behavior.
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
closeMemorySqliteWalMaintenance,
|
||||
configureMemorySqliteWalMaintenance,
|
||||
ensureDir,
|
||||
loadSqliteVecExtension,
|
||||
requireNodeSqlite,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import {
|
||||
ensureOpenClawAgentDatabaseSchema,
|
||||
runSqliteImmediateTransactionSync,
|
||||
} from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import {
|
||||
tryAcquireMemoryReindexLock,
|
||||
type MemoryReindexLockHandle,
|
||||
} from "./manager-reindex-lock.js";
|
||||
|
||||
const MEMORY_REINDEX_SCHEMA = "memory_reindex";
|
||||
const MEMORY_INDEX_STATE_ID = 1;
|
||||
const MEMORY_DATABASE_FILE_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const;
|
||||
const MEMORY_REINDEX_ENTRY_SUFFIXES = ["-wal", "-shm", "-journal", ""] as const;
|
||||
const MEMORY_REINDEX_UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const MEMORY_REINDEX_ORPHAN_MIN_AGE_MS = 24 * 60 * 60_000;
|
||||
|
||||
function resolveMemoryReindexBaseName(
|
||||
databaseBaseName: string,
|
||||
entryName: string,
|
||||
): string | undefined {
|
||||
for (const suffix of MEMORY_REINDEX_ENTRY_SUFFIXES) {
|
||||
if (!entryName.endsWith(suffix)) {
|
||||
continue;
|
||||
}
|
||||
const baseName = entryName.slice(0, entryName.length - suffix.length);
|
||||
const prefix = `${databaseBaseName}.memory-reindex-`;
|
||||
if (
|
||||
baseName.startsWith(prefix) &&
|
||||
MEMORY_REINDEX_UUID_PATTERN.test(baseName.slice(prefix.length))
|
||||
) {
|
||||
return baseName;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isRegularFile(filePath: string): boolean {
|
||||
try {
|
||||
return fs.statSync(filePath).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function tableExists(db: DatabaseSync, schema: string, tableName: string): boolean {
|
||||
const row = db
|
||||
.prepare(`SELECT 1 AS ok FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
|
||||
.get(tableName) as { ok?: unknown } | undefined;
|
||||
return row?.ok === 1;
|
||||
}
|
||||
|
||||
function readTableSql(db: DatabaseSync, schema: string, tableName: string): string | null {
|
||||
const row = db
|
||||
.prepare(`SELECT sql FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`)
|
||||
.get(tableName) as { sql?: unknown } | undefined;
|
||||
return typeof row?.sql === "string" && row.sql.trim() ? row.sql : null;
|
||||
}
|
||||
|
||||
function hasSqliteVecExtension(db: DatabaseSync): boolean {
|
||||
try {
|
||||
const row = db.prepare("SELECT vec_version() AS version").get() as
|
||||
| { version?: unknown }
|
||||
| undefined;
|
||||
return typeof row?.version === "string" && row.version.trim().length > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function readMemoryDatabaseRevision(db: DatabaseSync): number {
|
||||
const row = db
|
||||
.prepare("SELECT revision FROM memory_index_state WHERE id = ?")
|
||||
.get(MEMORY_INDEX_STATE_ID) as { revision?: unknown } | undefined;
|
||||
if (typeof row?.revision !== "number" || !Number.isSafeInteger(row.revision)) {
|
||||
throw new Error("Memory index revision is missing or invalid");
|
||||
}
|
||||
return row.revision;
|
||||
}
|
||||
|
||||
function replaceVirtualTable(params: {
|
||||
db: DatabaseSync;
|
||||
tableName: "memory_index_chunks_fts" | "memory_index_chunks_vec";
|
||||
columns: string;
|
||||
ignoreDropErrorWhenSourceMissing?: boolean;
|
||||
}): void {
|
||||
const { db, tableName, columns } = params;
|
||||
const createSql = readTableSql(db, MEMORY_REINDEX_SCHEMA, tableName);
|
||||
if (!createSql) {
|
||||
try {
|
||||
db.exec(`DROP TABLE IF EXISTS main.${tableName}`);
|
||||
} catch (err) {
|
||||
if (!params.ignoreDropErrorWhenSourceMissing) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
db.exec(`DROP TABLE IF EXISTS main.${tableName}`);
|
||||
db.exec(createSql);
|
||||
db.exec(
|
||||
`INSERT INTO main.${tableName} (${columns}) ` +
|
||||
`SELECT ${columns} FROM ${MEMORY_REINDEX_SCHEMA}.${tableName}`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Publish a completed shadow memory index without replacing the shared agent database file. */
|
||||
export async function publishMemoryDatabaseTables(params: {
|
||||
targetDb: DatabaseSync;
|
||||
sourcePath: string;
|
||||
metaKey: string;
|
||||
expectedRevision: number;
|
||||
vectorExtensionPath?: string;
|
||||
}): Promise<void> {
|
||||
params.targetDb.prepare(`ATTACH DATABASE ? AS ${MEMORY_REINDEX_SCHEMA}`).run(params.sourcePath);
|
||||
try {
|
||||
if (
|
||||
tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "memory_index_chunks_vec") &&
|
||||
!hasSqliteVecExtension(params.targetDb)
|
||||
) {
|
||||
const loaded = await loadSqliteVecExtension({
|
||||
db: params.targetDb,
|
||||
extensionPath: params.vectorExtensionPath,
|
||||
});
|
||||
if (!loaded.ok) {
|
||||
throw new Error(
|
||||
`Failed to load sqlite-vec before publishing the full memory reindex: ` +
|
||||
(loaded.error ?? "unknown sqlite-vec load error"),
|
||||
);
|
||||
}
|
||||
}
|
||||
runSqliteImmediateTransactionSync(params.targetDb, () => {
|
||||
const liveRevision = readMemoryDatabaseRevision(params.targetDb);
|
||||
if (liveRevision !== params.expectedRevision) {
|
||||
throw new Error(
|
||||
`Memory index changed while full reindex was building ` +
|
||||
`(expected revision ${params.expectedRevision}, found ${liveRevision}); retry the full reindex.`,
|
||||
);
|
||||
}
|
||||
params.targetDb
|
||||
.prepare("DELETE FROM main.memory_index_meta WHERE key = ?")
|
||||
.run(params.metaKey);
|
||||
params.targetDb
|
||||
.prepare(
|
||||
`INSERT INTO main.memory_index_meta (key, value)
|
||||
SELECT key, value FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_meta WHERE key = ?`,
|
||||
)
|
||||
.run(params.metaKey);
|
||||
|
||||
params.targetDb.exec(`
|
||||
DELETE FROM main.memory_index_sources;
|
||||
INSERT INTO main.memory_index_sources (path, source, hash, mtime, size)
|
||||
SELECT path, source, hash, mtime, size FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_sources;
|
||||
|
||||
DELETE FROM main.memory_index_chunks;
|
||||
INSERT INTO main.memory_index_chunks (
|
||||
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
)
|
||||
SELECT
|
||||
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_chunks;
|
||||
`);
|
||||
|
||||
if (tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "memory_embedding_cache")) {
|
||||
params.targetDb.exec(`
|
||||
DELETE FROM main.memory_embedding_cache;
|
||||
INSERT INTO main.memory_embedding_cache (
|
||||
provider, model, provider_key, hash, embedding, dims, updated_at
|
||||
)
|
||||
SELECT provider, model, provider_key, hash, embedding, dims, updated_at
|
||||
FROM ${MEMORY_REINDEX_SCHEMA}.memory_embedding_cache;
|
||||
`);
|
||||
}
|
||||
|
||||
replaceVirtualTable({
|
||||
db: params.targetDb,
|
||||
tableName: "memory_index_chunks_fts",
|
||||
columns: "text, id, path, source, model, start_line, end_line",
|
||||
});
|
||||
replaceVirtualTable({
|
||||
db: params.targetDb,
|
||||
tableName: "memory_index_chunks_vec",
|
||||
columns: "id, embedding",
|
||||
// A vector-disabled connection may not have sqlite-vec loaded and cannot
|
||||
// drop an old virtual table. Missing vector metadata forces a strict
|
||||
// rebuild before that table can be queried again.
|
||||
ignoreDropErrorWhenSourceMissing: true,
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
params.targetDb.exec(`DETACH DATABASE ${MEMORY_REINDEX_SCHEMA}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove one closed shadow memory database and its journal-mode sidecars. */
|
||||
export function removeMemoryDatabaseFiles(dbPath: string): void {
|
||||
for (const suffix of MEMORY_DATABASE_FILE_SUFFIXES) {
|
||||
fs.rmSync(`${dbPath}${suffix}`, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove crash-left shadow databases only when no full reindex is active. */
|
||||
export function cleanupAgedMemoryReindexTempFiles(dbPath: string, nowMs = Date.now()): void {
|
||||
if (!isRegularFile(dbPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
let reindexLock: MemoryReindexLockHandle | undefined;
|
||||
try {
|
||||
reindexLock = tryAcquireMemoryReindexLock(dbPath);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!reindexLock) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const dir = path.dirname(dbPath);
|
||||
const databaseBaseName = path.basename(dbPath);
|
||||
const shadowBaseNames = new Set<string>();
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
const shadowBaseName = resolveMemoryReindexBaseName(databaseBaseName, entry.name);
|
||||
if (shadowBaseName) {
|
||||
shadowBaseNames.add(shadowBaseName);
|
||||
}
|
||||
}
|
||||
|
||||
for (const shadowBaseName of shadowBaseNames) {
|
||||
const filePaths = MEMORY_DATABASE_FILE_SUFFIXES.map((suffix) =>
|
||||
path.join(dir, `${shadowBaseName}${suffix}`),
|
||||
);
|
||||
const stats: fs.Stats[] = [];
|
||||
let hasUnknownFileState = false;
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
stats.push(fs.statSync(filePath));
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
hasUnknownFileState = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hasUnknownFileState || stats.length === 0) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
nowMs - Math.max(...stats.map((stat) => stat.mtimeMs)) <
|
||||
MEMORY_REINDEX_ORPHAN_MIN_AGE_MS
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
fs.rmSync(filePath, { force: true });
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
reindexLock.release();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
export function openMemoryDatabaseAtPath(
|
||||
dbPath: string,
|
||||
allowExtension: boolean,
|
||||
agentId?: string,
|
||||
): DatabaseSync {
|
||||
ensureDir(path.dirname(dbPath));
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const db = new DatabaseSync(dbPath, { allowExtension });
|
||||
try {
|
||||
configureMemorySqliteWalMaintenance(db, {
|
||||
busyTimeoutMs: 5000,
|
||||
databasePath: dbPath,
|
||||
});
|
||||
if (agentId) {
|
||||
ensureOpenClawAgentDatabaseSchema(db, { agentId, path: dbPath, register: true });
|
||||
}
|
||||
return db;
|
||||
} catch (err) {
|
||||
try {
|
||||
closeMemorySqliteWalMaintenance(db);
|
||||
db.close();
|
||||
} catch {}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export function closeMemoryDatabase(db: DatabaseSync): void {
|
||||
closeMemorySqliteWalMaintenance(db);
|
||||
db.close();
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// Memory Core tests cover manager embedding cache plugin behavior.
|
||||
import {
|
||||
ensureMemoryIndexSchema,
|
||||
requireNodeSqlite,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
collectMemoryCachedEmbeddings,
|
||||
loadMemoryEmbeddingCache,
|
||||
upsertMemoryEmbeddingCache,
|
||||
} from "./manager-embedding-cache.js";
|
||||
|
||||
describe("memory embedding cache", () => {
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
|
||||
function createDb() {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled: true,
|
||||
ftsEnabled: false,
|
||||
ftsTokenizer: "unicode61",
|
||||
});
|
||||
return db;
|
||||
}
|
||||
|
||||
it("loads cached embeddings for the active provider key", () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
upsertMemoryEmbeddingCache({
|
||||
db,
|
||||
enabled: true,
|
||||
provider: { id: "openai", model: "text-embedding-3-small" },
|
||||
providerKey: "provider-key",
|
||||
entries: [
|
||||
{ hash: "a", embedding: [0.1, 0.2] },
|
||||
{ hash: "b", embedding: [0.3, 0.4] },
|
||||
],
|
||||
now: 123,
|
||||
});
|
||||
|
||||
const cached = loadMemoryEmbeddingCache({
|
||||
db,
|
||||
enabled: true,
|
||||
providerIdentities: [
|
||||
{
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
providerKey: "provider-key",
|
||||
},
|
||||
],
|
||||
hashes: ["a", "b", "a"],
|
||||
});
|
||||
|
||||
expect(cached).toEqual(
|
||||
new Map([
|
||||
["a", [0.1, 0.2]],
|
||||
["b", [0.3, 0.4]],
|
||||
]),
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("loads provider-declared alias cache rows without accepting arbitrary identities", () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
upsertMemoryEmbeddingCache({
|
||||
db,
|
||||
enabled: true,
|
||||
provider: { id: "local", model: "/cache/default.gguf" },
|
||||
providerKey: "provider-key-alias",
|
||||
entries: [{ hash: "alias", embedding: [0.1, 0.2] }],
|
||||
});
|
||||
upsertMemoryEmbeddingCache({
|
||||
db,
|
||||
enabled: true,
|
||||
provider: { id: "local", model: "/other/default.gguf" },
|
||||
providerKey: "provider-key-arbitrary",
|
||||
entries: [{ hash: "arbitrary", embedding: [0.3, 0.4] }],
|
||||
});
|
||||
|
||||
const cached = loadMemoryEmbeddingCache({
|
||||
db,
|
||||
enabled: true,
|
||||
providerIdentities: [
|
||||
{
|
||||
provider: "local",
|
||||
model: "hf:owner/default.gguf",
|
||||
providerKey: "provider-key-current",
|
||||
},
|
||||
{
|
||||
provider: "local",
|
||||
model: "/cache/default.gguf",
|
||||
providerKey: "provider-key-alias",
|
||||
},
|
||||
],
|
||||
hashes: ["alias", "arbitrary"],
|
||||
});
|
||||
|
||||
expect(cached).toEqual(new Map([["alias", [0.1, 0.2]]]));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses cached embeddings on forced reindex instead of scheduling new embeds", () => {
|
||||
const cached = new Map<string, number[]>([
|
||||
["alpha", [0.1, 0.2]],
|
||||
["beta", [0.3, 0.4]],
|
||||
]);
|
||||
const embedMissing = vi.fn();
|
||||
|
||||
const plan = collectMemoryCachedEmbeddings({
|
||||
chunks: [{ hash: "alpha" }, { hash: "beta" }],
|
||||
cached,
|
||||
});
|
||||
|
||||
if (plan.missing.length > 0) {
|
||||
embedMissing(plan.missing);
|
||||
}
|
||||
|
||||
expect(plan.embeddings).toEqual([
|
||||
[0.1, 0.2],
|
||||
[0.3, 0.4],
|
||||
]);
|
||||
expect(plan.missing).toHaveLength(0);
|
||||
expect(embedMissing).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
121
extensions/memory-core/src/memory/manager-embedding-cache.ts
Normal file
121
extensions/memory-core/src/memory/manager-embedding-cache.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
// Memory Core plugin module implements manager embedding cache behavior.
|
||||
import type { DatabaseSync, SQLInputValue } from "node:sqlite";
|
||||
import {
|
||||
parseEmbedding,
|
||||
type MemoryChunk,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
type EmbeddingCacheDb = Pick<DatabaseSync, "prepare">;
|
||||
|
||||
type EmbeddingProviderIdentity = {
|
||||
provider: string;
|
||||
model: string;
|
||||
providerKey: string;
|
||||
};
|
||||
|
||||
export function loadMemoryEmbeddingCache(params: {
|
||||
db: EmbeddingCacheDb;
|
||||
enabled: boolean;
|
||||
providerIdentities: EmbeddingProviderIdentity[];
|
||||
hashes: string[];
|
||||
tableName?: string;
|
||||
}): Map<string, number[]> {
|
||||
if (!params.enabled || params.providerIdentities.length === 0 || params.hashes.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
const unique: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const hash of params.hashes) {
|
||||
if (!hash || seen.has(hash)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(hash);
|
||||
unique.push(hash);
|
||||
}
|
||||
if (unique.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
|
||||
const tableName = params.tableName ?? "memory_embedding_cache";
|
||||
const out = new Map<string, number[]>();
|
||||
const batchSize = 400;
|
||||
for (const identity of params.providerIdentities) {
|
||||
const baseParams: SQLInputValue[] = [identity.provider, identity.model, identity.providerKey];
|
||||
for (let start = 0; start < unique.length; start += batchSize) {
|
||||
const batch = unique.slice(start, start + batchSize);
|
||||
const placeholders = batch.map(() => "?").join(", ");
|
||||
const rows = params.db
|
||||
.prepare(
|
||||
`SELECT hash, embedding FROM ${tableName}\n` +
|
||||
` WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${placeholders})`,
|
||||
)
|
||||
.all(...baseParams, ...batch) as Array<{ hash: string; embedding: string }>;
|
||||
for (const row of rows) {
|
||||
if (!out.has(row.hash)) {
|
||||
out.set(row.hash, parseEmbedding(row.embedding));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function upsertMemoryEmbeddingCache(params: {
|
||||
db: EmbeddingCacheDb;
|
||||
enabled: boolean;
|
||||
provider: { id: string; model: string } | null;
|
||||
providerKey: string | null;
|
||||
entries: Array<{ hash: string; embedding: number[] }>;
|
||||
now?: number;
|
||||
tableName?: string;
|
||||
}): void {
|
||||
const provider = params.provider;
|
||||
if (!params.enabled || !provider || !params.providerKey || params.entries.length === 0) {
|
||||
return;
|
||||
}
|
||||
const tableName = params.tableName ?? "memory_embedding_cache";
|
||||
const now = params.now ?? Date.now();
|
||||
const stmt = params.db.prepare(
|
||||
`INSERT INTO ${tableName} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` +
|
||||
` VALUES (?, ?, ?, ?, ?, ?, ?)\n` +
|
||||
` ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET\n` +
|
||||
` embedding=excluded.embedding,\n` +
|
||||
` dims=excluded.dims,\n` +
|
||||
` updated_at=excluded.updated_at`,
|
||||
);
|
||||
for (const entry of params.entries) {
|
||||
const embedding = entry.embedding ?? [];
|
||||
stmt.run(
|
||||
provider.id,
|
||||
provider.model,
|
||||
params.providerKey,
|
||||
entry.hash,
|
||||
JSON.stringify(embedding),
|
||||
embedding.length,
|
||||
now,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function collectMemoryCachedEmbeddings<T extends Pick<MemoryChunk, "hash">>(params: {
|
||||
chunks: T[];
|
||||
cached: Map<string, number[]>;
|
||||
}): {
|
||||
embeddings: number[][];
|
||||
missing: Array<{ index: number; chunk: T }>;
|
||||
} {
|
||||
const embeddings: number[][] = Array.from({ length: params.chunks.length }, () => []);
|
||||
const missing: Array<{ index: number; chunk: T }> = [];
|
||||
|
||||
for (let index = 0; index < params.chunks.length; index += 1) {
|
||||
const chunk = params.chunks[index];
|
||||
const hit = chunk?.hash ? params.cached.get(chunk.hash) : undefined;
|
||||
if (hit && hit.length > 0) {
|
||||
embeddings[index] = hit;
|
||||
} else if (chunk) {
|
||||
missing.push({ index, chunk });
|
||||
}
|
||||
}
|
||||
|
||||
return { embeddings, missing };
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// Memory Core plugin module implements manager embedding errors behavior.
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
export const MEMORY_EMBEDDING_OPERATION_ERROR_CODE = "MEMORY_EMBEDDING_OPERATION_FAILED";
|
||||
|
||||
export type MemoryEmbeddingOperationKind = "query" | "batch" | "structured-batch";
|
||||
|
||||
export type MemoryEmbeddingOperationError = Error & {
|
||||
code: typeof MEMORY_EMBEDDING_OPERATION_ERROR_CODE;
|
||||
operation: MemoryEmbeddingOperationKind;
|
||||
providerId?: string;
|
||||
cause?: unknown;
|
||||
};
|
||||
|
||||
export function createMemoryEmbeddingOperationError(params: {
|
||||
operation: MemoryEmbeddingOperationKind;
|
||||
providerId?: string;
|
||||
cause: unknown;
|
||||
}): MemoryEmbeddingOperationError {
|
||||
const message = formatErrorMessage(params.cause);
|
||||
const error = new Error(message) as MemoryEmbeddingOperationError;
|
||||
error.code = MEMORY_EMBEDDING_OPERATION_ERROR_CODE;
|
||||
error.operation = params.operation;
|
||||
if (params.providerId) {
|
||||
error.providerId = params.providerId;
|
||||
}
|
||||
error.cause = params.cause;
|
||||
return error;
|
||||
}
|
||||
|
||||
export function isMemoryEmbeddingOperationError(
|
||||
err: unknown,
|
||||
): err is MemoryEmbeddingOperationError {
|
||||
return (
|
||||
err instanceof Error &&
|
||||
(err as { code?: unknown }).code === MEMORY_EMBEDDING_OPERATION_ERROR_CODE
|
||||
);
|
||||
}
|
||||
1027
extensions/memory-core/src/memory/manager-embedding-ops.ts
Normal file
1027
extensions/memory-core/src/memory/manager-embedding-ops.ts
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,289 @@
|
||||
// Memory Core tests cover manager embedding policy plugin behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildMemoryEmbeddingBatches,
|
||||
filterNonEmptyMemoryChunks,
|
||||
isRetryableMemoryEmbeddingTransportError,
|
||||
isRetryableMemoryEmbeddingError,
|
||||
isSplittableMemoryEmbeddingTransportError,
|
||||
isStructuredInputTooLargeMemoryEmbeddingError,
|
||||
resolveMemoryEmbeddingRetryDelay,
|
||||
runMemoryEmbeddingBatchRetryWithSplit,
|
||||
runMemoryEmbeddingRetryLoop,
|
||||
} from "./manager-embedding-policy.js";
|
||||
|
||||
function chunk(text: string) {
|
||||
return {
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
text,
|
||||
hash: text,
|
||||
};
|
||||
}
|
||||
|
||||
describe("memory embedding policy", () => {
|
||||
it("splits large files across multiple embedding batches", () => {
|
||||
const line = "a".repeat(4200);
|
||||
const batches = buildMemoryEmbeddingBatches([chunk(line), chunk(line)], 8000);
|
||||
|
||||
expect(batches).toHaveLength(2);
|
||||
expect(batches.map((batch) => batch.length)).toEqual([1, 1]);
|
||||
});
|
||||
|
||||
it("keeps small files in a single embedding batch", () => {
|
||||
const line = "b".repeat(120);
|
||||
const batches = buildMemoryEmbeddingBatches(
|
||||
[chunk(line), chunk(line), chunk(line), chunk(line)],
|
||||
8000,
|
||||
);
|
||||
|
||||
expect(batches).toHaveLength(1);
|
||||
expect(batches[0]).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("filters empty chunks before embedding", () => {
|
||||
const chunks = filterNonEmptyMemoryChunks([chunk("\n\n"), chunk("hello"), chunk(" ")]);
|
||||
|
||||
expect(chunks.map((entry) => entry.text)).toEqual(["hello"]);
|
||||
});
|
||||
|
||||
it("retries transient rate limit and 5xx errors", async () => {
|
||||
const run = vi.fn(async () => {
|
||||
const call = run.mock.calls.length;
|
||||
if (call === 1) {
|
||||
throw new Error("openai embeddings failed: 429 rate limit");
|
||||
}
|
||||
if (call === 2) {
|
||||
throw new Error("openai embeddings failed: 502 Bad Gateway (cloudflare)");
|
||||
}
|
||||
return "ok";
|
||||
});
|
||||
const waits: number[] = [];
|
||||
|
||||
const result = await runMemoryEmbeddingRetryLoop({
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
waits.push(delayMs);
|
||||
},
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
});
|
||||
|
||||
expect(result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(3);
|
||||
expect(waits).toEqual([500, 1000]);
|
||||
});
|
||||
|
||||
it("stops retrying after the caller signal aborts, even for retryable-looking errors", async () => {
|
||||
const controller = new AbortController();
|
||||
const run = vi.fn(async () => {
|
||||
controller.abort(new Error("memory_search timed out after 15s"));
|
||||
// "timed out" matches the retryable transport pattern; abort must still win.
|
||||
throw new Error("memory embeddings query timed out after 60s");
|
||||
});
|
||||
const waitForRetry = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
runMemoryEmbeddingRetryLoop({
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry,
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow("memory embeddings query timed out after 60s");
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
expect(waitForRetry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries transient socket/network embedding errors", () => {
|
||||
const splittableMessages = [
|
||||
"TypeError: fetch failed | other side closed",
|
||||
"undici error: UND_ERR_SOCKET",
|
||||
"read ECONNRESET",
|
||||
"socket hang up",
|
||||
];
|
||||
|
||||
for (const message of splittableMessages) {
|
||||
expect(isRetryableMemoryEmbeddingError(message)).toBe(true);
|
||||
expect(isRetryableMemoryEmbeddingTransportError(message)).toBe(true);
|
||||
expect(isSplittableMemoryEmbeddingTransportError(message)).toBe(true);
|
||||
}
|
||||
expect(isRetryableMemoryEmbeddingTransportError("ECONNREFUSED")).toBe(true);
|
||||
expect(isSplittableMemoryEmbeddingTransportError("ECONNREFUSED")).toBe(false);
|
||||
expect(isRetryableMemoryEmbeddingTransportError("EHOSTUNREACH")).toBe(true);
|
||||
expect(isSplittableMemoryEmbeddingTransportError("EHOSTUNREACH")).toBe(false);
|
||||
expect(isRetryableMemoryEmbeddingTransportError("memory embeddings batch timed out")).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isSplittableMemoryEmbeddingTransportError("memory embeddings batch timed out")).toBe(
|
||||
false,
|
||||
);
|
||||
expect(isRetryableMemoryEmbeddingTransportError("worker terminated by user")).toBe(false);
|
||||
expect(isRetryableMemoryEmbeddingTransportError("embedding validation failed")).toBe(false);
|
||||
});
|
||||
|
||||
it("splits OpenAI 431 oversized embedding batches without retrying the same request", async () => {
|
||||
const run = vi.fn(async (items: string[]) => {
|
||||
if (items.length > 1) {
|
||||
throw new Error(
|
||||
"openai embeddings failed: 431 request_headers_too_large: Request Header Fields Too Large",
|
||||
);
|
||||
}
|
||||
return items.map((item) => [item.charCodeAt(0)]);
|
||||
});
|
||||
|
||||
const result = await runMemoryEmbeddingBatchRetryWithSplit({
|
||||
items: ["a", "b", "c", "d"],
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
isSplittable: isSplittableMemoryEmbeddingTransportError,
|
||||
waitForRetry: async () => {},
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
});
|
||||
|
||||
expect(result).toEqual([[97], [98], [99], [100]]);
|
||||
expect(run.mock.calls.map(([items]) => items.length)).toEqual([4, 2, 1, 1, 2, 1, 1]);
|
||||
expect(isRetryableMemoryEmbeddingError("431 request_headers_too_large")).toBe(false);
|
||||
expect(isSplittableMemoryEmbeddingTransportError("431 request_headers_too_large")).toBe(true);
|
||||
expect(
|
||||
isSplittableMemoryEmbeddingTransportError("embedding validation failed at item 4312"),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("retries too-many-tokens-per-day errors", async () => {
|
||||
let calls = 0;
|
||||
const waits: number[] = [];
|
||||
|
||||
const result = await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
throw new Error("AWS Bedrock embeddings failed: Too many tokens per day");
|
||||
}
|
||||
return "ok";
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
waits.push(delayMs);
|
||||
},
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
});
|
||||
|
||||
expect(result).toBe("ok");
|
||||
expect(calls).toBe(2);
|
||||
expect(waits).toEqual([500]);
|
||||
});
|
||||
|
||||
it("stops after the configured maximum attempts", async () => {
|
||||
const run = vi.fn(async () => {
|
||||
throw new Error("TypeError: fetch failed | other side closed");
|
||||
});
|
||||
const waits: number[] = [];
|
||||
|
||||
await expect(
|
||||
runMemoryEmbeddingRetryLoop({
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
waits.push(delayMs);
|
||||
},
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
}),
|
||||
).rejects.toThrow("fetch failed");
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(3);
|
||||
expect(waits).toEqual([500, 1000]);
|
||||
});
|
||||
|
||||
it("splits transport-failed batches after retries are exhausted", async () => {
|
||||
const waits: number[] = [];
|
||||
const splits: string[] = [];
|
||||
const run = vi.fn(async (items: string[]) => {
|
||||
if (items.length > 1) {
|
||||
throw new TypeError("fetch failed | other side closed");
|
||||
}
|
||||
return items.map((item) => [item.charCodeAt(0)]);
|
||||
});
|
||||
|
||||
const result = await runMemoryEmbeddingBatchRetryWithSplit({
|
||||
items: ["a", "b", "c", "d"],
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
isSplittable: isSplittableMemoryEmbeddingTransportError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
waits.push(delayMs);
|
||||
},
|
||||
maxAttempts: 2,
|
||||
baseDelayMs: 500,
|
||||
onSplit: ({ itemCount, splitAt }) => {
|
||||
splits.push(`${itemCount}:${splitAt}`);
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toEqual([[97], [98], [99], [100]]);
|
||||
expect(run.mock.calls.map(([items]) => items.length)).toEqual([4, 4, 2, 2, 1, 1, 2, 2, 1, 1]);
|
||||
expect(waits).toEqual([500, 500, 500]);
|
||||
expect(splits).toEqual(["4:2", "2:1", "2:1"]);
|
||||
});
|
||||
|
||||
it("does not split exhausted service retry errors", async () => {
|
||||
const run = vi.fn(async () => {
|
||||
throw new Error("openai embeddings failed: 429 rate limit");
|
||||
});
|
||||
|
||||
await expect(
|
||||
runMemoryEmbeddingBatchRetryWithSplit({
|
||||
items: ["a", "b"],
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
isSplittable: isSplittableMemoryEmbeddingTransportError,
|
||||
waitForRetry: async () => {},
|
||||
maxAttempts: 1,
|
||||
baseDelayMs: 500,
|
||||
}),
|
||||
).rejects.toThrow("429 rate limit");
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not split whole-endpoint transport outages", async () => {
|
||||
const run = vi.fn(async () => {
|
||||
throw new Error("connect ECONNREFUSED 127.0.0.1:11434");
|
||||
});
|
||||
|
||||
await expect(
|
||||
runMemoryEmbeddingBatchRetryWithSplit({
|
||||
items: ["a", "b"],
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
isSplittable: isSplittableMemoryEmbeddingTransportError,
|
||||
waitForRetry: async () => {},
|
||||
maxAttempts: 2,
|
||||
baseDelayMs: 500,
|
||||
}),
|
||||
).rejects.toThrow("ECONNREFUSED");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("classifies oversized structured-input errors", () => {
|
||||
expect(isStructuredInputTooLargeMemoryEmbeddingError("payload too large")).toBe(true);
|
||||
expect(
|
||||
isStructuredInputTooLargeMemoryEmbeddingError(
|
||||
"gemini embeddings failed: request size exceeded input limit",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(isStructuredInputTooLargeMemoryEmbeddingError("connection reset by peer")).toBe(false);
|
||||
});
|
||||
|
||||
it("caps retry jittered delays", () => {
|
||||
expect(resolveMemoryEmbeddingRetryDelay(500, 0, 8000)).toBe(500);
|
||||
expect(resolveMemoryEmbeddingRetryDelay(500, 1, 8000)).toBe(600);
|
||||
expect(resolveMemoryEmbeddingRetryDelay(10_000, 1, 8000)).toBe(8000);
|
||||
});
|
||||
});
|
||||
193
extensions/memory-core/src/memory/manager-embedding-policy.ts
Normal file
193
extensions/memory-core/src/memory/manager-embedding-policy.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
// Memory Core plugin module implements manager embedding policy behavior.
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
type MemoryEmbeddingTextPart = {
|
||||
type: "text";
|
||||
text: string;
|
||||
};
|
||||
|
||||
type MemoryEmbeddingInlineDataPart = {
|
||||
type: "inline-data";
|
||||
mimeType: string;
|
||||
data: string;
|
||||
};
|
||||
|
||||
type MemoryEmbeddingInput = {
|
||||
text: string;
|
||||
parts?: Array<MemoryEmbeddingTextPart | MemoryEmbeddingInlineDataPart>;
|
||||
};
|
||||
|
||||
type MemoryEmbeddingChunk = {
|
||||
text: string;
|
||||
embeddingInput?: MemoryEmbeddingInput;
|
||||
};
|
||||
|
||||
function estimateUtf8Bytes(text: string): number {
|
||||
if (!text) {
|
||||
return 0;
|
||||
}
|
||||
return Buffer.byteLength(text, "utf8");
|
||||
}
|
||||
|
||||
function estimateStructuredEmbeddingInputBytes(input: MemoryEmbeddingInput): number {
|
||||
if (!input.parts?.length) {
|
||||
return estimateUtf8Bytes(input.text);
|
||||
}
|
||||
let total = 0;
|
||||
for (const part of input.parts) {
|
||||
if (part.type === "text") {
|
||||
total += estimateUtf8Bytes(part.text);
|
||||
} else {
|
||||
total += estimateUtf8Bytes(part.mimeType);
|
||||
total += estimateUtf8Bytes(part.data);
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
export function filterNonEmptyMemoryChunks<T extends MemoryEmbeddingChunk>(chunks: T[]): T[] {
|
||||
return chunks.filter((chunk) => chunk.text.trim().length > 0);
|
||||
}
|
||||
|
||||
export function buildMemoryEmbeddingBatches<T extends MemoryEmbeddingChunk>(
|
||||
chunks: T[],
|
||||
maxTokens: number,
|
||||
): T[][] {
|
||||
const batches: T[][] = [];
|
||||
let current: T[] = [];
|
||||
let currentTokens = 0;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const estimate = chunk.embeddingInput
|
||||
? estimateStructuredEmbeddingInputBytes(chunk.embeddingInput)
|
||||
: estimateUtf8Bytes(chunk.text);
|
||||
const wouldExceed = current.length > 0 && currentTokens + estimate > maxTokens;
|
||||
if (wouldExceed) {
|
||||
batches.push(current);
|
||||
current = [];
|
||||
currentTokens = 0;
|
||||
}
|
||||
if (current.length === 0 && estimate > maxTokens) {
|
||||
batches.push([chunk]);
|
||||
continue;
|
||||
}
|
||||
current.push(chunk);
|
||||
currentTokens += estimate;
|
||||
}
|
||||
|
||||
if (current.length > 0) {
|
||||
batches.push(current);
|
||||
}
|
||||
return batches;
|
||||
}
|
||||
|
||||
const RETRYABLE_MEMORY_EMBEDDING_SERVICE_ERROR_RE =
|
||||
/(rate[_ ]limit|too many requests|429|resource has been exhausted|5\d\d|cloudflare|tokens per day)/i;
|
||||
|
||||
const RETRYABLE_MEMORY_EMBEDDING_TRANSPORT_ERROR_RE =
|
||||
/(fetch failed|other side closed|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|UND_ERR_|socket hang up|socket terminated|network error|read ECONN|timed out|connection (?:reset|refused|aborted|timed out)|EHOSTUNREACH|ENETUNREACH|ECONNABORTED|EAI_AGAIN)/i;
|
||||
|
||||
const SPLITTABLE_MEMORY_EMBEDDING_TRANSPORT_ERROR_RE =
|
||||
/(request_headers_too_large|request header fields too large|other side closed|ECONNRESET|EPIPE|UND_ERR_SOCKET|socket hang up|socket terminated|read ECONN|connection (?:reset|aborted))/i;
|
||||
|
||||
export function isRetryableMemoryEmbeddingTransportError(message: string): boolean {
|
||||
return RETRYABLE_MEMORY_EMBEDDING_TRANSPORT_ERROR_RE.test(message);
|
||||
}
|
||||
|
||||
export function isSplittableMemoryEmbeddingTransportError(message: string): boolean {
|
||||
return SPLITTABLE_MEMORY_EMBEDDING_TRANSPORT_ERROR_RE.test(message);
|
||||
}
|
||||
|
||||
export function isRetryableMemoryEmbeddingError(message: string): boolean {
|
||||
return (
|
||||
RETRYABLE_MEMORY_EMBEDDING_SERVICE_ERROR_RE.test(message) ||
|
||||
isRetryableMemoryEmbeddingTransportError(message)
|
||||
);
|
||||
}
|
||||
|
||||
export function isStructuredInputTooLargeMemoryEmbeddingError(message: string): boolean {
|
||||
return /(413|payload too large|request too large|input too large|too many tokens|input limit|request size)/i.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveMemoryEmbeddingRetryDelay(
|
||||
delayMs: number,
|
||||
randomValue: number,
|
||||
maxDelayMs: number,
|
||||
): number {
|
||||
return Math.min(maxDelayMs, Math.round(delayMs * (1 + randomValue * 0.2)));
|
||||
}
|
||||
|
||||
export async function runMemoryEmbeddingRetryLoop<T>(params: {
|
||||
run: () => Promise<T>;
|
||||
isRetryable: (message: string) => boolean;
|
||||
waitForRetry: (delayMs: number) => Promise<void>;
|
||||
maxAttempts: number;
|
||||
baseDelayMs: number;
|
||||
/** Caller-owned cancellation; an aborted caller stops the retry loop. */
|
||||
signal?: AbortSignal;
|
||||
}): Promise<T> {
|
||||
const attempts = Math.max(1, params.maxAttempts);
|
||||
for (const attempt of Array.from({ length: attempts }, (_, index) => index + 1)) {
|
||||
const delayMs = params.baseDelayMs * 2 ** (attempt - 1);
|
||||
try {
|
||||
return await params.run();
|
||||
} catch (err) {
|
||||
// Abort must win over retryable-looking failures: abort reasons often
|
||||
// carry "timed out" messages that match the retryable transport
|
||||
// patterns and would otherwise keep retrying for an absent caller.
|
||||
if (params.signal?.aborted) {
|
||||
throw err;
|
||||
}
|
||||
const message = formatErrorMessage(err);
|
||||
if (!params.isRetryable(message) || attempt >= params.maxAttempts) {
|
||||
throw err;
|
||||
}
|
||||
await params.waitForRetry(delayMs);
|
||||
}
|
||||
}
|
||||
throw new Error("retry loop exhausted");
|
||||
}
|
||||
|
||||
export async function runMemoryEmbeddingBatchRetryWithSplit<TInput, TOutput>(params: {
|
||||
items: TInput[];
|
||||
run: (items: TInput[]) => Promise<TOutput[]>;
|
||||
isRetryable: (message: string) => boolean;
|
||||
isSplittable: (message: string) => boolean;
|
||||
waitForRetry: (delayMs: number) => Promise<void>;
|
||||
maxAttempts: number;
|
||||
baseDelayMs: number;
|
||||
onSplit?: (info: { itemCount: number; splitAt: number; message: string }) => void;
|
||||
}): Promise<TOutput[]> {
|
||||
try {
|
||||
return await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => await params.run(params.items),
|
||||
isRetryable: params.isRetryable,
|
||||
waitForRetry: params.waitForRetry,
|
||||
maxAttempts: params.maxAttempts,
|
||||
baseDelayMs: params.baseDelayMs,
|
||||
});
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
if (params.items.length <= 1 || !params.isSplittable(message)) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
const splitAt = Math.ceil(params.items.length / 2);
|
||||
params.onSplit?.({ itemCount: params.items.length, splitAt, message });
|
||||
const left = await runMemoryEmbeddingBatchRetryWithSplit({
|
||||
...params,
|
||||
items: params.items.slice(0, splitAt),
|
||||
});
|
||||
const right = await runMemoryEmbeddingBatchRetryWithSplit({
|
||||
...params,
|
||||
items: params.items.slice(splitAt),
|
||||
});
|
||||
return [...left, ...right];
|
||||
}
|
||||
}
|
||||
|
||||
export function buildTextEmbeddingInputs(chunks: MemoryEmbeddingChunk[]): MemoryEmbeddingInput[] {
|
||||
return chunks.map((chunk) => chunk.embeddingInput ?? { text: chunk.text });
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
// Memory Core tests cover manager embedding timeout plugin behavior.
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
resolveEmbeddingTimeoutMs,
|
||||
resolveMemoryIndexConcurrency,
|
||||
runEmbeddingOperationWithTimeout,
|
||||
} from "./manager-embedding-ops.js";
|
||||
import {
|
||||
isLocalEmbeddingWorkerFailure,
|
||||
LOCAL_EMBEDDING_WORKER_ERROR_CODES,
|
||||
} from "./manager-local-worker-errors.js";
|
||||
|
||||
describe("memory embedding timeout resolution", () => {
|
||||
it("uses hosted defaults for inline embedding calls", () => {
|
||||
expect(resolveEmbeddingTimeoutMs({ kind: "query", providerId: "openai" })).toBe(60_000);
|
||||
expect(resolveEmbeddingTimeoutMs({ kind: "batch", providerId: "openai" })).toBe(120_000);
|
||||
});
|
||||
|
||||
it("uses local defaults for the builtin local provider", () => {
|
||||
expect(resolveEmbeddingTimeoutMs({ kind: "query", providerId: "local" })).toBe(300_000);
|
||||
expect(resolveEmbeddingTimeoutMs({ kind: "batch", providerId: "local" })).toBe(600_000);
|
||||
});
|
||||
|
||||
it("uses runtime batch defaults for local-server providers", () => {
|
||||
expect(
|
||||
resolveEmbeddingTimeoutMs({
|
||||
kind: "batch",
|
||||
providerId: "ollama",
|
||||
providerRuntime: { inlineBatchTimeoutMs: 600_000 },
|
||||
}),
|
||||
).toBe(600_000);
|
||||
});
|
||||
|
||||
it("lets configured batch timeout override provider defaults", () => {
|
||||
expect(
|
||||
resolveEmbeddingTimeoutMs({
|
||||
kind: "batch",
|
||||
providerId: "ollama",
|
||||
providerRuntime: { inlineBatchTimeoutMs: 600_000 },
|
||||
configuredBatchTimeoutSeconds: 45,
|
||||
}),
|
||||
).toBe(45_000);
|
||||
});
|
||||
|
||||
it("caps configured and runtime embedding timeouts to timer-safe values", () => {
|
||||
expect(
|
||||
resolveEmbeddingTimeoutMs({
|
||||
kind: "batch",
|
||||
providerId: "openai",
|
||||
configuredBatchTimeoutSeconds: Number.MAX_SAFE_INTEGER,
|
||||
}),
|
||||
).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
expect(
|
||||
resolveEmbeddingTimeoutMs({
|
||||
kind: "query",
|
||||
providerId: "openai",
|
||||
providerRuntime: { inlineQueryTimeoutMs: Number.MAX_SAFE_INTEGER },
|
||||
}),
|
||||
).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
expect(
|
||||
resolveEmbeddingTimeoutMs({
|
||||
kind: "batch",
|
||||
providerId: "openai",
|
||||
providerRuntime: { inlineBatchTimeoutMs: Number.MAX_SAFE_INTEGER },
|
||||
}),
|
||||
).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
});
|
||||
});
|
||||
|
||||
describe("local embedding worker failure detection", () => {
|
||||
it("matches structured local worker failure codes", () => {
|
||||
expect(
|
||||
isLocalEmbeddingWorkerFailure(
|
||||
Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), {
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
reason: "exit",
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isLocalEmbeddingWorkerFailure(
|
||||
Object.assign(new Error("Local embedding worker process failed"), {
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.processError,
|
||||
reason: "process-error",
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isLocalEmbeddingWorkerFailure(
|
||||
Object.assign(new Error("Local embedding request aborted"), {
|
||||
code: "ABORT_ERR",
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory embedding timeout abort", () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("aborts the provider operation when the timeout fires", async () => {
|
||||
vi.useFakeTimers();
|
||||
let signalSeen: AbortSignal | undefined;
|
||||
|
||||
const resultPromise = runEmbeddingOperationWithTimeout({
|
||||
timeoutMs: 1,
|
||||
message: "memory embeddings query timed out after 0s",
|
||||
run: async (signal) => {
|
||||
signalSeen = signal;
|
||||
return await new Promise<number[]>((resolve, reject) => {
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => reject(toLintErrorObject(signal.reason, "Non-Error rejection")),
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const rejection = expect(resultPromise).rejects.toThrow(
|
||||
"memory embeddings query timed out after 0s",
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await rejection;
|
||||
|
||||
expect(signalSeen?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
it("aborts the provider operation when the caller signal aborts before the watchdog", async () => {
|
||||
const external = new AbortController();
|
||||
let signalSeen: AbortSignal | undefined;
|
||||
|
||||
const resultPromise = runEmbeddingOperationWithTimeout({
|
||||
timeoutMs: 60_000,
|
||||
message: "memory embeddings query timed out after 60s",
|
||||
signal: external.signal,
|
||||
run: async (signal) => {
|
||||
signalSeen = signal;
|
||||
return await new Promise<number[]>((_resolve, reject) => {
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => reject(toLintErrorObject(signal.reason, "Non-Error rejection")),
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
external.abort(new Error("memory_search timed out after 15s"));
|
||||
await expect(resultPromise).rejects.toThrow("memory_search timed out after 15s");
|
||||
expect(signalSeen?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the timeout error when a provider abort listener rejects generically", async () => {
|
||||
vi.useFakeTimers();
|
||||
const resultPromise = runEmbeddingOperationWithTimeout({
|
||||
timeoutMs: 1,
|
||||
message: "memory embeddings batch timed out after 0s",
|
||||
run: async (signal) =>
|
||||
await new Promise<number[]>((_resolve, reject) => {
|
||||
signal.addEventListener("abort", () => reject(new Error("provider aborted")), {
|
||||
once: true,
|
||||
});
|
||||
}),
|
||||
});
|
||||
|
||||
const rejection = expect(resultPromise).rejects.toThrow(
|
||||
"memory embeddings batch timed out after 0s",
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await rejection;
|
||||
});
|
||||
|
||||
it("caps operation watchdog timers before scheduling", async () => {
|
||||
const timeoutSpy = vi
|
||||
.spyOn(globalThis, "setTimeout")
|
||||
.mockReturnValue(1 as unknown as ReturnType<typeof setTimeout>);
|
||||
|
||||
try {
|
||||
await runEmbeddingOperationWithTimeout({
|
||||
timeoutMs: Number.MAX_SAFE_INTEGER,
|
||||
message: "memory embeddings query timed out",
|
||||
run: async () => [1, 2, 3],
|
||||
});
|
||||
expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS);
|
||||
} finally {
|
||||
timeoutSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory index concurrency resolution", () => {
|
||||
it("uses the default index concurrency when batch mode is disabled and unconfigured", () => {
|
||||
expect(
|
||||
resolveMemoryIndexConcurrency({
|
||||
batch: { enabled: false, concurrency: 2 },
|
||||
}),
|
||||
).toBe(4);
|
||||
});
|
||||
|
||||
it("respects configured non-batch concurrency when batch mode is disabled", () => {
|
||||
expect(
|
||||
resolveMemoryIndexConcurrency({
|
||||
batch: { enabled: false, concurrency: 1 },
|
||||
configuredNonBatchConcurrency: 1,
|
||||
}),
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("clamps configured non-batch concurrency to a positive integer", () => {
|
||||
expect(
|
||||
resolveMemoryIndexConcurrency({
|
||||
batch: { enabled: false, concurrency: 2 },
|
||||
configuredNonBatchConcurrency: 2.8,
|
||||
}),
|
||||
).toBe(2);
|
||||
expect(
|
||||
resolveMemoryIndexConcurrency({
|
||||
batch: { enabled: false, concurrency: 2 },
|
||||
configuredNonBatchConcurrency: 0,
|
||||
}),
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("uses conservative non-batch concurrency for Ollama by default", () => {
|
||||
expect(
|
||||
resolveMemoryIndexConcurrency({
|
||||
batch: { enabled: false, concurrency: 2 },
|
||||
providerId: "ollama",
|
||||
}),
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("uses resolved batch concurrency when batch mode is enabled", () => {
|
||||
expect(
|
||||
resolveMemoryIndexConcurrency({
|
||||
batch: { enabled: true, concurrency: 3 },
|
||||
}),
|
||||
).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
function toLintErrorObject(value: unknown, fallbackMessage: string): Error {
|
||||
if (value instanceof Error) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return new Error(value);
|
||||
}
|
||||
const error = new Error(fallbackMessage, { cause: value });
|
||||
if ((typeof value === "object" && value !== null) || typeof value === "function") {
|
||||
Object.assign(error, value);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
83
extensions/memory-core/src/memory/manager-fts-state.test.ts
Normal file
83
extensions/memory-core/src/memory/manager-fts-state.test.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
// Memory Core tests cover manager fts state plugin behavior.
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { deleteMemoryFtsRows } from "./manager-fts-state.js";
|
||||
|
||||
describe("memory FTS state", () => {
|
||||
let db: DatabaseSync | null = null;
|
||||
|
||||
afterEach(() => {
|
||||
db?.close();
|
||||
db = null;
|
||||
});
|
||||
|
||||
it("removes rows for all models when a provider is active", () => {
|
||||
db = new DatabaseSync(":memory:");
|
||||
db.exec("CREATE TABLE memory_index_chunks_fts (path TEXT, source TEXT, model TEXT)");
|
||||
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"mock-embed",
|
||||
);
|
||||
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"other-model",
|
||||
);
|
||||
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-13.md",
|
||||
"memory",
|
||||
"other-model",
|
||||
);
|
||||
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"sessions",
|
||||
"other-model",
|
||||
);
|
||||
|
||||
deleteMemoryFtsRows({
|
||||
db,
|
||||
path: "memory/2026-01-12.md",
|
||||
source: "memory",
|
||||
currentModel: "mock-embed",
|
||||
});
|
||||
|
||||
const rows = db
|
||||
.prepare("SELECT path, source, model FROM memory_index_chunks_fts ORDER BY path, source")
|
||||
.all() as Array<{
|
||||
path: string;
|
||||
source: string;
|
||||
model: string;
|
||||
}>;
|
||||
expect(rows).toEqual([
|
||||
{ path: "memory/2026-01-12.md", source: "sessions", model: "other-model" },
|
||||
{ path: "memory/2026-01-13.md", source: "memory", model: "other-model" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("removes all rows for the path in FTS-only mode", () => {
|
||||
db = new DatabaseSync(":memory:");
|
||||
db.exec("CREATE TABLE memory_index_chunks_fts (path TEXT, source TEXT, model TEXT)");
|
||||
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"mock-embed",
|
||||
);
|
||||
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"fts-only",
|
||||
);
|
||||
|
||||
deleteMemoryFtsRows({
|
||||
db,
|
||||
path: "memory/2026-01-12.md",
|
||||
source: "memory",
|
||||
});
|
||||
|
||||
const count = db.prepare("SELECT COUNT(*) as c FROM memory_index_chunks_fts").get() as {
|
||||
c: number;
|
||||
};
|
||||
expect(count.c).toBe(0);
|
||||
});
|
||||
});
|
||||
18
extensions/memory-core/src/memory/manager-fts-state.ts
Normal file
18
extensions/memory-core/src/memory/manager-fts-state.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
// Memory Core plugin module implements manager fts state behavior.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export function deleteMemoryFtsRows(params: {
|
||||
db: DatabaseSync;
|
||||
tableName?: string;
|
||||
path: string;
|
||||
source: MemorySource;
|
||||
currentModel?: string;
|
||||
}): void {
|
||||
const tableName = params.tableName ?? "memory_index_chunks_fts";
|
||||
// Lexical search is model-agnostic, so refreshed/deleted files must not
|
||||
// leave old-model FTS rows behind for the same path/source.
|
||||
params.db
|
||||
.prepare(`DELETE FROM ${tableName} WHERE path = ? AND source = ?`)
|
||||
.run(params.path, params.source);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Memory Core plugin module implements manager local worker errors behavior.
|
||||
export const LOCAL_EMBEDDING_WORKER_ERROR_CODES = {
|
||||
exited: "LOCAL_EMBEDDING_WORKER_EXITED",
|
||||
processError: "LOCAL_EMBEDDING_WORKER_PROCESS_ERROR",
|
||||
ipcError: "LOCAL_EMBEDDING_WORKER_IPC_ERROR",
|
||||
} as const;
|
||||
|
||||
export type LocalEmbeddingWorkerFailureCode =
|
||||
(typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES)[keyof typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES];
|
||||
|
||||
export type LocalEmbeddingWorkerFailureError = Error & {
|
||||
code: LocalEmbeddingWorkerFailureCode;
|
||||
};
|
||||
|
||||
const LOCAL_EMBEDDING_WORKER_FAILURE_CODES = new Set<string>(
|
||||
Object.values(LOCAL_EMBEDDING_WORKER_ERROR_CODES),
|
||||
);
|
||||
|
||||
export function isLocalEmbeddingWorkerFailure(
|
||||
err: unknown,
|
||||
): err is LocalEmbeddingWorkerFailureError {
|
||||
return (
|
||||
err instanceof Error &&
|
||||
LOCAL_EMBEDDING_WORKER_FAILURE_CODES.has(String((err as { code?: unknown }).code))
|
||||
);
|
||||
}
|
||||
219
extensions/memory-core/src/memory/manager-provider-state.ts
Normal file
219
extensions/memory-core/src/memory/manager-provider-state.ts
Normal file
@@ -0,0 +1,219 @@
|
||||
// Memory Core provider module implements model/runtime integration.
|
||||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import {
|
||||
resolveEmbeddingProviderFallbackModel,
|
||||
type EmbeddingProvider,
|
||||
type EmbeddingProviderResult,
|
||||
type EmbeddingProviderRuntime,
|
||||
} from "./embeddings.js";
|
||||
|
||||
type MemoryResolvedProviderState = {
|
||||
provider: EmbeddingProvider | null;
|
||||
fallbackFrom?: string;
|
||||
fallbackReason?: string;
|
||||
providerUnavailableReason?: string;
|
||||
providerRuntime?: EmbeddingProviderRuntime;
|
||||
lifecycle: MemoryProviderLifecycleState;
|
||||
};
|
||||
|
||||
export type MemoryProviderLifecycleState =
|
||||
| {
|
||||
mode: "pending";
|
||||
requestedProvider: string;
|
||||
}
|
||||
| {
|
||||
mode: "active";
|
||||
providerId: string;
|
||||
}
|
||||
| {
|
||||
mode: "degraded";
|
||||
providerId: string;
|
||||
reason: string;
|
||||
code?: string;
|
||||
}
|
||||
| {
|
||||
mode: "fallback-active";
|
||||
providerId: string;
|
||||
fallbackFrom: string;
|
||||
reason: string;
|
||||
}
|
||||
| {
|
||||
mode: "fts-only";
|
||||
reason: string;
|
||||
attemptedProviderId?: string;
|
||||
};
|
||||
|
||||
export function createPendingMemoryProviderLifecycle(
|
||||
requestedProvider: string,
|
||||
): MemoryProviderLifecycleState {
|
||||
return { mode: "pending", requestedProvider };
|
||||
}
|
||||
|
||||
export function createDegradedMemoryProviderLifecycle(params: {
|
||||
providerId: string;
|
||||
reason: string;
|
||||
code?: string;
|
||||
}): MemoryProviderLifecycleState {
|
||||
return {
|
||||
mode: "degraded",
|
||||
providerId: params.providerId,
|
||||
reason: params.reason,
|
||||
...(params.code ? { code: params.code } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveProviderLifecycle(
|
||||
result: Pick<
|
||||
EmbeddingProviderResult,
|
||||
| "provider"
|
||||
| "fallbackFrom"
|
||||
| "fallbackReason"
|
||||
| "providerUnavailableReason"
|
||||
| "requestedProvider"
|
||||
>,
|
||||
): MemoryProviderLifecycleState {
|
||||
if (result.provider && result.fallbackFrom) {
|
||||
return {
|
||||
mode: "fallback-active",
|
||||
providerId: result.provider.id,
|
||||
fallbackFrom: result.fallbackFrom,
|
||||
reason: result.fallbackReason ?? "fallback activated",
|
||||
};
|
||||
}
|
||||
if (result.provider) {
|
||||
return { mode: "active", providerId: result.provider.id };
|
||||
}
|
||||
return {
|
||||
mode: "fts-only",
|
||||
reason: result.providerUnavailableReason ?? "No embedding provider available",
|
||||
attemptedProviderId: result.requestedProvider,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveFallbackCurrentProviderId(params: {
|
||||
provider: EmbeddingProvider | null;
|
||||
lifecycle: MemoryProviderLifecycleState;
|
||||
}): string | null {
|
||||
if (params.provider) {
|
||||
return params.provider.id;
|
||||
}
|
||||
if (params.lifecycle.mode === "degraded") {
|
||||
return params.lifecycle.providerId;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveMemoryPrimaryProviderRequest(params: {
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
}): {
|
||||
provider: string;
|
||||
model: string;
|
||||
remote: ResolvedMemorySearchConfig["remote"];
|
||||
inputType: ResolvedMemorySearchConfig["inputType"];
|
||||
queryInputType: ResolvedMemorySearchConfig["queryInputType"];
|
||||
documentInputType: ResolvedMemorySearchConfig["documentInputType"];
|
||||
outputDimensionality: ResolvedMemorySearchConfig["outputDimensionality"];
|
||||
fallback: ResolvedMemorySearchConfig["fallback"];
|
||||
local: ResolvedMemorySearchConfig["local"];
|
||||
} {
|
||||
return {
|
||||
provider: params.settings.provider,
|
||||
model: params.settings.model,
|
||||
remote: params.settings.remote,
|
||||
inputType: params.settings.inputType,
|
||||
queryInputType: params.settings.queryInputType,
|
||||
documentInputType: params.settings.documentInputType,
|
||||
outputDimensionality: params.settings.outputDimensionality,
|
||||
fallback: params.settings.fallback,
|
||||
local: params.settings.local,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveMemoryProviderState(
|
||||
result: Pick<
|
||||
EmbeddingProviderResult,
|
||||
| "provider"
|
||||
| "fallbackFrom"
|
||||
| "fallbackReason"
|
||||
| "providerUnavailableReason"
|
||||
| "runtime"
|
||||
| "requestedProvider"
|
||||
>,
|
||||
): MemoryResolvedProviderState {
|
||||
return {
|
||||
provider: result.provider,
|
||||
fallbackFrom: result.fallbackFrom,
|
||||
fallbackReason: result.fallbackReason,
|
||||
providerUnavailableReason: result.providerUnavailableReason,
|
||||
providerRuntime: result.runtime,
|
||||
lifecycle: resolveProviderLifecycle(result),
|
||||
};
|
||||
}
|
||||
|
||||
export function applyMemoryFallbackProviderState(params: {
|
||||
current: MemoryResolvedProviderState;
|
||||
fallbackFrom: string;
|
||||
reason: string;
|
||||
result: Pick<EmbeddingProviderResult, "provider" | "runtime">;
|
||||
}): MemoryResolvedProviderState {
|
||||
return {
|
||||
...params.current,
|
||||
fallbackFrom: params.fallbackFrom,
|
||||
fallbackReason: params.reason,
|
||||
providerUnavailableReason: undefined,
|
||||
provider: params.result.provider,
|
||||
providerRuntime: params.result.runtime,
|
||||
lifecycle: params.result.provider
|
||||
? {
|
||||
mode: "fallback-active",
|
||||
providerId: params.result.provider.id,
|
||||
fallbackFrom: params.fallbackFrom,
|
||||
reason: params.reason,
|
||||
}
|
||||
: {
|
||||
mode: "fts-only",
|
||||
reason: params.reason,
|
||||
attemptedProviderId: params.fallbackFrom,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveMemoryFallbackProviderRequest(params: {
|
||||
cfg: OpenClawConfig;
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
currentProviderId: string | null;
|
||||
}): {
|
||||
provider: string;
|
||||
model: string;
|
||||
remote: ResolvedMemorySearchConfig["remote"];
|
||||
inputType: ResolvedMemorySearchConfig["inputType"];
|
||||
queryInputType: ResolvedMemorySearchConfig["queryInputType"];
|
||||
documentInputType: ResolvedMemorySearchConfig["documentInputType"];
|
||||
outputDimensionality: ResolvedMemorySearchConfig["outputDimensionality"];
|
||||
fallback: "none";
|
||||
local: ResolvedMemorySearchConfig["local"];
|
||||
} | null {
|
||||
const fallback = params.settings.fallback;
|
||||
if (
|
||||
!fallback ||
|
||||
fallback === "none" ||
|
||||
!params.currentProviderId ||
|
||||
fallback === params.currentProviderId
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
provider: fallback,
|
||||
model: resolveEmbeddingProviderFallbackModel(fallback, params.settings.model, params.cfg),
|
||||
remote: params.settings.remote,
|
||||
inputType: params.settings.inputType,
|
||||
queryInputType: params.settings.queryInputType,
|
||||
documentInputType: params.settings.documentInputType,
|
||||
outputDimensionality: params.settings.outputDimensionality,
|
||||
fallback: "none",
|
||||
local: params.settings.local,
|
||||
};
|
||||
}
|
||||
84
extensions/memory-core/src/memory/manager-reindex-lock.ts
Normal file
84
extensions/memory-core/src/memory/manager-reindex-lock.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
// Memory Core plugin module serializes full memory reindex builds across processes.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { requireNodeSqlite } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export type MemoryReindexLockHandle = {
|
||||
release: () => void;
|
||||
};
|
||||
|
||||
export function resolveMemoryReindexLockPath(dbPath: string): string {
|
||||
return `${dbPath}.reindex-lock.sqlite`;
|
||||
}
|
||||
|
||||
function isSqliteBusyError(err: unknown): boolean {
|
||||
const code = (err as { code?: unknown }).code;
|
||||
if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED") {
|
||||
return true;
|
||||
}
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return /SQLITE_(?:BUSY|LOCKED)|database is locked/i.test(message);
|
||||
}
|
||||
|
||||
function openMemoryLockDatabase(lockPath: string): DatabaseSync {
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const lockDb = new DatabaseSync(lockPath);
|
||||
try {
|
||||
lockDb.exec("PRAGMA busy_timeout = 0");
|
||||
return lockDb;
|
||||
} catch (err) {
|
||||
try {
|
||||
lockDb.close();
|
||||
} catch {}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function createMemoryReindexLockHandle(lockDb: DatabaseSync): MemoryReindexLockHandle {
|
||||
return {
|
||||
release: () => {
|
||||
let releaseError: unknown;
|
||||
try {
|
||||
lockDb.exec("ROLLBACK");
|
||||
} catch (err) {
|
||||
releaseError = err;
|
||||
}
|
||||
try {
|
||||
lockDb.close();
|
||||
} catch (err) {
|
||||
releaseError ??= err;
|
||||
}
|
||||
if (releaseError) {
|
||||
throw new Error("Failed to release memory reindex lock", { cause: releaseError });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Try to acquire the build lock without locking readers of the live agent database. */
|
||||
export function tryAcquireMemoryReindexLock(dbPath: string): MemoryReindexLockHandle | undefined {
|
||||
const lockDb = openMemoryLockDatabase(resolveMemoryReindexLockPath(dbPath));
|
||||
try {
|
||||
lockDb.exec("BEGIN EXCLUSIVE");
|
||||
} catch (err) {
|
||||
lockDb.close();
|
||||
if (isSqliteBusyError(err)) {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return createMemoryReindexLockHandle(lockDb);
|
||||
}
|
||||
|
||||
/** Acquire an exclusive build lock without locking readers of the live agent database. */
|
||||
export function acquireMemoryReindexLock(dbPath: string): MemoryReindexLockHandle {
|
||||
const lock = tryAcquireMemoryReindexLock(dbPath);
|
||||
if (lock) {
|
||||
return lock;
|
||||
}
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
`Memory reindex lock is held at ${resolveMemoryReindexLockPath(dbPath)}; another reindex is active.`,
|
||||
),
|
||||
{ code: "SQLITE_BUSY" },
|
||||
);
|
||||
}
|
||||
311
extensions/memory-core/src/memory/manager-reindex-state.test.ts
Normal file
311
extensions/memory-core/src/memory/manager-reindex-state.test.ts
Normal file
@@ -0,0 +1,311 @@
|
||||
// Memory Core tests cover manager reindex state plugin behavior.
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveConfiguredScopeHash,
|
||||
resolveConfiguredSourcesForMeta,
|
||||
resolveMemoryIndexProviderIdentities,
|
||||
resolveMemoryIndexIdentityState,
|
||||
isMemoryIndexIdentityDirty,
|
||||
type MemoryIndexMeta,
|
||||
} from "./manager-reindex-state.js";
|
||||
|
||||
function createMeta(overrides: Partial<MemoryIndexMeta> = {}): MemoryIndexMeta {
|
||||
return {
|
||||
model: "mock-embed-v1",
|
||||
provider: "openai",
|
||||
providerKey: "provider-key-v1",
|
||||
sources: ["memory"],
|
||||
scopeHash: "scope-v1",
|
||||
chunkTokens: 4000,
|
||||
chunkOverlap: 0,
|
||||
ftsTokenizer: "unicode61",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createIdentityParams(
|
||||
overrides: {
|
||||
meta?: MemoryIndexMeta | null;
|
||||
provider?: { id: string; model: string } | null;
|
||||
providerKey?: string;
|
||||
providerAliases?: Array<{ model: string; providerKey: string }>;
|
||||
providerKeyKnown?: boolean;
|
||||
configuredSources?: MemorySource[];
|
||||
configuredScopeHash?: string;
|
||||
chunkTokens?: number;
|
||||
chunkOverlap?: number;
|
||||
vectorReady?: boolean;
|
||||
hasIndexedChunks?: boolean;
|
||||
ftsTokenizer?: string;
|
||||
} = {},
|
||||
) {
|
||||
return {
|
||||
meta: createMeta(),
|
||||
provider: { id: "openai", model: "mock-embed-v1" },
|
||||
providerKey: "provider-key-v1",
|
||||
configuredSources: ["memory"] as MemorySource[],
|
||||
configuredScopeHash: "scope-v1",
|
||||
chunkTokens: 4000,
|
||||
chunkOverlap: 0,
|
||||
vectorReady: false,
|
||||
hasIndexedChunks: true,
|
||||
ftsTokenizer: "unicode61",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("memory reindex state", () => {
|
||||
it("retains the primary provider identity when its model is empty", () => {
|
||||
expect(
|
||||
resolveMemoryIndexProviderIdentities({
|
||||
provider: { id: "empty-model-provider", model: "" },
|
||||
}),
|
||||
).toMatchObject([{ provider: "empty-model-provider", model: "" }]);
|
||||
});
|
||||
|
||||
it("marks identity dirty when the embedding model changes", () => {
|
||||
expect(
|
||||
isMemoryIndexIdentityDirty(
|
||||
createIdentityParams({
|
||||
provider: { id: "openai", model: "mock-embed-v2" },
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns a mismatch reason when provider identity changes", () => {
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
provider: { id: "ollama", model: "mock-embed-v1" },
|
||||
providerKey: "provider-key-ollama",
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
status: "mismatched",
|
||||
reason: "index was built for provider openai, expected ollama",
|
||||
});
|
||||
});
|
||||
|
||||
it("marks identity dirty when the provider cache key changes", () => {
|
||||
expect(
|
||||
isMemoryIndexIdentityDirty(
|
||||
createIdentityParams({
|
||||
provider: { id: "gemini", model: "gemini-embedding-2-preview" },
|
||||
providerKey: "provider-key-dims-768",
|
||||
meta: createMeta({
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
providerKey: "provider-key-dims-3072",
|
||||
}),
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("can defer provider key comparison until provider initialization", () => {
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
providerKey: undefined,
|
||||
providerKeyKnown: false,
|
||||
}),
|
||||
),
|
||||
).toEqual({ status: "valid" });
|
||||
});
|
||||
|
||||
it("keeps model identity strict when paths share a basename", () => {
|
||||
const indexedModel = "/models/default/model.gguf";
|
||||
const currentModel = "/models/custom/model.gguf";
|
||||
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
provider: { id: "local", model: currentModel },
|
||||
providerKey: "provider-key-current",
|
||||
meta: createMeta({
|
||||
provider: "local",
|
||||
model: indexedModel,
|
||||
providerKey: "provider-key-indexed",
|
||||
vectorDims: 768,
|
||||
}),
|
||||
vectorReady: true,
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
status: "mismatched",
|
||||
reason: `index was built for model ${indexedModel}, expected ${currentModel}`,
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts only provider-declared model and provider-key alias pairs", () => {
|
||||
const alias = {
|
||||
model: "/models/default/model.gguf",
|
||||
providerKey: "provider-key-alias",
|
||||
};
|
||||
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
provider: { id: "local", model: "hf:owner/default/model.gguf" },
|
||||
providerKey: "provider-key-current",
|
||||
providerAliases: [alias],
|
||||
meta: createMeta({
|
||||
provider: "local",
|
||||
model: alias.model,
|
||||
providerKey: alias.providerKey,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
).toEqual({ status: "valid" });
|
||||
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
provider: { id: "local", model: "hf:owner/default/model.gguf" },
|
||||
providerKey: "provider-key-current",
|
||||
providerAliases: [alias],
|
||||
meta: createMeta({
|
||||
provider: "local",
|
||||
model: alias.model,
|
||||
providerKey: "provider-key-arbitrary",
|
||||
}),
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
status: "mismatched",
|
||||
reason: "index provider settings changed",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not mark identity dirty for vector dimensions before chunks exist", () => {
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
vectorReady: true,
|
||||
hasIndexedChunks: false,
|
||||
meta: createMeta({ vectorDims: undefined }),
|
||||
}),
|
||||
),
|
||||
).toEqual({ status: "valid" });
|
||||
});
|
||||
|
||||
it("marks identity dirty when extraPaths change", () => {
|
||||
const workspaceDir = "/tmp/workspace";
|
||||
const firstScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/a"],
|
||||
multimodal: {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
const secondScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/b"],
|
||||
multimodal: {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
isMemoryIndexIdentityDirty(
|
||||
createIdentityParams({
|
||||
meta: createMeta({ scopeHash: firstScopeHash }),
|
||||
configuredScopeHash: secondScopeHash,
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("marks identity dirty when configured sources add sessions", () => {
|
||||
expect(
|
||||
isMemoryIndexIdentityDirty(
|
||||
createIdentityParams({
|
||||
configuredSources: ["memory", "sessions"],
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("marks identity dirty when multimodal settings change", () => {
|
||||
const workspaceDir = "/tmp/workspace";
|
||||
const firstScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/media"],
|
||||
multimodal: {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
const secondScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/media"],
|
||||
multimodal: {
|
||||
enabled: true,
|
||||
modalities: ["image"],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
isMemoryIndexIdentityDirty(
|
||||
createIdentityParams({
|
||||
meta: createMeta({ scopeHash: firstScopeHash }),
|
||||
configuredScopeHash: secondScopeHash,
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps older indexes with missing sources compatible with memory-only config", () => {
|
||||
expect(
|
||||
isMemoryIndexIdentityDirty(
|
||||
createIdentityParams({
|
||||
meta: createMeta({ sources: undefined }),
|
||||
configuredSources: resolveConfiguredSourcesForMeta(new Set(["memory"])),
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back to fts-only when provider.model is an empty string", () => {
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
provider: { id: "openai", model: "" },
|
||||
meta: createMeta({ model: "fts-only" }),
|
||||
}),
|
||||
),
|
||||
).toEqual({ status: "valid" });
|
||||
});
|
||||
|
||||
it("reports mismatch when empty-string expected model is compared to a non-fts index", () => {
|
||||
const state = resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
provider: { id: "openai", model: "" },
|
||||
meta: createMeta({ model: "text-embedding-3-small" }),
|
||||
}),
|
||||
);
|
||||
expect(state.status).toBe("mismatched");
|
||||
if (state.status === "mismatched") {
|
||||
expect(state.reason).toContain("expected fts-only");
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to fts-only when provider.model is whitespace-only", () => {
|
||||
expect(
|
||||
resolveMemoryIndexIdentityState(
|
||||
createIdentityParams({
|
||||
provider: { id: "openai", model: " " },
|
||||
meta: createMeta({ model: "fts-only" }),
|
||||
}),
|
||||
),
|
||||
).toEqual({ status: "valid" });
|
||||
});
|
||||
});
|
||||
225
extensions/memory-core/src/memory/manager-reindex-state.ts
Normal file
225
extensions/memory-core/src/memory/manager-reindex-state.ts
Normal file
@@ -0,0 +1,225 @@
|
||||
// Memory Core plugin module implements manager reindex state behavior.
|
||||
import {
|
||||
hashText,
|
||||
normalizeExtraMemoryPaths,
|
||||
type MemorySource,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export type MemoryIndexMeta = {
|
||||
model: string;
|
||||
provider: string;
|
||||
providerKey?: string;
|
||||
sources?: MemorySource[];
|
||||
scopeHash?: string;
|
||||
chunkTokens: number;
|
||||
chunkOverlap: number;
|
||||
vectorDims?: number;
|
||||
ftsTokenizer?: string;
|
||||
};
|
||||
|
||||
export type MemoryIndexIdentityState =
|
||||
| {
|
||||
status: "valid";
|
||||
}
|
||||
| {
|
||||
status: "missing";
|
||||
reason: string;
|
||||
}
|
||||
| {
|
||||
status: "mismatched";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export type MemoryIndexProviderIdentity = {
|
||||
provider: string;
|
||||
model: string;
|
||||
providerKey: string;
|
||||
};
|
||||
|
||||
export function resolveMemoryIndexProviderIdentities(params: {
|
||||
provider: { id: string; model: string } | null;
|
||||
cacheKeyData?: Record<string, unknown>;
|
||||
aliases?: Array<{ model: string; cacheKeyData: Record<string, unknown> }>;
|
||||
}): MemoryIndexProviderIdentity[] {
|
||||
const provider = params.provider ?? { id: "none", model: "fts-only" };
|
||||
const candidates = [
|
||||
{
|
||||
model: provider.model,
|
||||
cacheKeyData: params.cacheKeyData ?? { provider: provider.id, model: provider.model },
|
||||
},
|
||||
...(params.provider ? (params.aliases ?? []) : []),
|
||||
];
|
||||
const seen = new Set<string>();
|
||||
const identities: MemoryIndexProviderIdentity[] = [];
|
||||
for (const [index, candidate] of candidates.entries()) {
|
||||
const providerKey = hashText(JSON.stringify(candidate.cacheKeyData));
|
||||
const key = `${candidate.model}\u0000${providerKey}`;
|
||||
if ((index > 0 && !candidate.model) || seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
identities.push({
|
||||
provider: provider.id,
|
||||
model: candidate.model,
|
||||
providerKey,
|
||||
});
|
||||
}
|
||||
return identities;
|
||||
}
|
||||
|
||||
export function resolveConfiguredSourcesForMeta(sources: Iterable<MemorySource>): MemorySource[] {
|
||||
const normalized = Array.from(sources)
|
||||
.filter((source): source is MemorySource => source === "memory" || source === "sessions")
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
return normalized.length > 0 ? normalized : ["memory"];
|
||||
}
|
||||
|
||||
function normalizeMetaSources(meta: MemoryIndexMeta): MemorySource[] {
|
||||
if (!Array.isArray(meta.sources)) {
|
||||
// Backward compatibility for older indexes that did not persist sources.
|
||||
return ["memory"];
|
||||
}
|
||||
const normalized = Array.from(
|
||||
new Set(
|
||||
meta.sources.filter(
|
||||
(source): source is MemorySource => source === "memory" || source === "sessions",
|
||||
),
|
||||
),
|
||||
).toSorted((left, right) => left.localeCompare(right));
|
||||
return normalized.length > 0 ? normalized : ["memory"];
|
||||
}
|
||||
|
||||
function configuredMetaSourcesDiffer(params: {
|
||||
meta: MemoryIndexMeta;
|
||||
configuredSources: MemorySource[];
|
||||
}): boolean {
|
||||
const metaSources = normalizeMetaSources(params.meta);
|
||||
if (metaSources.length !== params.configuredSources.length) {
|
||||
return true;
|
||||
}
|
||||
return metaSources.some((source, index) => source !== params.configuredSources[index]);
|
||||
}
|
||||
|
||||
export function resolveConfiguredScopeHash(params: {
|
||||
workspaceDir: string;
|
||||
extraPaths?: string[];
|
||||
multimodal: {
|
||||
enabled: boolean;
|
||||
modalities: string[];
|
||||
maxFileBytes: number;
|
||||
};
|
||||
}): string {
|
||||
const extraPaths = normalizeExtraMemoryPaths(params.workspaceDir, params.extraPaths)
|
||||
.map((value) => value.replace(/\\/g, "/"))
|
||||
.toSorted();
|
||||
return hashText(
|
||||
JSON.stringify({
|
||||
extraPaths,
|
||||
multimodal: {
|
||||
enabled: params.multimodal.enabled,
|
||||
modalities: [...params.multimodal.modalities].toSorted(),
|
||||
maxFileBytes: params.multimodal.maxFileBytes,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function isMemoryIndexIdentityDirty(params: {
|
||||
meta: MemoryIndexMeta | null;
|
||||
provider: { id: string; model: string } | null;
|
||||
providerKey?: string;
|
||||
providerAliases?: Array<Pick<MemoryIndexProviderIdentity, "model" | "providerKey">>;
|
||||
providerKeyKnown?: boolean;
|
||||
configuredSources: MemorySource[];
|
||||
configuredScopeHash: string;
|
||||
chunkTokens: number;
|
||||
chunkOverlap: number;
|
||||
vectorReady: boolean;
|
||||
hasIndexedChunks?: boolean;
|
||||
ftsTokenizer: string;
|
||||
}): boolean {
|
||||
return resolveMemoryIndexIdentityState(params).status !== "valid";
|
||||
}
|
||||
|
||||
export function resolveMemoryIndexIdentityState(params: {
|
||||
meta: MemoryIndexMeta | null;
|
||||
provider: { id: string; model: string } | null;
|
||||
providerKey?: string;
|
||||
providerAliases?: Array<Pick<MemoryIndexProviderIdentity, "model" | "providerKey">>;
|
||||
providerKeyKnown?: boolean;
|
||||
configuredSources: MemorySource[];
|
||||
configuredScopeHash: string;
|
||||
chunkTokens: number;
|
||||
chunkOverlap: number;
|
||||
vectorReady: boolean;
|
||||
hasIndexedChunks?: boolean;
|
||||
ftsTokenizer: string;
|
||||
}): MemoryIndexIdentityState {
|
||||
const { meta } = params;
|
||||
if (!meta) {
|
||||
return { status: "missing", reason: "index metadata is missing" };
|
||||
}
|
||||
const expectedModel = params.provider?.model?.trim() || "fts-only";
|
||||
const matchingModelIdentities = [
|
||||
{ model: expectedModel, providerKey: params.providerKey },
|
||||
...(params.providerAliases ?? []),
|
||||
].filter((identity) => identity.model === meta.model);
|
||||
if (matchingModelIdentities.length === 0) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: `index was built for model ${meta.model}, expected ${expectedModel}`,
|
||||
};
|
||||
}
|
||||
const expectedProvider = params.provider ? params.provider.id : "none";
|
||||
if (meta.provider !== expectedProvider) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: `index was built for provider ${meta.provider}, expected ${expectedProvider}`,
|
||||
};
|
||||
}
|
||||
if (
|
||||
params.providerKeyKnown !== false &&
|
||||
!matchingModelIdentities.some((identity) => identity.providerKey === meta.providerKey)
|
||||
) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: "index provider settings changed",
|
||||
};
|
||||
}
|
||||
if (
|
||||
configuredMetaSourcesDiffer({
|
||||
meta,
|
||||
configuredSources: params.configuredSources,
|
||||
})
|
||||
) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: "index sources changed",
|
||||
};
|
||||
}
|
||||
if (meta.scopeHash !== params.configuredScopeHash) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: "index scope changed",
|
||||
};
|
||||
}
|
||||
if (meta.chunkTokens !== params.chunkTokens || meta.chunkOverlap !== params.chunkOverlap) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: "index chunking changed",
|
||||
};
|
||||
}
|
||||
if (params.vectorReady && params.hasIndexedChunks !== false && !meta.vectorDims) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: "index vector dimensions are missing",
|
||||
};
|
||||
}
|
||||
if ((meta.ftsTokenizer ?? "unicode61") !== params.ftsTokenizer) {
|
||||
return {
|
||||
status: "mismatched",
|
||||
reason: "index FTS tokenizer changed",
|
||||
};
|
||||
}
|
||||
return { status: "valid" };
|
||||
}
|
||||
6
extensions/memory-core/src/memory/manager-runtime.ts
Normal file
6
extensions/memory-core/src/memory/manager-runtime.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
// Memory Core plugin module implements manager runtime behavior.
|
||||
export {
|
||||
closeAllMemoryIndexManagers,
|
||||
closeMemoryIndexManagersForAgent,
|
||||
MemoryIndexManager,
|
||||
} from "./manager.js";
|
||||
@@ -0,0 +1,44 @@
|
||||
// Memory Core tests cover manager search preflight plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveMemorySearchPreflight } from "./manager-search-preflight.js";
|
||||
|
||||
describe("memory manager search preflight", () => {
|
||||
it("skips search and provider init for blank queries", () => {
|
||||
expect(
|
||||
resolveMemorySearchPreflight({
|
||||
query: " ",
|
||||
hasIndexedContent: true,
|
||||
}),
|
||||
).toEqual({
|
||||
normalizedQuery: "",
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("skips provider init when the index is empty", () => {
|
||||
expect(
|
||||
resolveMemorySearchPreflight({
|
||||
query: "hello",
|
||||
hasIndexedContent: false,
|
||||
}),
|
||||
).toEqual({
|
||||
normalizedQuery: "hello",
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("allows provider init when query and indexed content are present", () => {
|
||||
expect(
|
||||
resolveMemorySearchPreflight({
|
||||
query: " hello ",
|
||||
hasIndexedContent: true,
|
||||
}),
|
||||
).toEqual({
|
||||
normalizedQuery: "hello",
|
||||
shouldInitializeProvider: true,
|
||||
shouldSearch: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
// Memory Core plugin module implements manager search preflight behavior.
|
||||
export function resolveMemorySearchPreflight(params: {
|
||||
query: string;
|
||||
hasIndexedContent: boolean;
|
||||
}):
|
||||
| {
|
||||
normalizedQuery: string;
|
||||
shouldInitializeProvider: boolean;
|
||||
shouldSearch: true;
|
||||
}
|
||||
| {
|
||||
normalizedQuery: string;
|
||||
shouldInitializeProvider: false;
|
||||
shouldSearch: false;
|
||||
} {
|
||||
const normalizedQuery = params.query.trim();
|
||||
if (!normalizedQuery) {
|
||||
return {
|
||||
normalizedQuery,
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
};
|
||||
}
|
||||
if (!params.hasIndexedContent) {
|
||||
return {
|
||||
normalizedQuery,
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
normalizedQuery,
|
||||
shouldInitializeProvider: true,
|
||||
shouldSearch: true,
|
||||
};
|
||||
}
|
||||
1066
extensions/memory-core/src/memory/manager-search.test.ts
Normal file
1066
extensions/memory-core/src/memory/manager-search.test.ts
Normal file
File diff suppressed because it is too large
Load Diff
436
extensions/memory-core/src/memory/manager-search.ts
Normal file
436
extensions/memory-core/src/memory/manager-search.ts
Normal file
@@ -0,0 +1,436 @@
|
||||
// Memory Core plugin module implements manager search behavior.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { truncateUtf16Safe } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import {
|
||||
cosineSimilarity,
|
||||
parseEmbedding,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import {
|
||||
normalizeStringEntries,
|
||||
normalizeStringEntriesLower,
|
||||
uniqueStrings,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { vectorToBlob } from "./vector-blob.js";
|
||||
|
||||
const FTS_QUERY_TOKEN_RE = /[\p{L}\p{N}_]+/gu;
|
||||
const SHORT_CJK_TRIGRAM_RE = /[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\u3131-\u3163]/u;
|
||||
const VECTOR_KNN_OVERSAMPLE_FACTOR = 8;
|
||||
|
||||
// Scan fallback vector rows in bounded batches so large chunk tables (no usable
|
||||
// vec0 index) cannot pin the main thread for multi-second windows and starve
|
||||
// channel I/O / liveness signals. Matches the session-indexing yield pattern
|
||||
// introduced in #76978 for the same class of bug. Issue #81172.
|
||||
const FALLBACK_VECTOR_BATCH_SIZE = 256;
|
||||
|
||||
function yieldToEventLoop(): Promise<void> {
|
||||
return new Promise<void>((resolve) => {
|
||||
setImmediate(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
type SearchSource = string;
|
||||
|
||||
type SearchRowResult = {
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: SearchSource;
|
||||
};
|
||||
|
||||
function normalizeSearchTokens(raw: string): string[] {
|
||||
return normalizeStringEntriesLower(raw.match(FTS_QUERY_TOKEN_RE) ?? []);
|
||||
}
|
||||
|
||||
function scoreFallbackKeywordResult(params: {
|
||||
query: string;
|
||||
path: string;
|
||||
text: string;
|
||||
ftsScore: number;
|
||||
}): number {
|
||||
const queryTokens = uniqueStrings(normalizeSearchTokens(params.query));
|
||||
if (queryTokens.length === 0) {
|
||||
return params.ftsScore;
|
||||
}
|
||||
|
||||
const textTokens = normalizeSearchTokens(params.text);
|
||||
const textTokenSet = new Set(textTokens);
|
||||
const pathLower = params.path.toLowerCase();
|
||||
const overlap = queryTokens.filter((token) => textTokenSet.has(token)).length;
|
||||
const uniqueQueryOverlap = overlap / Math.max(new Set(queryTokens).size, 1);
|
||||
const density = overlap / Math.max(textTokenSet.size, 1);
|
||||
const pathBoost = queryTokens.reduce(
|
||||
(score, token) => score + (pathLower.includes(token) ? 0.18 : 0),
|
||||
0,
|
||||
);
|
||||
const textLengthBoost = Math.min(params.text.length / 160, 0.18);
|
||||
|
||||
const lexicalBoost = uniqueQueryOverlap * 0.45 + density * 0.2 + pathBoost + textLengthBoost;
|
||||
return Math.min(1, params.ftsScore + lexicalBoost);
|
||||
}
|
||||
|
||||
function escapeLikePattern(term: string): string {
|
||||
return term.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
|
||||
}
|
||||
|
||||
function buildMatchQueryFromTerms(terms: string[]): string | null {
|
||||
if (terms.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const quoted = terms.map((term) => `"${term.replaceAll('"', "")}"`);
|
||||
return quoted.join(" AND ");
|
||||
}
|
||||
|
||||
function readCount(row: { count?: number | bigint } | undefined): number {
|
||||
if (typeof row?.count === "bigint") {
|
||||
return Number(row.count);
|
||||
}
|
||||
if (typeof row?.count === "number") {
|
||||
return row.count;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function resolveProviderModels(primary: string, aliases: string[] | undefined): string[] {
|
||||
return Array.from(new Set([primary, ...(aliases ?? []).filter(Boolean)]));
|
||||
}
|
||||
|
||||
function buildModelFilter(column: string, models: string[]): string {
|
||||
return models.length === 1
|
||||
? `${column} = ?`
|
||||
: `${column} IN (${models.map(() => "?").join(", ")})`;
|
||||
}
|
||||
|
||||
function planKeywordSearch(params: {
|
||||
query: string;
|
||||
ftsTokenizer?: "unicode61" | "trigram";
|
||||
buildFtsQuery: (raw: string) => string | null;
|
||||
}): { matchQuery: string | null; substringTerms: string[] } {
|
||||
if (params.ftsTokenizer !== "trigram") {
|
||||
return {
|
||||
matchQuery: params.buildFtsQuery(params.query),
|
||||
substringTerms: [],
|
||||
};
|
||||
}
|
||||
|
||||
const tokens = normalizeStringEntries(params.query.match(FTS_QUERY_TOKEN_RE) ?? []);
|
||||
if (tokens.length === 0) {
|
||||
return { matchQuery: null, substringTerms: [] };
|
||||
}
|
||||
|
||||
const matchTerms: string[] = [];
|
||||
const substringTerms: string[] = [];
|
||||
for (const token of tokens) {
|
||||
if (SHORT_CJK_TRIGRAM_RE.test(token) && Array.from(token).length < 3) {
|
||||
substringTerms.push(token);
|
||||
continue;
|
||||
}
|
||||
matchTerms.push(token);
|
||||
}
|
||||
|
||||
return {
|
||||
matchQuery: buildMatchQueryFromTerms(matchTerms),
|
||||
substringTerms,
|
||||
};
|
||||
}
|
||||
|
||||
export async function searchVector(params: {
|
||||
db: DatabaseSync;
|
||||
vectorTable: string;
|
||||
providerModel: string;
|
||||
providerModelAliases?: string[];
|
||||
queryVec: number[];
|
||||
limit: number;
|
||||
snippetMaxChars: number;
|
||||
ensureVectorReady: (dimensions: number) => Promise<boolean>;
|
||||
sourceFilterVec: { sql: string; params: SearchSource[] };
|
||||
sourceFilterChunks: { sql: string; params: SearchSource[] };
|
||||
}): Promise<SearchRowResult[]> {
|
||||
if (params.queryVec.length === 0 || params.limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
const providerModels = resolveProviderModels(params.providerModel, params.providerModelAliases);
|
||||
const vectorModelFilter = buildModelFilter("c.model", providerModels);
|
||||
if (await params.ensureVectorReady(params.queryVec.length)) {
|
||||
// Use sqlite-vec's native KNN (MATCH ? AND k = ?) for candidate selection,
|
||||
// which runs in ~O(log N + k) via the vec0 index, instead of the previous
|
||||
// full-table scan over vec_distance_cosine(). Keep vec_distance_cosine() in
|
||||
// the SELECT so `score = 1 - dist` stays in the cosine [0, 1] range the
|
||||
// downstream merge/minScore pipeline expects. (memory_index_chunks_vec is created with
|
||||
// sqlite-vec's default L2 distance, so v.distance cannot be used directly
|
||||
// for scoring.)
|
||||
const qBlob = vectorToBlob(params.queryVec);
|
||||
const runVectorQuery = (candidateLimit: number) =>
|
||||
params.db
|
||||
.prepare(
|
||||
`SELECT c.id, c.path, c.start_line, c.end_line, c.text,\n` +
|
||||
` c.source,\n` +
|
||||
` vec_distance_cosine(v.embedding, ?) AS dist\n` +
|
||||
` FROM ${params.vectorTable} v\n` +
|
||||
` JOIN memory_index_chunks c ON c.id = v.id\n` +
|
||||
` WHERE v.embedding MATCH ? AND k = ? AND ${vectorModelFilter}${params.sourceFilterVec.sql}\n` +
|
||||
` ORDER BY dist ASC\n` +
|
||||
` LIMIT ?`,
|
||||
)
|
||||
.all(
|
||||
qBlob,
|
||||
qBlob,
|
||||
candidateLimit,
|
||||
...providerModels,
|
||||
...params.sourceFilterVec.params,
|
||||
params.limit,
|
||||
) as Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
text: string;
|
||||
source: SearchSource;
|
||||
dist: number;
|
||||
}>;
|
||||
|
||||
const candidateLimit = params.limit * VECTOR_KNN_OVERSAMPLE_FACTOR;
|
||||
let rows = runVectorQuery(candidateLimit);
|
||||
if (rows.length < params.limit) {
|
||||
const matchingChunkCount = readCount(
|
||||
params.db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM memory_index_chunks c WHERE ${vectorModelFilter}${params.sourceFilterVec.sql}`,
|
||||
)
|
||||
.get(...providerModels, ...params.sourceFilterVec.params) as
|
||||
| { count?: number | bigint }
|
||||
| undefined,
|
||||
);
|
||||
if (matchingChunkCount > rows.length) {
|
||||
const vectorCount = readCount(
|
||||
params.db.prepare(`SELECT COUNT(*) AS count FROM ${params.vectorTable}`).get() as
|
||||
| { count?: number | bigint }
|
||||
| undefined,
|
||||
);
|
||||
if (vectorCount > candidateLimit) {
|
||||
rows = runVectorQuery(vectorCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
path: row.path,
|
||||
startLine: row.start_line,
|
||||
endLine: row.end_line,
|
||||
score: 1 - row.dist,
|
||||
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
||||
source: row.source,
|
||||
}));
|
||||
}
|
||||
|
||||
return await searchChunksByEmbedding({
|
||||
db: params.db,
|
||||
providerModel: params.providerModel,
|
||||
providerModelAliases: params.providerModelAliases,
|
||||
sourceFilter: params.sourceFilterChunks,
|
||||
queryVec: params.queryVec,
|
||||
limit: params.limit,
|
||||
snippetMaxChars: params.snippetMaxChars,
|
||||
});
|
||||
}
|
||||
|
||||
async function searchChunksByEmbedding(params: {
|
||||
db: DatabaseSync;
|
||||
providerModel: string;
|
||||
providerModelAliases?: string[];
|
||||
sourceFilter: { sql: string; params: SearchSource[] };
|
||||
queryVec: number[];
|
||||
limit: number;
|
||||
snippetMaxChars: number;
|
||||
}): Promise<SearchRowResult[]> {
|
||||
if (params.limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
const providerModels = resolveProviderModels(params.providerModel, params.providerModelAliases);
|
||||
const modelFilter = buildModelFilter("model", providerModels);
|
||||
// Keep batches bounded instead of calling `.all()` across the entire chunks
|
||||
// table, and do not hold a sqlite iterator open across the setImmediate yield
|
||||
// below. The rowid cursor keeps memory bounded without OFFSET rescans.
|
||||
const stmt = params.db.prepare(
|
||||
`SELECT rowid, id, path, start_line, end_line, text, embedding, source\n` +
|
||||
` FROM memory_index_chunks\n` +
|
||||
` WHERE ${modelFilter} AND rowid > ?${params.sourceFilter.sql}\n` +
|
||||
` ORDER BY rowid ASC\n` +
|
||||
` LIMIT ?`,
|
||||
);
|
||||
type ChunkEmbeddingRow = {
|
||||
rowid: number | bigint;
|
||||
id: string;
|
||||
path: string;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
text: string;
|
||||
embedding: string;
|
||||
source: SearchSource;
|
||||
};
|
||||
|
||||
const topResults: SearchRowResult[] = [];
|
||||
let lastRowid = 0;
|
||||
while (true) {
|
||||
const batch = stmt.all(
|
||||
...providerModels,
|
||||
lastRowid,
|
||||
...params.sourceFilter.params,
|
||||
FALLBACK_VECTOR_BATCH_SIZE,
|
||||
) as ChunkEmbeddingRow[];
|
||||
if (batch.length === 0) {
|
||||
break;
|
||||
}
|
||||
for (const row of batch) {
|
||||
const score = cosineSimilarity(params.queryVec, parseEmbedding(row.embedding));
|
||||
if (Number.isFinite(score)) {
|
||||
const result: SearchRowResult = {
|
||||
id: row.id,
|
||||
path: row.path,
|
||||
startLine: row.start_line,
|
||||
endLine: row.end_line,
|
||||
score,
|
||||
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
||||
source: row.source,
|
||||
};
|
||||
if (topResults.length < params.limit) {
|
||||
topResults.push(result);
|
||||
if (topResults.length === params.limit) {
|
||||
topResults.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
} else {
|
||||
const lowest = topResults.at(-1);
|
||||
if (lowest && result.score > lowest.score) {
|
||||
topResults[topResults.length - 1] = result;
|
||||
topResults.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const nextRowid = batch.at(-1)?.rowid;
|
||||
lastRowid = typeof nextRowid === "bigint" ? Number(nextRowid) : (nextRowid ?? lastRowid);
|
||||
if (batch.length < FALLBACK_VECTOR_BATCH_SIZE) {
|
||||
break;
|
||||
}
|
||||
await yieldToEventLoop();
|
||||
}
|
||||
topResults.sort((a, b) => b.score - a.score);
|
||||
return topResults;
|
||||
}
|
||||
|
||||
export async function searchKeyword(params: {
|
||||
db: DatabaseSync;
|
||||
ftsTable: string;
|
||||
query: string;
|
||||
ftsTokenizer?: "unicode61" | "trigram";
|
||||
limit: number;
|
||||
snippetMaxChars: number;
|
||||
sourceFilter: { sql: string; params: SearchSource[] };
|
||||
buildFtsQuery: (raw: string) => string | null;
|
||||
bm25RankToScore: (rank: number) => number;
|
||||
boostFallbackRanking?: boolean;
|
||||
}): Promise<Array<SearchRowResult & { textScore: number }>> {
|
||||
if (params.limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
const plan = planKeywordSearch({
|
||||
query: params.query,
|
||||
ftsTokenizer: params.ftsTokenizer,
|
||||
buildFtsQuery: params.buildFtsQuery,
|
||||
});
|
||||
if (!plan.matchQuery && plan.substringTerms.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Lexical FTS is model-agnostic (issue #48300), but old databases may
|
||||
// already contain orphaned FTS rows from prior model-scoped cleanup.
|
||||
const liveChunkClause = ` AND EXISTS (SELECT 1 FROM memory_index_chunks c WHERE c.id = ${params.ftsTable}.id)`;
|
||||
const substringClause = plan.substringTerms.map(() => " AND text LIKE ? ESCAPE '\\'").join("");
|
||||
const substringParams = plan.substringTerms.map((term) => `%${escapeLikePattern(term)}%`);
|
||||
|
||||
let rows: Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
source: SearchSource;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
text: string;
|
||||
rank: number;
|
||||
}>;
|
||||
let usedMatch = false;
|
||||
|
||||
if (plan.matchQuery) {
|
||||
try {
|
||||
rows = params.db
|
||||
.prepare(
|
||||
`SELECT id, path, source, start_line, end_line, text,\n` +
|
||||
` bm25(${params.ftsTable}) AS rank\n` +
|
||||
` FROM ${params.ftsTable}\n` +
|
||||
` WHERE ${params.ftsTable} MATCH ?${substringClause}${liveChunkClause}${params.sourceFilter.sql}\n` +
|
||||
` ORDER BY rank ASC\n` +
|
||||
` LIMIT ?`,
|
||||
)
|
||||
.all(
|
||||
plan.matchQuery,
|
||||
...substringParams,
|
||||
...params.sourceFilter.params,
|
||||
params.limit,
|
||||
) as typeof rows;
|
||||
usedMatch = true;
|
||||
} catch (matchErr) {
|
||||
// FTS5 MATCH can fail on certain token patterns depending on the
|
||||
// Node.js sqlite runtime and tokenizer (e.g. unicode61 vs trigram).
|
||||
// Log the root cause, then fall back to per-token LIKE-based substring
|
||||
// search so results are still returned instead of being silently dropped.
|
||||
console.warn(`memory search: FTS5 MATCH failed, falling back to LIKE: ${String(matchErr)}`);
|
||||
const queryTokens = normalizeStringEntries(params.query.match(FTS_QUERY_TOKEN_RE) ?? []);
|
||||
const allTerms = uniqueStrings([...queryTokens, ...plan.substringTerms]);
|
||||
const fallbackLikeClause = allTerms.map(() => " AND text LIKE ? ESCAPE '\\'").join("");
|
||||
const fallbackLikeParams = allTerms.map((term) => `%${escapeLikePattern(term)}%`);
|
||||
rows = params.db
|
||||
.prepare(
|
||||
`SELECT id, path, source, start_line, end_line, text,\n` +
|
||||
` 0 AS rank\n` +
|
||||
` FROM ${params.ftsTable}\n` +
|
||||
` WHERE 1=1${fallbackLikeClause}${liveChunkClause}${params.sourceFilter.sql}\n` +
|
||||
` LIMIT ?`,
|
||||
)
|
||||
.all(...fallbackLikeParams, ...params.sourceFilter.params, params.limit) as typeof rows;
|
||||
}
|
||||
} else {
|
||||
rows = params.db
|
||||
.prepare(
|
||||
`SELECT id, path, source, start_line, end_line, text,\n` +
|
||||
` 0 AS rank\n` +
|
||||
` FROM ${params.ftsTable}\n` +
|
||||
` WHERE 1=1${substringClause}${liveChunkClause}${params.sourceFilter.sql}\n` +
|
||||
` LIMIT ?`,
|
||||
)
|
||||
.all(...substringParams, ...params.sourceFilter.params, params.limit) as typeof rows;
|
||||
}
|
||||
|
||||
return rows.map((row) => {
|
||||
const textScore = usedMatch ? params.bm25RankToScore(row.rank) : 1;
|
||||
const score = params.boostFallbackRanking
|
||||
? scoreFallbackKeywordResult({
|
||||
query: params.query,
|
||||
path: row.path,
|
||||
text: row.text,
|
||||
ftsScore: textScore,
|
||||
})
|
||||
: textScore;
|
||||
return {
|
||||
id: row.id,
|
||||
path: row.path,
|
||||
startLine: row.start_line,
|
||||
endLine: row.end_line,
|
||||
score,
|
||||
textScore,
|
||||
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
||||
source: row.source,
|
||||
};
|
||||
});
|
||||
}
|
||||
35
extensions/memory-core/src/memory/manager-session-reindex.ts
Normal file
35
extensions/memory-core/src/memory/manager-session-reindex.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
// Memory Core plugin module implements manager session reindex behavior.
|
||||
import type { MemorySyncParams } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export function shouldSyncSessionsForReindex(params: {
|
||||
hasSessionSource: boolean;
|
||||
sessionsDirty: boolean;
|
||||
sessionsFullRetryDirty?: boolean;
|
||||
dirtySessionFileCount: number;
|
||||
sync?: MemorySyncParams;
|
||||
needsFullReindex?: boolean;
|
||||
}): boolean {
|
||||
if (!params.hasSessionSource) {
|
||||
return false;
|
||||
}
|
||||
if (params.sync?.sessions?.some((session) => session.sessionId.trim().length > 0)) {
|
||||
return true;
|
||||
}
|
||||
if (params.sync?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) {
|
||||
return true;
|
||||
}
|
||||
if (params.sync?.force) {
|
||||
return true;
|
||||
}
|
||||
if (params.needsFullReindex) {
|
||||
return true;
|
||||
}
|
||||
if (params.sessionsFullRetryDirty) {
|
||||
return true;
|
||||
}
|
||||
const reason = params.sync?.reason;
|
||||
if (reason === "session-start" || reason === "watch") {
|
||||
return false;
|
||||
}
|
||||
return params.sessionsDirty && params.dirtySessionFileCount > 0;
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Memory Core tests cover manager session sync state plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveMemorySessionStartupDirtyFiles,
|
||||
resolveMemorySessionSyncPlan,
|
||||
} from "./manager-session-sync-state.js";
|
||||
|
||||
describe("memory session sync state", () => {
|
||||
it("tracks active paths and bulk hashes for full scans", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
files: ["/tmp/a.jsonl", "/tmp/b.jsonl"],
|
||||
targetSessionFiles: null,
|
||||
sessionsDirtyFiles: new Set(),
|
||||
existingRows: [
|
||||
{ path: "sessions/a.jsonl", hash: "hash-a" },
|
||||
{ path: "sessions/b.jsonl", hash: "hash-b" },
|
||||
],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(true);
|
||||
expect(plan.activePaths).toEqual(new Set(["sessions/a.jsonl", "sessions/b.jsonl"]));
|
||||
expect(plan.existingRows).toEqual([
|
||||
{ path: "sessions/a.jsonl", hash: "hash-a" },
|
||||
{ path: "sessions/b.jsonl", hash: "hash-b" },
|
||||
]);
|
||||
expect(plan.existingHashes).toEqual(
|
||||
new Map([
|
||||
["sessions/a.jsonl", "hash-a"],
|
||||
["sessions/b.jsonl", "hash-b"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("treats targeted session syncs as refresh-only and skips unrelated pruning", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
files: ["/tmp/targeted-first.jsonl"],
|
||||
targetSessionFiles: new Set(["/tmp/targeted-first.jsonl"]),
|
||||
sessionsDirtyFiles: new Set(["/tmp/targeted-first.jsonl"]),
|
||||
existingRows: [
|
||||
{ path: "sessions/targeted-first.jsonl", hash: "hash-first" },
|
||||
{ path: "sessions/targeted-second.jsonl", hash: "hash-second" },
|
||||
],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(true);
|
||||
expect(plan.activePaths).toBeNull();
|
||||
expect(plan.existingRows).toBeNull();
|
||||
expect(plan.existingHashes).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps dirty-only incremental mode when no targeted sync is requested", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
files: ["/tmp/incremental.jsonl"],
|
||||
targetSessionFiles: null,
|
||||
sessionsDirtyFiles: new Set(["/tmp/incremental.jsonl"]),
|
||||
existingRows: [],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(false);
|
||||
expect(plan.activePaths).toEqual(new Set(["sessions/incremental.jsonl"]));
|
||||
});
|
||||
|
||||
it("marks identity-targeted syncs as session work", async () => {
|
||||
const { shouldSyncSessionsForReindex } = await import("./manager-session-reindex.js");
|
||||
|
||||
expect(
|
||||
shouldSyncSessionsForReindex({
|
||||
hasSessionSource: true,
|
||||
sessionsDirty: false,
|
||||
dirtySessionFileCount: 0,
|
||||
sync: { sessions: [{ agentId: "main", sessionId: "targeted" }] },
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("marks missing and changed startup session files dirty", () => {
|
||||
const dirtyFiles = resolveMemorySessionStartupDirtyFiles({
|
||||
files: [
|
||||
{
|
||||
absPath: "/tmp/sessions/unchanged.jsonl",
|
||||
path: "sessions/unchanged.jsonl",
|
||||
mtimeMs: 100,
|
||||
size: 10,
|
||||
},
|
||||
{
|
||||
absPath: "/tmp/sessions/newer.jsonl",
|
||||
path: "sessions/newer.jsonl",
|
||||
mtimeMs: 250,
|
||||
size: 20,
|
||||
},
|
||||
{
|
||||
absPath: "/tmp/sessions/resized.jsonl",
|
||||
path: "sessions/resized.jsonl",
|
||||
mtimeMs: 300,
|
||||
size: 31,
|
||||
},
|
||||
{
|
||||
absPath: "/tmp/sessions/missing.jsonl",
|
||||
path: "sessions/missing.jsonl",
|
||||
mtimeMs: 400,
|
||||
size: 40,
|
||||
},
|
||||
],
|
||||
existingRows: [
|
||||
{ path: "sessions/unchanged.jsonl", hash: "hash-unchanged", mtime: 100, size: 10 },
|
||||
{ path: "sessions/newer.jsonl", hash: "hash-newer", mtime: 200, size: 20 },
|
||||
{ path: "sessions/resized.jsonl", hash: "hash-resized", mtime: 300, size: 30 },
|
||||
],
|
||||
});
|
||||
|
||||
expect(dirtyFiles).toEqual([
|
||||
"/tmp/sessions/newer.jsonl",
|
||||
"/tmp/sessions/resized.jsonl",
|
||||
"/tmp/sessions/missing.jsonl",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
// Memory Core plugin module implements manager session sync state behavior.
|
||||
import type { MemorySourceFileStateRow } from "./manager-source-state.js";
|
||||
|
||||
export type MemorySessionStartupFileState = {
|
||||
absPath: string;
|
||||
path: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export function resolveMemorySessionStartupDirtyFiles(params: {
|
||||
files: MemorySessionStartupFileState[];
|
||||
existingRows?: MemorySourceFileStateRow[] | null;
|
||||
}): string[] {
|
||||
const indexedRows = new Map((params.existingRows ?? []).map((row) => [row.path, row]));
|
||||
const dirtyFiles: string[] = [];
|
||||
for (const file of params.files) {
|
||||
const existing = indexedRows.get(file.path);
|
||||
if (!existing) {
|
||||
dirtyFiles.push(file.absPath);
|
||||
continue;
|
||||
}
|
||||
const indexedMtimeMs = Number(existing.mtime);
|
||||
const indexedSize = Number(existing.size);
|
||||
if (!Number.isFinite(indexedMtimeMs) || !Number.isFinite(indexedSize)) {
|
||||
dirtyFiles.push(file.absPath);
|
||||
continue;
|
||||
}
|
||||
if (file.size !== indexedSize || file.mtimeMs > indexedMtimeMs) {
|
||||
dirtyFiles.push(file.absPath);
|
||||
}
|
||||
}
|
||||
return dirtyFiles;
|
||||
}
|
||||
|
||||
export function resolveMemorySessionSyncPlan(params: {
|
||||
needsFullReindex: boolean;
|
||||
files: string[];
|
||||
targetSessionFiles: Set<string> | null;
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
existingRows?: MemorySourceFileStateRow[] | null;
|
||||
sessionPathForFile: (file: string) => string;
|
||||
}): {
|
||||
activePaths: Set<string> | null;
|
||||
existingRows: MemorySourceFileStateRow[] | null;
|
||||
existingHashes: Map<string, string> | null;
|
||||
indexAll: boolean;
|
||||
} {
|
||||
const activePaths = params.targetSessionFiles
|
||||
? null
|
||||
: new Set(params.files.map((file) => params.sessionPathForFile(file)));
|
||||
const existingRows = activePaths === null ? null : (params.existingRows ?? []);
|
||||
return {
|
||||
activePaths,
|
||||
existingRows,
|
||||
existingHashes: existingRows ? new Map(existingRows.map((row) => [row.path, row.hash])) : null,
|
||||
indexAll:
|
||||
params.needsFullReindex ||
|
||||
Boolean(params.targetSessionFiles) ||
|
||||
params.sessionsDirtyFiles.size === 0,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
// Memory Core tests cover manager source state plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
loadMemorySourceFileState,
|
||||
MEMORY_SOURCE_FILE_HASH_SQL,
|
||||
MEMORY_SOURCE_FILE_STATE_SQL,
|
||||
resolveMemorySourceExistingHash,
|
||||
} from "./manager-source-state.js";
|
||||
|
||||
describe("memory source state", () => {
|
||||
it("loads source hashes with one bulk query", () => {
|
||||
const calls: Array<{ sql: string; args: unknown[] }> = [];
|
||||
const state = loadMemorySourceFileState({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: (...args) => {
|
||||
calls.push({ sql, args });
|
||||
return [
|
||||
{ path: "memory/one.md", hash: "hash-1", mtime: 100, size: 10 },
|
||||
{ path: "memory/two.md", hash: "hash-2", mtime: 200, size: 20 },
|
||||
];
|
||||
},
|
||||
get: () => undefined,
|
||||
}),
|
||||
},
|
||||
source: "memory",
|
||||
});
|
||||
|
||||
expect(calls).toEqual([{ sql: MEMORY_SOURCE_FILE_STATE_SQL, args: ["memory"] }]);
|
||||
expect(state.rows).toEqual([
|
||||
{ path: "memory/one.md", hash: "hash-1", mtime: 100, size: 10 },
|
||||
{ path: "memory/two.md", hash: "hash-2", mtime: 200, size: 20 },
|
||||
]);
|
||||
expect(state.hashes).toEqual(
|
||||
new Map([
|
||||
["memory/one.md", "hash-1"],
|
||||
["memory/two.md", "hash-2"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("uses bulk snapshot hashes when present", () => {
|
||||
const calls: Array<{ sql: string; args: unknown[] }> = [];
|
||||
const hash = resolveMemorySourceExistingHash({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: () => [],
|
||||
get: (...args) => {
|
||||
calls.push({ sql, args });
|
||||
return { hash: "unexpected" };
|
||||
},
|
||||
}),
|
||||
},
|
||||
source: "sessions",
|
||||
path: "sessions/thread.jsonl",
|
||||
existingHashes: new Map([["sessions/thread.jsonl", "hash-from-snapshot"]]),
|
||||
});
|
||||
|
||||
expect(hash).toBe("hash-from-snapshot");
|
||||
expect(calls).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("falls back to per-file lookups without a bulk snapshot", () => {
|
||||
const calls: Array<{ sql: string; args: unknown[] }> = [];
|
||||
const hash = resolveMemorySourceExistingHash({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: () => [],
|
||||
get: (...args) => {
|
||||
calls.push({ sql, args });
|
||||
return { hash: "hash-from-row" };
|
||||
},
|
||||
}),
|
||||
},
|
||||
source: "sessions",
|
||||
path: "sessions/thread.jsonl",
|
||||
existingHashes: null,
|
||||
});
|
||||
|
||||
expect(hash).toBe("hash-from-row");
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
sql: MEMORY_SOURCE_FILE_HASH_SQL,
|
||||
args: ["sessions/thread.jsonl", "sessions"],
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
53
extensions/memory-core/src/memory/manager-source-state.ts
Normal file
53
extensions/memory-core/src/memory/manager-source-state.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
// Memory Core plugin module implements manager source state behavior.
|
||||
import type { SQLInputValue } from "node:sqlite";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export type MemorySourceFileStateRow = {
|
||||
path: string;
|
||||
hash: string;
|
||||
mtime?: number;
|
||||
size?: number;
|
||||
};
|
||||
|
||||
type MemorySourceStateDb = {
|
||||
prepare: (sql: string) => {
|
||||
all: (...args: SQLInputValue[]) => unknown;
|
||||
get: (...args: SQLInputValue[]) => unknown;
|
||||
};
|
||||
};
|
||||
|
||||
export const MEMORY_SOURCE_FILE_STATE_SQL = `SELECT path, hash, mtime, size FROM memory_index_sources WHERE source = ?`;
|
||||
export const MEMORY_SOURCE_FILE_HASH_SQL = `SELECT hash FROM memory_index_sources WHERE path = ? AND source = ?`;
|
||||
|
||||
export function loadMemorySourceFileState(params: {
|
||||
db: MemorySourceStateDb;
|
||||
source: MemorySource;
|
||||
}): {
|
||||
rows: MemorySourceFileStateRow[];
|
||||
hashes: Map<string, string>;
|
||||
} {
|
||||
const rows = params.db.prepare(MEMORY_SOURCE_FILE_STATE_SQL).all(params.source) as
|
||||
| MemorySourceFileStateRow[]
|
||||
| undefined;
|
||||
const normalizedRows = rows ?? [];
|
||||
return {
|
||||
rows: normalizedRows,
|
||||
hashes: new Map(normalizedRows.map((row) => [row.path, row.hash])),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveMemorySourceExistingHash(params: {
|
||||
db: MemorySourceStateDb;
|
||||
source: MemorySource;
|
||||
path: string;
|
||||
existingHashes?: Map<string, string> | null;
|
||||
}): string | undefined {
|
||||
if (params.existingHashes) {
|
||||
return params.existingHashes.get(params.path);
|
||||
}
|
||||
return (
|
||||
params.db.prepare(MEMORY_SOURCE_FILE_HASH_SQL).get(params.path, params.source) as
|
||||
| { hash: string }
|
||||
| undefined
|
||||
)?.hash;
|
||||
}
|
||||
109
extensions/memory-core/src/memory/manager-status-state.test.ts
Normal file
109
extensions/memory-core/src/memory/manager-status-state.test.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
// Memory Core tests cover manager status state plugin behavior.
|
||||
import type { SQLInputValue } from "node:sqlite";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
collectMemoryStatusAggregate,
|
||||
MEMORY_STATUS_AGGREGATE_SQL,
|
||||
resolveInitialMemoryDirty,
|
||||
resolveStatusProviderInfo,
|
||||
} from "./manager-status-state.js";
|
||||
|
||||
describe("memory manager status state", () => {
|
||||
it("keeps memory clean for status-only managers after prior indexing", () => {
|
||||
expect(
|
||||
resolveInitialMemoryDirty({
|
||||
hasMemorySource: true,
|
||||
statusOnly: true,
|
||||
hasIndexedMeta: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("marks status-only managers dirty when no prior index metadata exists", () => {
|
||||
expect(
|
||||
resolveInitialMemoryDirty({
|
||||
hasMemorySource: true,
|
||||
statusOnly: true,
|
||||
hasIndexedMeta: false,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("marks status-only managers dirty when index identity mismatches", () => {
|
||||
expect(
|
||||
resolveInitialMemoryDirty({
|
||||
hasMemorySource: false,
|
||||
statusOnly: true,
|
||||
hasIndexedMeta: true,
|
||||
indexIdentityMismatched: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the requested provider before provider initialization", () => {
|
||||
expect(
|
||||
resolveStatusProviderInfo({
|
||||
provider: null,
|
||||
providerInitialized: false,
|
||||
requestedProvider: "openai",
|
||||
configuredModel: "mock-embed",
|
||||
}),
|
||||
).toEqual({
|
||||
provider: "openai",
|
||||
model: "mock-embed",
|
||||
searchMode: "hybrid",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports fts-only mode when initialization finished without a provider", () => {
|
||||
expect(
|
||||
resolveStatusProviderInfo({
|
||||
provider: null,
|
||||
providerInitialized: true,
|
||||
requestedProvider: "openai",
|
||||
configuredModel: "mock-embed",
|
||||
}),
|
||||
).toEqual({
|
||||
provider: "none",
|
||||
model: undefined,
|
||||
searchMode: "fts-only",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses one aggregation query for status counts and source breakdowns", () => {
|
||||
const calls: Array<{ sql: string; params: SQLInputValue[] }> = [];
|
||||
const aggregate = collectMemoryStatusAggregate({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: (...params) => {
|
||||
calls.push({ sql, params });
|
||||
return [
|
||||
{ kind: "files" as const, source: "memory" as const, c: 2 },
|
||||
{ kind: "chunks" as const, source: "memory" as const, c: 5 },
|
||||
{ kind: "files" as const, source: "sessions" as const, c: 1 },
|
||||
{ kind: "chunks" as const, source: "sessions" as const, c: 3 },
|
||||
];
|
||||
},
|
||||
}),
|
||||
},
|
||||
sources: ["memory", "sessions"],
|
||||
sourceFilterSql: " AND source IN (?, ?)",
|
||||
sourceFilterParams: ["memory", "sessions"],
|
||||
});
|
||||
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
sql: MEMORY_STATUS_AGGREGATE_SQL.replaceAll("__FILTER__", " AND source IN (?, ?)"),
|
||||
params: ["memory", "sessions", "memory", "sessions"],
|
||||
},
|
||||
]);
|
||||
expect(aggregate).toEqual({
|
||||
files: 3,
|
||||
chunks: 8,
|
||||
sourceCounts: [
|
||||
{ source: "memory", files: 2, chunks: 5 },
|
||||
{ source: "sessions", files: 1, chunks: 3 },
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
109
extensions/memory-core/src/memory/manager-status-state.ts
Normal file
109
extensions/memory-core/src/memory/manager-status-state.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
// Memory Core plugin module implements manager status state behavior.
|
||||
import type { SQLInputValue } from "node:sqlite";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
type StatusProvider = {
|
||||
id: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
type StatusAggregateRow = {
|
||||
kind: "files" | "chunks";
|
||||
source: MemorySource;
|
||||
c: number;
|
||||
};
|
||||
|
||||
type StatusAggregateDb = {
|
||||
prepare: (sql: string) => {
|
||||
all: (...args: SQLInputValue[]) => StatusAggregateRow[];
|
||||
};
|
||||
};
|
||||
|
||||
export const MEMORY_STATUS_AGGREGATE_SQL =
|
||||
`SELECT 'files' AS kind, source, COUNT(*) as c FROM memory_index_sources WHERE 1=1__FILTER__ GROUP BY source\n` +
|
||||
`UNION ALL\n` +
|
||||
`SELECT 'chunks' AS kind, source, COUNT(*) as c FROM memory_index_chunks WHERE 1=1__FILTER__ GROUP BY source`;
|
||||
|
||||
export function resolveInitialMemoryDirty(params: {
|
||||
hasMemorySource: boolean;
|
||||
statusOnly: boolean;
|
||||
hasIndexedMeta: boolean;
|
||||
indexIdentityMismatched?: boolean;
|
||||
}): boolean {
|
||||
return (
|
||||
Boolean(params.indexIdentityMismatched) ||
|
||||
(params.hasMemorySource && (params.statusOnly ? !params.hasIndexedMeta : true))
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveStatusProviderInfo(params: {
|
||||
provider: StatusProvider | null;
|
||||
providerInitialized: boolean;
|
||||
requestedProvider: string;
|
||||
configuredModel?: string;
|
||||
}): {
|
||||
provider: string;
|
||||
model?: string;
|
||||
searchMode: "hybrid" | "fts-only";
|
||||
} {
|
||||
if (params.provider) {
|
||||
return {
|
||||
provider: params.provider.id,
|
||||
model: params.provider.model,
|
||||
searchMode: "hybrid",
|
||||
};
|
||||
}
|
||||
if (params.providerInitialized) {
|
||||
return {
|
||||
provider: "none",
|
||||
model: undefined,
|
||||
searchMode: "fts-only",
|
||||
};
|
||||
}
|
||||
return {
|
||||
provider: params.requestedProvider,
|
||||
model: params.configuredModel || undefined,
|
||||
searchMode: "hybrid",
|
||||
};
|
||||
}
|
||||
|
||||
export function collectMemoryStatusAggregate(params: {
|
||||
db: StatusAggregateDb;
|
||||
sources: Iterable<MemorySource>;
|
||||
sourceFilterSql?: string;
|
||||
sourceFilterParams?: MemorySource[];
|
||||
}): {
|
||||
files: number;
|
||||
chunks: number;
|
||||
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
|
||||
} {
|
||||
const sources = Array.from(params.sources);
|
||||
const bySource = new Map<MemorySource, { files: number; chunks: number }>();
|
||||
for (const source of sources) {
|
||||
bySource.set(source, { files: 0, chunks: 0 });
|
||||
}
|
||||
const sourceFilterSql = params.sourceFilterSql ?? "";
|
||||
const sourceFilterParams = params.sourceFilterParams ?? [];
|
||||
const aggregateRows = params.db
|
||||
.prepare(MEMORY_STATUS_AGGREGATE_SQL.replaceAll("__FILTER__", sourceFilterSql))
|
||||
.all(...sourceFilterParams, ...sourceFilterParams);
|
||||
let files = 0;
|
||||
let chunks = 0;
|
||||
for (const row of aggregateRows) {
|
||||
const count = row.c ?? 0;
|
||||
const entry = bySource.get(row.source) ?? { files: 0, chunks: 0 };
|
||||
if (row.kind === "files") {
|
||||
entry.files = count;
|
||||
files += count;
|
||||
} else {
|
||||
entry.chunks = count;
|
||||
chunks += count;
|
||||
}
|
||||
bySource.set(row.source, entry);
|
||||
}
|
||||
return {
|
||||
files,
|
||||
chunks,
|
||||
sourceCounts: sources.map((source) => Object.assign({ source }, bySource.get(source)!)),
|
||||
};
|
||||
}
|
||||
194
extensions/memory-core/src/memory/manager-sync-control.ts
Normal file
194
extensions/memory-core/src/memory/manager-sync-control.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
// Memory Core plugin module implements manager sync control behavior.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { createSubsystemLogger } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import type {
|
||||
MemorySessionSyncTarget,
|
||||
MemorySyncParams,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
|
||||
export type MemoryReadonlyRecoveryState = {
|
||||
closed: boolean;
|
||||
db: DatabaseSync;
|
||||
vector: {
|
||||
dims?: number;
|
||||
};
|
||||
readonlyRecoveryAttempts: number;
|
||||
readonlyRecoverySuccesses: number;
|
||||
readonlyRecoveryFailures: number;
|
||||
readonlyRecoveryLastError?: string;
|
||||
runSync: (params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessions?: MemorySessionSyncTarget[];
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}) => Promise<void>;
|
||||
openDatabase: () => DatabaseSync;
|
||||
closeDatabase: (db: DatabaseSync) => void;
|
||||
resetVectorState: () => void;
|
||||
ensureSchema: () => void;
|
||||
readMeta: () => { vectorDims?: number } | undefined;
|
||||
};
|
||||
|
||||
export function isMemoryReadonlyDbError(err: unknown): boolean {
|
||||
const readonlyPattern =
|
||||
/attempt to write a readonly database|database is read-only|SQLITE_READONLY/i;
|
||||
const messages = new Set<string>();
|
||||
|
||||
const pushValue = (value: unknown): void => {
|
||||
if (typeof value !== "string") {
|
||||
return;
|
||||
}
|
||||
const normalized = value.trim();
|
||||
if (!normalized) {
|
||||
return;
|
||||
}
|
||||
messages.add(normalized);
|
||||
};
|
||||
|
||||
pushValue(formatErrorMessage(err));
|
||||
if (err && typeof err === "object") {
|
||||
const record = err as Record<string, unknown>;
|
||||
pushValue(record.message);
|
||||
pushValue(record.code);
|
||||
pushValue(record.name);
|
||||
if (record.cause && typeof record.cause === "object") {
|
||||
const cause = record.cause as Record<string, unknown>;
|
||||
pushValue(cause.message);
|
||||
pushValue(cause.code);
|
||||
pushValue(cause.name);
|
||||
}
|
||||
}
|
||||
|
||||
return [...messages].some((value) => readonlyPattern.test(value));
|
||||
}
|
||||
|
||||
export function extractMemoryErrorReason(err: unknown): string {
|
||||
if (err instanceof Error && err.message.trim()) {
|
||||
return err.message;
|
||||
}
|
||||
if (err && typeof err === "object") {
|
||||
const record = err as Record<string, unknown>;
|
||||
if (typeof record.message === "string" && record.message.trim()) {
|
||||
return record.message;
|
||||
}
|
||||
if (typeof record.code === "string" && record.code.trim()) {
|
||||
return record.code;
|
||||
}
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
|
||||
export async function runMemorySyncWithReadonlyRecovery(
|
||||
state: MemoryReadonlyRecoveryState,
|
||||
params?: MemorySyncParams,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await state.runSync(params);
|
||||
} catch (err) {
|
||||
if (!isMemoryReadonlyDbError(err) || state.closed) {
|
||||
throw err;
|
||||
}
|
||||
const reason = extractMemoryErrorReason(err);
|
||||
state.readonlyRecoveryAttempts += 1;
|
||||
state.readonlyRecoveryLastError = reason;
|
||||
log.warn(`memory sync readonly handle detected; reopening sqlite connection`, { reason });
|
||||
try {
|
||||
state.closeDatabase(state.db);
|
||||
} catch {}
|
||||
const previousVectorDims = state.vector.dims;
|
||||
state.db = state.openDatabase();
|
||||
state.resetVectorState();
|
||||
state.ensureSchema();
|
||||
const meta = state.readMeta();
|
||||
state.vector.dims = meta?.vectorDims ?? previousVectorDims;
|
||||
try {
|
||||
await state.runSync(params);
|
||||
state.readonlyRecoverySuccesses += 1;
|
||||
} catch (retryErr) {
|
||||
state.readonlyRecoveryFailures += 1;
|
||||
throw retryErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function enqueueMemoryTargetedSessionSync(
|
||||
state: {
|
||||
isClosed: () => boolean;
|
||||
getSyncing: () => Promise<void> | null;
|
||||
getQueuedSessionFiles: () => Set<string>;
|
||||
getQueuedSessions: () => Map<string, MemorySessionSyncTarget>;
|
||||
getQueuedSessionSync: () => Promise<void> | null;
|
||||
setQueuedSessionSync: (value: Promise<void> | null) => void;
|
||||
sync: (params?: MemorySyncParams) => Promise<void>;
|
||||
},
|
||||
targets?: Pick<MemorySyncParams, "sessions" | "sessionFiles">,
|
||||
): Promise<void> {
|
||||
const queuedSessionFiles = state.getQueuedSessionFiles();
|
||||
for (const sessionFile of targets?.sessionFiles ?? []) {
|
||||
const trimmed = sessionFile.trim();
|
||||
if (trimmed) {
|
||||
queuedSessionFiles.add(trimmed);
|
||||
}
|
||||
}
|
||||
const queuedSessions = state.getQueuedSessions();
|
||||
for (const session of targets?.sessions ?? []) {
|
||||
const normalized = normalizeQueuedMemorySessionSyncTarget(session);
|
||||
if (normalized) {
|
||||
queuedSessions.set(memorySessionSyncTargetKey(normalized), normalized);
|
||||
}
|
||||
}
|
||||
if (queuedSessionFiles.size === 0 && queuedSessions.size === 0) {
|
||||
return state.getSyncing() ?? Promise.resolve();
|
||||
}
|
||||
if (!state.getQueuedSessionSync()) {
|
||||
state.setQueuedSessionSync(
|
||||
(async () => {
|
||||
try {
|
||||
await state.getSyncing()?.catch(() => undefined);
|
||||
while (
|
||||
!state.isClosed() &&
|
||||
(state.getQueuedSessionFiles().size > 0 || state.getQueuedSessions().size > 0)
|
||||
) {
|
||||
const pendingSessionFiles = Array.from(state.getQueuedSessionFiles());
|
||||
const pendingSessions = Array.from(state.getQueuedSessions().values());
|
||||
state.getQueuedSessionFiles().clear();
|
||||
state.getQueuedSessions().clear();
|
||||
await state.sync({
|
||||
reason: "queued-sessions",
|
||||
sessions: pendingSessions,
|
||||
sessionFiles: pendingSessionFiles,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
state.setQueuedSessionSync(null);
|
||||
}
|
||||
})(),
|
||||
);
|
||||
}
|
||||
return state.getQueuedSessionSync() ?? Promise.resolve();
|
||||
}
|
||||
|
||||
function normalizeQueuedMemorySessionSyncTarget(
|
||||
target: MemorySessionSyncTarget,
|
||||
): MemorySessionSyncTarget | null {
|
||||
const sessionId = target.sessionId.trim();
|
||||
if (!sessionId) {
|
||||
return null;
|
||||
}
|
||||
const agentId = target.agentId?.trim();
|
||||
const sessionKey = target.sessionKey?.trim();
|
||||
return {
|
||||
...(agentId ? { agentId } : {}),
|
||||
sessionId,
|
||||
...(sessionKey ? { sessionKey } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function memorySessionSyncTargetKey(target: MemorySessionSyncTarget): string {
|
||||
return [target.agentId ?? "", target.sessionId, target.sessionKey ?? ""].join("\0");
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// Memory Core tests cover manager sync ops.archive delta bypass plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import type {
|
||||
MemorySource,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { MemoryManagerSyncOps } from "./manager-sync-ops.js";
|
||||
|
||||
type MemoryIndexEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
hash: string;
|
||||
content?: string;
|
||||
};
|
||||
|
||||
type SyncParams = {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
forceSessions?: boolean;
|
||||
sessionFile?: string;
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
};
|
||||
|
||||
class SessionDeltaHarness extends MemoryManagerSyncOps {
|
||||
protected readonly cfg = {} as OpenClawConfig;
|
||||
protected readonly agentId = "main";
|
||||
protected readonly workspaceDir = "/tmp/openclaw-test-workspace";
|
||||
protected readonly settings = {
|
||||
sync: {
|
||||
sessions: {
|
||||
deltaBytes: 100_000,
|
||||
deltaMessages: 50,
|
||||
postCompactionForce: true,
|
||||
},
|
||||
},
|
||||
} as ResolvedMemorySearchConfig;
|
||||
protected readonly batch = {
|
||||
enabled: false,
|
||||
wait: false,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 0,
|
||||
timeoutMs: 0,
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db = null as unknown as DatabaseSync;
|
||||
|
||||
readonly syncCalls: SyncParams[] = [];
|
||||
|
||||
addPendingSessionFile(sessionFile: string) {
|
||||
this.sessionPendingFiles.add(sessionFile);
|
||||
}
|
||||
|
||||
getDirtySessionFiles(): string[] {
|
||||
return Array.from(this.sessionsDirtyFiles);
|
||||
}
|
||||
|
||||
isSessionsDirty(): boolean {
|
||||
return this.sessionsDirty;
|
||||
}
|
||||
|
||||
async processPendingSessionDeltas(): Promise<void> {
|
||||
await (
|
||||
this as unknown as {
|
||||
processSessionDeltaBatch: () => Promise<void>;
|
||||
}
|
||||
).processSessionDeltaBatch();
|
||||
}
|
||||
|
||||
protected computeProviderKey(): string {
|
||||
return "test";
|
||||
}
|
||||
|
||||
protected resolveProviderIndexIdentities() {
|
||||
return [];
|
||||
}
|
||||
|
||||
protected async sync(params?: SyncParams): Promise<void> {
|
||||
this.syncCalls.push(params ?? {});
|
||||
}
|
||||
|
||||
protected async withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
_timeoutMs: number,
|
||||
_message: string,
|
||||
): Promise<T> {
|
||||
return await promise;
|
||||
}
|
||||
|
||||
protected getIndexConcurrency(): number {
|
||||
return 1;
|
||||
}
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {}
|
||||
|
||||
protected resetProviderInitializationForRetry(): void {}
|
||||
|
||||
protected assertRequiredProviderAvailable(): void {}
|
||||
|
||||
protected async indexFile(
|
||||
_entry: MemoryIndexEntry,
|
||||
_options: { source: MemorySource; content?: string },
|
||||
): Promise<void> {}
|
||||
}
|
||||
|
||||
describe("session archive delta bypass", () => {
|
||||
let tmpDir = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-archive-delta-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeSessionFile(name: string): Promise<string> {
|
||||
const filePath = path.join(tmpDir, name);
|
||||
await fs.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "short archived session" },
|
||||
}) + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
return filePath;
|
||||
}
|
||||
|
||||
it.each(["reset", "deleted"] as const)(
|
||||
"marks below-threshold %s archives dirty immediately",
|
||||
async (reason) => {
|
||||
const archivePath = await writeSessionFile(
|
||||
`session-a.jsonl.${reason}.2026-05-03T05-38-59.000Z`,
|
||||
);
|
||||
const harness = new SessionDeltaHarness();
|
||||
harness.addPendingSessionFile(archivePath);
|
||||
|
||||
await harness.processPendingSessionDeltas();
|
||||
|
||||
expect(harness.getDirtySessionFiles()).toEqual([archivePath]);
|
||||
expect(harness.isSessionsDirty()).toBe(true);
|
||||
expect(harness.syncCalls).toEqual([{ reason: "session-delta" }]);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps .jsonl.bak archives on the normal below-threshold delta path", async () => {
|
||||
const bakPath = await writeSessionFile("session-a.jsonl.bak.2026-05-03T05-38-59.000Z");
|
||||
const harness = new SessionDeltaHarness();
|
||||
harness.addPendingSessionFile(bakPath);
|
||||
|
||||
await harness.processPendingSessionDeltas();
|
||||
|
||||
expect(harness.getDirtySessionFiles()).toStrictEqual([]);
|
||||
expect(harness.isSessionsDirty()).toBe(false);
|
||||
expect(harness.syncCalls).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("keeps live transcripts below the configured thresholds", async () => {
|
||||
const livePath = await writeSessionFile("session-a.jsonl");
|
||||
const harness = new SessionDeltaHarness();
|
||||
harness.addPendingSessionFile(livePath);
|
||||
|
||||
await harness.processPendingSessionDeltas();
|
||||
|
||||
expect(harness.getDirtySessionFiles()).toStrictEqual([]);
|
||||
expect(harness.isSessionsDirty()).toBe(false);
|
||||
expect(harness.syncCalls).toStrictEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
// Memory Core tests cover manager sync ops.interval plugin behavior.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { MemoryManagerSyncOps } from "./manager-sync-ops.js";
|
||||
|
||||
type MemoryIndexEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
hash: string;
|
||||
content?: string;
|
||||
};
|
||||
|
||||
class IntervalSyncHarness extends MemoryManagerSyncOps {
|
||||
protected readonly cfg = {} as OpenClawConfig;
|
||||
protected readonly agentId = "main";
|
||||
protected readonly workspaceDir = "/tmp/openclaw-memory-interval-test";
|
||||
protected readonly settings: ResolvedMemorySearchConfig;
|
||||
protected readonly batch = {
|
||||
enabled: false,
|
||||
wait: false,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 0,
|
||||
timeoutMs: 0,
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db = {} as DatabaseSync;
|
||||
|
||||
constructor(params: { intervalMinutes?: number; batchTimeoutMinutes?: number }) {
|
||||
super();
|
||||
this.settings = {
|
||||
sync: { intervalMinutes: params.intervalMinutes ?? 0 },
|
||||
remote: {
|
||||
batch: {
|
||||
enabled: true,
|
||||
timeoutMinutes: params.batchTimeoutMinutes,
|
||||
},
|
||||
},
|
||||
} as ResolvedMemorySearchConfig;
|
||||
}
|
||||
|
||||
arm(): void {
|
||||
this.ensureIntervalSync();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.intervalTimer) {
|
||||
clearInterval(this.intervalTimer);
|
||||
this.intervalTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
batchConfig(): ReturnType<MemoryManagerSyncOps["resolveBatchConfig"]> {
|
||||
return this.resolveBatchConfig();
|
||||
}
|
||||
|
||||
protected computeProviderKey(): string {
|
||||
return "test";
|
||||
}
|
||||
|
||||
protected resolveProviderIndexIdentities() {
|
||||
return [];
|
||||
}
|
||||
|
||||
protected async sync(): Promise<void> {}
|
||||
|
||||
protected async withTimeout<T>(promise: Promise<T>): Promise<T> {
|
||||
return await promise;
|
||||
}
|
||||
|
||||
protected getIndexConcurrency(): number {
|
||||
return 1;
|
||||
}
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {}
|
||||
|
||||
protected resetProviderInitializationForRetry(): void {}
|
||||
|
||||
protected assertRequiredProviderAvailable(): void {}
|
||||
|
||||
protected async indexFile(
|
||||
_entry: MemoryIndexEntry,
|
||||
_options: { source: MemorySource; content?: string },
|
||||
): Promise<void> {}
|
||||
}
|
||||
|
||||
describe("MemoryManagerSyncOps interval sync", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("clamps oversized interval sync timers", () => {
|
||||
vi.useFakeTimers();
|
||||
const setIntervalSpy = vi.spyOn(globalThis, "setInterval");
|
||||
const harness = new IntervalSyncHarness({ intervalMinutes: Number.MAX_SAFE_INTEGER });
|
||||
|
||||
harness.arm();
|
||||
|
||||
expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS);
|
||||
harness.stop();
|
||||
});
|
||||
|
||||
it("clamps oversized batch timeout minutes", () => {
|
||||
const harness = new IntervalSyncHarness({
|
||||
batchTimeoutMinutes: Number.MAX_SAFE_INTEGER,
|
||||
});
|
||||
|
||||
expect(harness.batchConfig().timeoutMs).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,722 @@
|
||||
// Memory Core tests cover manager sync ops.startup catchup plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { emitSessionTranscriptUpdate } from "openclaw/plugin-sdk/agent-harness-runtime";
|
||||
import {
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
type OpenClawConfig,
|
||||
type ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import type {
|
||||
MemorySource,
|
||||
MemorySyncParams,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import {
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
} from "openclaw/plugin-sdk/runtime-config-snapshot";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { MemoryManagerSyncOps } from "./manager-sync-ops.js";
|
||||
|
||||
type MemoryIndexEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
hash: string;
|
||||
content?: string;
|
||||
};
|
||||
|
||||
type SyncParams = {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessions?: MemorySyncParams["sessions"];
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
};
|
||||
|
||||
type MemorySessionTranscriptUpdate = {
|
||||
agentId?: string;
|
||||
sessionFile?: string;
|
||||
sessionKey?: string;
|
||||
target?: {
|
||||
agentId: string;
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
};
|
||||
};
|
||||
|
||||
type MemoryTranscriptUpdateSubscriber = (
|
||||
listener: (update: MemorySessionTranscriptUpdate) => void,
|
||||
) => () => void;
|
||||
|
||||
const MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY = Symbol.for(
|
||||
"openclaw.memoryCore.sessionTranscriptUpdateSubscriber",
|
||||
);
|
||||
const originalStartupStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
const originalStartupConfigPath = process.env.OPENCLAW_CONFIG_PATH;
|
||||
|
||||
type SourceStateRow = { path: string; hash: string; mtime: number; size: number };
|
||||
|
||||
function setStartupStateDir(stateDir: string): void {
|
||||
Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir);
|
||||
}
|
||||
|
||||
function setStartupConfigPath(configPath: string): void {
|
||||
Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath);
|
||||
}
|
||||
|
||||
function restoreStartupEnv(): void {
|
||||
if (originalStartupStateDir === undefined) {
|
||||
Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR");
|
||||
} else {
|
||||
Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalStartupStateDir);
|
||||
}
|
||||
if (originalStartupConfigPath === undefined) {
|
||||
Reflect.deleteProperty(process.env, "OPENCLAW_CONFIG_PATH");
|
||||
} else {
|
||||
Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", originalStartupConfigPath);
|
||||
}
|
||||
}
|
||||
|
||||
class SessionStartupCatchupHarness extends MemoryManagerSyncOps {
|
||||
protected readonly cfg = {} as OpenClawConfig;
|
||||
protected readonly agentId = "main";
|
||||
protected readonly workspaceDir = "/tmp/openclaw-test-workspace";
|
||||
protected readonly settings = {
|
||||
chunking: {
|
||||
overlap: 0,
|
||||
tokens: 256,
|
||||
},
|
||||
extraPaths: [],
|
||||
multimodal: {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 0,
|
||||
},
|
||||
provider: "none",
|
||||
store: {
|
||||
fts: {
|
||||
tokenizer: "unicode61",
|
||||
},
|
||||
vector: {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
sync: {
|
||||
sessions: {
|
||||
deltaBytes: 100_000,
|
||||
deltaMessages: 50,
|
||||
postCompactionForce: true,
|
||||
},
|
||||
},
|
||||
} as unknown as ResolvedMemorySearchConfig;
|
||||
protected readonly batch = {
|
||||
enabled: false,
|
||||
wait: false,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 0,
|
||||
timeoutMs: 0,
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db: DatabaseSync;
|
||||
|
||||
readonly syncCalls: SyncParams[] = [];
|
||||
readonly indexedPaths: string[] = [];
|
||||
readonly indexedContents: string[] = [];
|
||||
|
||||
constructor(sourceRows: SourceStateRow[]) {
|
||||
super();
|
||||
this.sources.add("sessions");
|
||||
this.db = {
|
||||
prepare: () => ({
|
||||
all: () => sourceRows,
|
||||
get: () => undefined,
|
||||
run: () => undefined,
|
||||
}),
|
||||
} as unknown as DatabaseSync;
|
||||
}
|
||||
|
||||
async catchUp(): Promise<string[]> {
|
||||
return await this.runSessionStartupCatchup();
|
||||
}
|
||||
|
||||
async markStartupDirtyFiles(): Promise<string[]> {
|
||||
return await this.markSessionStartupCatchupDirtyFiles();
|
||||
}
|
||||
|
||||
async runSyncForTest(params?: MemorySyncParams): Promise<void> {
|
||||
await this.runSync(params);
|
||||
}
|
||||
|
||||
getDirtySessionFiles(): string[] {
|
||||
return Array.from(this.sessionsDirtyFiles);
|
||||
}
|
||||
|
||||
getPendingSessionTargets(): MemorySyncParams["sessions"] {
|
||||
return Array.from(this.sessionPendingTargets.values());
|
||||
}
|
||||
|
||||
getPendingSessionFiles(): string[] {
|
||||
return Array.from(this.sessionPendingFiles);
|
||||
}
|
||||
|
||||
addPendingSessionTarget(target: NonNullable<MemorySyncParams["sessions"]>[number]): void {
|
||||
this.sessionPendingTargets.set(
|
||||
[target.agentId ?? "", target.sessionId, target.sessionKey ?? ""].join("\0"),
|
||||
target,
|
||||
);
|
||||
}
|
||||
|
||||
async processPendingSessionDeltas(): Promise<void> {
|
||||
await (
|
||||
this as unknown as {
|
||||
processSessionDeltaBatch: () => Promise<void>;
|
||||
}
|
||||
).processSessionDeltaBatch();
|
||||
}
|
||||
|
||||
async combineTargetSessionFilesForTest(params: {
|
||||
sessions?: MemorySyncParams["sessions"];
|
||||
sessionFiles?: string[];
|
||||
}): Promise<Set<string> | null> {
|
||||
return await (
|
||||
this as unknown as {
|
||||
combineTargetSessionFiles: (params: {
|
||||
sessions?: MemorySyncParams["sessions"];
|
||||
sessionFiles?: string[];
|
||||
}) => Promise<Set<string> | null>;
|
||||
}
|
||||
).combineTargetSessionFiles(params);
|
||||
}
|
||||
|
||||
isSessionsDirty(): boolean {
|
||||
return this.sessionsDirty;
|
||||
}
|
||||
|
||||
startTranscriptListener(): void {
|
||||
this.ensureSessionListener();
|
||||
}
|
||||
|
||||
stopTranscriptListener(): void {
|
||||
this.sessionUnsubscribe?.();
|
||||
this.sessionUnsubscribe = null;
|
||||
}
|
||||
|
||||
protected computeProviderKey(): string {
|
||||
return "test";
|
||||
}
|
||||
|
||||
protected resolveProviderIndexIdentities() {
|
||||
return [];
|
||||
}
|
||||
|
||||
protected async sync(params?: MemorySyncParams): Promise<void> {
|
||||
this.syncCalls.push(params ?? {});
|
||||
}
|
||||
|
||||
protected async withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
_timeoutMs: number,
|
||||
_message: string,
|
||||
): Promise<T> {
|
||||
return await promise;
|
||||
}
|
||||
|
||||
protected getIndexConcurrency(): number {
|
||||
return 1;
|
||||
}
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {}
|
||||
|
||||
protected resetProviderInitializationForRetry(): void {}
|
||||
|
||||
protected assertRequiredProviderAvailable(): void {}
|
||||
|
||||
protected async indexFile(
|
||||
entry: MemoryIndexEntry,
|
||||
options: { source: MemorySource; content?: string },
|
||||
): Promise<void> {
|
||||
this.indexedPaths.push(entry.path);
|
||||
this.indexedContents.push(options.content ?? "");
|
||||
}
|
||||
}
|
||||
|
||||
describe("session startup catch-up", () => {
|
||||
let stateDir = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-startup-"));
|
||||
setStartupStateDir(stateDir);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
restoreStartupEnv();
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeSessionFile(
|
||||
name: string,
|
||||
): Promise<{ filePath: string; size: number; mtimeMs: number }> {
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent("main");
|
||||
await fs.mkdir(sessionsDir, { recursive: true });
|
||||
const filePath = path.join(sessionsDir, name);
|
||||
await fs.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({ type: "message", message: { role: "user", content: "startup catchup" } }) +
|
||||
"\n",
|
||||
"utf-8",
|
||||
);
|
||||
const stat = await fs.stat(filePath);
|
||||
return { filePath, size: stat.size, mtimeMs: stat.mtimeMs };
|
||||
}
|
||||
|
||||
it("marks stale indexed session files dirty and schedules catch-up sync", async () => {
|
||||
const session = await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([
|
||||
{
|
||||
path: "sessions/main/thread.jsonl",
|
||||
hash: "old-hash",
|
||||
mtime: session.mtimeMs - 1000,
|
||||
size: session.size,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(harness.catchUp()).resolves.toEqual([session.filePath]);
|
||||
expect(harness.getDirtySessionFiles()).toEqual([session.filePath]);
|
||||
expect(harness.isSessionsDirty()).toBe(true);
|
||||
expect(harness.syncCalls).toEqual([{ reason: "session-startup-catchup" }]);
|
||||
});
|
||||
|
||||
it("retries transient session transcript reads during session indexing", async () => {
|
||||
const session = await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
|
||||
const realOpen = fs.open;
|
||||
let attempts = 0;
|
||||
const openSpy = vi
|
||||
.spyOn(fs, "open")
|
||||
.mockImplementation(async (...args: Parameters<typeof realOpen>) => {
|
||||
const [target, flags, mode] = args;
|
||||
if (
|
||||
typeof target === "string" &&
|
||||
path.resolve(target) === session.filePath &&
|
||||
attempts++ === 0
|
||||
) {
|
||||
const err = new Error(
|
||||
"Unknown system error -11: Unknown system error -11, open",
|
||||
) as NodeJS.ErrnoException;
|
||||
err.code = "UNKNOWN";
|
||||
err.errno = -11;
|
||||
throw err;
|
||||
}
|
||||
return await realOpen(target, flags, mode);
|
||||
});
|
||||
|
||||
try {
|
||||
await (harness as any).syncSessionFiles({ needsFullReindex: true });
|
||||
expect(attempts).toBe(2);
|
||||
} finally {
|
||||
openSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("can mark startup catch-up files without scheduling background sync", async () => {
|
||||
const session = await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([
|
||||
{
|
||||
path: "sessions/main/thread.jsonl",
|
||||
hash: "old-hash",
|
||||
mtime: session.mtimeMs - 1000,
|
||||
size: session.size,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(harness.markStartupDirtyFiles()).resolves.toEqual([session.filePath]);
|
||||
expect(harness.getDirtySessionFiles()).toEqual([session.filePath]);
|
||||
expect(harness.isSessionsDirty()).toBe(true);
|
||||
expect(harness.syncCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it("leaves unchanged indexed session files clean", async () => {
|
||||
const session = await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([
|
||||
{
|
||||
path: "sessions/main/thread.jsonl",
|
||||
hash: "current-hash",
|
||||
mtime: session.mtimeMs,
|
||||
size: session.size,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(harness.catchUp()).resolves.toEqual([]);
|
||||
expect(harness.getDirtySessionFiles()).toEqual([]);
|
||||
expect(harness.isSessionsDirty()).toBe(false);
|
||||
expect(harness.syncCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "read",
|
||||
fileName: "delta-read.jsonl",
|
||||
failOn: "read" as const,
|
||||
code: "EWOULDBLOCK",
|
||||
},
|
||||
{
|
||||
name: "open",
|
||||
fileName: "delta-open.jsonl",
|
||||
failOn: "open" as const,
|
||||
code: "EAGAIN",
|
||||
},
|
||||
])("retries transient session transcript $name failures during delta updates", async (params) => {
|
||||
const session = await writeSessionFile(params.fileName);
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
let attempts = 0;
|
||||
const sessionBuffer = await fs.readFile(session.filePath);
|
||||
const openSpy = vi
|
||||
.spyOn(fs, "open")
|
||||
.mockImplementation(async (...args: Parameters<typeof fs.open>) => {
|
||||
const [target] = args;
|
||||
if (
|
||||
params.failOn === "open" &&
|
||||
typeof target === "string" &&
|
||||
path.resolve(target) === session.filePath &&
|
||||
attempts++ === 0
|
||||
) {
|
||||
const err = new Error(
|
||||
"Unknown system error -11: Unknown system error -11, open",
|
||||
) as NodeJS.ErrnoException;
|
||||
err.code = params.code;
|
||||
err.errno = -11;
|
||||
throw err;
|
||||
}
|
||||
|
||||
return {
|
||||
read: async (buffer: Buffer, offset: number, length: number, position: number | null) => {
|
||||
if (params.failOn === "read" && attempts++ === 0) {
|
||||
const err = new Error(
|
||||
"Unknown system error -11: Unknown system error -11, read",
|
||||
) as NodeJS.ErrnoException;
|
||||
err.code = params.code;
|
||||
err.errno = -11;
|
||||
throw err;
|
||||
}
|
||||
const start = position ?? 0;
|
||||
const chunk = sessionBuffer.subarray(start, start + length);
|
||||
chunk.copy(buffer, offset);
|
||||
return { bytesRead: chunk.length, buffer };
|
||||
},
|
||||
close: async () => {},
|
||||
} as unknown as Awaited<ReturnType<typeof fs.open>>;
|
||||
});
|
||||
|
||||
try {
|
||||
const delta = await (harness as any).updateSessionDelta(session.filePath);
|
||||
expect(delta).toMatchObject({
|
||||
pendingBytes: session.size,
|
||||
pendingMessages: 1,
|
||||
});
|
||||
expect(attempts).toBe(2);
|
||||
} finally {
|
||||
openSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not fall back to full session sync when identity targets normalize away", async () => {
|
||||
await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
|
||||
await harness.runSyncForTest({
|
||||
reason: "queued-sessions",
|
||||
sessions: [{ agentId: "other", sessionId: "thread" }],
|
||||
});
|
||||
|
||||
expect(harness.indexedPaths).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not fall back to full session sync for malformed identity session ids", async () => {
|
||||
await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
|
||||
await harness.runSyncForTest({
|
||||
reason: "queued-sessions",
|
||||
sessions: [{ agentId: "main", sessionId: "bad/nested" }],
|
||||
});
|
||||
|
||||
expect(harness.indexedPaths).toEqual([]);
|
||||
});
|
||||
|
||||
it("resolves identity-targeted delta sync through a custom session store", async () => {
|
||||
const storeDir = path.join(stateDir, "custom-sessions");
|
||||
const sessionFile = path.join(storeDir, "custom-thread.jsonl");
|
||||
const storePath = path.join(storeDir, "sessions.json");
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
await fs.mkdir(storeDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
sessionFile,
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "custom store target" },
|
||||
}) + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
storePath,
|
||||
JSON.stringify({
|
||||
"agent:main:chat:custom": {
|
||||
sessionFile: "custom-thread.jsonl",
|
||||
sessionId: "custom-thread",
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8");
|
||||
setStartupConfigPath(configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
(harness as unknown as { settings: ResolvedMemorySearchConfig }).settings.sync.sessions = {
|
||||
deltaBytes: 1,
|
||||
deltaMessages: 1,
|
||||
postCompactionForce: true,
|
||||
};
|
||||
harness.addPendingSessionTarget({
|
||||
agentId: "main",
|
||||
sessionId: "custom-thread",
|
||||
sessionKey: "agent:main:chat:custom",
|
||||
});
|
||||
|
||||
await harness.processPendingSessionDeltas();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(harness.getDirtySessionFiles()).toEqual([sessionFile]);
|
||||
expect(harness.syncCalls).toEqual([{ reason: "session-delta" }]);
|
||||
});
|
||||
|
||||
it("keeps explicit custom-store session file targets at the sync gate", async () => {
|
||||
const storeDir = path.join(stateDir, "custom-sessions");
|
||||
const sessionFile = path.join(storeDir, "explicit-target.jsonl");
|
||||
const storePath = path.join(storeDir, "sessions.json");
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
await fs.mkdir(storeDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
sessionFile,
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "explicit target" },
|
||||
}) + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
storePath,
|
||||
JSON.stringify({
|
||||
"agent:main:chat:explicit-target": {
|
||||
sessionFile: "explicit-target.jsonl",
|
||||
sessionId: "explicit-target",
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8");
|
||||
setStartupConfigPath(configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
|
||||
await expect(
|
||||
harness.combineTargetSessionFilesForTest({ sessionFiles: [sessionFile] }),
|
||||
).resolves.toEqual(new Set([sessionFile]));
|
||||
});
|
||||
|
||||
it("preserves generated-session classification during targeted custom-store indexing", async () => {
|
||||
const storeDir = path.join(stateDir, "custom-sessions");
|
||||
const sessionFile = path.join(storeDir, "cron-thread.jsonl");
|
||||
const otherSessionFile = path.join(storeDir, "other-thread.jsonl");
|
||||
const storePath = path.join(storeDir, "sessions.json");
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
await fs.mkdir(storeDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
sessionFile,
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "Internal cron output that must stay out." },
|
||||
}) + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
otherSessionFile,
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "Other custom-store content" },
|
||||
}) + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
storePath,
|
||||
JSON.stringify({
|
||||
"agent:main:cron:job-1:run:run-1": {
|
||||
sessionFile: "cron-thread.jsonl",
|
||||
sessionId: "cron-thread",
|
||||
},
|
||||
"agent:main:chat:other": {
|
||||
sessionFile: "other-thread.jsonl",
|
||||
sessionId: "other-thread",
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8");
|
||||
setStartupConfigPath(configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
|
||||
await (
|
||||
harness as unknown as {
|
||||
syncSessionFiles: (params: {
|
||||
needsFullReindex: boolean;
|
||||
targetSessionFiles: string[];
|
||||
}) => Promise<void>;
|
||||
}
|
||||
).syncSessionFiles({
|
||||
needsFullReindex: false,
|
||||
targetSessionFiles: [sessionFile],
|
||||
});
|
||||
|
||||
expect(harness.indexedPaths).toEqual(["sessions/cron-thread.jsonl"]);
|
||||
expect(harness.indexedContents).toEqual([""]);
|
||||
});
|
||||
|
||||
it("queues transcript update identity without requiring a session file", async () => {
|
||||
vi.useFakeTimers();
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
const originalSubscriber = (globalThis as Record<symbol, unknown>)[
|
||||
MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY
|
||||
];
|
||||
let transcriptListener: ((update: MemorySessionTranscriptUpdate) => void) | undefined;
|
||||
(globalThis as Record<symbol, unknown>)[MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY] = ((
|
||||
listener,
|
||||
) => {
|
||||
transcriptListener = listener;
|
||||
return () => {
|
||||
if (transcriptListener === listener) {
|
||||
transcriptListener = undefined;
|
||||
}
|
||||
};
|
||||
}) satisfies MemoryTranscriptUpdateSubscriber;
|
||||
harness.startTranscriptListener();
|
||||
|
||||
try {
|
||||
transcriptListener?.({
|
||||
target: {
|
||||
agentId: "main",
|
||||
sessionId: "thread",
|
||||
sessionKey: "agent:main:thread",
|
||||
},
|
||||
});
|
||||
|
||||
expect(harness.getPendingSessionTargets()).toEqual([
|
||||
{ agentId: "main", sessionId: "thread", sessionKey: "agent:main:thread" },
|
||||
]);
|
||||
} finally {
|
||||
harness.stopTranscriptListener();
|
||||
if (originalSubscriber === undefined) {
|
||||
delete (globalThis as Record<symbol, unknown>)[
|
||||
MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY
|
||||
];
|
||||
} else {
|
||||
(globalThis as Record<symbol, unknown>)[MEMORY_CORE_TRANSCRIPT_UPDATE_SUBSCRIBER_KEY] =
|
||||
originalSubscriber;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps canonical path transcript update compatibility", async () => {
|
||||
vi.useFakeTimers();
|
||||
const session = await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
harness.startTranscriptListener();
|
||||
|
||||
emitSessionTranscriptUpdate({
|
||||
sessionFile: session.filePath,
|
||||
sessionKey: "agent:main:thread",
|
||||
});
|
||||
|
||||
expect(harness.getPendingSessionFiles()).toEqual([session.filePath]);
|
||||
expect(harness.getPendingSessionTargets()).toEqual([]);
|
||||
harness.stopTranscriptListener();
|
||||
});
|
||||
|
||||
it("queues file-only transcript updates from a custom session store", async () => {
|
||||
vi.useFakeTimers();
|
||||
const storeDir = path.join(stateDir, "custom-sessions");
|
||||
const sessionFile = path.join(storeDir, "custom-update.jsonl");
|
||||
const storePath = path.join(storeDir, "sessions.json");
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
await fs.mkdir(storeDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
sessionFile,
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "custom update" },
|
||||
}) + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
storePath,
|
||||
JSON.stringify({
|
||||
"agent:main:chat:custom-update": {
|
||||
sessionFile: "custom-update.jsonl",
|
||||
sessionId: "custom-update",
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(configPath, JSON.stringify({ session: { store: storePath } }), "utf-8");
|
||||
setStartupConfigPath(configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
harness.startTranscriptListener();
|
||||
|
||||
emitSessionTranscriptUpdate({
|
||||
sessionFile,
|
||||
sessionKey: "agent:main:chat:custom-update",
|
||||
});
|
||||
await Promise.resolve();
|
||||
|
||||
expect(harness.getPendingSessionFiles()).toEqual([sessionFile]);
|
||||
expect(harness.getPendingSessionTargets()).toEqual([]);
|
||||
harness.stopTranscriptListener();
|
||||
});
|
||||
|
||||
it("prefers transcript update path compatibility before identity", async () => {
|
||||
vi.useFakeTimers();
|
||||
const session = await writeSessionFile("thread.jsonl");
|
||||
const harness = new SessionStartupCatchupHarness([]);
|
||||
harness.startTranscriptListener();
|
||||
|
||||
emitSessionTranscriptUpdate({
|
||||
sessionFile: session.filePath,
|
||||
target: {
|
||||
agentId: "main",
|
||||
sessionId: "identity-target",
|
||||
sessionKey: "agent:main:identity-target",
|
||||
},
|
||||
});
|
||||
|
||||
expect(harness.getPendingSessionFiles()).toEqual([session.filePath]);
|
||||
expect(harness.getPendingSessionTargets()).toEqual([]);
|
||||
harness.stopTranscriptListener();
|
||||
});
|
||||
});
|
||||
2864
extensions/memory-core/src/memory/manager-sync-ops.ts
Normal file
2864
extensions/memory-core/src/memory/manager-sync-ops.ts
Normal file
File diff suppressed because it is too large
Load Diff
226
extensions/memory-core/src/memory/manager-sync-yield.test.ts
Normal file
226
extensions/memory-core/src/memory/manager-sync-yield.test.ts
Normal file
@@ -0,0 +1,226 @@
|
||||
// Memory Core tests cover manager sync yield plugin behavior.
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
type OpenClawConfig,
|
||||
type ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { buildSessionEntryMock } = vi.hoisted(() => ({
|
||||
buildSessionEntryMock: vi.fn(),
|
||||
}));
|
||||
const originalSyncYieldStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
|
||||
function setSyncYieldStateDir(): void {
|
||||
Reflect.set(
|
||||
process.env,
|
||||
"OPENCLAW_STATE_DIR",
|
||||
path.join(os.tmpdir(), "openclaw-session-sync-yield"),
|
||||
);
|
||||
}
|
||||
|
||||
function restoreSyncYieldStateDir(): void {
|
||||
if (originalSyncYieldStateDir === undefined) {
|
||||
Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR");
|
||||
} else {
|
||||
Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalSyncYieldStateDir);
|
||||
}
|
||||
}
|
||||
|
||||
vi.mock("undici", async () => {
|
||||
const actual = await vi.importActual<typeof import("undici")>("undici");
|
||||
return {
|
||||
...actual,
|
||||
Agent: vi.fn(),
|
||||
EnvHttpProxyAgent: vi.fn(),
|
||||
ProxyAgent: vi.fn(),
|
||||
fetch: vi.fn(),
|
||||
getGlobalDispatcher: vi.fn(),
|
||||
setGlobalDispatcher: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/memory-core-host-engine-qmd", () => {
|
||||
const basename = (filePath: string) => filePath.split(/[\\/]/).pop() ?? filePath;
|
||||
return {
|
||||
buildSessionEntry: buildSessionEntryMock,
|
||||
isSessionArchiveArtifactName: (fileName: string) => /\.jsonl\.(reset|deleted)\./.test(fileName),
|
||||
isUsageCountedSessionTranscriptFileName: (fileName: string) => fileName.endsWith(".jsonl"),
|
||||
listSessionFilesForAgent: vi.fn(async () => []),
|
||||
listSessionTranscriptCorpusEntriesForAgent: vi.fn(async () => []),
|
||||
parseCanonicalSessionSyncTargetFromPath: (filePath: string) => ({
|
||||
agentId: "main",
|
||||
sessionId: basename(filePath).replace(/\.jsonl$/, ""),
|
||||
}),
|
||||
resolveSessionFileForSyncTarget: (target: { agentId?: string; sessionId: string }) => ({
|
||||
agentId: target.agentId ?? "main",
|
||||
sessionFile: `/tmp/${target.sessionId}.jsonl`,
|
||||
sessionId: target.sessionId,
|
||||
}),
|
||||
sessionPathForFile: (filePath: string) => `sessions/${basename(filePath)}`,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
resolveEmbeddingProviderAdapterId: (providerId: string) => providerId,
|
||||
resolveEmbeddingProviderAdapterTransport: (providerId: string) =>
|
||||
providerId === "local" ? "local" : "remote",
|
||||
resolveEmbeddingProviderIndexIdentity: () => undefined,
|
||||
createEmbeddingProvider: vi.fn(),
|
||||
}));
|
||||
|
||||
import { MemoryManagerSyncOps } from "./manager-sync-ops.js";
|
||||
|
||||
type MemoryIndexEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
hash: string;
|
||||
content?: string;
|
||||
};
|
||||
|
||||
function createDbMock(): DatabaseSync {
|
||||
return {
|
||||
prepare: vi.fn(() => ({
|
||||
all: vi.fn(() => []),
|
||||
get: vi.fn(() => undefined),
|
||||
run: vi.fn(),
|
||||
})),
|
||||
} as unknown as DatabaseSync;
|
||||
}
|
||||
|
||||
class SessionSyncYieldHarness extends MemoryManagerSyncOps {
|
||||
protected readonly cfg = {} as OpenClawConfig;
|
||||
protected readonly agentId = "main";
|
||||
protected readonly workspaceDir = "/tmp/openclaw-test-workspace";
|
||||
protected readonly settings = {
|
||||
sync: {
|
||||
sessions: {
|
||||
deltaBytes: 100_000,
|
||||
deltaMessages: 50,
|
||||
postCompactionForce: true,
|
||||
},
|
||||
},
|
||||
} as ResolvedMemorySearchConfig;
|
||||
protected readonly batch = {
|
||||
enabled: false,
|
||||
wait: false,
|
||||
concurrency: 1,
|
||||
pollIntervalMs: 0,
|
||||
timeoutMs: 0,
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db = createDbMock();
|
||||
|
||||
readonly indexedPaths: string[] = [];
|
||||
|
||||
constructor(private readonly onIndexFile: (count: number) => void) {
|
||||
super();
|
||||
}
|
||||
|
||||
async syncTargetSessionFiles(files: string[]): Promise<void> {
|
||||
await (
|
||||
this as unknown as {
|
||||
syncSessionFiles: (params: {
|
||||
needsFullReindex: boolean;
|
||||
targetSessionFiles: string[];
|
||||
}) => Promise<void>;
|
||||
}
|
||||
).syncSessionFiles({
|
||||
needsFullReindex: false,
|
||||
targetSessionFiles: files,
|
||||
});
|
||||
}
|
||||
|
||||
protected computeProviderKey(): string {
|
||||
return "test";
|
||||
}
|
||||
|
||||
protected resolveProviderIndexIdentities() {
|
||||
return [];
|
||||
}
|
||||
|
||||
protected async sync(): Promise<void> {}
|
||||
|
||||
protected async withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
_timeoutMs: number,
|
||||
_message: string,
|
||||
): Promise<T> {
|
||||
return await promise;
|
||||
}
|
||||
|
||||
protected getIndexConcurrency(): number {
|
||||
return 1;
|
||||
}
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {}
|
||||
|
||||
protected resetProviderInitializationForRetry(): void {}
|
||||
|
||||
protected assertRequiredProviderAvailable(): void {}
|
||||
|
||||
protected async indexFile(
|
||||
entry: MemoryIndexEntry,
|
||||
_options: { source: MemorySource; content?: string },
|
||||
): Promise<void> {
|
||||
this.indexedPaths.push(entry.path);
|
||||
this.onIndexFile(this.indexedPaths.length);
|
||||
}
|
||||
}
|
||||
|
||||
describe("session sync responsiveness", () => {
|
||||
beforeEach(() => {
|
||||
setSyncYieldStateDir();
|
||||
buildSessionEntryMock.mockImplementation(async (absPath: string) => {
|
||||
const name = path.basename(absPath);
|
||||
return {
|
||||
path: `sessions/${name}`,
|
||||
absPath,
|
||||
mtimeMs: 1,
|
||||
size: 1,
|
||||
hash: `hash-${name}`,
|
||||
content: `user message for ${name}`,
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
restoreSyncYieldStateDir();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("yields to the event loop between session file batches", async () => {
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent("main");
|
||||
const files = Array.from({ length: 11 }, (_value, index) =>
|
||||
path.join(sessionsDir, `session-${index}.jsonl`),
|
||||
);
|
||||
let immediateRan = false;
|
||||
const immediate = new Promise<void>((resolve) => {
|
||||
setImmediate(() => {
|
||||
immediateRan = true;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const observedBeforeLastFile: boolean[] = [];
|
||||
const harness = new SessionSyncYieldHarness((count) => {
|
||||
if (count === 11) {
|
||||
observedBeforeLastFile.push(immediateRan);
|
||||
}
|
||||
});
|
||||
|
||||
await harness.syncTargetSessionFiles(files);
|
||||
|
||||
expect(harness.indexedPaths).toHaveLength(files.length);
|
||||
expect(observedBeforeLastFile).toEqual([true]);
|
||||
await immediate;
|
||||
});
|
||||
});
|
||||
126
extensions/memory-core/src/memory/manager-targeted-sync.test.ts
Normal file
126
extensions/memory-core/src/memory/manager-targeted-sync.test.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
// Memory Core tests cover manager targeted sync plugin behavior.
|
||||
import type { MemorySessionSyncTarget } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { enqueueMemoryTargetedSessionSync } from "./manager-sync-control.js";
|
||||
import {
|
||||
clearMemorySyncedSessionFiles,
|
||||
markMemoryTargetSessionFilesDirty,
|
||||
runMemoryTargetedSessionSync,
|
||||
} from "./manager-targeted-sync.js";
|
||||
|
||||
describe("memory targeted session sync", () => {
|
||||
it("preserves unrelated dirty sessions after targeted cleanup", () => {
|
||||
const secondSessionPath = "/tmp/targeted-dirty-second.jsonl";
|
||||
const sessionsDirtyFiles = new Set(["/tmp/targeted-dirty-first.jsonl", secondSessionPath]);
|
||||
|
||||
const sessionsDirty = clearMemorySyncedSessionFiles({
|
||||
sessionsDirtyFiles,
|
||||
targetSessionFiles: ["/tmp/targeted-dirty-first.jsonl"],
|
||||
});
|
||||
|
||||
expect(sessionsDirtyFiles.has(secondSessionPath)).toBe(true);
|
||||
expect(sessionsDirty).toBe(true);
|
||||
});
|
||||
|
||||
it("marks target sessions dirty while identity sync is paused", () => {
|
||||
const targetSessionPath = "/tmp/paused-target.jsonl";
|
||||
const sessionsDirtyFiles = new Set(["/tmp/other-dirty.jsonl"]);
|
||||
|
||||
const sessionsDirty = markMemoryTargetSessionFilesDirty({
|
||||
sessionsDirtyFiles,
|
||||
targetSessionFiles: [targetSessionPath],
|
||||
});
|
||||
|
||||
expect(sessionsDirty).toBe(true);
|
||||
expect(sessionsDirtyFiles.has(targetSessionPath)).toBe(true);
|
||||
expect(sessionsDirtyFiles.has("/tmp/other-dirty.jsonl")).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves targeted sessions dirty after fallback activates during targeted sync", async () => {
|
||||
const activateFallbackProvider = vi.fn(async () => true);
|
||||
const syncSessionFiles = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("embedding backend failed"))
|
||||
.mockResolvedValueOnce(undefined);
|
||||
const sessionsDirtyFiles = new Set(["/tmp/targeted-fallback.jsonl", "/tmp/other-dirty.jsonl"]);
|
||||
|
||||
const result = await runMemoryTargetedSessionSync({
|
||||
hasSessionSource: true,
|
||||
targetSessionFiles: new Set(["/tmp/targeted-fallback.jsonl"]),
|
||||
reason: "post-compaction",
|
||||
progress: undefined,
|
||||
sessionsDirtyFiles,
|
||||
syncSessionFiles,
|
||||
shouldFallbackOnError: () => true,
|
||||
activateFallbackProvider,
|
||||
});
|
||||
|
||||
expect(activateFallbackProvider).toHaveBeenCalledWith("embedding backend failed");
|
||||
expect(syncSessionFiles).toHaveBeenCalledTimes(1);
|
||||
expect(syncSessionFiles).toHaveBeenCalledWith({
|
||||
needsFullReindex: false,
|
||||
targetSessionFiles: ["/tmp/targeted-fallback.jsonl"],
|
||||
progress: undefined,
|
||||
});
|
||||
expect(result).toEqual({ handled: true, sessionsDirty: true });
|
||||
expect(sessionsDirtyFiles.has("/tmp/targeted-fallback.jsonl")).toBe(true);
|
||||
expect(sessionsDirtyFiles.has("/tmp/other-dirty.jsonl")).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves the full-retry dirty marker after targeted cleanup", async () => {
|
||||
const syncSessionFiles = vi.fn(async () => undefined);
|
||||
const sessionsDirtyFiles = new Set(["/tmp/targeted-full-retry.jsonl"]);
|
||||
|
||||
const result = await runMemoryTargetedSessionSync({
|
||||
hasSessionSource: true,
|
||||
targetSessionFiles: new Set(["/tmp/targeted-full-retry.jsonl"]),
|
||||
reason: "post-compaction",
|
||||
progress: undefined,
|
||||
sessionsFullRetryDirty: true,
|
||||
sessionsDirtyFiles,
|
||||
syncSessionFiles,
|
||||
shouldFallbackOnError: () => false,
|
||||
activateFallbackProvider: async () => false,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ handled: true, sessionsDirty: true });
|
||||
expect(sessionsDirtyFiles.size).toBe(0);
|
||||
});
|
||||
|
||||
it("queues identity session targets while a sync is already running", async () => {
|
||||
let resolveSyncing: (() => void) | undefined;
|
||||
const syncing = new Promise<void>((resolve) => {
|
||||
resolveSyncing = resolve;
|
||||
});
|
||||
const queuedSessionFiles = new Set<string>();
|
||||
const queuedSessions = new Map<string, MemorySessionSyncTarget>();
|
||||
let queuedSessionSync: Promise<void> | null = null;
|
||||
const sync = vi.fn(async () => {});
|
||||
|
||||
const queued = enqueueMemoryTargetedSessionSync(
|
||||
{
|
||||
isClosed: () => false,
|
||||
getSyncing: () => syncing,
|
||||
getQueuedSessionFiles: () => queuedSessionFiles,
|
||||
getQueuedSessions: () => queuedSessions,
|
||||
getQueuedSessionSync: () => queuedSessionSync,
|
||||
setQueuedSessionSync: (value) => {
|
||||
queuedSessionSync = value;
|
||||
},
|
||||
sync,
|
||||
},
|
||||
{
|
||||
sessions: [{ agentId: "main", sessionId: "targeted", sessionKey: "agent:main:targeted" }],
|
||||
},
|
||||
);
|
||||
|
||||
resolveSyncing?.();
|
||||
await queued;
|
||||
|
||||
expect(sync).toHaveBeenCalledWith({
|
||||
reason: "queued-sessions",
|
||||
sessions: [{ agentId: "main", sessionId: "targeted", sessionKey: "agent:main:targeted" }],
|
||||
sessionFiles: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
90
extensions/memory-core/src/memory/manager-targeted-sync.ts
Normal file
90
extensions/memory-core/src/memory/manager-targeted-sync.ts
Normal file
@@ -0,0 +1,90 @@
|
||||
// Memory Core plugin module implements manager targeted sync behavior.
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import type { MemorySyncProgressUpdate } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
type TargetedSyncProgress = {
|
||||
completed: number;
|
||||
total: number;
|
||||
label?: string;
|
||||
report: (update: MemorySyncProgressUpdate) => void;
|
||||
};
|
||||
|
||||
export function clearMemorySyncedSessionFiles(params: {
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
targetSessionFiles?: Iterable<string> | null;
|
||||
}): boolean {
|
||||
if (!params.targetSessionFiles) {
|
||||
params.sessionsDirtyFiles.clear();
|
||||
} else {
|
||||
for (const targetSessionFile of params.targetSessionFiles) {
|
||||
params.sessionsDirtyFiles.delete(targetSessionFile);
|
||||
}
|
||||
}
|
||||
return params.sessionsDirtyFiles.size > 0;
|
||||
}
|
||||
|
||||
export function markMemoryTargetSessionFilesDirty(params: {
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
targetSessionFiles?: Iterable<string> | null;
|
||||
}): boolean {
|
||||
if (params.targetSessionFiles) {
|
||||
for (const targetSessionFile of params.targetSessionFiles) {
|
||||
params.sessionsDirtyFiles.add(targetSessionFile);
|
||||
}
|
||||
}
|
||||
return params.sessionsDirtyFiles.size > 0;
|
||||
}
|
||||
|
||||
export async function runMemoryTargetedSessionSync(params: {
|
||||
hasSessionSource: boolean;
|
||||
targetSessionFiles: Set<string> | null;
|
||||
reason?: string;
|
||||
progress?: TargetedSyncProgress;
|
||||
sessionsFullRetryDirty?: boolean;
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
syncSessionFiles: (params: {
|
||||
needsFullReindex: boolean;
|
||||
targetSessionFiles?: string[];
|
||||
progress?: TargetedSyncProgress;
|
||||
}) => Promise<void>;
|
||||
shouldFallbackOnError: (err: unknown) => boolean;
|
||||
activateFallbackProvider: (reason: string) => Promise<boolean>;
|
||||
}): Promise<{ handled: boolean; sessionsDirty: boolean }> {
|
||||
if (!params.hasSessionSource || !params.targetSessionFiles) {
|
||||
return {
|
||||
handled: false,
|
||||
sessionsDirty: Boolean(params.sessionsFullRetryDirty) || params.sessionsDirtyFiles.size > 0,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await params.syncSessionFiles({
|
||||
needsFullReindex: false,
|
||||
targetSessionFiles: Array.from(params.targetSessionFiles),
|
||||
progress: params.progress,
|
||||
});
|
||||
const remainingSessionsDirty = clearMemorySyncedSessionFiles({
|
||||
sessionsDirtyFiles: params.sessionsDirtyFiles,
|
||||
targetSessionFiles: params.targetSessionFiles,
|
||||
});
|
||||
return {
|
||||
handled: true,
|
||||
sessionsDirty: Boolean(params.sessionsFullRetryDirty) || remainingSessionsDirty,
|
||||
};
|
||||
} catch (err) {
|
||||
const reason = formatErrorMessage(err);
|
||||
const activated =
|
||||
params.shouldFallbackOnError(err) && (await params.activateFallbackProvider(reason));
|
||||
if (!activated) {
|
||||
throw err;
|
||||
}
|
||||
const remainingSessionsDirty = markMemoryTargetSessionFilesDirty({
|
||||
sessionsDirtyFiles: params.sessionsDirtyFiles,
|
||||
targetSessionFiles: params.targetSessionFiles,
|
||||
});
|
||||
return {
|
||||
handled: true,
|
||||
sessionsDirty: Boolean(params.sessionsFullRetryDirty) || remainingSessionsDirty,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Memory Core tests cover manager vector warning plugin behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { logMemoryVectorDegradedWrite } from "./manager-vector-warning.js";
|
||||
|
||||
describe("memory vector degradation warnings", () => {
|
||||
it("emits the degraded warning only once for a manager", () => {
|
||||
const warn = vi.fn();
|
||||
|
||||
const first = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: true,
|
||||
vectorReady: false,
|
||||
chunkCount: 3,
|
||||
warningShown: false,
|
||||
loadError: "load failed",
|
||||
warn,
|
||||
});
|
||||
const second = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: true,
|
||||
vectorReady: false,
|
||||
chunkCount: 2,
|
||||
warningShown: first,
|
||||
loadError: "load failed",
|
||||
warn,
|
||||
});
|
||||
|
||||
expect(first).toBe(true);
|
||||
expect(second).toBe(true);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
"memory_index_chunks_vec not updated — sqlite-vec unavailable: load failed. Vector recall degraded. Further duplicate warnings suppressed.",
|
||||
);
|
||||
});
|
||||
|
||||
it("blames embedding readiness when sqlite-vec loaded but no dimensions resolved", () => {
|
||||
const warn = vi.fn();
|
||||
|
||||
const shown = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: true,
|
||||
vectorReady: false,
|
||||
chunkCount: 3,
|
||||
warningShown: false,
|
||||
warn,
|
||||
});
|
||||
|
||||
expect(shown).toBe(true);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
"memory_index_chunks_vec not updated — semantic vector embeddings unavailable — no vector dimensions resolved. Vector recall degraded. Further duplicate warnings suppressed.",
|
||||
);
|
||||
});
|
||||
|
||||
it("skips the warning when vector writes are available", () => {
|
||||
const warn = vi.fn();
|
||||
|
||||
const shown = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: true,
|
||||
vectorReady: true,
|
||||
chunkCount: 1,
|
||||
warningShown: false,
|
||||
warn,
|
||||
});
|
||||
|
||||
expect(shown).toBe(false);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
28
extensions/memory-core/src/memory/manager-vector-warning.ts
Normal file
28
extensions/memory-core/src/memory/manager-vector-warning.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
// Memory Core plugin module implements manager vector warning behavior.
|
||||
export function formatMemoryVectorDegradedWriteReason(loadError?: string): string {
|
||||
return loadError
|
||||
? `sqlite-vec unavailable: ${loadError}`
|
||||
: "semantic vector embeddings unavailable — no vector dimensions resolved";
|
||||
}
|
||||
|
||||
export function logMemoryVectorDegradedWrite(params: {
|
||||
vectorEnabled: boolean;
|
||||
vectorReady: boolean;
|
||||
chunkCount: number;
|
||||
warningShown: boolean;
|
||||
loadError?: string;
|
||||
warn: (message: string) => void;
|
||||
}): boolean {
|
||||
if (
|
||||
!params.vectorEnabled ||
|
||||
params.vectorReady ||
|
||||
params.chunkCount <= 0 ||
|
||||
params.warningShown
|
||||
) {
|
||||
return params.warningShown;
|
||||
}
|
||||
params.warn(
|
||||
`memory_index_chunks_vec not updated — ${formatMemoryVectorDegradedWriteReason(params.loadError)}. Vector recall degraded. Further duplicate warnings suppressed.`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
24
extensions/memory-core/src/memory/manager-vector-write.ts
Normal file
24
extensions/memory-core/src/memory/manager-vector-write.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
// Memory Core plugin module implements manager vector write behavior.
|
||||
import type { SQLInputValue } from "node:sqlite";
|
||||
import { vectorToBlob } from "./vector-blob.js";
|
||||
|
||||
type VectorWriteDb = {
|
||||
prepare: (sql: string) => {
|
||||
run: (...params: SQLInputValue[]) => unknown;
|
||||
};
|
||||
};
|
||||
|
||||
export function replaceMemoryVectorRow(params: {
|
||||
db: VectorWriteDb;
|
||||
id: string;
|
||||
embedding: number[];
|
||||
tableName?: string;
|
||||
}): void {
|
||||
const tableName = params.tableName ?? "memory_index_chunks_vec";
|
||||
try {
|
||||
params.db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(params.id);
|
||||
} catch {}
|
||||
params.db
|
||||
.prepare(`INSERT INTO ${tableName} (id, embedding) VALUES (?, ?)`)
|
||||
.run(params.id, vectorToBlob(params.embedding));
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
// Memory Core tests cover manager.async search plugin behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { awaitPendingManagerWork, startAsyncSearchSync } from "./manager-async-state.js";
|
||||
import { MemoryIndexManager } from "./manager.js";
|
||||
|
||||
describe("memory search async sync", () => {
|
||||
it("waits for dirty sync before querying", async () => {
|
||||
let releaseSync = () => {};
|
||||
const pendingSync = new Promise<void>((resolve) => {
|
||||
releaseSync = () => resolve();
|
||||
});
|
||||
const syncMock = vi.fn(async () => {
|
||||
return pendingSync;
|
||||
});
|
||||
const queryMock = vi.fn(async () => []);
|
||||
const manager = Object.create(MemoryIndexManager.prototype) as MemoryIndexManager;
|
||||
Object.assign(manager as unknown as Record<string, unknown>, {
|
||||
providerRequirement: { mode: "fts-only", provider: "none" },
|
||||
hasIndexedContent: () => true,
|
||||
settings: {
|
||||
sync: { onSearch: true },
|
||||
query: {
|
||||
minScore: 0,
|
||||
maxResults: 5,
|
||||
hybrid: {
|
||||
enabled: true,
|
||||
candidateMultiplier: 2,
|
||||
temporalDecay: { enabled: false, halfLifeDays: 30 },
|
||||
},
|
||||
},
|
||||
},
|
||||
warmSession: vi.fn(),
|
||||
ensureProviderInitialized: vi.fn(async () => {}),
|
||||
assertRequiredProviderAvailable: vi.fn(),
|
||||
dirty: true,
|
||||
sessionsDirty: false,
|
||||
sync: syncMock,
|
||||
provider: null,
|
||||
providerLifecycle: { mode: "fts-only", reason: "test" },
|
||||
refreshIndexIdentityDirty: () => ({ status: "valid" }),
|
||||
sources: new Set(["memory"]),
|
||||
fts: { enabled: true, available: true },
|
||||
searchKeywordWithFallback: queryMock,
|
||||
workspaceDir: "",
|
||||
});
|
||||
|
||||
const searchPromise = manager.search("current memory");
|
||||
await vi.waitFor(() => expect(syncMock).toHaveBeenCalledWith({ reason: "search" }));
|
||||
expect(queryMock).not.toHaveBeenCalled();
|
||||
|
||||
expect(syncMock).toHaveBeenCalledTimes(1);
|
||||
releaseSync();
|
||||
await searchPromise;
|
||||
expect(queryMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("waits for in-flight search sync during close", async () => {
|
||||
let releaseSync = () => {};
|
||||
const pendingSync = new Promise<void>((resolve) => {
|
||||
releaseSync = () => resolve();
|
||||
});
|
||||
|
||||
let closed = false;
|
||||
const closePromise = awaitPendingManagerWork({ pendingSync }).then(() => {
|
||||
closed = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(closed).toBe(false);
|
||||
|
||||
releaseSync();
|
||||
await closePromise;
|
||||
});
|
||||
|
||||
it("skips background search sync when search-triggered sync is disabled", async () => {
|
||||
const syncMock = vi.fn(async () => {});
|
||||
await startAsyncSearchSync({
|
||||
enabled: false,
|
||||
dirty: true,
|
||||
sessionsDirty: false,
|
||||
sync: syncMock,
|
||||
onError: vi.fn(),
|
||||
});
|
||||
expect(syncMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,227 @@
|
||||
// Memory Core tests cover manager.fts only reindex plugin behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { resolveOpenClawAgentSqlitePath } from "openclaw/plugin-sdk/sqlite-runtime";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js";
|
||||
import type { MemoryIndexMeta } from "./manager-reindex-state.js";
|
||||
import type { MemoryIndexManager } from "./manager.js";
|
||||
import "./test-runtime-mocks.js";
|
||||
|
||||
const createEmbeddingProviderMock = vi.hoisted(() =>
|
||||
vi.fn(async () => ({
|
||||
requestedProvider: "auto",
|
||||
provider: null,
|
||||
providerUnavailableReason: "No embeddings provider available.",
|
||||
})),
|
||||
);
|
||||
const originalFtsOnlyStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
|
||||
function setFtsOnlyStateDir(stateDir: string): void {
|
||||
Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir);
|
||||
}
|
||||
|
||||
function restoreFtsOnlyStateDir(): void {
|
||||
if (originalFtsOnlyStateDir === undefined) {
|
||||
Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR");
|
||||
} else {
|
||||
Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalFtsOnlyStateDir);
|
||||
}
|
||||
}
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
createEmbeddingProvider: createEmbeddingProviderMock,
|
||||
resolveEmbeddingProviderAdapterId: (providerId: string) => providerId,
|
||||
resolveEmbeddingProviderAdapterTransport: (providerId: string) =>
|
||||
providerId === "local" ? "local" : "remote",
|
||||
resolveEmbeddingProviderIndexIdentity: () => undefined,
|
||||
resolveEmbeddingProviderFallbackModel: () => "fts-only",
|
||||
}));
|
||||
|
||||
describe("memory manager FTS-only reindex", () => {
|
||||
let fixtureRoot = "";
|
||||
let caseId = 0;
|
||||
let workspaceDir = "";
|
||||
let indexPath = "";
|
||||
let manager: MemoryIndexManager | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fts-only-"));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
createEmbeddingProviderMock.mockClear();
|
||||
workspaceDir = path.join(fixtureRoot, `case-${caseId++}`);
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "Alpha topic\n\nKeep this note.");
|
||||
setFtsOnlyStateDir(path.join(workspaceDir, "state"));
|
||||
indexPath = resolveOpenClawAgentSqlitePath({ agentId: "main" });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (manager) {
|
||||
await manager.close();
|
||||
manager = null;
|
||||
}
|
||||
await closeAllMemorySearchManagers();
|
||||
restoreFtsOnlyStateDir();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await closeAllMemorySearchManagers();
|
||||
if (fixtureRoot) {
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
async function createManager(
|
||||
params: { provider?: string; vectorEnabled?: boolean } = {},
|
||||
): Promise<MemoryIndexManager> {
|
||||
const store =
|
||||
params.vectorEnabled === undefined
|
||||
? undefined
|
||||
: { vector: { enabled: params.vectorEnabled } };
|
||||
const cfg = {
|
||||
memory: {
|
||||
backend: "builtin",
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: params.provider ?? "auto",
|
||||
model: "",
|
||||
store,
|
||||
cache: { enabled: false },
|
||||
sync: { watch: false, onSessionStart: false, onSearch: false },
|
||||
},
|
||||
},
|
||||
list: [{ id: "main", default: true }],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
if (!result.manager) {
|
||||
throw new Error(result.error ?? "manager missing");
|
||||
}
|
||||
manager = result.manager as unknown as MemoryIndexManager;
|
||||
return manager;
|
||||
}
|
||||
|
||||
function countChunksContaining(term: string): number {
|
||||
const db = new DatabaseSync(indexPath);
|
||||
try {
|
||||
const row = db
|
||||
.prepare(`SELECT COUNT(*) as c FROM memory_index_chunks WHERE text LIKE ?`)
|
||||
.get(`%${term}%`) as { c: number } | undefined;
|
||||
return row?.c ?? 0;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeExistingMeta(memoryManager: MemoryIndexManager, model: string): void {
|
||||
const metaWriter = memoryManager as unknown as {
|
||||
writeMeta(meta: MemoryIndexMeta): void;
|
||||
};
|
||||
metaWriter.writeMeta({
|
||||
model,
|
||||
provider: "openai",
|
||||
chunkTokens: 600,
|
||||
chunkOverlap: 120,
|
||||
sources: ["memory"],
|
||||
});
|
||||
}
|
||||
|
||||
it("preserves indexed chunks across forced reindex in FTS-only mode", async () => {
|
||||
const memoryManager = await createManager();
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
const firstStatus = memoryManager.status();
|
||||
expect(firstStatus.chunks).toBeGreaterThan(0);
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
const secondStatus = memoryManager.status();
|
||||
expect(secondStatus.chunks).toBeGreaterThan(0);
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("syncs explicit provider-none memory without resolving an embedding provider", async () => {
|
||||
const memoryManager = await createManager({ provider: "none", vectorEnabled: false });
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
expect(createEmbeddingProviderMock).not.toHaveBeenCalled();
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
expect(memoryManager.status().custom?.indexIdentity).toEqual({ status: "valid" });
|
||||
expect(memoryManager.status().custom?.providerState).toEqual({
|
||||
mode: "fts-only",
|
||||
reason: "No embedding provider available (FTS-only mode)",
|
||||
attemptedProviderId: "none",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports explicit provider-none probes as FTS-only without resolving providers", async () => {
|
||||
const memoryManager = await createManager({ provider: "none", vectorEnabled: false });
|
||||
|
||||
await expect(memoryManager.probeEmbeddingAvailability()).resolves.toEqual({
|
||||
ok: false,
|
||||
error: "No embedding provider available (FTS-only mode)",
|
||||
});
|
||||
|
||||
expect(createEmbeddingProviderMock).not.toHaveBeenCalled();
|
||||
expect(memoryManager.status().custom?.providerState).toEqual({
|
||||
mode: "fts-only",
|
||||
reason: "No embedding provider available (FTS-only mode)",
|
||||
attemptedProviderId: "none",
|
||||
});
|
||||
});
|
||||
|
||||
it("forces provider-none memory to FTS-only when vector config is omitted", async () => {
|
||||
const memoryManager = await createManager({ provider: "none" });
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
const status = memoryManager.status();
|
||||
expect(createEmbeddingProviderMock).not.toHaveBeenCalled();
|
||||
expect(status.vector).toMatchObject({ enabled: false });
|
||||
expect(status.custom?.indexIdentity).toEqual({ status: "valid" });
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("still initializes configured providers when vector storage is disabled", async () => {
|
||||
const memoryManager = await createManager({ provider: "auto", vectorEnabled: false });
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
expect(createEmbeddingProviderMock).toHaveBeenCalledOnce();
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("refreshes FTS-only indexed content after memory file updates", async () => {
|
||||
const memoryManager = await createManager();
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "MEMORY.md"),
|
||||
"Beta refresh marker\n\nUpdated memory content.",
|
||||
);
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
expect(countChunksContaining("refresh marker")).toBeGreaterThan(0);
|
||||
expect(countChunksContaining("Alpha topic")).toBe(0);
|
||||
});
|
||||
|
||||
it("aborts instead of downgrading an existing semantic index to FTS-only", async () => {
|
||||
const memoryManager = await createManager();
|
||||
writeExistingMeta(memoryManager, "mock-embed");
|
||||
|
||||
await expect(memoryManager.sync({ force: true })).rejects.toThrow(
|
||||
"Refusing to run sync in fts-only fallback mode to protect existing vector index (current model: mock-embed).",
|
||||
);
|
||||
expect(memoryManager.status().provider).toBe("openai");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,210 @@
|
||||
// Memory Core tests cover manager.mistral provider plugin behavior.
|
||||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
applyMemoryFallbackProviderState,
|
||||
resolveMemoryFallbackProviderRequest,
|
||||
resolveMemoryPrimaryProviderRequest,
|
||||
resolveMemoryProviderState,
|
||||
} from "./manager-provider-state.js";
|
||||
|
||||
const DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text";
|
||||
const DEFAULT_LMSTUDIO_EMBEDDING_MODEL = "text-embedding-nomic-embed-text-v1.5";
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
resolveEmbeddingProviderIndexIdentity: () => undefined,
|
||||
resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) =>
|
||||
providerId === "ollama"
|
||||
? DEFAULT_OLLAMA_EMBEDDING_MODEL
|
||||
: providerId === "lmstudio"
|
||||
? DEFAULT_LMSTUDIO_EMBEDDING_MODEL
|
||||
: fallbackSourceModel,
|
||||
}));
|
||||
|
||||
type EmbeddingProvider = {
|
||||
id: string;
|
||||
model: string;
|
||||
embedQuery: (text: string) => Promise<number[]>;
|
||||
embedBatch: (texts: string[]) => Promise<number[][]>;
|
||||
};
|
||||
|
||||
type EmbeddingProviderRuntime = {
|
||||
id: string;
|
||||
cacheKeyData: { provider: string; model: string };
|
||||
};
|
||||
|
||||
function createProvider(id: string): EmbeddingProvider {
|
||||
return {
|
||||
id,
|
||||
model: `${id}-model`,
|
||||
embedQuery: async () => [0.1, 0.2, 0.3],
|
||||
embedBatch: async (texts: string[]) => texts.map(() => [0.1, 0.2, 0.3]),
|
||||
};
|
||||
}
|
||||
|
||||
function createSettings(params: {
|
||||
provider: "openai" | "mistral";
|
||||
fallback?: "none" | "mistral" | "ollama" | "lmstudio";
|
||||
}): ResolvedMemorySearchConfig {
|
||||
return {
|
||||
provider: params.provider,
|
||||
model: params.provider === "mistral" ? "mistral/mistral-embed" : "text-embedding-3-small",
|
||||
fallback: params.fallback ?? "none",
|
||||
remote: undefined,
|
||||
outputDimensionality: undefined,
|
||||
local: undefined,
|
||||
} as unknown as ResolvedMemorySearchConfig;
|
||||
}
|
||||
|
||||
type MemoryFallbackProviderRequest = NonNullable<
|
||||
ReturnType<typeof resolveMemoryFallbackProviderRequest>
|
||||
>;
|
||||
|
||||
function expectMemoryFallbackRequest(
|
||||
request: ReturnType<typeof resolveMemoryFallbackProviderRequest>,
|
||||
): MemoryFallbackProviderRequest {
|
||||
if (!request) {
|
||||
throw new Error("Expected memory fallback provider request");
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
describe("memory manager mistral provider wiring", () => {
|
||||
it("stores mistral client when mistral provider is selected", () => {
|
||||
const mistralProvider = createProvider("mistral");
|
||||
const mistralRuntime: EmbeddingProviderRuntime = {
|
||||
id: "mistral",
|
||||
cacheKeyData: { provider: "mistral", model: "mistral-embed" },
|
||||
};
|
||||
|
||||
const state = resolveMemoryProviderState({
|
||||
provider: mistralProvider,
|
||||
requestedProvider: "mistral",
|
||||
runtime: mistralRuntime,
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
providerUnavailableReason: undefined,
|
||||
});
|
||||
|
||||
expect(state.provider).toBe(mistralProvider);
|
||||
expect(state.providerRuntime).toBe(mistralRuntime);
|
||||
});
|
||||
|
||||
it("stores mistral client after fallback activation", () => {
|
||||
const openAiRuntime: EmbeddingProviderRuntime = {
|
||||
id: "openai",
|
||||
cacheKeyData: { provider: "openai", model: "text-embedding-3-small" },
|
||||
};
|
||||
const mistralRuntime: EmbeddingProviderRuntime = {
|
||||
id: "mistral",
|
||||
cacheKeyData: { provider: "mistral", model: "mistral-embed" },
|
||||
};
|
||||
const mistralProvider = createProvider("mistral");
|
||||
const current = resolveMemoryProviderState({
|
||||
provider: createProvider("openai"),
|
||||
requestedProvider: "openai",
|
||||
runtime: openAiRuntime,
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
providerUnavailableReason: undefined,
|
||||
});
|
||||
|
||||
const fallbackState = applyMemoryFallbackProviderState({
|
||||
current,
|
||||
fallbackFrom: "openai",
|
||||
reason: "forced test",
|
||||
result: {
|
||||
provider: mistralProvider,
|
||||
runtime: mistralRuntime,
|
||||
},
|
||||
});
|
||||
|
||||
expect(fallbackState.fallbackFrom).toBe("openai");
|
||||
expect(fallbackState.fallbackReason).toBe("forced test");
|
||||
expect(fallbackState.provider).toBe(mistralProvider);
|
||||
expect(fallbackState.providerRuntime).toBe(mistralRuntime);
|
||||
});
|
||||
|
||||
it("clears provider unavailable reason after fallback activation", () => {
|
||||
const fallbackState = applyMemoryFallbackProviderState({
|
||||
current: resolveMemoryProviderState({
|
||||
provider: null,
|
||||
requestedProvider: "local",
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
providerUnavailableReason: "Local embeddings degraded: worker crashed",
|
||||
runtime: undefined,
|
||||
}),
|
||||
fallbackFrom: "local",
|
||||
reason: "worker crashed",
|
||||
result: {
|
||||
provider: createProvider("openai"),
|
||||
runtime: {
|
||||
id: "openai",
|
||||
cacheKeyData: { provider: "openai", model: "text-embedding-3-small" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(fallbackState.providerUnavailableReason).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses default ollama model when activating ollama fallback", () => {
|
||||
const request = resolveMemoryFallbackProviderRequest({
|
||||
cfg: {} as OpenClawConfig,
|
||||
settings: createSettings({ provider: "openai", fallback: "ollama" }),
|
||||
currentProviderId: "openai",
|
||||
});
|
||||
|
||||
const fallbackRequest = expectMemoryFallbackRequest(request);
|
||||
expect(fallbackRequest.provider).toBe("ollama");
|
||||
expect(fallbackRequest.model).toBe(DEFAULT_OLLAMA_EMBEDDING_MODEL);
|
||||
expect(fallbackRequest.fallback).toBe("none");
|
||||
});
|
||||
|
||||
it("includes outputDimensionality in the primary provider request", () => {
|
||||
const request = resolveMemoryPrimaryProviderRequest({
|
||||
settings: {
|
||||
...createSettings({ provider: "mistral" }),
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
outputDimensionality: 1536,
|
||||
} as ResolvedMemorySearchConfig,
|
||||
});
|
||||
|
||||
expect(request.provider).toBe("gemini");
|
||||
expect(request.model).toBe("gemini-embedding-2-preview");
|
||||
expect(request.outputDimensionality).toBe(1536);
|
||||
});
|
||||
|
||||
it("includes memory input_type fields in the primary provider request", () => {
|
||||
const request = resolveMemoryPrimaryProviderRequest({
|
||||
settings: {
|
||||
...createSettings({ provider: "openai" }),
|
||||
inputType: "passage",
|
||||
queryInputType: "query",
|
||||
documentInputType: "document",
|
||||
} as ResolvedMemorySearchConfig,
|
||||
});
|
||||
|
||||
expect(request.inputType).toBe("passage");
|
||||
expect(request.queryInputType).toBe("query");
|
||||
expect(request.documentInputType).toBe("document");
|
||||
});
|
||||
|
||||
it("uses default lmstudio model when activating lmstudio fallback", () => {
|
||||
const request = resolveMemoryFallbackProviderRequest({
|
||||
cfg: {} as OpenClawConfig,
|
||||
settings: createSettings({ provider: "openai", fallback: "lmstudio" }),
|
||||
currentProviderId: "openai",
|
||||
});
|
||||
|
||||
const fallbackRequest = expectMemoryFallbackRequest(request);
|
||||
expect(fallbackRequest.provider).toBe("lmstudio");
|
||||
expect(fallbackRequest.model).toBe(DEFAULT_LMSTUDIO_EMBEDDING_MODEL);
|
||||
expect(fallbackRequest.fallback).toBe("none");
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user