Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
21
packages/memory-host-sdk/package.json
Normal file
21
packages/memory-host-sdk/package.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "@openclaw/memory-host-sdk",
|
||||
"version": "0.0.0-private",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"exports": {
|
||||
"./runtime": "./src/runtime.ts",
|
||||
"./runtime-core": "./src/runtime-core.ts",
|
||||
"./runtime-cli": "./src/runtime-cli.ts",
|
||||
"./runtime-files": "./src/runtime-files.ts",
|
||||
"./engine": "./src/engine.ts",
|
||||
"./engine-foundation": "./src/engine-foundation.ts",
|
||||
"./engine-storage": "./src/engine-storage.ts",
|
||||
"./engine-embeddings": "./src/engine-embeddings.ts",
|
||||
"./engine-qmd": "./src/engine-qmd.ts",
|
||||
"./multimodal": "./src/multimodal.ts",
|
||||
"./query": "./src/query.ts",
|
||||
"./secret": "./src/secret.ts",
|
||||
"./status": "./src/status.ts"
|
||||
}
|
||||
}
|
||||
74
packages/memory-host-sdk/src/engine-embeddings.ts
Normal file
74
packages/memory-host-sdk/src/engine-embeddings.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
// Real workspace contract for memory embedding providers and batch helpers.
|
||||
|
||||
export {
|
||||
getMemoryEmbeddingProvider,
|
||||
listRegisteredMemoryEmbeddingProviders,
|
||||
listMemoryEmbeddingProviders,
|
||||
listRegisteredMemoryEmbeddingProviderAdapters,
|
||||
} from "./host/openclaw-runtime-memory.js";
|
||||
export type {
|
||||
MemoryEmbeddingBatchChunk,
|
||||
MemoryEmbeddingBatchOptions,
|
||||
MemoryEmbeddingProvider,
|
||||
MemoryEmbeddingProviderAdapter,
|
||||
MemoryEmbeddingProviderCallOptions,
|
||||
MemoryEmbeddingProviderCreateOptions,
|
||||
MemoryEmbeddingProviderCreateResult,
|
||||
MemoryEmbeddingProviderRuntime,
|
||||
} from "./host/openclaw-runtime-memory.js";
|
||||
export { createLocalEmbeddingProvider, DEFAULT_LOCAL_MODEL } from "./host/embeddings.js";
|
||||
export { extractBatchErrorMessage, formatUnavailableBatchError } from "./host/batch-error-utils.js";
|
||||
export { postJsonWithRetry } from "./host/batch-http.js";
|
||||
export { applyEmbeddingBatchOutputLine } from "./host/batch-output.js";
|
||||
export {
|
||||
EMBEDDING_BATCH_ENDPOINT,
|
||||
type EmbeddingBatchStatus,
|
||||
type ProviderBatchOutputLine,
|
||||
} from "./host/batch-provider-common.js";
|
||||
export {
|
||||
buildEmbeddingBatchGroupOptions,
|
||||
runEmbeddingBatchGroups,
|
||||
type EmbeddingBatchExecutionParams,
|
||||
} from "./host/batch-runner.js";
|
||||
export {
|
||||
resolveBatchCompletionFromStatus,
|
||||
resolveCompletedBatchResult,
|
||||
throwIfBatchTerminalFailure,
|
||||
type BatchCompletionResult,
|
||||
} from "./host/batch-status.js";
|
||||
export { uploadBatchJsonlFile } from "./host/batch-upload.js";
|
||||
export {
|
||||
buildBatchHeaders,
|
||||
normalizeBatchBaseUrl,
|
||||
type BatchHttpClientConfig,
|
||||
} from "./host/batch-utils.js";
|
||||
export { enforceEmbeddingMaxInputTokens } from "./host/embedding-chunk-limits.js";
|
||||
export {
|
||||
isMissingEmbeddingApiKeyError,
|
||||
mapBatchEmbeddingsByIndex,
|
||||
sanitizeEmbeddingCacheHeaders,
|
||||
} from "./host/embedding-provider-adapter-utils.js";
|
||||
export { sanitizeAndNormalizeEmbedding } from "./host/embedding-vectors.js";
|
||||
export { debugEmbeddingsLog } from "./host/embeddings-debug.js";
|
||||
export { normalizeEmbeddingModelWithPrefixes } from "./host/embeddings-model-normalize.js";
|
||||
export {
|
||||
resolveRemoteEmbeddingBearerClient,
|
||||
type RemoteEmbeddingProviderId,
|
||||
} from "./host/embeddings-remote-client.js";
|
||||
export {
|
||||
createRemoteEmbeddingProvider,
|
||||
resolveRemoteEmbeddingClient,
|
||||
type RemoteEmbeddingClient,
|
||||
} from "./host/embeddings-remote-provider.js";
|
||||
export { fetchRemoteEmbeddingVectors } from "./host/embeddings-remote-fetch.js";
|
||||
export {
|
||||
estimateStructuredEmbeddingInputBytes,
|
||||
estimateUtf8Bytes,
|
||||
} from "./host/embedding-input-limits.js";
|
||||
export { hasNonTextEmbeddingParts, type EmbeddingInput } from "./host/embedding-inputs.js";
|
||||
export { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./host/remote-http.js";
|
||||
export {
|
||||
buildCaseInsensitiveExtensionGlob,
|
||||
classifyMemoryMultimodalPath,
|
||||
getMemoryMultimodalExtensions,
|
||||
} from "./host/multimodal.js";
|
||||
49
packages/memory-host-sdk/src/engine-foundation.ts
Normal file
49
packages/memory-host-sdk/src/engine-foundation.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
// Real workspace contract for memory engine foundation concerns.
|
||||
|
||||
export {
|
||||
resolveAgentContextLimits,
|
||||
resolveAgentDir,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveDefaultAgentId,
|
||||
resolveSessionAgentId,
|
||||
} from "./host/openclaw-runtime-agent.js";
|
||||
export {
|
||||
resolveMemorySearchConfig,
|
||||
resolveMemorySearchSyncConfig,
|
||||
type ResolvedMemorySearchConfig,
|
||||
type ResolvedMemorySearchSyncConfig,
|
||||
} from "./host/openclaw-runtime-agent.js";
|
||||
export { parseDurationMs } from "./host/openclaw-runtime-config.js";
|
||||
export { loadConfig } from "./host/openclaw-runtime-config.js";
|
||||
export { resolveStateDir } from "./host/openclaw-runtime-config.js";
|
||||
export { resolveSessionTranscriptsDirForAgent } from "./host/openclaw-runtime-config.js";
|
||||
export {
|
||||
hasConfiguredSecretInput,
|
||||
normalizeResolvedSecretInputString,
|
||||
} from "./host/openclaw-runtime-config.js";
|
||||
export { root } from "./host/openclaw-runtime-io.js";
|
||||
export { isPathInside } from "./host/fs-utils.js";
|
||||
export { createSubsystemLogger } from "./host/openclaw-runtime-io.js";
|
||||
export { detectMime } from "./host/openclaw-runtime-io.js";
|
||||
export { resolveGlobalSingleton } from "./host/openclaw-runtime-io.js";
|
||||
export { onSessionTranscriptUpdate } from "./host/openclaw-runtime-session.js";
|
||||
export { splitShellArgs } from "./host/openclaw-runtime-io.js";
|
||||
export { runTasksWithConcurrency } from "./host/openclaw-runtime-io.js";
|
||||
export {
|
||||
shortenHomeInString,
|
||||
shortenHomePath,
|
||||
resolveUserPath,
|
||||
truncateUtf16Safe,
|
||||
} from "./host/openclaw-runtime-io.js";
|
||||
export type { OpenClawConfig } from "./host/openclaw-runtime-config.js";
|
||||
export type { SessionSendPolicyConfig } from "./host/openclaw-runtime-config.js";
|
||||
export type { SecretInput } from "./host/openclaw-runtime-config.js";
|
||||
export type {
|
||||
MemoryBackend,
|
||||
MemoryCitationsMode,
|
||||
MemoryQmdConfig,
|
||||
MemoryQmdIndexPath,
|
||||
MemoryQmdMcporterConfig,
|
||||
MemoryQmdSearchMode,
|
||||
} from "./host/openclaw-runtime-config.js";
|
||||
export type { MemorySearchConfig } from "./host/openclaw-runtime-config.js";
|
||||
41
packages/memory-host-sdk/src/engine-qmd.ts
Normal file
41
packages/memory-host-sdk/src/engine-qmd.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
// Real workspace contract for QMD/session/query helpers used by the memory engine.
|
||||
|
||||
export { extractKeywords, isQueryStopWordToken } from "./host/query-expansion.js";
|
||||
export {
|
||||
buildSessionEntry,
|
||||
listSessionFilesForAgent,
|
||||
listSessionTranscriptCorpusEntriesForAgent,
|
||||
loadDreamingNarrativeTranscriptPathSetForAgent,
|
||||
loadSessionTranscriptClassificationForAgent,
|
||||
normalizeSessionTranscriptPathForComparison,
|
||||
parseCanonicalSessionSyncTargetFromPath,
|
||||
resolveSessionIdentityForTranscriptFile,
|
||||
resolveSessionFileForSyncTarget,
|
||||
sessionPathForFile,
|
||||
type BuildSessionEntryOptions,
|
||||
type ResolvedMemorySessionSyncTarget,
|
||||
type ResolvedSessionTranscriptIdentity,
|
||||
type SessionFileEntry,
|
||||
type SessionTranscriptClassification,
|
||||
type SessionTranscriptCorpusEntry,
|
||||
} from "./host/session-files.js";
|
||||
export {
|
||||
isSessionArchiveArtifactName,
|
||||
isUsageCountedSessionTranscriptFileName,
|
||||
parseUsageCountedSessionIdFromFileName,
|
||||
} from "./host/openclaw-runtime-session.js";
|
||||
export { parseQmdQueryJson, type QmdQueryResult } from "./host/qmd-query-parser.js";
|
||||
export {
|
||||
deriveQmdScopeChannel,
|
||||
deriveQmdScopeChatType,
|
||||
isQmdScopeAllowed,
|
||||
} from "./host/qmd-scope.js";
|
||||
export {
|
||||
checkQmdBinaryAvailability,
|
||||
resolveCliSpawnInvocation,
|
||||
resolveQmdBinaryUnavailableReason,
|
||||
runCliCommand,
|
||||
type QmdBinaryAvailability,
|
||||
type QmdBinaryUnavailable,
|
||||
type QmdBinaryUnavailableReason,
|
||||
} from "./host/qmd-process.js";
|
||||
60
packages/memory-host-sdk/src/engine-storage.ts
Normal file
60
packages/memory-host-sdk/src/engine-storage.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
// Real workspace contract for memory engine storage/index helpers.
|
||||
|
||||
export {
|
||||
buildFileEntry,
|
||||
buildMultimodalChunkForIndexing,
|
||||
chunkMarkdown,
|
||||
cosineSimilarity,
|
||||
ensureDir,
|
||||
hashText,
|
||||
listMemoryFiles,
|
||||
normalizeExtraMemoryPaths,
|
||||
parseEmbedding,
|
||||
remapChunkLines,
|
||||
runWithConcurrency,
|
||||
type MemoryChunk,
|
||||
type MemoryFileEntry,
|
||||
} from "./host/internal.js";
|
||||
export { readMemoryFile } from "./host/read-file.js";
|
||||
export { isTransientMemoryReadError, retryTransientMemoryRead } from "./host/read-retry.js";
|
||||
export {
|
||||
buildMemoryReadResult,
|
||||
buildMemoryReadResultFromSlice,
|
||||
DEFAULT_MEMORY_READ_LINES,
|
||||
DEFAULT_MEMORY_READ_MAX_CHARS,
|
||||
type MemoryReadResult,
|
||||
} from "./host/read-file-shared.js";
|
||||
export { resolveMemoryBackendConfig } from "./host/backend-config.js";
|
||||
export type {
|
||||
ResolvedMemoryBackendConfig,
|
||||
ResolvedQmdConfig,
|
||||
ResolvedQmdMcporterConfig,
|
||||
} from "./host/backend-config.js";
|
||||
export type {
|
||||
MemoryEmbeddingProbeResult,
|
||||
MemoryProviderStatus,
|
||||
MemorySearchManager,
|
||||
MemorySearchRuntimeDebug,
|
||||
MemorySearchResult,
|
||||
MemorySessionSyncTarget,
|
||||
MemorySource,
|
||||
MemorySyncParams,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "./host/types.js";
|
||||
export {
|
||||
ensureMemoryIndexSchema,
|
||||
MEMORY_EMBEDDING_CACHE_TABLE,
|
||||
MEMORY_INDEX_CHUNKS_TABLE,
|
||||
MEMORY_INDEX_FTS_TABLE,
|
||||
MEMORY_INDEX_META_TABLE,
|
||||
MEMORY_INDEX_SOURCES_TABLE,
|
||||
MEMORY_INDEX_STATE_TABLE,
|
||||
MEMORY_INDEX_VECTOR_TABLE,
|
||||
} from "./host/memory-schema.js";
|
||||
export { loadSqliteVecExtension } from "./host/sqlite-vec.js";
|
||||
export {
|
||||
closeMemorySqliteWalMaintenance,
|
||||
configureMemorySqliteWalMaintenance,
|
||||
requireNodeSqlite,
|
||||
} from "./host/sqlite.js";
|
||||
export { isFileMissingError, statRegularFile } from "./host/fs-utils.js";
|
||||
7
packages/memory-host-sdk/src/engine.ts
Normal file
7
packages/memory-host-sdk/src/engine.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
// Aggregate workspace contract for the memory engine surface.
|
||||
// Keep focused subpaths preferred for new code.
|
||||
|
||||
export * from "./engine-foundation.js";
|
||||
export * from "./engine-storage.js";
|
||||
export * from "./engine-embeddings.js";
|
||||
export * from "./engine-qmd.js";
|
||||
772
packages/memory-host-sdk/src/host/backend-config.test.ts
Normal file
772
packages/memory-host-sdk/src/host/backend-config.test.ts
Normal file
@@ -0,0 +1,772 @@
|
||||
// Memory Host SDK tests cover backend config behavior.
|
||||
import syncFs from "node:fs";
|
||||
import type { Dirent } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { resolveMemoryBackendConfig } from "./backend-config.js";
|
||||
import type { OpenClawConfig } from "./config-utils.js";
|
||||
|
||||
type ResolvedMemoryBackendConfig = ReturnType<typeof resolveMemoryBackendConfig>;
|
||||
|
||||
const resolveComparablePath = (value: string, workspaceDir = "/workspace/root"): string =>
|
||||
path.isAbsolute(value) ? path.resolve(value) : path.resolve(workspaceDir, value);
|
||||
|
||||
const memoryFileEntry = (name: string): Dirent =>
|
||||
({
|
||||
name,
|
||||
isFile: () => true,
|
||||
isSymbolicLink: () => false,
|
||||
}) as Dirent;
|
||||
|
||||
const withMemoryRootEntries = <T>(entries: Dirent[], test: () => T): T => {
|
||||
const readdirSpy = vi
|
||||
.spyOn(syncFs, "readdirSync")
|
||||
.mockReturnValue(entries as unknown as ReturnType<typeof syncFs.readdirSync>);
|
||||
try {
|
||||
return test();
|
||||
} finally {
|
||||
readdirSpy.mockRestore();
|
||||
}
|
||||
};
|
||||
|
||||
const rootMemoryConfig = (workspaceDir: string): OpenClawConfig =>
|
||||
({
|
||||
agents: {
|
||||
defaults: { workspace: workspaceDir },
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {},
|
||||
},
|
||||
}) as OpenClawConfig;
|
||||
|
||||
const collectionNames = (resolved: ResolvedMemoryBackendConfig): string[] =>
|
||||
(resolved.qmd?.collections ?? []).map((collection) => collection.name).toSorted();
|
||||
|
||||
function requireQmdConfig(
|
||||
resolved: ResolvedMemoryBackendConfig,
|
||||
): NonNullable<ResolvedMemoryBackendConfig["qmd"]> {
|
||||
if (!resolved.qmd) {
|
||||
throw new Error("expected qmd memory backend config");
|
||||
}
|
||||
return resolved.qmd;
|
||||
}
|
||||
|
||||
function requireQmdCollection(
|
||||
resolved: ResolvedMemoryBackendConfig,
|
||||
name: string,
|
||||
): NonNullable<ResolvedMemoryBackendConfig["qmd"]>["collections"][number] {
|
||||
const collection = requireQmdConfig(resolved).collections.find(
|
||||
(candidate) => candidate.name === name,
|
||||
);
|
||||
if (!collection) {
|
||||
throw new Error(`expected qmd collection ${name}`);
|
||||
}
|
||||
return collection;
|
||||
}
|
||||
|
||||
const customQmdCollections = (
|
||||
resolved: ResolvedMemoryBackendConfig,
|
||||
): NonNullable<ResolvedMemoryBackendConfig["qmd"]>["collections"] =>
|
||||
(resolved.qmd?.collections ?? []).filter((collection) => collection.kind === "custom");
|
||||
|
||||
const customCollectionPaths = (resolved: ResolvedMemoryBackendConfig): string[] =>
|
||||
customQmdCollections(resolved)
|
||||
.map((collection) => collection.path)
|
||||
.toSorted();
|
||||
|
||||
let fixtureRoot: string;
|
||||
let fixtureId = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "qmd-backend-config-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function createFixtureDir(name: string): Promise<string> {
|
||||
const dir = path.join(fixtureRoot, `${name}-${fixtureId++}`);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
describe("resolveMemoryBackendConfig", () => {
|
||||
it("defaults to builtin backend when config missing", () => {
|
||||
const cfg = { agents: { defaults: { workspace: "/tmp/memory-test" } } } as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(resolved.backend).toBe("builtin");
|
||||
expect(resolved.citations).toBe("auto");
|
||||
expect(resolved.qmd).toBeUndefined();
|
||||
});
|
||||
|
||||
it("resolves qmd backend with default collections", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(resolved.backend).toBe("qmd");
|
||||
const qmd = requireQmdConfig(resolved);
|
||||
expect(qmd.collections.length).toBe(2);
|
||||
expect(qmd.command).toBe("qmd");
|
||||
expect(qmd.searchMode).toBe("search");
|
||||
expect(qmd.update.intervalMs).toBe(300_000);
|
||||
expect(qmd.update.debounceMs).toBe(15_000);
|
||||
expect(qmd.update.onBoot).toBe(true);
|
||||
expect(qmd.update.startup).toBe("off");
|
||||
expect(qmd.update.startupDelayMs).toBe(120_000);
|
||||
expect(qmd.update.waitForBootSync).toBe(false);
|
||||
expect(qmd.update.embedIntervalMs).toBe(3_600_000);
|
||||
expect(qmd.update.commandTimeoutMs).toBe(30_000);
|
||||
expect(qmd.update.updateTimeoutMs).toBe(120_000);
|
||||
expect(qmd.update.embedTimeoutMs).toBe(120_000);
|
||||
expect(collectionNames(resolved)).toStrictEqual(["memory-dir-main", "memory-root-main"]);
|
||||
expect(requireQmdCollection(resolved, "memory-root-main").pattern).toBe("MEMORY.md");
|
||||
});
|
||||
|
||||
it("keeps uppercase MEMORY.md as the root pattern when only lowercase memory.md exists", () => {
|
||||
const workspaceDir = "/workspace/root";
|
||||
withMemoryRootEntries([memoryFileEntry("memory.md")], () => {
|
||||
const cfg = rootMemoryConfig(workspaceDir);
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(requireQmdCollection(resolved, "memory-root-main").pattern).toBe("MEMORY.md");
|
||||
expect(collectionNames(resolved)).toStrictEqual(["memory-dir-main", "memory-root-main"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers MEMORY.md over legacy memory.md when both root files exist", () => {
|
||||
const workspaceDir = "/workspace/root";
|
||||
withMemoryRootEntries([memoryFileEntry("MEMORY.md"), memoryFileEntry("memory.md")], () => {
|
||||
const cfg = rootMemoryConfig(workspaceDir);
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(requireQmdCollection(resolved, "memory-root-main").pattern).toBe("MEMORY.md");
|
||||
expect(collectionNames(resolved)).toStrictEqual(["memory-dir-main", "memory-root-main"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("parses quoted qmd command paths", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
command: '"/Applications/QMD Tools/qmd" --flag',
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(requireQmdConfig(resolved).command).toBe("/Applications/QMD Tools/qmd");
|
||||
});
|
||||
|
||||
it("preserves unquoted Windows absolute qmd command paths", () => {
|
||||
const command = String.raw`C:\Users\penny\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js`;
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: String.raw`C:\Users\penny\.openclaw\workspace` } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
command,
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
|
||||
expect(requireQmdConfig(resolved).command).toBe(command);
|
||||
});
|
||||
|
||||
it("preserves Windows wrapper command paths before extra args", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: String.raw`C:\Users\penny\.openclaw\workspace` } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
command: String.raw`C:\Program Files\qmd\qmd.cmd --json`,
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
|
||||
expect(requireQmdConfig(resolved).command).toBe(String.raw`C:\Program Files\qmd\qmd.cmd`);
|
||||
});
|
||||
|
||||
it("preserves unquoted UNC qmd command paths", () => {
|
||||
const command = String.raw`\\fileserver\tools\qmd\dist\cli\qmd.js`;
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: String.raw`C:\Users\penny\.openclaw\workspace` } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
command,
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
|
||||
expect(requireQmdConfig(resolved).command).toBe(command);
|
||||
});
|
||||
|
||||
it("resolves custom paths relative to workspace", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { workspace: "/workspace/root" },
|
||||
list: [{ id: "main", workspace: "/workspace/root" }],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
paths: [
|
||||
{
|
||||
path: "notes",
|
||||
name: "custom-notes",
|
||||
pattern: "**/*.md",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const custom = requireQmdConfig(resolved).collections.find((c) =>
|
||||
c.name.startsWith("custom-notes"),
|
||||
);
|
||||
if (!custom) {
|
||||
throw new Error("expected custom-notes qmd collection");
|
||||
}
|
||||
expect(custom.path).toBe(path.resolve("/workspace/root", "notes"));
|
||||
});
|
||||
|
||||
it("normalizes direct file qmd paths to escaped exact-file patterns", async () => {
|
||||
const workspaceDir = await createFixtureDir("direct-file-path");
|
||||
const notesPath = path.join(workspaceDir, "notes{a,b}[1].md");
|
||||
await fs.writeFile(notesPath, "# Notes\n", "utf8");
|
||||
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { workspace: workspaceDir },
|
||||
list: [{ id: "main", workspace: workspaceDir }],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
paths: [{ path: "notes{a,b}[1].md", name: "direct-note", pattern: "**/*.md" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const custom = resolved.qmd?.collections.find((c) => c.name.startsWith("direct-note"));
|
||||
expect(custom).toMatchObject({
|
||||
path: workspaceDir,
|
||||
pattern: String.raw`notes\{a,b\}\[1\].md`,
|
||||
});
|
||||
});
|
||||
|
||||
it("scopes qmd collection names per agent", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { workspace: "/workspace/root" },
|
||||
list: [
|
||||
{ id: "main", default: true, workspace: "/workspace/root" },
|
||||
{ id: "dev", workspace: "/workspace/dev" },
|
||||
],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: true,
|
||||
paths: [{ path: "notes", name: "workspace", pattern: "**/*.md" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const mainResolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const devResolved = resolveMemoryBackendConfig({ cfg, agentId: "dev" });
|
||||
const mainNames = collectionNames(mainResolved);
|
||||
const devNames = collectionNames(devResolved);
|
||||
expect(mainNames).toStrictEqual(["memory-dir-main", "memory-root-main", "workspace-main"]);
|
||||
expect(devNames).toStrictEqual(["memory-dir-dev", "memory-root-dev", "workspace-dev"]);
|
||||
});
|
||||
|
||||
it("merges default and per-agent qmd extra collections", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
qmd: {
|
||||
extraCollections: [
|
||||
{
|
||||
path: "/shared/team-notes",
|
||||
name: "team-notes",
|
||||
pattern: "**/*.md",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
list: [
|
||||
{
|
||||
id: "main",
|
||||
default: true,
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
qmd: {
|
||||
extraCollections: [
|
||||
{
|
||||
path: "notes",
|
||||
name: "notes",
|
||||
pattern: "**/*.md",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const names = collectionNames(resolved);
|
||||
expect(names).toStrictEqual(["notes-main", "team-notes"]);
|
||||
});
|
||||
|
||||
it("preserves explicit custom collection names for paths outside the workspace", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { workspace: "/workspace/root" },
|
||||
list: [
|
||||
{ id: "main", default: true, workspace: "/workspace/root" },
|
||||
{ id: "dev", workspace: "/workspace/dev" },
|
||||
],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: true,
|
||||
paths: [{ path: "/shared/notion-mirror", name: "notion-mirror", pattern: "**/*.md" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const mainResolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const devResolved = resolveMemoryBackendConfig({ cfg, agentId: "dev" });
|
||||
const mainNames = collectionNames(mainResolved);
|
||||
const devNames = collectionNames(devResolved);
|
||||
expect(mainNames).toStrictEqual(["memory-dir-main", "memory-root-main", "notion-mirror"]);
|
||||
expect(devNames).toStrictEqual(["memory-dir-dev", "memory-root-dev", "notion-mirror"]);
|
||||
});
|
||||
|
||||
it("keeps symlinked workspace paths agent-scoped when deciding custom collection names", async () => {
|
||||
const tmpRoot = await createFixtureDir("symlinked-workspace");
|
||||
const workspaceDir = path.join(tmpRoot, "workspace");
|
||||
const workspaceAliasDir = path.join(tmpRoot, "workspace-alias");
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.symlink(workspaceDir, workspaceAliasDir);
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { workspace: workspaceDir },
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
paths: [{ path: workspaceAliasDir, name: "workspace", pattern: "**/*.md" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const names = collectionNames(resolved);
|
||||
expect(names).toStrictEqual(["workspace-main"]);
|
||||
});
|
||||
|
||||
it("keeps unresolved child paths under a symlinked workspace agent-scoped", async () => {
|
||||
const tmpRoot = await createFixtureDir("symlinked-child");
|
||||
const realRootDir = path.join(tmpRoot, "real-root");
|
||||
const aliasRootDir = path.join(tmpRoot, "alias-root");
|
||||
const workspaceDir = path.join(realRootDir, "workspace");
|
||||
const workspaceAliasDir = path.join(aliasRootDir, "workspace");
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.symlink(realRootDir, aliasRootDir);
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { workspace: workspaceDir },
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
paths: [
|
||||
{ path: path.join(workspaceAliasDir, "notes"), name: "notes", pattern: "**/*.md" },
|
||||
],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const names = collectionNames(resolved);
|
||||
expect(names).toStrictEqual(["notes-main"]);
|
||||
});
|
||||
|
||||
it("resolves qmd update timeout overrides", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
update: {
|
||||
waitForBootSync: true,
|
||||
commandTimeoutMs: 12_000,
|
||||
updateTimeoutMs: 480_000,
|
||||
embedTimeoutMs: 360_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const update = requireQmdConfig(resolved).update;
|
||||
expect(update.waitForBootSync).toBe(true);
|
||||
expect(update.commandTimeoutMs).toBe(12_000);
|
||||
expect(update.updateTimeoutMs).toBe(480_000);
|
||||
expect(update.embedTimeoutMs).toBe(360_000);
|
||||
});
|
||||
|
||||
it("keeps sub-unit positive qmd numeric overrides usable", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
sessions: {
|
||||
enabled: true,
|
||||
retentionDays: 0.5,
|
||||
},
|
||||
update: {
|
||||
commandTimeoutMs: 0.5,
|
||||
updateTimeoutMs: 0.5,
|
||||
embedTimeoutMs: 0.5,
|
||||
},
|
||||
limits: {
|
||||
maxResults: 0.5,
|
||||
maxSnippetChars: 0.5,
|
||||
maxInjectedChars: 0.5,
|
||||
timeoutMs: 0.5,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const qmd = requireQmdConfig(resolved);
|
||||
|
||||
expect(qmd.sessions.retentionDays).toBe(1);
|
||||
expect(qmd.update.commandTimeoutMs).toBe(1);
|
||||
expect(qmd.update.updateTimeoutMs).toBe(1);
|
||||
expect(qmd.update.embedTimeoutMs).toBe(1);
|
||||
expect(qmd.limits).toMatchObject({
|
||||
maxResults: 1,
|
||||
maxSnippetChars: 1,
|
||||
maxInjectedChars: 1,
|
||||
timeoutMs: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back for non-finite qmd numeric overrides", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
sessions: {
|
||||
enabled: true,
|
||||
retentionDays: Number.NaN,
|
||||
},
|
||||
update: {
|
||||
commandTimeoutMs: Number.POSITIVE_INFINITY,
|
||||
updateTimeoutMs: Number.NaN,
|
||||
embedTimeoutMs: Number.NEGATIVE_INFINITY,
|
||||
},
|
||||
limits: {
|
||||
maxResults: Number.NaN,
|
||||
maxSnippetChars: Number.POSITIVE_INFINITY,
|
||||
maxInjectedChars: Number.NEGATIVE_INFINITY,
|
||||
timeoutMs: Number.NaN,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const qmd = requireQmdConfig(resolved);
|
||||
|
||||
expect(qmd.sessions.retentionDays).toBeUndefined();
|
||||
expect(qmd.update.commandTimeoutMs).toBe(30_000);
|
||||
expect(qmd.update.updateTimeoutMs).toBe(120_000);
|
||||
expect(qmd.update.embedTimeoutMs).toBe(120_000);
|
||||
expect(qmd.limits).toMatchObject({
|
||||
maxResults: 4,
|
||||
maxSnippetChars: 450,
|
||||
maxInjectedChars: 2_200,
|
||||
timeoutMs: 4_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves qmd startup refresh overrides", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
update: {
|
||||
startup: "idle",
|
||||
startupDelayMs: 45_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const update = requireQmdConfig(resolved).update;
|
||||
expect(update.startup).toBe("idle");
|
||||
expect(update.startupDelayMs).toBe(45_000);
|
||||
expect(update.onBoot).toBe(true);
|
||||
});
|
||||
|
||||
it("resolves qmd search mode override", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
searchMode: "vsearch",
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(requireQmdConfig(resolved).searchMode).toBe("vsearch");
|
||||
});
|
||||
|
||||
it("resolves qmd rerank override", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
searchMode: "query",
|
||||
rerank: false,
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const qmd = requireQmdConfig(resolved);
|
||||
expect(qmd.searchMode).toBe("query");
|
||||
expect(qmd.rerank).toBe(false);
|
||||
});
|
||||
|
||||
it("resolves qmd mcporter search tool override", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
searchMode: "query",
|
||||
searchTool: " hybrid_search ",
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const qmd = requireQmdConfig(resolved);
|
||||
expect(qmd.searchMode).toBe("query");
|
||||
expect(qmd.searchTool).toBe("hybrid_search");
|
||||
});
|
||||
});
|
||||
|
||||
describe("memorySearch.extraPaths integration", () => {
|
||||
it("maps agents.defaults.memorySearch.extraPaths to QMD collections", () => {
|
||||
const cfg = {
|
||||
memory: { backend: "qmd" },
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
extraPaths: ["/home/user/docs", "/home/user/vault"],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const result = resolveMemoryBackendConfig({ cfg, agentId: "test-agent" });
|
||||
expect(result.backend).toBe("qmd");
|
||||
const paths = customCollectionPaths(result);
|
||||
expect(paths).toStrictEqual([
|
||||
resolveComparablePath("/home/user/docs"),
|
||||
resolveComparablePath("/home/user/vault"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("merges default and per-agent memorySearch.extraPaths for QMD collections", () => {
|
||||
const cfg = {
|
||||
memory: { backend: "qmd" },
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
extraPaths: ["/default/path"],
|
||||
},
|
||||
},
|
||||
list: [
|
||||
{
|
||||
id: "my-agent",
|
||||
memorySearch: {
|
||||
extraPaths: ["/agent/specific/path"],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const result = resolveMemoryBackendConfig({ cfg, agentId: "my-agent" });
|
||||
expect(result.backend).toBe("qmd");
|
||||
const paths = customCollectionPaths(result);
|
||||
expect(paths).toStrictEqual([
|
||||
resolveComparablePath("/agent/specific/path"),
|
||||
resolveComparablePath("/default/path"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("falls back to defaults when agent has no overrides", () => {
|
||||
const cfg = {
|
||||
memory: { backend: "qmd" },
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
extraPaths: ["/default/path"],
|
||||
},
|
||||
},
|
||||
list: [
|
||||
{
|
||||
id: "other-agent",
|
||||
memorySearch: {
|
||||
extraPaths: ["/other/path"],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const result = resolveMemoryBackendConfig({ cfg, agentId: "my-agent" });
|
||||
expect(result.backend).toBe("qmd");
|
||||
const paths = customCollectionPaths(result);
|
||||
expect(paths).toStrictEqual([resolveComparablePath("/default/path")]);
|
||||
});
|
||||
|
||||
it("deduplicates merged memorySearch.extraPaths for QMD collections", () => {
|
||||
const cfg = {
|
||||
memory: { backend: "qmd" },
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
extraPaths: ["/shared/path", " /shared/path "],
|
||||
},
|
||||
},
|
||||
list: [
|
||||
{
|
||||
id: "my-agent",
|
||||
memorySearch: {
|
||||
extraPaths: ["/shared/path", "/agent-only"],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const result = resolveMemoryBackendConfig({ cfg, agentId: "my-agent" });
|
||||
const paths = customCollectionPaths(result);
|
||||
|
||||
expect(paths).toStrictEqual([
|
||||
resolveComparablePath("/agent-only"),
|
||||
resolveComparablePath("/shared/path"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps unnamed extra paths agent-scoped even when they resolve outside the workspace", () => {
|
||||
const cfg = {
|
||||
memory: { backend: "qmd" },
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
extraPaths: ["/shared/path"],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const result = resolveMemoryBackendConfig({ cfg, agentId: "my-agent" });
|
||||
expect(customQmdCollections(result).map((collection) => collection.name)).toStrictEqual([
|
||||
"custom-1-my-agent",
|
||||
]);
|
||||
});
|
||||
|
||||
it("matches per-agent memorySearch.extraPaths using normalized agent ids", () => {
|
||||
const cfg = {
|
||||
memory: { backend: "qmd" },
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
},
|
||||
list: [
|
||||
{
|
||||
id: "My-Agent",
|
||||
memorySearch: {
|
||||
extraPaths: ["/agent/mixed-case"],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const result = resolveMemoryBackendConfig({ cfg, agentId: "my-agent" });
|
||||
|
||||
expect(customCollectionPaths(result)).toStrictEqual([
|
||||
resolveComparablePath("/agent/mixed-case"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("deduplicates identical roots shared by memory.qmd.paths and memorySearch.extraPaths", () => {
|
||||
const cfg = {
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
paths: [{ path: "docs", pattern: "**/*.md", name: "workspace-docs" }],
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/workspace/root",
|
||||
memorySearch: {
|
||||
extraPaths: ["./docs"],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const result = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const docsCollections = customQmdCollections(result).filter(
|
||||
(collection) =>
|
||||
collection.path === resolveComparablePath("./docs") && collection.pattern === "**/*.md",
|
||||
);
|
||||
|
||||
expect(docsCollections).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
510
packages/memory-host-sdk/src/host/backend-config.ts
Normal file
510
packages/memory-host-sdk/src/host/backend-config.ts
Normal file
@@ -0,0 +1,510 @@
|
||||
// Memory Host SDK module implements backend config behavior.
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import {
|
||||
CANONICAL_ROOT_MEMORY_FILENAME,
|
||||
type MemoryBackend,
|
||||
type MemoryCitationsMode,
|
||||
type MemoryQmdConfig,
|
||||
type MemoryQmdIndexPath,
|
||||
type MemoryQmdMcporterConfig,
|
||||
type MemoryQmdSearchMode,
|
||||
type MemoryQmdStartupMode,
|
||||
type OpenClawConfig,
|
||||
parseDurationMs,
|
||||
resolveAgentWorkspaceDir,
|
||||
normalizeAgentId,
|
||||
resolveUserPath,
|
||||
type SessionSendPolicyConfig,
|
||||
splitShellArgs,
|
||||
} from "./config-utils.js";
|
||||
import { isPathInside } from "./fs-utils.js";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeStringEntries,
|
||||
uniqueStrings,
|
||||
} from "./string-utils.js";
|
||||
|
||||
function escapeQmdExactFilePattern(fileName: string): string {
|
||||
return fileName.replace(/[\\*?[\]{}()!+@]/g, "\\$&");
|
||||
}
|
||||
|
||||
const WINDOWS_COMMAND_EXTENSION_RE =
|
||||
/^((?:[A-Za-z]:[\\/]|\\\\[^\\/]+[\\/][^\\/]+[\\/]).*?\.(?:bat|cmd|cjs|exe|js|mjs|ps1))(?:\s+|$)/i;
|
||||
|
||||
function resolveQmdCommand(rawCommand: string): string {
|
||||
const trimmedCommand = rawCommand.trim();
|
||||
const windowsCommand = resolveWindowsAbsoluteCommand(trimmedCommand);
|
||||
if (windowsCommand) {
|
||||
return windowsCommand;
|
||||
}
|
||||
|
||||
const parsedCommand = splitShellArgs(trimmedCommand);
|
||||
return parsedCommand?.[0] || trimmedCommand.split(/\s+/)[0] || "qmd";
|
||||
}
|
||||
|
||||
function resolveWindowsAbsoluteCommand(rawCommand: string): string | undefined {
|
||||
if (!path.win32.isAbsolute(rawCommand)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const extensionMatch = WINDOWS_COMMAND_EXTENSION_RE.exec(rawCommand);
|
||||
if (extensionMatch) {
|
||||
return extensionMatch[1];
|
||||
}
|
||||
|
||||
const firstWhitespace = rawCommand.search(/\s/);
|
||||
return firstWhitespace === -1 ? rawCommand : rawCommand.slice(0, firstWhitespace);
|
||||
}
|
||||
|
||||
export type ResolvedMemoryBackendConfig = {
|
||||
backend: MemoryBackend;
|
||||
citations: MemoryCitationsMode;
|
||||
qmd?: ResolvedQmdConfig;
|
||||
};
|
||||
|
||||
export type ResolvedQmdCollection = {
|
||||
name: string;
|
||||
path: string;
|
||||
pattern: string;
|
||||
kind: "memory" | "custom" | "sessions";
|
||||
};
|
||||
|
||||
export type ResolvedQmdUpdateConfig = {
|
||||
intervalMs: number;
|
||||
debounceMs: number;
|
||||
onBoot: boolean;
|
||||
startup: MemoryQmdStartupMode;
|
||||
startupDelayMs: number;
|
||||
waitForBootSync: boolean;
|
||||
embedIntervalMs: number;
|
||||
commandTimeoutMs: number;
|
||||
updateTimeoutMs: number;
|
||||
embedTimeoutMs: number;
|
||||
};
|
||||
|
||||
export type ResolvedQmdLimitsConfig = {
|
||||
maxResults: number;
|
||||
maxSnippetChars: number;
|
||||
maxInjectedChars: number;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
export type ResolvedQmdSessionConfig = {
|
||||
enabled: boolean;
|
||||
exportDir?: string;
|
||||
retentionDays?: number;
|
||||
};
|
||||
|
||||
export type ResolvedQmdMcporterConfig = {
|
||||
enabled: boolean;
|
||||
serverName: string;
|
||||
startDaemon: boolean;
|
||||
};
|
||||
|
||||
export type ResolvedQmdConfig = {
|
||||
command: string;
|
||||
mcporter: ResolvedQmdMcporterConfig;
|
||||
searchMode: MemoryQmdSearchMode;
|
||||
rerank?: boolean;
|
||||
searchTool?: string;
|
||||
collections: ResolvedQmdCollection[];
|
||||
sessions: ResolvedQmdSessionConfig;
|
||||
update: ResolvedQmdUpdateConfig;
|
||||
limits: ResolvedQmdLimitsConfig;
|
||||
includeDefaultMemory: boolean;
|
||||
scope?: SessionSendPolicyConfig;
|
||||
};
|
||||
|
||||
const DEFAULT_BACKEND: MemoryBackend = "builtin";
|
||||
const DEFAULT_CITATIONS: MemoryCitationsMode = "auto";
|
||||
const DEFAULT_QMD_INTERVAL = "5m";
|
||||
const DEFAULT_QMD_DEBOUNCE_MS = 15_000;
|
||||
const DEFAULT_QMD_TIMEOUT_MS = 4_000;
|
||||
// Defaulting to `query` can be extremely slow on CPU-only systems (query expansion + rerank).
|
||||
// Prefer a faster mode for interactive use; users can opt into `query` for best recall.
|
||||
const DEFAULT_QMD_SEARCH_MODE: MemoryQmdSearchMode = "search";
|
||||
const DEFAULT_QMD_STARTUP: MemoryQmdStartupMode = "off";
|
||||
const DEFAULT_QMD_STARTUP_DELAY_MS = 120_000;
|
||||
const DEFAULT_QMD_EMBED_INTERVAL = "60m";
|
||||
const DEFAULT_QMD_COMMAND_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_QMD_UPDATE_TIMEOUT_MS = 120_000;
|
||||
const DEFAULT_QMD_EMBED_TIMEOUT_MS = 120_000;
|
||||
const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = {
|
||||
maxResults: 4,
|
||||
maxSnippetChars: 450,
|
||||
maxInjectedChars: 2_200,
|
||||
timeoutMs: DEFAULT_QMD_TIMEOUT_MS,
|
||||
};
|
||||
const DEFAULT_QMD_MCPORTER: ResolvedQmdMcporterConfig = {
|
||||
enabled: false,
|
||||
serverName: "qmd",
|
||||
startDaemon: true,
|
||||
};
|
||||
|
||||
const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = {
|
||||
default: "deny",
|
||||
rules: [
|
||||
{
|
||||
action: "allow",
|
||||
match: { chatType: "direct" },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function sanitizeName(input: string): string {
|
||||
const lower = normalizeLowercaseStringOrEmpty(input).replace(/[^a-z0-9-]+/g, "-");
|
||||
const trimmed = lower.replace(/^-+|-+$/g, "");
|
||||
return trimmed || "collection";
|
||||
}
|
||||
|
||||
function scopeCollectionBase(base: string, agentId: string): string {
|
||||
return `${base}-${sanitizeName(agentId)}`;
|
||||
}
|
||||
|
||||
function canonicalizePathForContainment(rawPath: string): string {
|
||||
const resolved = path.resolve(rawPath);
|
||||
let current = resolved;
|
||||
const suffix: string[] = [];
|
||||
while (true) {
|
||||
try {
|
||||
const canonical = path.normalize(fs.realpathSync.native(current));
|
||||
return path.normalize(path.join(canonical, ...suffix));
|
||||
} catch {
|
||||
const parent = path.dirname(current);
|
||||
if (parent === current) {
|
||||
return path.normalize(resolved);
|
||||
}
|
||||
suffix.unshift(path.basename(current));
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isPathInsideRoot(candidatePath: string, rootPath: string): boolean {
|
||||
return isPathInside(
|
||||
canonicalizePathForContainment(rootPath),
|
||||
canonicalizePathForContainment(candidatePath),
|
||||
);
|
||||
}
|
||||
|
||||
function ensureUniqueName(base: string, existing: Set<string>): string {
|
||||
const name = sanitizeName(base);
|
||||
if (!existing.has(name)) {
|
||||
existing.add(name);
|
||||
return name;
|
||||
}
|
||||
let suffix = 2;
|
||||
while (existing.has(`${name}-${suffix}`)) {
|
||||
suffix += 1;
|
||||
}
|
||||
const unique = `${name}-${suffix}`;
|
||||
existing.add(unique);
|
||||
return unique;
|
||||
}
|
||||
|
||||
function resolvePath(raw: string, workspaceDir: string): string {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
if (trimmed.startsWith("~") || path.isAbsolute(trimmed)) {
|
||||
return path.normalize(resolveUserPath(trimmed));
|
||||
}
|
||||
return path.normalize(path.resolve(workspaceDir, trimmed));
|
||||
}
|
||||
|
||||
function resolveIntervalMs(raw: string | undefined): number {
|
||||
const value = raw?.trim();
|
||||
if (!value) {
|
||||
return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" });
|
||||
}
|
||||
try {
|
||||
return parseDurationMs(value, { defaultUnit: "m" });
|
||||
} catch {
|
||||
return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" });
|
||||
}
|
||||
}
|
||||
|
||||
function resolveEmbedIntervalMs(raw: string | undefined): number {
|
||||
const value = raw?.trim();
|
||||
if (!value) {
|
||||
return parseDurationMs(DEFAULT_QMD_EMBED_INTERVAL, { defaultUnit: "m" });
|
||||
}
|
||||
try {
|
||||
return parseDurationMs(value, { defaultUnit: "m" });
|
||||
} catch {
|
||||
return parseDurationMs(DEFAULT_QMD_EMBED_INTERVAL, { defaultUnit: "m" });
|
||||
}
|
||||
}
|
||||
|
||||
function resolveDebounceMs(raw: number | undefined): number {
|
||||
if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) {
|
||||
return Math.floor(raw);
|
||||
}
|
||||
return DEFAULT_QMD_DEBOUNCE_MS;
|
||||
}
|
||||
|
||||
function resolveTimeoutMs(raw: number | undefined, fallback: number): number {
|
||||
return resolvePositiveIntegerConfig(raw, fallback);
|
||||
}
|
||||
|
||||
function resolvePositiveIntegerConfig(raw: number | undefined, fallback: number): number;
|
||||
function resolvePositiveIntegerConfig(raw: number | undefined): number | undefined;
|
||||
function resolvePositiveIntegerConfig(
|
||||
raw: number | undefined,
|
||||
fallback?: number,
|
||||
): number | undefined {
|
||||
if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) {
|
||||
return fallback;
|
||||
}
|
||||
return Math.max(1, Math.floor(raw));
|
||||
}
|
||||
|
||||
function resolveStartupMode(raw: MemoryQmdConfig["update"]): MemoryQmdStartupMode {
|
||||
const value = raw?.startup;
|
||||
if (value === "idle" || value === "immediate" || value === "off") {
|
||||
return value;
|
||||
}
|
||||
return DEFAULT_QMD_STARTUP;
|
||||
}
|
||||
|
||||
function resolveStartupDelayMs(raw: number | undefined): number {
|
||||
if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) {
|
||||
return Math.floor(raw);
|
||||
}
|
||||
return DEFAULT_QMD_STARTUP_DELAY_MS;
|
||||
}
|
||||
|
||||
function resolveLimits(raw?: MemoryQmdConfig["limits"]): ResolvedQmdLimitsConfig {
|
||||
return {
|
||||
maxResults: resolvePositiveIntegerConfig(raw?.maxResults, DEFAULT_QMD_LIMITS.maxResults),
|
||||
maxSnippetChars: resolvePositiveIntegerConfig(
|
||||
raw?.maxSnippetChars,
|
||||
DEFAULT_QMD_LIMITS.maxSnippetChars,
|
||||
),
|
||||
maxInjectedChars: resolvePositiveIntegerConfig(
|
||||
raw?.maxInjectedChars,
|
||||
DEFAULT_QMD_LIMITS.maxInjectedChars,
|
||||
),
|
||||
timeoutMs: resolvePositiveIntegerConfig(raw?.timeoutMs, DEFAULT_QMD_LIMITS.timeoutMs),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveSearchMode(raw?: MemoryQmdConfig["searchMode"]): MemoryQmdSearchMode {
|
||||
if (raw === "search" || raw === "vsearch" || raw === "query") {
|
||||
return raw;
|
||||
}
|
||||
return DEFAULT_QMD_SEARCH_MODE;
|
||||
}
|
||||
|
||||
function resolveSearchTool(raw?: MemoryQmdConfig["searchTool"]): string | undefined {
|
||||
const value = raw?.trim();
|
||||
return value ? value : undefined;
|
||||
}
|
||||
|
||||
function resolveSessionConfig(
|
||||
cfg: MemoryQmdConfig["sessions"],
|
||||
workspaceDir: string,
|
||||
): ResolvedQmdSessionConfig {
|
||||
const enabled = Boolean(cfg?.enabled);
|
||||
const exportDirRaw = cfg?.exportDir?.trim();
|
||||
const exportDir = exportDirRaw ? resolvePath(exportDirRaw, workspaceDir) : undefined;
|
||||
const retentionDays = resolvePositiveIntegerConfig(cfg?.retentionDays);
|
||||
return {
|
||||
enabled,
|
||||
exportDir,
|
||||
retentionDays,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCustomPaths(
|
||||
rawPaths: MemoryQmdIndexPath[] | undefined,
|
||||
workspaceDir: string,
|
||||
existing: Set<string>,
|
||||
agentId: string,
|
||||
): ResolvedQmdCollection[] {
|
||||
if (!rawPaths?.length) {
|
||||
return [];
|
||||
}
|
||||
const collections: ResolvedQmdCollection[] = [];
|
||||
const seenRoots = new Set<string>();
|
||||
rawPaths.forEach((entry, index) => {
|
||||
const trimmedPath = entry?.path?.trim();
|
||||
if (!trimmedPath) {
|
||||
return;
|
||||
}
|
||||
let resolved: string;
|
||||
let collectionPath: string;
|
||||
try {
|
||||
resolved = resolvePath(trimmedPath, workspaceDir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
collectionPath = resolved;
|
||||
let pattern = entry.pattern?.trim() || "**/*.md";
|
||||
try {
|
||||
const stat = fs.statSync(resolved);
|
||||
if (stat.isFile()) {
|
||||
// When the configured path points directly to a file, normalize into a
|
||||
// parent-directory collection with an exact-filename pattern, regardless
|
||||
// of any user-supplied glob (a glob does not apply to a single file).
|
||||
collectionPath = path.dirname(resolved);
|
||||
pattern = escapeQmdExactFilePattern(path.basename(resolved));
|
||||
}
|
||||
} catch {
|
||||
// not a file or can't stat, use as-is
|
||||
}
|
||||
const dedupeKey = `${collectionPath}\u0000${pattern}`;
|
||||
if (seenRoots.has(dedupeKey)) {
|
||||
return;
|
||||
}
|
||||
seenRoots.add(dedupeKey);
|
||||
const explicitName = entry.name?.trim();
|
||||
const baseName =
|
||||
explicitName && !isPathInsideRoot(collectionPath, workspaceDir)
|
||||
? explicitName
|
||||
: scopeCollectionBase(explicitName || `custom-${index + 1}`, agentId);
|
||||
const name = ensureUniqueName(baseName, existing);
|
||||
collections.push({
|
||||
name,
|
||||
path: collectionPath,
|
||||
pattern,
|
||||
kind: "custom",
|
||||
});
|
||||
});
|
||||
return collections;
|
||||
}
|
||||
|
||||
function resolveMcporterConfig(raw?: MemoryQmdMcporterConfig): ResolvedQmdMcporterConfig {
|
||||
const parsed: ResolvedQmdMcporterConfig = { ...DEFAULT_QMD_MCPORTER };
|
||||
if (!raw) {
|
||||
return parsed;
|
||||
}
|
||||
if (raw.enabled !== undefined) {
|
||||
parsed.enabled = raw.enabled;
|
||||
}
|
||||
if (typeof raw.serverName === "string" && raw.serverName.trim()) {
|
||||
parsed.serverName = raw.serverName.trim();
|
||||
}
|
||||
if (raw.startDaemon !== undefined) {
|
||||
parsed.startDaemon = raw.startDaemon;
|
||||
}
|
||||
// When enabled, default startDaemon to true.
|
||||
if (parsed.enabled && raw.startDaemon === undefined) {
|
||||
parsed.startDaemon = true;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function resolveDefaultCollections(
|
||||
include: boolean,
|
||||
workspaceDir: string,
|
||||
existing: Set<string>,
|
||||
agentId: string,
|
||||
): ResolvedQmdCollection[] {
|
||||
if (!include) {
|
||||
return [];
|
||||
}
|
||||
const entries: Array<{ path: string; pattern: string; base: string }> = [
|
||||
{ path: workspaceDir, pattern: CANONICAL_ROOT_MEMORY_FILENAME, base: "memory-root" },
|
||||
{ path: path.join(workspaceDir, "memory"), pattern: "**/*.md", base: "memory-dir" },
|
||||
];
|
||||
return entries.map((entry) => ({
|
||||
name: ensureUniqueName(scopeCollectionBase(entry.base, agentId), existing),
|
||||
path: entry.path,
|
||||
pattern: entry.pattern,
|
||||
kind: "memory",
|
||||
}));
|
||||
}
|
||||
|
||||
export function resolveMemoryBackendConfig(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
}): ResolvedMemoryBackendConfig {
|
||||
const normalizedAgentId = normalizeAgentId(params.agentId);
|
||||
const backend = params.cfg.memory?.backend ?? DEFAULT_BACKEND;
|
||||
const citations = params.cfg.memory?.citations ?? DEFAULT_CITATIONS;
|
||||
if (backend !== "qmd") {
|
||||
return { backend: "builtin", citations };
|
||||
}
|
||||
|
||||
const workspaceDir = resolveAgentWorkspaceDir(params.cfg, normalizedAgentId);
|
||||
const qmdCfg = params.cfg.memory?.qmd;
|
||||
const includeDefaultMemory = qmdCfg?.includeDefaultMemory !== false;
|
||||
const nameSet = new Set<string>();
|
||||
const agentEntry = params.cfg.agents?.list?.find(
|
||||
(entry) => normalizeAgentId(entry?.id) === normalizedAgentId,
|
||||
);
|
||||
const mergedExtraPaths = normalizeStringEntries(
|
||||
[
|
||||
...(params.cfg.agents?.defaults?.memorySearch?.extraPaths ?? []),
|
||||
...(agentEntry?.memorySearch?.extraPaths ?? []),
|
||||
].filter((value): value is string => typeof value === "string"),
|
||||
);
|
||||
const dedupedExtraPaths = uniqueStrings(mergedExtraPaths);
|
||||
const searchExtraPaths = dedupedExtraPaths.map(
|
||||
(pathValue): { path: string; pattern?: string; name?: string } => ({ path: pathValue }),
|
||||
);
|
||||
const mergedExtraCollections = [
|
||||
...(params.cfg.agents?.defaults?.memorySearch?.qmd?.extraCollections ?? []),
|
||||
...(agentEntry?.memorySearch?.qmd?.extraCollections ?? []),
|
||||
].filter(
|
||||
(value): value is MemoryQmdIndexPath =>
|
||||
value !== null && typeof value === "object" && typeof value.path === "string",
|
||||
);
|
||||
|
||||
// Combine QMD-specific paths with extraPaths and per-agent cross-agent collections.
|
||||
const allQmdPaths: MemoryQmdIndexPath[] = [
|
||||
...(qmdCfg?.paths ?? []),
|
||||
...searchExtraPaths,
|
||||
...mergedExtraCollections,
|
||||
];
|
||||
|
||||
const collections = [
|
||||
...resolveDefaultCollections(includeDefaultMemory, workspaceDir, nameSet, normalizedAgentId),
|
||||
...resolveCustomPaths(allQmdPaths, workspaceDir, nameSet, normalizedAgentId),
|
||||
];
|
||||
|
||||
const rawCommand = qmdCfg?.command?.trim() || "qmd";
|
||||
const command = resolveQmdCommand(rawCommand);
|
||||
const resolved: ResolvedQmdConfig = {
|
||||
command,
|
||||
mcporter: resolveMcporterConfig(qmdCfg?.mcporter),
|
||||
searchMode: resolveSearchMode(qmdCfg?.searchMode),
|
||||
rerank: qmdCfg?.rerank,
|
||||
searchTool: resolveSearchTool(qmdCfg?.searchTool),
|
||||
collections,
|
||||
includeDefaultMemory,
|
||||
sessions: resolveSessionConfig(qmdCfg?.sessions, workspaceDir),
|
||||
update: {
|
||||
intervalMs: resolveIntervalMs(qmdCfg?.update?.interval),
|
||||
debounceMs: resolveDebounceMs(qmdCfg?.update?.debounceMs),
|
||||
onBoot: qmdCfg?.update?.onBoot !== false,
|
||||
startup: resolveStartupMode(qmdCfg?.update),
|
||||
startupDelayMs: resolveStartupDelayMs(qmdCfg?.update?.startupDelayMs),
|
||||
waitForBootSync: qmdCfg?.update?.waitForBootSync === true,
|
||||
embedIntervalMs: resolveEmbedIntervalMs(qmdCfg?.update?.embedInterval),
|
||||
commandTimeoutMs: resolveTimeoutMs(
|
||||
qmdCfg?.update?.commandTimeoutMs,
|
||||
DEFAULT_QMD_COMMAND_TIMEOUT_MS,
|
||||
),
|
||||
updateTimeoutMs: resolveTimeoutMs(
|
||||
qmdCfg?.update?.updateTimeoutMs,
|
||||
DEFAULT_QMD_UPDATE_TIMEOUT_MS,
|
||||
),
|
||||
embedTimeoutMs: resolveTimeoutMs(
|
||||
qmdCfg?.update?.embedTimeoutMs,
|
||||
DEFAULT_QMD_EMBED_TIMEOUT_MS,
|
||||
),
|
||||
},
|
||||
limits: resolveLimits(qmdCfg?.limits),
|
||||
scope: qmdCfg?.scope ?? DEFAULT_QMD_SCOPE,
|
||||
};
|
||||
|
||||
return {
|
||||
backend: "qmd",
|
||||
citations,
|
||||
qmd: resolved,
|
||||
};
|
||||
}
|
||||
33
packages/memory-host-sdk/src/host/batch-error-utils.test.ts
Normal file
33
packages/memory-host-sdk/src/host/batch-error-utils.test.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
// Memory Host SDK tests cover batch error utils behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { extractBatchErrorMessage, formatUnavailableBatchError } from "./batch-error-utils.js";
|
||||
|
||||
describe("extractBatchErrorMessage", () => {
|
||||
it("returns the first top-level error message", () => {
|
||||
expect(
|
||||
extractBatchErrorMessage([
|
||||
{ response: { body: { error: { message: "nested" } } } },
|
||||
{ error: { message: "top-level" } },
|
||||
]),
|
||||
).toBe("nested");
|
||||
});
|
||||
|
||||
it("falls back to nested response error message", () => {
|
||||
expect(
|
||||
extractBatchErrorMessage([{ response: { body: { error: { message: "nested-only" } } } }, {}]),
|
||||
).toBe("nested-only");
|
||||
});
|
||||
|
||||
it("accepts plain string response bodies", () => {
|
||||
expect(extractBatchErrorMessage([{ response: { body: "provider plain-text error" } }])).toBe(
|
||||
"provider plain-text error",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatUnavailableBatchError", () => {
|
||||
it("formats errors and non-error values", () => {
|
||||
expect(formatUnavailableBatchError(new Error("boom"))).toBe("error file unavailable: boom");
|
||||
expect(formatUnavailableBatchError("unreachable")).toBe("error file unavailable: unreachable");
|
||||
});
|
||||
});
|
||||
40
packages/memory-host-sdk/src/host/batch-error-utils.ts
Normal file
40
packages/memory-host-sdk/src/host/batch-error-utils.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
// Memory Host SDK helper module supports batch error utils behavior.
|
||||
import { formatErrorMessage } from "./error-utils.js";
|
||||
|
||||
// Extracts provider batch error text from output and unavailable error files.
|
||||
|
||||
/** Minimal batch output line shape that can carry provider error messages. */
|
||||
type BatchOutputErrorLike = {
|
||||
error?: { message?: string };
|
||||
response?: {
|
||||
body?:
|
||||
| string
|
||||
| {
|
||||
error?: { message?: string };
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
/** Pull a nested response error message without assuming a fixed provider body shape. */
|
||||
function getResponseErrorMessage(line: BatchOutputErrorLike | undefined): string | undefined {
|
||||
const body = line?.response?.body;
|
||||
if (typeof body === "string") {
|
||||
return body || undefined;
|
||||
}
|
||||
if (!body || typeof body !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return typeof body.error?.message === "string" ? body.error.message : undefined;
|
||||
}
|
||||
|
||||
/** Return the first useful error message from batch output lines. */
|
||||
export function extractBatchErrorMessage(lines: BatchOutputErrorLike[]): string | undefined {
|
||||
const first = lines.find((line) => line.error?.message || getResponseErrorMessage(line));
|
||||
return first?.error?.message ?? getResponseErrorMessage(first);
|
||||
}
|
||||
|
||||
/** Format a failed error-file read without hiding the underlying read problem. */
|
||||
export function formatUnavailableBatchError(err: unknown): string | undefined {
|
||||
const message = formatErrorMessage(err);
|
||||
return message ? `error file unavailable: ${message}` : undefined;
|
||||
}
|
||||
114
packages/memory-host-sdk/src/host/batch-http.test.ts
Normal file
114
packages/memory-host-sdk/src/host/batch-http.test.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
// Memory Host SDK tests cover batch http behavior.
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("./post-json.js", () => ({
|
||||
postJson: vi.fn(),
|
||||
}));
|
||||
|
||||
type RetryOptions = {
|
||||
attempts: number;
|
||||
minDelayMs: number;
|
||||
maxDelayMs: number;
|
||||
shouldRetry: (err: unknown) => boolean;
|
||||
};
|
||||
|
||||
type PostJsonParams = {
|
||||
url?: unknown;
|
||||
headers?: unknown;
|
||||
body?: unknown;
|
||||
errorPrefix?: unknown;
|
||||
attachStatus?: unknown;
|
||||
};
|
||||
|
||||
function requirePostJsonParams(
|
||||
postJsonMock: ReturnType<typeof vi.mocked<typeof import("./post-json.js").postJson>>,
|
||||
): PostJsonParams {
|
||||
const [call] = postJsonMock.mock.calls;
|
||||
if (!call) {
|
||||
throw new Error("expected postJson call");
|
||||
}
|
||||
const [params] = call;
|
||||
if (typeof params !== "object" || params === null || Array.isArray(params)) {
|
||||
throw new Error("expected postJson params to be an object");
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
function requireFirstRetryOptions(retryAsyncMock: ReturnType<typeof vi.fn>): RetryOptions {
|
||||
const call = retryAsyncMock.mock.calls[0];
|
||||
const options = call?.[1] as RetryOptions | undefined;
|
||||
if (!options) {
|
||||
throw new Error("expected retry options");
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
describe("postJsonWithRetry", () => {
|
||||
let postJsonMock: ReturnType<typeof vi.mocked<typeof import("./post-json.js").postJson>>;
|
||||
let postJsonWithRetry: typeof import("./batch-http.js").postJsonWithRetry;
|
||||
let retryAsyncMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ postJsonWithRetry } = await import("./batch-http.js"));
|
||||
const postJsonModule = await import("./post-json.js");
|
||||
postJsonMock = vi.mocked(postJsonModule.postJson);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
retryAsyncMock = vi.fn(async (run: () => Promise<unknown>) => await run());
|
||||
});
|
||||
|
||||
it("posts JSON and returns parsed response payload", async () => {
|
||||
postJsonMock.mockImplementationOnce(async (params) => {
|
||||
return await params.parse({ ok: true, ids: [1, 2] });
|
||||
});
|
||||
|
||||
const result = await postJsonWithRetry<{ ok: boolean; ids: number[] }>({
|
||||
url: "https://memory.example/v1/batch",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
body: { chunks: ["a", "b"] },
|
||||
errorPrefix: "memory batch failed",
|
||||
retryImpl: retryAsyncMock as typeof import("./retry-utils.js").retryAsync,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ ok: true, ids: [1, 2] });
|
||||
const postJsonParams = requirePostJsonParams(postJsonMock);
|
||||
expect(postJsonParams.url).toBe("https://memory.example/v1/batch");
|
||||
expect(postJsonParams.headers).toEqual({ Authorization: "Bearer test" });
|
||||
expect(postJsonParams.body).toEqual({ chunks: ["a", "b"] });
|
||||
expect(postJsonParams.errorPrefix).toBe("memory batch failed");
|
||||
expect(postJsonParams.attachStatus).toBe(true);
|
||||
|
||||
const retryOptions = requireFirstRetryOptions(retryAsyncMock);
|
||||
expect(retryOptions.attempts).toBe(3);
|
||||
expect(retryOptions.minDelayMs).toBe(300);
|
||||
expect(retryOptions.maxDelayMs).toBe(2000);
|
||||
expect(retryOptions.shouldRetry({ status: 429 })).toBe(true);
|
||||
expect(retryOptions.shouldRetry({ status: 503 })).toBe(true);
|
||||
expect(retryOptions.shouldRetry({ status: 400 })).toBe(false);
|
||||
});
|
||||
|
||||
it("attaches status to non-ok errors", async () => {
|
||||
postJsonMock.mockRejectedValueOnce(
|
||||
Object.assign(new Error("memory batch failed: 503 backend down"), { status: 503 }),
|
||||
);
|
||||
|
||||
let error: unknown;
|
||||
try {
|
||||
await postJsonWithRetry({
|
||||
url: "https://memory.example/v1/batch",
|
||||
headers: {},
|
||||
body: { chunks: [] },
|
||||
errorPrefix: "memory batch failed",
|
||||
retryImpl: retryAsyncMock as typeof import("./retry-utils.js").retryAsync,
|
||||
});
|
||||
} catch (caught) {
|
||||
error = caught;
|
||||
}
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as Error).message).toBe("memory batch failed: 503 backend down");
|
||||
expect((error as { status?: unknown }).status).toBe(503);
|
||||
});
|
||||
});
|
||||
43
packages/memory-host-sdk/src/host/batch-http.ts
Normal file
43
packages/memory-host-sdk/src/host/batch-http.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
// Memory Host SDK module implements batch http behavior.
|
||||
import { postJson } from "./post-json.js";
|
||||
import { retryAsync } from "./retry-utils.js";
|
||||
import type { SsrFPolicy } from "./ssrf-policy.js";
|
||||
|
||||
// JSON POST helper for batch APIs with provider-style transient retry.
|
||||
|
||||
/** POST JSON and retry provider 429/5xx failures with bounded backoff. */
|
||||
export async function postJsonWithRetry<T>(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
retryImpl?: typeof retryAsync;
|
||||
body: unknown;
|
||||
errorPrefix: string;
|
||||
}): Promise<T> {
|
||||
const retry = params.retryImpl ?? retryAsync;
|
||||
return await retry(
|
||||
async () => {
|
||||
return await postJson<T>({
|
||||
url: params.url,
|
||||
headers: params.headers,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
fetchImpl: params.fetchImpl,
|
||||
body: params.body,
|
||||
errorPrefix: params.errorPrefix,
|
||||
attachStatus: true,
|
||||
parse: async (payload) => payload as T,
|
||||
});
|
||||
},
|
||||
{
|
||||
attempts: 3,
|
||||
minDelayMs: 300,
|
||||
maxDelayMs: 2000,
|
||||
jitter: 0.2,
|
||||
shouldRetry: (err) => {
|
||||
const status = (err as { status?: number }).status;
|
||||
return status === 429 || (typeof status === "number" && status >= 500);
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
83
packages/memory-host-sdk/src/host/batch-output.test.ts
Normal file
83
packages/memory-host-sdk/src/host/batch-output.test.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
// Memory Host SDK tests cover batch output behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyEmbeddingBatchOutputLine } from "./batch-output.js";
|
||||
|
||||
describe("applyEmbeddingBatchOutputLine", () => {
|
||||
it("stores embedding for successful response", () => {
|
||||
const remaining = new Set(["req-1"]);
|
||||
const errors: string[] = [];
|
||||
const byCustomId = new Map<string, number[]>();
|
||||
|
||||
applyEmbeddingBatchOutputLine({
|
||||
line: {
|
||||
custom_id: "req-1",
|
||||
response: {
|
||||
status_code: 200,
|
||||
body: { data: [{ embedding: [0.1, 0.2] }] },
|
||||
},
|
||||
},
|
||||
remaining,
|
||||
errors,
|
||||
byCustomId,
|
||||
});
|
||||
|
||||
expect(remaining.has("req-1")).toBe(false);
|
||||
expect(errors).toStrictEqual([]);
|
||||
expect(byCustomId.get("req-1")).toEqual([0.1, 0.2]);
|
||||
});
|
||||
|
||||
it("records provider error from line.error", () => {
|
||||
const remaining = new Set(["req-2"]);
|
||||
const errors: string[] = [];
|
||||
const byCustomId = new Map<string, number[]>();
|
||||
|
||||
applyEmbeddingBatchOutputLine({
|
||||
line: {
|
||||
custom_id: "req-2",
|
||||
error: { message: "provider failed" },
|
||||
},
|
||||
remaining,
|
||||
errors,
|
||||
byCustomId,
|
||||
});
|
||||
|
||||
expect(remaining.has("req-2")).toBe(false);
|
||||
expect(errors).toEqual(["req-2: provider failed"]);
|
||||
expect(byCustomId.size).toBe(0);
|
||||
});
|
||||
|
||||
it("records non-2xx response errors and empty embedding errors", () => {
|
||||
const remaining = new Set(["req-3", "req-4"]);
|
||||
const errors: string[] = [];
|
||||
const byCustomId = new Map<string, number[]>();
|
||||
|
||||
applyEmbeddingBatchOutputLine({
|
||||
line: {
|
||||
custom_id: "req-3",
|
||||
response: {
|
||||
status_code: 500,
|
||||
body: { error: { message: "internal" } },
|
||||
},
|
||||
},
|
||||
remaining,
|
||||
errors,
|
||||
byCustomId,
|
||||
});
|
||||
|
||||
applyEmbeddingBatchOutputLine({
|
||||
line: {
|
||||
custom_id: "req-4",
|
||||
response: {
|
||||
status_code: 200,
|
||||
body: { data: [] },
|
||||
},
|
||||
},
|
||||
remaining,
|
||||
errors,
|
||||
byCustomId,
|
||||
});
|
||||
|
||||
expect(errors).toEqual(["req-3: internal", "req-4: empty embedding"]);
|
||||
expect(byCustomId.size).toBe(0);
|
||||
});
|
||||
});
|
||||
59
packages/memory-host-sdk/src/host/batch-output.ts
Normal file
59
packages/memory-host-sdk/src/host/batch-output.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
// Parses provider batch output lines into the custom-id embedding map.
|
||||
|
||||
/** Minimal OpenAI-compatible embedding batch output line. */
|
||||
export type EmbeddingBatchOutputLine = {
|
||||
custom_id?: string;
|
||||
error?: { message?: string };
|
||||
response?: {
|
||||
status_code?: number;
|
||||
body?:
|
||||
| {
|
||||
data?: Array<{
|
||||
embedding?: number[];
|
||||
}>;
|
||||
error?: { message?: string };
|
||||
}
|
||||
| string;
|
||||
};
|
||||
};
|
||||
|
||||
/** Apply one output line, collecting errors and successful embeddings by custom id. */
|
||||
export function applyEmbeddingBatchOutputLine(params: {
|
||||
line: EmbeddingBatchOutputLine;
|
||||
remaining: Set<string>;
|
||||
errors: string[];
|
||||
byCustomId: Map<string, number[]>;
|
||||
}) {
|
||||
const customId = params.line.custom_id;
|
||||
if (!customId) {
|
||||
return;
|
||||
}
|
||||
params.remaining.delete(customId);
|
||||
|
||||
const errorMessage = params.line.error?.message;
|
||||
if (errorMessage) {
|
||||
params.errors.push(`${customId}: ${errorMessage}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const response = params.line.response;
|
||||
const statusCode = response?.status_code ?? 0;
|
||||
if (statusCode >= 400) {
|
||||
const messageFromObject =
|
||||
response?.body && typeof response.body === "object"
|
||||
? (response.body as { error?: { message?: string } }).error?.message
|
||||
: undefined;
|
||||
const messageFromString = typeof response?.body === "string" ? response.body : undefined;
|
||||
params.errors.push(`${customId}: ${messageFromObject ?? messageFromString ?? "unknown error"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const data =
|
||||
response?.body && typeof response.body === "object" ? (response.body.data ?? []) : [];
|
||||
const embedding = data[0]?.embedding ?? [];
|
||||
if (embedding.length === 0) {
|
||||
params.errors.push(`${customId}: empty embedding`);
|
||||
return;
|
||||
}
|
||||
params.byCustomId.set(customId, embedding);
|
||||
}
|
||||
18
packages/memory-host-sdk/src/host/batch-provider-common.ts
Normal file
18
packages/memory-host-sdk/src/host/batch-provider-common.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
// Memory Host SDK helper module supports batch provider common behavior.
|
||||
import type { EmbeddingBatchOutputLine } from "./batch-output.js";
|
||||
|
||||
// Common OpenAI-compatible batch shapes shared by remote embedding providers.
|
||||
|
||||
/** Minimal provider batch status payload used by polling code. */
|
||||
export type EmbeddingBatchStatus = {
|
||||
id?: string;
|
||||
status?: string;
|
||||
output_file_id?: string | null;
|
||||
error_file_id?: string | null;
|
||||
};
|
||||
|
||||
/** Provider output line after an embedding batch file is read. */
|
||||
export type ProviderBatchOutputLine = EmbeddingBatchOutputLine;
|
||||
|
||||
/** OpenAI-compatible endpoint used inside embedding batch request lines. */
|
||||
export const EMBEDDING_BATCH_ENDPOINT = "/v1/embeddings";
|
||||
157
packages/memory-host-sdk/src/host/batch-runner.test.ts
Normal file
157
packages/memory-host-sdk/src/host/batch-runner.test.ts
Normal file
@@ -0,0 +1,157 @@
|
||||
// Memory Host SDK tests cover batch runner behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MAX_SAFE_TIMEOUT_DELAY_MS } from "../../../gateway-client/src/timeouts.js";
|
||||
import { buildEmbeddingBatchGroupOptions, runEmbeddingBatchGroups } from "./batch-runner.js";
|
||||
|
||||
const jsonlEncoder = new TextEncoder();
|
||||
|
||||
function jsonlLineBytes(value: unknown): number {
|
||||
return jsonlEncoder.encode(JSON.stringify(value)).byteLength;
|
||||
}
|
||||
|
||||
describe("buildEmbeddingBatchGroupOptions", () => {
|
||||
it("clamps oversized embedding batch poll intervals to the timeout budget", () => {
|
||||
const options = buildEmbeddingBatchGroupOptions(
|
||||
{
|
||||
requests: ["request-1"],
|
||||
wait: true,
|
||||
pollIntervalMs: Number.MAX_SAFE_INTEGER,
|
||||
timeoutMs: 60_000,
|
||||
concurrency: 1,
|
||||
},
|
||||
{
|
||||
maxRequests: 100,
|
||||
debugLabel: "embedding batch submit",
|
||||
},
|
||||
);
|
||||
|
||||
expect(options.pollIntervalMs).toBe(60_000);
|
||||
});
|
||||
|
||||
it("passes clamped poll intervals into batch group runners", async () => {
|
||||
const runGroup = vi.fn(async () => {});
|
||||
|
||||
await runEmbeddingBatchGroups({
|
||||
requests: ["request-1"],
|
||||
maxRequests: 100,
|
||||
wait: true,
|
||||
pollIntervalMs: Number.MAX_SAFE_INTEGER,
|
||||
timeoutMs: 60_000,
|
||||
concurrency: 1,
|
||||
debugLabel: "embedding batch submit",
|
||||
runGroup,
|
||||
});
|
||||
|
||||
expect(runGroup).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
pollIntervalMs: 60_000,
|
||||
timeoutMs: 60_000,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps timeout-safe oversized embedding batch poll intervals bounded", () => {
|
||||
const options = buildEmbeddingBatchGroupOptions(
|
||||
{
|
||||
requests: ["request-1"],
|
||||
wait: true,
|
||||
pollIntervalMs: Number.MAX_SAFE_INTEGER,
|
||||
timeoutMs: Number.MAX_SAFE_INTEGER,
|
||||
concurrency: 1,
|
||||
},
|
||||
{
|
||||
maxRequests: 100,
|
||||
debugLabel: "embedding batch submit",
|
||||
},
|
||||
);
|
||||
|
||||
expect(options.pollIntervalMs).toBe(MAX_SAFE_TIMEOUT_DELAY_MS);
|
||||
});
|
||||
|
||||
it("splits embedding batch groups by serialized JSONL bytes", async () => {
|
||||
const requests = [
|
||||
{ id: "one", body: { input: "alpha" } },
|
||||
{ id: "two", body: { input: "βeta" } },
|
||||
{ id: "three", body: { input: "gamma" } },
|
||||
];
|
||||
const maxJsonlBytes = jsonlLineBytes(requests[0]) + 1 + jsonlLineBytes(requests[1]);
|
||||
const groups: string[][] = [];
|
||||
|
||||
await runEmbeddingBatchGroups({
|
||||
requests,
|
||||
maxRequests: 100,
|
||||
maxJsonlBytes,
|
||||
wait: true,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
concurrency: 1,
|
||||
debugLabel: "embedding batch submit",
|
||||
runGroup: async ({ group }) => {
|
||||
groups.push(group.map((request) => request.id));
|
||||
},
|
||||
});
|
||||
|
||||
expect(groups).toEqual([["one", "two"], ["three"]]);
|
||||
});
|
||||
|
||||
it("splits provider-rejected batch groups when the error is splittable", async () => {
|
||||
const uploadTooLarge = new Error("batch upload failed: 413 payload too large");
|
||||
const calls: string[][] = [];
|
||||
const onSplitGroup = vi.fn();
|
||||
|
||||
await runEmbeddingBatchGroups({
|
||||
requests: ["one", "two", "three", "four"],
|
||||
maxRequests: 100,
|
||||
wait: true,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
concurrency: 1,
|
||||
debugLabel: "embedding batch submit",
|
||||
shouldSplitGroupOnError: (error) => error === uploadTooLarge,
|
||||
onSplitGroup,
|
||||
runGroup: async ({ group }) => {
|
||||
calls.push([...group]);
|
||||
if (group.length === 4) {
|
||||
throw uploadTooLarge;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
expect(calls).toEqual([
|
||||
["one", "two", "three", "four"],
|
||||
["one", "two"],
|
||||
["three", "four"],
|
||||
]);
|
||||
expect(onSplitGroup).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
error: uploadTooLarge,
|
||||
group: ["one", "two", "three", "four"],
|
||||
parts: [
|
||||
["one", "two"],
|
||||
["three", "four"],
|
||||
],
|
||||
depth: 0,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not split a single rejected batch request", async () => {
|
||||
const uploadTooLarge = new Error("batch upload failed: 413 payload too large");
|
||||
|
||||
await expect(
|
||||
runEmbeddingBatchGroups({
|
||||
requests: ["one"],
|
||||
maxRequests: 100,
|
||||
wait: true,
|
||||
pollIntervalMs: 1000,
|
||||
timeoutMs: 60_000,
|
||||
concurrency: 1,
|
||||
debugLabel: "embedding batch submit",
|
||||
shouldSplitGroupOnError: () => true,
|
||||
runGroup: async () => {
|
||||
throw uploadTooLarge;
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(uploadTooLarge);
|
||||
});
|
||||
});
|
||||
141
packages/memory-host-sdk/src/host/batch-runner.ts
Normal file
141
packages/memory-host-sdk/src/host/batch-runner.ts
Normal file
@@ -0,0 +1,141 @@
|
||||
// Memory Host SDK module implements batch runner behavior.
|
||||
import { resolveSafeTimeoutDelayMs } from "../../../gateway-client/src/timeouts.js";
|
||||
import { splitBatchRequestsByLimits } from "./batch-utils.js";
|
||||
import { runWithConcurrency } from "./internal.js";
|
||||
|
||||
// Shared runner for splitting and executing remote embedding batch groups.
|
||||
|
||||
/** Execution controls for provider embedding batch submissions and polling. */
|
||||
export type EmbeddingBatchExecutionParams = {
|
||||
wait: boolean;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
concurrency: number;
|
||||
debug?: (message: string, data?: Record<string, unknown>) => void;
|
||||
};
|
||||
|
||||
type EmbeddingBatchGroupRunArgs<TRequest> = {
|
||||
group: TRequest[];
|
||||
groupIndex: number;
|
||||
groups: number;
|
||||
byCustomId: Map<string, number[]>;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
type EmbeddingBatchSplitArgs<TRequest> = {
|
||||
error: unknown;
|
||||
group: TRequest[];
|
||||
parts: TRequest[][];
|
||||
groupIndex: number;
|
||||
groups: number;
|
||||
depth: number;
|
||||
};
|
||||
|
||||
/** Clamp polling to both configured poll interval and total timeout budget. */
|
||||
function resolveEmbeddingBatchPollIntervalMs(params: {
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
}): number {
|
||||
const safePollIntervalMs = resolveSafeTimeoutDelayMs(params.pollIntervalMs);
|
||||
const safeTimeoutMs =
|
||||
typeof params.timeoutMs === "number" &&
|
||||
Number.isFinite(params.timeoutMs) &&
|
||||
params.timeoutMs > 0
|
||||
? resolveSafeTimeoutDelayMs(params.timeoutMs)
|
||||
: safePollIntervalMs;
|
||||
return Math.min(safePollIntervalMs, safeTimeoutMs);
|
||||
}
|
||||
|
||||
/** Run request groups with bounded concurrency and return embeddings by custom id. */
|
||||
export async function runEmbeddingBatchGroups<TRequest>(params: {
|
||||
requests: TRequest[];
|
||||
maxRequests: number;
|
||||
maxJsonlBytes?: number;
|
||||
wait: EmbeddingBatchExecutionParams["wait"];
|
||||
pollIntervalMs: EmbeddingBatchExecutionParams["pollIntervalMs"];
|
||||
timeoutMs: EmbeddingBatchExecutionParams["timeoutMs"];
|
||||
concurrency: EmbeddingBatchExecutionParams["concurrency"];
|
||||
debugLabel: string;
|
||||
debug?: EmbeddingBatchExecutionParams["debug"];
|
||||
shouldSplitGroupOnError?: (error: unknown, group: TRequest[]) => boolean;
|
||||
onSplitGroup?: (args: EmbeddingBatchSplitArgs<TRequest>) => void;
|
||||
runGroup: (args: EmbeddingBatchGroupRunArgs<TRequest>) => Promise<void>;
|
||||
}): Promise<Map<string, number[]>> {
|
||||
if (params.requests.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
const groups = splitBatchRequestsByLimits(params.requests, {
|
||||
maxRequests: params.maxRequests,
|
||||
maxJsonlBytes: params.maxJsonlBytes,
|
||||
});
|
||||
const byCustomId = new Map<string, number[]>();
|
||||
const pollIntervalMs = resolveEmbeddingBatchPollIntervalMs(params);
|
||||
const runGroup = async (group: TRequest[], groupIndex: number, depth = 0): Promise<void> => {
|
||||
try {
|
||||
await params.runGroup({
|
||||
group,
|
||||
groupIndex,
|
||||
groups: groups.length,
|
||||
byCustomId,
|
||||
pollIntervalMs,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
} catch (error) {
|
||||
if (group.length <= 1 || !params.shouldSplitGroupOnError?.(error, group)) {
|
||||
throw error;
|
||||
}
|
||||
const splitAt = Math.ceil(group.length / 2);
|
||||
const parts = [group.slice(0, splitAt), group.slice(splitAt)].filter(
|
||||
(part) => part.length > 0,
|
||||
);
|
||||
params.onSplitGroup?.({
|
||||
error,
|
||||
group,
|
||||
parts,
|
||||
groupIndex,
|
||||
groups: groups.length,
|
||||
depth,
|
||||
});
|
||||
for (const part of parts) {
|
||||
await runGroup(part, groupIndex, depth + 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
const tasks = groups.map((group, groupIndex) => async () => {
|
||||
await runGroup(group, groupIndex);
|
||||
});
|
||||
|
||||
params.debug?.(params.debugLabel, {
|
||||
requests: params.requests.length,
|
||||
groups: groups.length,
|
||||
maxRequests: params.maxRequests,
|
||||
maxJsonlBytes: params.maxJsonlBytes,
|
||||
wait: params.wait,
|
||||
concurrency: params.concurrency,
|
||||
pollIntervalMs,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
|
||||
await runWithConcurrency(tasks, params.concurrency);
|
||||
return byCustomId;
|
||||
}
|
||||
|
||||
/** Build normalized batch-group options for provider-specific runners. */
|
||||
export function buildEmbeddingBatchGroupOptions<TRequest>(
|
||||
params: { requests: TRequest[] } & EmbeddingBatchExecutionParams,
|
||||
options: { maxRequests: number; maxJsonlBytes?: number; debugLabel: string },
|
||||
) {
|
||||
const pollIntervalMs = resolveEmbeddingBatchPollIntervalMs(params);
|
||||
return {
|
||||
requests: params.requests,
|
||||
maxRequests: options.maxRequests,
|
||||
maxJsonlBytes: options.maxJsonlBytes,
|
||||
wait: params.wait,
|
||||
pollIntervalMs,
|
||||
timeoutMs: params.timeoutMs,
|
||||
concurrency: params.concurrency,
|
||||
debug: params.debug,
|
||||
debugLabel: options.debugLabel,
|
||||
};
|
||||
}
|
||||
61
packages/memory-host-sdk/src/host/batch-status.test.ts
Normal file
61
packages/memory-host-sdk/src/host/batch-status.test.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
// Memory Host SDK tests cover batch status behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveBatchCompletionFromStatus,
|
||||
resolveCompletedBatchResult,
|
||||
throwIfBatchTerminalFailure,
|
||||
} from "./batch-status.js";
|
||||
|
||||
describe("batch-status helpers", () => {
|
||||
it("resolves completion payload from completed status", () => {
|
||||
expect(
|
||||
resolveBatchCompletionFromStatus({
|
||||
provider: "openai",
|
||||
batchId: "b1",
|
||||
status: {
|
||||
output_file_id: "out-1",
|
||||
error_file_id: "err-1",
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
outputFileId: "out-1",
|
||||
errorFileId: "err-1",
|
||||
});
|
||||
});
|
||||
|
||||
it("throws for terminal failure states", async () => {
|
||||
await expect(
|
||||
throwIfBatchTerminalFailure({
|
||||
provider: "voyage",
|
||||
status: { id: "b2", status: "failed", error_file_id: "err-file" },
|
||||
readError: async () => "bad input",
|
||||
}),
|
||||
).rejects.toThrow("voyage batch b2 failed: bad input");
|
||||
});
|
||||
|
||||
it("returns completed result directly without waiting", async () => {
|
||||
const waitForBatch = async () => ({ outputFileId: "out-2" });
|
||||
const result = await resolveCompletedBatchResult({
|
||||
provider: "openai",
|
||||
status: {
|
||||
id: "b3",
|
||||
status: "completed",
|
||||
output_file_id: "out-3",
|
||||
},
|
||||
wait: false,
|
||||
waitForBatch,
|
||||
});
|
||||
expect(result).toEqual({ outputFileId: "out-3", errorFileId: undefined });
|
||||
});
|
||||
|
||||
it("throws when wait disabled and batch is not complete", async () => {
|
||||
await expect(
|
||||
resolveCompletedBatchResult({
|
||||
provider: "openai",
|
||||
status: { id: "b4", status: "pending" },
|
||||
wait: false,
|
||||
waitForBatch: async () => ({ outputFileId: "out" }),
|
||||
}),
|
||||
).rejects.toThrow("openai batch b4 submitted; enable remote.batch.wait to await completion");
|
||||
});
|
||||
});
|
||||
76
packages/memory-host-sdk/src/host/batch-status.ts
Normal file
76
packages/memory-host-sdk/src/host/batch-status.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
// Batch status helpers shared by remote embedding providers.
|
||||
|
||||
const TERMINAL_FAILURE_STATES = new Set(["failed", "expired", "cancelled", "canceled"]);
|
||||
|
||||
/** Minimal provider batch status used for completion and terminal-failure checks. */
|
||||
type BatchStatusLike = {
|
||||
id?: string;
|
||||
status?: string;
|
||||
output_file_id?: string | null;
|
||||
error_file_id?: string | null;
|
||||
};
|
||||
|
||||
/** File ids returned once a batch has completed. */
|
||||
export type BatchCompletionResult = {
|
||||
outputFileId: string;
|
||||
errorFileId?: string;
|
||||
};
|
||||
|
||||
/** Convert a completed provider status payload into output/error file ids. */
|
||||
export function resolveBatchCompletionFromStatus(params: {
|
||||
provider: string;
|
||||
batchId: string;
|
||||
status: BatchStatusLike;
|
||||
}): BatchCompletionResult {
|
||||
if (!params.status.output_file_id) {
|
||||
throw new Error(`${params.provider} batch ${params.batchId} completed without output file`);
|
||||
}
|
||||
return {
|
||||
outputFileId: params.status.output_file_id,
|
||||
errorFileId: params.status.error_file_id ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Throw when a provider reports a terminal failure, including error-file detail if available. */
|
||||
export async function throwIfBatchTerminalFailure(params: {
|
||||
provider: string;
|
||||
status: BatchStatusLike;
|
||||
readError: (errorFileId: string) => Promise<string | undefined>;
|
||||
}): Promise<void> {
|
||||
const state = params.status.status ?? "unknown";
|
||||
if (!TERMINAL_FAILURE_STATES.has(state)) {
|
||||
return;
|
||||
}
|
||||
const detail = params.status.error_file_id
|
||||
? await params.readError(params.status.error_file_id)
|
||||
: undefined;
|
||||
const suffix = detail ? `: ${detail}` : "";
|
||||
throw new Error(`${params.provider} batch ${params.status.id ?? "<unknown>"} ${state}${suffix}`);
|
||||
}
|
||||
|
||||
/** Resolve the completed batch files, optionally waiting according to caller policy. */
|
||||
export async function resolveCompletedBatchResult(params: {
|
||||
provider: string;
|
||||
status: BatchStatusLike;
|
||||
wait: boolean;
|
||||
waitForBatch: () => Promise<BatchCompletionResult>;
|
||||
}): Promise<BatchCompletionResult> {
|
||||
const batchId = params.status.id ?? "<unknown>";
|
||||
if (!params.wait && params.status.status !== "completed") {
|
||||
throw new Error(
|
||||
`${params.provider} batch ${batchId} submitted; enable remote.batch.wait to await completion`,
|
||||
);
|
||||
}
|
||||
const completed =
|
||||
params.status.status === "completed"
|
||||
? resolveBatchCompletionFromStatus({
|
||||
provider: params.provider,
|
||||
batchId,
|
||||
status: params.status,
|
||||
})
|
||||
: await params.waitForBatch();
|
||||
if (!completed.outputFileId) {
|
||||
throw new Error(`${params.provider} batch ${batchId} completed without output file`);
|
||||
}
|
||||
return completed;
|
||||
}
|
||||
194
packages/memory-host-sdk/src/host/batch-upload.test.ts
Normal file
194
packages/memory-host-sdk/src/host/batch-upload.test.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
// Memory Host SDK tests cover batch upload behavior.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { uploadBatchJsonlFile } from "./batch-upload.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
vi.mock("./remote-http.js", () => ({
|
||||
withRemoteHttpResponse: vi.fn(),
|
||||
}));
|
||||
|
||||
const remoteHttpMock = vi.mocked(withRemoteHttpResponse);
|
||||
|
||||
function textResponse(body: string, status: number): Response {
|
||||
return new Response(body, { status });
|
||||
}
|
||||
|
||||
function streamingTextResponse(params: {
|
||||
body: string;
|
||||
status: number;
|
||||
headers?: HeadersInit;
|
||||
onCancel: () => void;
|
||||
}): Response {
|
||||
const encoded = new TextEncoder().encode(params.body);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoded);
|
||||
},
|
||||
cancel() {
|
||||
params.onCancel();
|
||||
},
|
||||
});
|
||||
return new Response(stream, { status: params.status, headers: params.headers });
|
||||
}
|
||||
|
||||
function stallingResponse(params: { status: number; onCancel: () => void }): Response {
|
||||
const reader = {
|
||||
read: () => new Promise<ReadableStreamReadResult<Uint8Array>>(() => {}),
|
||||
cancel: async () => {
|
||||
params.onCancel();
|
||||
},
|
||||
releaseLock: () => undefined,
|
||||
} as ReadableStreamDefaultReader<Uint8Array>;
|
||||
|
||||
return {
|
||||
status: params.status,
|
||||
ok: params.status >= 200 && params.status < 300,
|
||||
headers: new Headers(),
|
||||
body: { getReader: () => reader },
|
||||
} as Response;
|
||||
}
|
||||
|
||||
describe("uploadBatchJsonlFile", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("wraps malformed file-upload JSON with the request error prefix", async () => {
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(textResponse("{ nope", 200));
|
||||
});
|
||||
|
||||
await expect(
|
||||
uploadBatchJsonlFile({
|
||||
client: {
|
||||
baseUrl: "https://memory.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
},
|
||||
requests: [{ input: "one" }],
|
||||
errorPrefix: "file upload failed",
|
||||
}),
|
||||
).rejects.toThrow("file upload failed: malformed JSON response");
|
||||
});
|
||||
|
||||
it("bounds non-ok file-upload response bodies before formatting the error", async () => {
|
||||
let canceled = false;
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
streamingTextResponse({
|
||||
body: "x".repeat(12_000),
|
||||
status: 413,
|
||||
onCancel: () => {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
await expect(
|
||||
uploadBatchJsonlFile({
|
||||
client: {
|
||||
baseUrl: "https://memory.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
},
|
||||
requests: [{ input: "one" }],
|
||||
errorPrefix: "file upload failed",
|
||||
}),
|
||||
).rejects.toThrow(`file upload failed: 413 ${"x".repeat(1_000)}... [truncated]`);
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects oversized successful file-upload JSON before parsing", async () => {
|
||||
let canceled = false;
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
streamingTextResponse({
|
||||
body: '{"id":"file_123"}',
|
||||
status: 200,
|
||||
headers: { "content-length": "64" },
|
||||
onCancel: () => {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
await expect(
|
||||
uploadBatchJsonlFile({
|
||||
client: {
|
||||
baseUrl: "https://memory.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
},
|
||||
requests: [{ input: "one" }],
|
||||
errorPrefix: "file upload failed",
|
||||
maxResponseBytes: 8,
|
||||
}),
|
||||
).rejects.toThrow("file upload failed: response body too large: 64 bytes (limit: 8 bytes)");
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
|
||||
it("passes caller abort signals through non-ok file-upload response snippets", async () => {
|
||||
let canceled = false;
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
stallingResponse({
|
||||
status: 500,
|
||||
onCancel: () => {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const upload = uploadBatchJsonlFile({
|
||||
client: {
|
||||
baseUrl: "https://memory.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
},
|
||||
requests: [{ input: "one" }],
|
||||
errorPrefix: "file upload failed",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 0);
|
||||
});
|
||||
controller.abort(new Error("upload aborted"));
|
||||
|
||||
await expect(upload).rejects.toThrow("upload aborted");
|
||||
expect(canceled).toBe(true);
|
||||
expect(remoteHttpMock.mock.calls[0]?.[0].signal).toBe(controller.signal);
|
||||
});
|
||||
|
||||
it("passes caller abort signals through successful file-upload JSON reads", async () => {
|
||||
let canceled = false;
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
stallingResponse({
|
||||
status: 200,
|
||||
onCancel: () => {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const upload = uploadBatchJsonlFile({
|
||||
client: {
|
||||
baseUrl: "https://memory.example/v1",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
},
|
||||
requests: [{ input: "one" }],
|
||||
errorPrefix: "file upload failed",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 0);
|
||||
});
|
||||
controller.abort(new Error("upload json aborted"));
|
||||
|
||||
await expect(upload).rejects.toThrow("upload json aborted");
|
||||
expect(canceled).toBe(true);
|
||||
expect(remoteHttpMock.mock.calls[0]?.[0].signal).toBe(controller.signal);
|
||||
});
|
||||
});
|
||||
57
packages/memory-host-sdk/src/host/batch-upload.ts
Normal file
57
packages/memory-host-sdk/src/host/batch-upload.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
// Memory Host SDK module implements batch upload behavior.
|
||||
import {
|
||||
buildBatchHeaders,
|
||||
normalizeBatchBaseUrl,
|
||||
type BatchHttpClientConfig,
|
||||
} from "./batch-utils.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
import { readResponseJsonWithLimit, readResponseTextSnippet } from "./response-snippet.js";
|
||||
|
||||
// Uploads provider batch JSONL payloads through the shared remote HTTP guard.
|
||||
|
||||
/** Upload embedding batch requests and return the provider file id. */
|
||||
export async function uploadBatchJsonlFile(params: {
|
||||
client: BatchHttpClientConfig;
|
||||
requests: unknown[];
|
||||
errorPrefix: string;
|
||||
maxResponseBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.client);
|
||||
const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
|
||||
const form = new FormData();
|
||||
form.append("purpose", "batch");
|
||||
form.append(
|
||||
"file",
|
||||
new Blob([jsonl], { type: "application/jsonl" }),
|
||||
`memory-embeddings.${hashText(String(Date.now()))}.jsonl`,
|
||||
);
|
||||
|
||||
const filePayload = await withRemoteHttpResponse({
|
||||
url: `${baseUrl}/files`,
|
||||
ssrfPolicy: params.client.ssrfPolicy,
|
||||
fetchImpl: params.client.fetchImpl,
|
||||
signal: params.signal,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: buildBatchHeaders(params.client, { json: false }),
|
||||
body: form,
|
||||
},
|
||||
onResponse: async (fileRes) => {
|
||||
if (!fileRes.ok) {
|
||||
const text = await readResponseTextSnippet(fileRes, { signal: params.signal });
|
||||
throw new Error(`${params.errorPrefix}: ${fileRes.status} ${text}`);
|
||||
}
|
||||
return (await readResponseJsonWithLimit(fileRes, {
|
||||
errorPrefix: params.errorPrefix,
|
||||
maxBytes: params.maxResponseBytes,
|
||||
signal: params.signal,
|
||||
})) as { id?: string };
|
||||
},
|
||||
});
|
||||
if (!filePayload.id) {
|
||||
throw new Error(`${params.errorPrefix}: missing file id`);
|
||||
}
|
||||
return filePayload.id;
|
||||
}
|
||||
94
packages/memory-host-sdk/src/host/batch-utils.ts
Normal file
94
packages/memory-host-sdk/src/host/batch-utils.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
// Memory Host SDK helper module supports batch utils behavior.
|
||||
import type { SsrFPolicy } from "./ssrf-policy.js";
|
||||
|
||||
// Common HTTP and grouping helpers for remote embedding batch clients.
|
||||
|
||||
/** Minimal HTTP client config needed by batch providers. */
|
||||
export type BatchHttpClientConfig = {
|
||||
baseUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
};
|
||||
|
||||
/** Normalize batch API base URLs by removing one trailing slash. */
|
||||
export function normalizeBatchBaseUrl(client: BatchHttpClientConfig): string {
|
||||
return client.baseUrl?.replace(/\/$/, "") ?? "";
|
||||
}
|
||||
|
||||
/** Build request headers, preserving caller auth and controlling JSON/form content type. */
|
||||
export function buildBatchHeaders(
|
||||
client: Pick<BatchHttpClientConfig, "headers">,
|
||||
params: { json: boolean },
|
||||
): Record<string, string> {
|
||||
const headers = client.headers ? { ...client.headers } : {};
|
||||
if (params.json) {
|
||||
if (!headers["Content-Type"] && !headers["content-type"]) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
} else {
|
||||
delete headers["Content-Type"];
|
||||
delete headers["content-type"];
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
const jsonlEncoder = new TextEncoder();
|
||||
|
||||
function estimateJsonlLineBytes(request: unknown): number {
|
||||
return jsonlEncoder.encode(JSON.stringify(request) ?? "").byteLength;
|
||||
}
|
||||
|
||||
function normalizePositiveInteger(value: number | undefined): number | undefined {
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
return Math.floor(value);
|
||||
}
|
||||
|
||||
/** Split provider requests into max-sized groups while preserving order. */
|
||||
export function splitBatchRequests<T>(requests: T[], maxRequests: number): T[][] {
|
||||
const limit = normalizePositiveInteger(maxRequests) ?? 1;
|
||||
if (requests.length <= limit) {
|
||||
return [requests];
|
||||
}
|
||||
const groups: T[][] = [];
|
||||
for (let i = 0; i < requests.length; i += limit) {
|
||||
groups.push(requests.slice(i, i + limit));
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
export function splitBatchRequestsByLimits<T>(
|
||||
requests: T[],
|
||||
limits: { maxRequests: number; maxJsonlBytes?: number },
|
||||
): T[][] {
|
||||
const maxRequests = normalizePositiveInteger(limits.maxRequests) ?? 1;
|
||||
const maxJsonlBytes = normalizePositiveInteger(limits.maxJsonlBytes);
|
||||
if (!maxJsonlBytes) {
|
||||
return splitBatchRequests(requests, maxRequests);
|
||||
}
|
||||
|
||||
const groups: T[][] = [];
|
||||
let current: T[] = [];
|
||||
let currentBytes = 0;
|
||||
for (const request of requests) {
|
||||
const requestBytes = estimateJsonlLineBytes(request);
|
||||
const separatorBytes = current.length === 0 ? 0 : 1;
|
||||
const wouldExceedRequests = current.length >= maxRequests;
|
||||
const wouldExceedBytes =
|
||||
current.length > 0 && currentBytes + separatorBytes + requestBytes > maxJsonlBytes;
|
||||
if (current.length > 0 && (wouldExceedRequests || wouldExceedBytes)) {
|
||||
groups.push(current);
|
||||
current = [];
|
||||
currentBytes = 0;
|
||||
}
|
||||
|
||||
currentBytes += (current.length === 0 ? 0 : 1) + requestBytes;
|
||||
current.push(request);
|
||||
}
|
||||
if (current.length > 0) {
|
||||
groups.push(current);
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
15
packages/memory-host-sdk/src/host/config-utils.test.ts
Normal file
15
packages/memory-host-sdk/src/host/config-utils.test.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
// Memory Host SDK tests cover config utils behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseDurationMs } from "./config-utils.js";
|
||||
|
||||
describe("parseDurationMs", () => {
|
||||
it("parses decimal durations into milliseconds", () => {
|
||||
expect(parseDurationMs("1.5s")).toBe(1_500);
|
||||
expect(parseDurationMs("1h30m")).toBe(5_400_000);
|
||||
});
|
||||
|
||||
it("rejects unsafe millisecond results", () => {
|
||||
expect(() => parseDurationMs("9007199254740993ms")).toThrow(/invalid duration/u);
|
||||
expect(() => parseDurationMs("9007199254740990ms10ms")).toThrow(/invalid duration/u);
|
||||
});
|
||||
});
|
||||
423
packages/memory-host-sdk/src/host/config-utils.ts
Normal file
423
packages/memory-host-sdk/src/host/config-utils.ts
Normal file
@@ -0,0 +1,423 @@
|
||||
// Memory Host SDK helper module supports config utils behavior.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
normalizeStringEntries,
|
||||
uniqueStrings,
|
||||
} from "./string-utils.js";
|
||||
export { splitShellArgs } from "./openclaw-runtime-io.js";
|
||||
|
||||
// Shared OpenClaw config helpers used by memory host, QMD, and agent context code.
|
||||
|
||||
/** Chat shape used by memory send-policy matching. */
|
||||
export type ChatType = "direct" | "group" | "channel";
|
||||
/** Memory backend selected by user config. */
|
||||
export type MemoryBackend = "builtin" | "qmd";
|
||||
/** Citation injection behavior for memory search results. */
|
||||
export type MemoryCitationsMode = "auto" | "on" | "off";
|
||||
/** QMD command mode used for search calls. */
|
||||
export type MemoryQmdSearchMode = "query" | "search" | "vsearch";
|
||||
/** QMD startup policy for background indexing. */
|
||||
export type MemoryQmdStartupMode = "off" | "idle" | "immediate";
|
||||
|
||||
/** Action returned by a session send-policy rule. */
|
||||
export type SessionSendPolicyAction = "allow" | "deny";
|
||||
/** Match criteria for one memory send-policy rule. */
|
||||
export type SessionSendPolicyMatch = {
|
||||
channel?: string;
|
||||
chatType?: ChatType;
|
||||
keyPrefix?: string;
|
||||
rawKeyPrefix?: string;
|
||||
};
|
||||
/** One ordered rule in session send-policy config. */
|
||||
export type SessionSendPolicyRule = {
|
||||
action: SessionSendPolicyAction;
|
||||
match?: SessionSendPolicyMatch;
|
||||
};
|
||||
/** Memory send-policy config with default action and ordered rules. */
|
||||
export type SessionSendPolicyConfig = {
|
||||
default?: SessionSendPolicyAction;
|
||||
rules?: SessionSendPolicyRule[];
|
||||
};
|
||||
|
||||
/** QMD collection path plus optional display name and glob pattern. */
|
||||
export type MemoryQmdIndexPath = {
|
||||
path: string;
|
||||
name?: string;
|
||||
pattern?: string;
|
||||
};
|
||||
|
||||
/** QMD mcporter daemon integration config. */
|
||||
export type MemoryQmdMcporterConfig = {
|
||||
enabled?: boolean;
|
||||
serverName?: string;
|
||||
startDaemon?: boolean;
|
||||
};
|
||||
|
||||
/** QMD session export config. */
|
||||
export type MemoryQmdSessionConfig = {
|
||||
enabled?: boolean;
|
||||
exportDir?: string;
|
||||
retentionDays?: number;
|
||||
};
|
||||
|
||||
/** QMD update, debounce, startup, and timeout config. */
|
||||
export type MemoryQmdUpdateConfig = {
|
||||
interval?: string;
|
||||
debounceMs?: number;
|
||||
onBoot?: boolean;
|
||||
startup?: MemoryQmdStartupMode;
|
||||
startupDelayMs?: number;
|
||||
waitForBootSync?: boolean;
|
||||
embedInterval?: string;
|
||||
commandTimeoutMs?: number;
|
||||
updateTimeoutMs?: number;
|
||||
embedTimeoutMs?: number;
|
||||
};
|
||||
|
||||
/** Search and injection limits for QMD memory results. */
|
||||
export type MemoryQmdLimitsConfig = {
|
||||
maxResults?: number;
|
||||
maxSnippetChars?: number;
|
||||
maxInjectedChars?: number;
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
/** Full QMD-backed memory config. */
|
||||
export type MemoryQmdConfig = {
|
||||
command?: string;
|
||||
mcporter?: MemoryQmdMcporterConfig;
|
||||
searchMode?: MemoryQmdSearchMode;
|
||||
rerank?: boolean;
|
||||
searchTool?: string;
|
||||
includeDefaultMemory?: boolean;
|
||||
paths?: MemoryQmdIndexPath[];
|
||||
sessions?: MemoryQmdSessionConfig;
|
||||
update?: MemoryQmdUpdateConfig;
|
||||
limits?: MemoryQmdLimitsConfig;
|
||||
scope?: SessionSendPolicyConfig;
|
||||
};
|
||||
|
||||
/** Top-level memory config shared by host and runtime callers. */
|
||||
export type MemoryConfig = {
|
||||
backend?: MemoryBackend;
|
||||
citations?: MemoryCitationsMode;
|
||||
qmd?: MemoryQmdConfig;
|
||||
};
|
||||
|
||||
/** Per-agent memory search enablement and extra collection paths. */
|
||||
export type MemorySearchConfig = {
|
||||
enabled?: boolean;
|
||||
extraPaths?: string[];
|
||||
qmd?: {
|
||||
extraCollections?: MemoryQmdIndexPath[];
|
||||
};
|
||||
};
|
||||
|
||||
/** Agent context limits that bound memory file reads. */
|
||||
export type AgentContextLimitsConfig = {
|
||||
memoryGetMaxChars?: number;
|
||||
memoryGetDefaultLines?: number;
|
||||
};
|
||||
|
||||
/** Secret reference accepted by provider header config. */
|
||||
export type SecretInput =
|
||||
| string
|
||||
| {
|
||||
source: string;
|
||||
provider: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
/** Agent-level config fields consumed by memory host helpers. */
|
||||
type AgentConfig = {
|
||||
id?: string;
|
||||
default?: boolean;
|
||||
workspace?: string;
|
||||
memorySearch?: MemorySearchConfig;
|
||||
contextLimits?: AgentContextLimitsConfig;
|
||||
};
|
||||
|
||||
/** Narrow OpenClaw config shape consumed by memory host utilities. */
|
||||
export type OpenClawConfig = {
|
||||
agents?: {
|
||||
defaults?: {
|
||||
workspace?: string;
|
||||
memorySearch?: MemorySearchConfig;
|
||||
contextLimits?: AgentContextLimitsConfig;
|
||||
};
|
||||
list?: AgentConfig[];
|
||||
};
|
||||
memory?: MemoryConfig;
|
||||
models?: {
|
||||
providers?: Record<
|
||||
string,
|
||||
{
|
||||
api?: string;
|
||||
baseUrl?: string;
|
||||
headers?: Record<string, SecretInput>;
|
||||
}
|
||||
>;
|
||||
};
|
||||
};
|
||||
|
||||
/** Root memory filename used in agent workspaces. */
|
||||
export const CANONICAL_ROOT_MEMORY_FILENAME = "MEMORY.md";
|
||||
|
||||
const DEFAULT_AGENT_ID = "main";
|
||||
const VALID_ID_RE = /^[a-z0-9][a-z0-9_-]{0,63}$/i;
|
||||
const INVALID_CHARS_RE = /[^a-z0-9_-]+/g;
|
||||
const LEADING_DASH_RE = /^-+/;
|
||||
const TRAILING_DASH_RE = /-+$/;
|
||||
const LEGACY_STATE_DIRNAMES = [".clawdbot"] as const;
|
||||
const NEW_STATE_DIRNAME = ".openclaw";
|
||||
const DURATION_MULTIPLIERS: Record<string, number> = {
|
||||
ms: 1,
|
||||
s: 1000,
|
||||
m: 60_000,
|
||||
h: 3_600_000,
|
||||
d: 86_400_000,
|
||||
};
|
||||
|
||||
/** Round parsed durations and reject values outside the safe integer range. */
|
||||
function roundDurationMs(raw: string, value: number): number {
|
||||
const rounded = Math.round(value);
|
||||
if (!Number.isSafeInteger(rounded)) {
|
||||
throw new Error(`invalid duration: ${raw}`);
|
||||
}
|
||||
return rounded;
|
||||
}
|
||||
|
||||
/** Normalize user or config agent ids to the filesystem-safe canonical form. */
|
||||
export function normalizeAgentId(value: string | undefined | null): string {
|
||||
const trimmed = (value ?? "").trim();
|
||||
if (!trimmed) {
|
||||
return DEFAULT_AGENT_ID;
|
||||
}
|
||||
const normalized = normalizeLowercaseStringOrEmpty(trimmed);
|
||||
if (VALID_ID_RE.test(trimmed)) {
|
||||
return normalized;
|
||||
}
|
||||
return (
|
||||
normalized
|
||||
.replace(INVALID_CHARS_RE, "-")
|
||||
.replace(LEADING_DASH_RE, "")
|
||||
.replace(TRAILING_DASH_RE, "")
|
||||
.slice(0, 64) || DEFAULT_AGENT_ID
|
||||
);
|
||||
}
|
||||
|
||||
/** Treat shell-placeholder home values as absent. */
|
||||
function normalizeHomeValue(value: string | undefined): string | undefined {
|
||||
const trimmed = normalizeOptionalString(value);
|
||||
if (!trimmed || trimmed === "undefined" || trimmed === "null") {
|
||||
return undefined;
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
/** Resolve the underlying OS home before applying OpenClaw-specific overrides. */
|
||||
function resolveRawOsHomeDir(env: NodeJS.ProcessEnv, homedir: () => string): string | undefined {
|
||||
return (
|
||||
normalizeHomeValue(env.HOME) ??
|
||||
normalizeHomeValue(env.USERPROFILE) ??
|
||||
normalizeHomeValue(homedir())
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolve OPENCLAW_HOME or the OS home, falling back to cwd for hermetic tests. */
|
||||
function resolveRequiredHomeDir(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
homedir: () => string = os.homedir,
|
||||
): string {
|
||||
const explicitHome = normalizeHomeValue(env.OPENCLAW_HOME);
|
||||
const rawHome = explicitHome
|
||||
? explicitHome.replace(/^~(?=$|[\\/])/, resolveRawOsHomeDir(env, homedir) ?? "")
|
||||
: resolveRawOsHomeDir(env, homedir);
|
||||
return rawHome ? path.resolve(rawHome) : path.resolve(process.cwd());
|
||||
}
|
||||
|
||||
/** Resolve absolute user paths, including "~" against the effective OpenClaw home. */
|
||||
export function resolveUserPath(
|
||||
input: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
homedir: () => string = os.homedir,
|
||||
): string {
|
||||
const trimmed = input.trim();
|
||||
if (!trimmed) {
|
||||
return trimmed;
|
||||
}
|
||||
if (trimmed.startsWith("~")) {
|
||||
return path.resolve(trimmed.replace(/^~(?=$|[\\/])/, resolveRequiredHomeDir(env, homedir)));
|
||||
}
|
||||
return path.resolve(trimmed);
|
||||
}
|
||||
|
||||
/** Return legacy state roots in priority order. */
|
||||
function legacyStateDirs(homedir: () => string): string[] {
|
||||
return LEGACY_STATE_DIRNAMES.map((dir) => path.join(homedir(), dir));
|
||||
}
|
||||
|
||||
/** Resolve the current state root while preserving shipped legacy installs when present. */
|
||||
export function resolveStateDir(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
homedir: () => string = os.homedir,
|
||||
): string {
|
||||
const override = env.OPENCLAW_STATE_DIR?.trim();
|
||||
if (override) {
|
||||
return resolveUserPath(override, env, homedir);
|
||||
}
|
||||
const effectiveHome = () => resolveRequiredHomeDir(env, homedir);
|
||||
const nextDir = path.join(effectiveHome(), NEW_STATE_DIRNAME);
|
||||
if (env.OPENCLAW_TEST_FAST === "1" || fs.existsSync(nextDir)) {
|
||||
return nextDir;
|
||||
}
|
||||
// Existing legacy state remains authoritative until an explicit migration creates .openclaw.
|
||||
const existingLegacy = legacyStateDirs(effectiveHome).find((dir) => {
|
||||
try {
|
||||
return fs.existsSync(dir);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
return existingLegacy ?? nextDir;
|
||||
}
|
||||
|
||||
/** Resolve the default agent workspace, partitioned by OPENCLAW_PROFILE when set. */
|
||||
function resolveDefaultAgentWorkspaceDir(env: NodeJS.ProcessEnv = process.env): string {
|
||||
const home = resolveRequiredHomeDir(env, os.homedir);
|
||||
const profile = env.OPENCLAW_PROFILE?.trim();
|
||||
if (profile && normalizeLowercaseStringOrEmpty(profile) !== "default") {
|
||||
return path.join(home, ".openclaw", `workspace-${profile}`);
|
||||
}
|
||||
return path.join(home, ".openclaw", "workspace");
|
||||
}
|
||||
|
||||
/** Return configured agent entries after dropping nullish placeholders. */
|
||||
function listAgentEntries(cfg: OpenClawConfig): AgentConfig[] {
|
||||
return Array.isArray(cfg.agents?.list)
|
||||
? cfg.agents.list.filter((entry): entry is AgentConfig => Boolean(entry))
|
||||
: [];
|
||||
}
|
||||
|
||||
/** Resolve the default agent id from explicit default marker or first agent entry. */
|
||||
function resolveDefaultAgentId(cfg: OpenClawConfig): string {
|
||||
const agents = listAgentEntries(cfg);
|
||||
if (agents.length === 0) {
|
||||
return DEFAULT_AGENT_ID;
|
||||
}
|
||||
const chosen = (agents.find((agent) => agent.default) ?? agents[0])?.id;
|
||||
return normalizeAgentId(chosen || DEFAULT_AGENT_ID);
|
||||
}
|
||||
|
||||
/** Find one agent config by canonical id. */
|
||||
function resolveAgentConfig(cfg: OpenClawConfig, agentId: string): AgentConfig | undefined {
|
||||
const id = normalizeAgentId(agentId);
|
||||
return listAgentEntries(cfg).find((entry) => normalizeAgentId(entry.id) === id);
|
||||
}
|
||||
|
||||
/** Remove null bytes before paths are handed to filesystem APIs. */
|
||||
function stripNullBytes(value: string): string {
|
||||
return value.replaceAll("\0", "");
|
||||
}
|
||||
|
||||
/** Resolve the workspace directory for an agent id and config defaults. */
|
||||
export function resolveAgentWorkspaceDir(
|
||||
cfg: OpenClawConfig,
|
||||
agentId: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): string {
|
||||
const id = normalizeAgentId(agentId);
|
||||
const configured = resolveAgentConfig(cfg, id)?.workspace?.trim();
|
||||
if (configured) {
|
||||
return stripNullBytes(resolveUserPath(configured, env));
|
||||
}
|
||||
const fallback = cfg.agents?.defaults?.workspace?.trim();
|
||||
if (id === resolveDefaultAgentId(cfg)) {
|
||||
return stripNullBytes(
|
||||
fallback ? resolveUserPath(fallback, env) : resolveDefaultAgentWorkspaceDir(env),
|
||||
);
|
||||
}
|
||||
if (fallback) {
|
||||
return stripNullBytes(path.join(resolveUserPath(fallback, env), id));
|
||||
}
|
||||
return stripNullBytes(path.join(resolveStateDir(env), `workspace-${id}`));
|
||||
}
|
||||
|
||||
/** Resolve context limits for an agent with defaults fallback. */
|
||||
export function resolveAgentContextLimits(
|
||||
cfg: OpenClawConfig | undefined,
|
||||
agentId?: string | null,
|
||||
): AgentContextLimitsConfig | undefined {
|
||||
const defaults = cfg?.agents?.defaults?.contextLimits;
|
||||
if (!cfg || !agentId) {
|
||||
return defaults;
|
||||
}
|
||||
return resolveAgentConfig(cfg, agentId)?.contextLimits ?? defaults;
|
||||
}
|
||||
|
||||
/** Resolve enabled memory search config plus deduplicated extra paths for an agent. */
|
||||
export function resolveMemorySearchConfig(
|
||||
cfg: OpenClawConfig,
|
||||
agentId: string,
|
||||
): { enabled: boolean; extraPaths: string[] } | null {
|
||||
const defaults = cfg.agents?.defaults?.memorySearch;
|
||||
const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch;
|
||||
const enabled = overrides?.enabled ?? defaults?.enabled ?? true;
|
||||
if (!enabled) {
|
||||
return null;
|
||||
}
|
||||
const rawPaths = normalizeStringEntries([
|
||||
...(defaults?.extraPaths ?? []),
|
||||
...(overrides?.extraPaths ?? []),
|
||||
]);
|
||||
return {
|
||||
enabled,
|
||||
extraPaths: uniqueStrings(rawPaths),
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse compact duration strings such as "500ms", "5s", or "1h30m" into milliseconds. */
|
||||
export function parseDurationMs(
|
||||
raw: string,
|
||||
opts?: { defaultUnit?: "ms" | "s" | "m" | "h" | "d" },
|
||||
): number {
|
||||
const trimmed = normalizeLowercaseStringOrEmpty(normalizeOptionalString(raw) ?? "");
|
||||
if (!trimmed) {
|
||||
throw new Error("invalid duration (empty)");
|
||||
}
|
||||
const single = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)?$/.exec(trimmed);
|
||||
if (single) {
|
||||
const value = Number(single[1]);
|
||||
if (!Number.isFinite(value) || value < 0) {
|
||||
throw new Error(`invalid duration: ${raw}`);
|
||||
}
|
||||
const unit = single[2] ?? opts?.defaultUnit ?? "ms";
|
||||
return roundDurationMs(raw, value * (DURATION_MULTIPLIERS[unit] ?? 1));
|
||||
}
|
||||
|
||||
let totalMs = 0;
|
||||
let consumed = 0;
|
||||
const tokenRe = /(\d+(?:\.\d+)?)(ms|s|m|h|d)/g;
|
||||
for (const match of trimmed.matchAll(tokenRe)) {
|
||||
const [full, valueRaw, unitRaw] = match;
|
||||
const index = match.index ?? -1;
|
||||
if (!full || !valueRaw || !unitRaw || index !== consumed) {
|
||||
throw new Error(`invalid duration: ${raw}`);
|
||||
}
|
||||
const value = Number(valueRaw);
|
||||
const multiplier = DURATION_MULTIPLIERS[unitRaw];
|
||||
if (!Number.isFinite(value) || value < 0 || !multiplier) {
|
||||
throw new Error(`invalid duration: ${raw}`);
|
||||
}
|
||||
totalMs += value * multiplier;
|
||||
consumed += full.length;
|
||||
}
|
||||
if (consumed !== trimmed.length || consumed === 0) {
|
||||
throw new Error(`invalid duration: ${raw}`);
|
||||
}
|
||||
return roundDurationMs(raw, totalMs);
|
||||
}
|
||||
142
packages/memory-host-sdk/src/host/embedding-chunk-limits.test.ts
Normal file
142
packages/memory-host-sdk/src/host/embedding-chunk-limits.test.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
// Memory Host SDK tests cover embedding chunk limits behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { enforceEmbeddingMaxInputTokens } from "./embedding-chunk-limits.js";
|
||||
import { estimateUtf8Bytes } from "./embedding-input-limits.js";
|
||||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
|
||||
function createProvider(maxInputTokens: number): EmbeddingProvider {
|
||||
return {
|
||||
id: "mock",
|
||||
model: "mock-embed",
|
||||
maxInputTokens,
|
||||
embedQuery: async () => [0],
|
||||
embedBatch: async () => [[0]],
|
||||
};
|
||||
}
|
||||
|
||||
function createProviderWithoutMaxInputTokens(params: {
|
||||
id: string;
|
||||
model: string;
|
||||
}): EmbeddingProvider {
|
||||
return {
|
||||
id: params.id,
|
||||
model: params.model,
|
||||
embedQuery: async () => [0],
|
||||
embedBatch: async () => [[0]],
|
||||
};
|
||||
}
|
||||
|
||||
type EmbeddingChunks = ReturnType<typeof enforceEmbeddingMaxInputTokens>;
|
||||
|
||||
function expectChunksWithinUtf8Bytes(chunks: EmbeddingChunks, maxBytes: number) {
|
||||
const oversized: Array<{ index: number; bytes: number }> = [];
|
||||
for (const [index, chunk] of chunks.entries()) {
|
||||
const bytes = estimateUtf8Bytes(chunk.text);
|
||||
if (bytes > maxBytes) {
|
||||
oversized.push({ index, bytes });
|
||||
}
|
||||
}
|
||||
expect(oversized).toStrictEqual([]);
|
||||
}
|
||||
|
||||
function expectChunksLineRange(chunks: EmbeddingChunks, startLine: number, endLine: number) {
|
||||
const unexpectedRanges: Array<{ index: number; startLine: number; endLine: number }> = [];
|
||||
for (const [index, chunk] of chunks.entries()) {
|
||||
if (chunk.startLine !== startLine || chunk.endLine !== endLine) {
|
||||
unexpectedRanges.push({ index, startLine: chunk.startLine, endLine: chunk.endLine });
|
||||
}
|
||||
}
|
||||
expect(unexpectedRanges).toStrictEqual([]);
|
||||
}
|
||||
|
||||
function expectChunksHaveHashes(chunks: EmbeddingChunks) {
|
||||
const invalidHashes: Array<{ index: number; hash: unknown }> = [];
|
||||
for (const [index, chunk] of chunks.entries()) {
|
||||
if (typeof chunk.hash !== "string" || chunk.hash.length === 0) {
|
||||
invalidHashes.push({ index, hash: chunk.hash });
|
||||
}
|
||||
}
|
||||
expect(invalidHashes).toStrictEqual([]);
|
||||
}
|
||||
|
||||
function joinedChunkText(chunks: EmbeddingChunks): string {
|
||||
let text = "";
|
||||
for (const chunk of chunks) {
|
||||
text += chunk.text;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
describe("embedding chunk limits", () => {
|
||||
it("splits oversized chunks so each embedding input stays <= maxInputTokens bytes", () => {
|
||||
const provider = createProvider(8192);
|
||||
const input = {
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
text: "x".repeat(9000),
|
||||
hash: "ignored",
|
||||
};
|
||||
|
||||
const out = enforceEmbeddingMaxInputTokens(provider, [input]);
|
||||
expect(out.length).toBeGreaterThan(1);
|
||||
expect(joinedChunkText(out)).toBe(input.text);
|
||||
expectChunksWithinUtf8Bytes(out, 8192);
|
||||
expectChunksLineRange(out, 1, 1);
|
||||
expectChunksHaveHashes(out);
|
||||
});
|
||||
|
||||
it("does not split inside surrogate pairs (emoji)", () => {
|
||||
const provider = createProvider(8192);
|
||||
const emoji = "😀";
|
||||
const inputText = `${emoji.repeat(2100)}\n${emoji.repeat(2100)}`;
|
||||
|
||||
const out = enforceEmbeddingMaxInputTokens(provider, [
|
||||
{ startLine: 1, endLine: 2, text: inputText, hash: "ignored" },
|
||||
]);
|
||||
|
||||
expect(out.length).toBeGreaterThan(1);
|
||||
expect(joinedChunkText(out)).toBe(inputText);
|
||||
expectChunksWithinUtf8Bytes(out, 8192);
|
||||
|
||||
// If we split inside surrogate pairs we'd likely end up with replacement chars.
|
||||
expect(joinedChunkText(out)).not.toContain("\uFFFD");
|
||||
});
|
||||
|
||||
it("uses conservative fallback limits for local providers without declared maxInputTokens", () => {
|
||||
const provider = createProviderWithoutMaxInputTokens({
|
||||
id: "local",
|
||||
model: "unknown-local-embedding",
|
||||
});
|
||||
|
||||
const out = enforceEmbeddingMaxInputTokens(provider, [
|
||||
{
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
text: "x".repeat(3000),
|
||||
hash: "ignored",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(out.length).toBeGreaterThan(1);
|
||||
expectChunksWithinUtf8Bytes(out, 2048);
|
||||
});
|
||||
|
||||
it("honors hard safety caps lower than provider maxInputTokens", () => {
|
||||
const provider = createProvider(8192);
|
||||
const out = enforceEmbeddingMaxInputTokens(
|
||||
provider,
|
||||
[
|
||||
{
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
text: "x".repeat(8100),
|
||||
hash: "ignored",
|
||||
},
|
||||
],
|
||||
8000,
|
||||
);
|
||||
|
||||
expect(out.length).toBeGreaterThan(1);
|
||||
expectChunksWithinUtf8Bytes(out, 8000);
|
||||
});
|
||||
});
|
||||
51
packages/memory-host-sdk/src/host/embedding-chunk-limits.ts
Normal file
51
packages/memory-host-sdk/src/host/embedding-chunk-limits.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
// Memory Host SDK module implements embedding chunk limits behavior.
|
||||
import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-limits.js";
|
||||
import { hasNonTextEmbeddingParts } from "./embedding-inputs.js";
|
||||
import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js";
|
||||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import type { MemoryChunk } from "./internal.js";
|
||||
|
||||
// Enforces provider byte budgets before chunks reach embedding workers.
|
||||
|
||||
/**
|
||||
* Split text-only chunks to the provider's effective input limit.
|
||||
*
|
||||
* Structured multimodal chunks are preserved because only the provider can decide how to count
|
||||
* non-text parts.
|
||||
*/
|
||||
export function enforceEmbeddingMaxInputTokens(
|
||||
provider: EmbeddingProvider,
|
||||
chunks: MemoryChunk[],
|
||||
hardMaxInputTokens?: number,
|
||||
): MemoryChunk[] {
|
||||
const providerMaxInputTokens = resolveEmbeddingMaxInputTokens(provider);
|
||||
const maxInputTokens =
|
||||
typeof hardMaxInputTokens === "number" && hardMaxInputTokens > 0
|
||||
? Math.min(providerMaxInputTokens, hardMaxInputTokens)
|
||||
: providerMaxInputTokens;
|
||||
const out: MemoryChunk[] = [];
|
||||
|
||||
for (const chunk of chunks) {
|
||||
if (hasNonTextEmbeddingParts(chunk.embeddingInput)) {
|
||||
out.push(chunk);
|
||||
continue;
|
||||
}
|
||||
if (estimateUtf8Bytes(chunk.text) <= maxInputTokens) {
|
||||
out.push(chunk);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const text of splitTextToUtf8ByteLimit(chunk.text, maxInputTokens)) {
|
||||
out.push({
|
||||
startLine: chunk.startLine,
|
||||
endLine: chunk.endLine,
|
||||
text,
|
||||
hash: hashText(text),
|
||||
embeddingInput: { text },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
5
packages/memory-host-sdk/src/host/embedding-defaults.ts
Normal file
5
packages/memory-host-sdk/src/host/embedding-defaults.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
// Shared embedding model defaults for builtin memory providers.
|
||||
|
||||
/** Default local embedding model used when config omits an explicit model. */
|
||||
export const DEFAULT_LOCAL_MODEL =
|
||||
"hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf";
|
||||
86
packages/memory-host-sdk/src/host/embedding-input-limits.ts
Normal file
86
packages/memory-host-sdk/src/host/embedding-input-limits.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
// Memory Host SDK module implements embedding input limits behavior.
|
||||
import type { EmbeddingInput } from "./embedding-inputs.js";
|
||||
|
||||
// Helpers for enforcing embedding model input size limits.
|
||||
//
|
||||
// We use UTF-8 byte length as a conservative upper bound for tokenizer output.
|
||||
// Tokenizers operate over bytes; a token must contain at least one byte, so
|
||||
// token_count <= utf8_byte_length.
|
||||
|
||||
export function estimateUtf8Bytes(text: string): number {
|
||||
if (!text) {
|
||||
return 0;
|
||||
}
|
||||
return Buffer.byteLength(text, "utf8");
|
||||
}
|
||||
|
||||
export function estimateStructuredEmbeddingInputBytes(input: EmbeddingInput): number {
|
||||
if (!input.parts?.length) {
|
||||
return estimateUtf8Bytes(input.text);
|
||||
}
|
||||
let total = 0;
|
||||
for (const part of input.parts) {
|
||||
if (part.type === "text") {
|
||||
total += estimateUtf8Bytes(part.text);
|
||||
continue;
|
||||
}
|
||||
total += estimateUtf8Bytes(part.mimeType);
|
||||
total += estimateUtf8Bytes(part.data);
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
export function splitTextToUtf8ByteLimit(text: string, maxUtf8Bytes: number): string[] {
|
||||
if (maxUtf8Bytes <= 0) {
|
||||
return [text];
|
||||
}
|
||||
if (estimateUtf8Bytes(text) <= maxUtf8Bytes) {
|
||||
return [text];
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
let cursor = 0;
|
||||
while (cursor < text.length) {
|
||||
// The number of UTF-16 code units is always <= the number of UTF-8 bytes.
|
||||
// This makes `cursor + maxUtf8Bytes` a safe upper bound on the next split point.
|
||||
let low = cursor + 1;
|
||||
let high = Math.min(text.length, cursor + maxUtf8Bytes);
|
||||
let best = cursor;
|
||||
|
||||
while (low <= high) {
|
||||
const mid = Math.floor((low + high) / 2);
|
||||
const bytes = estimateUtf8Bytes(text.slice(cursor, mid));
|
||||
if (bytes <= maxUtf8Bytes) {
|
||||
best = mid;
|
||||
low = mid + 1;
|
||||
} else {
|
||||
high = mid - 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (best <= cursor) {
|
||||
best = Math.min(text.length, cursor + 1);
|
||||
}
|
||||
|
||||
// Avoid splitting inside a surrogate pair.
|
||||
if (
|
||||
best < text.length &&
|
||||
best > cursor &&
|
||||
text.charCodeAt(best - 1) >= 0xd800 &&
|
||||
text.charCodeAt(best - 1) <= 0xdbff &&
|
||||
text.charCodeAt(best) >= 0xdc00 &&
|
||||
text.charCodeAt(best) <= 0xdfff
|
||||
) {
|
||||
best -= 1;
|
||||
}
|
||||
|
||||
const part = text.slice(cursor, best);
|
||||
if (!part) {
|
||||
break;
|
||||
}
|
||||
parts.push(part);
|
||||
cursor = best;
|
||||
}
|
||||
|
||||
return parts;
|
||||
}
|
||||
43
packages/memory-host-sdk/src/host/embedding-inputs.ts
Normal file
43
packages/memory-host-sdk/src/host/embedding-inputs.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
// Public embedding input contract for text and inline multimodal parts.
|
||||
|
||||
/** Text part passed through embedding providers that support structured input. */
|
||||
export type EmbeddingInputTextPart = {
|
||||
type: "text";
|
||||
text: string;
|
||||
};
|
||||
|
||||
/** Inline binary payload encoded for providers with multimodal embedding support. */
|
||||
export type EmbeddingInputInlineDataPart = {
|
||||
type: "inline-data";
|
||||
mimeType: string;
|
||||
data: string;
|
||||
};
|
||||
|
||||
/** Single structured embedding input part. */
|
||||
export type EmbeddingInputPart = EmbeddingInputTextPart | EmbeddingInputInlineDataPart;
|
||||
|
||||
/** Provider-facing input while preserving the plain text fallback. */
|
||||
export type EmbeddingInput = {
|
||||
text: string;
|
||||
parts?: EmbeddingInputPart[];
|
||||
};
|
||||
|
||||
/** Build the common text-only embedding input shape. */
|
||||
export function buildTextEmbeddingInput(text: string): EmbeddingInput {
|
||||
return { text };
|
||||
}
|
||||
|
||||
/** Narrow an embedding part to an inline-data payload. */
|
||||
export function isInlineDataEmbeddingInputPart(
|
||||
part: EmbeddingInputPart,
|
||||
): part is EmbeddingInputInlineDataPart {
|
||||
return part.type === "inline-data";
|
||||
}
|
||||
|
||||
/** Return true when a chunk needs structured provider handling, not text splitting. */
|
||||
export function hasNonTextEmbeddingParts(input: EmbeddingInput | undefined): boolean {
|
||||
if (!input?.parts?.length) {
|
||||
return false;
|
||||
}
|
||||
return input.parts.some((part) => isInlineDataEmbeddingInputPart(part));
|
||||
}
|
||||
20
packages/memory-host-sdk/src/host/embedding-model-limits.ts
Normal file
20
packages/memory-host-sdk/src/host/embedding-model-limits.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
// Memory Host SDK module implements embedding model limits behavior.
|
||||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
|
||||
// Provider input limits are byte-based approximations for pre-embedding chunk splitting.
|
||||
|
||||
const DEFAULT_EMBEDDING_MAX_INPUT_TOKENS = 8192;
|
||||
const DEFAULT_LOCAL_EMBEDDING_MAX_INPUT_TOKENS = 2048;
|
||||
|
||||
/** Resolve the effective embedding input limit for a provider. */
|
||||
export function resolveEmbeddingMaxInputTokens(provider: EmbeddingProvider): number {
|
||||
if (typeof provider.maxInputTokens === "number") {
|
||||
return provider.maxInputTokens;
|
||||
}
|
||||
|
||||
if (provider.id === "local") {
|
||||
return DEFAULT_LOCAL_EMBEDDING_MAX_INPUT_TOKENS;
|
||||
}
|
||||
|
||||
return DEFAULT_EMBEDDING_MAX_INPUT_TOKENS;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Memory Host SDK helper module supports embedding provider adapter utils behavior.
|
||||
import { normalizeLowercaseStringOrEmpty } from "./string-utils.js";
|
||||
|
||||
// Adapter helpers shared by remote embedding provider implementations.
|
||||
|
||||
/** Detect missing API key errors from provider auth resolution. */
|
||||
export function isMissingEmbeddingApiKeyError(err: unknown): boolean {
|
||||
return err instanceof Error && err.message.includes("No API key found for provider");
|
||||
}
|
||||
|
||||
/** Return stable cache headers after removing provider-specific secret headers. */
|
||||
export function sanitizeEmbeddingCacheHeaders(
|
||||
headers: Record<string, string>,
|
||||
excludedHeaderNames: string[],
|
||||
): Array<[string, string]> {
|
||||
const excluded = new Set(
|
||||
excludedHeaderNames.map((name) => normalizeLowercaseStringOrEmpty(name)),
|
||||
);
|
||||
return Object.entries(headers)
|
||||
.filter(([key]) => !excluded.has(normalizeLowercaseStringOrEmpty(key)))
|
||||
.toSorted(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, value]) => [key, value]);
|
||||
}
|
||||
|
||||
/** Convert custom-id keyed batch embeddings back to request-index order. */
|
||||
export function mapBatchEmbeddingsByIndex(
|
||||
byCustomId: Map<string, number[]>,
|
||||
count: number,
|
||||
): number[][] {
|
||||
const embeddings: number[][] = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
embeddings.push(byCustomId.get(String(index)) ?? []);
|
||||
}
|
||||
return embeddings;
|
||||
}
|
||||
11
packages/memory-host-sdk/src/host/embedding-vectors.ts
Normal file
11
packages/memory-host-sdk/src/host/embedding-vectors.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
// Vector normalization helpers used before embedding similarity search.
|
||||
|
||||
/** Replace invalid coordinates and L2-normalize non-empty vectors. */
|
||||
export function sanitizeAndNormalizeEmbedding(vec: number[]): number[] {
|
||||
const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0));
|
||||
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
||||
if (magnitude < 1e-10) {
|
||||
return sanitized;
|
||||
}
|
||||
return sanitized.map((value) => value / magnitude);
|
||||
}
|
||||
41
packages/memory-host-sdk/src/host/embedding-worker-errors.ts
Normal file
41
packages/memory-host-sdk/src/host/embedding-worker-errors.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
// Typed local embedding worker failures for process and IPC lifecycle handling.
|
||||
|
||||
/** Stable error codes emitted by the local embedding worker supervisor. */
|
||||
export const LOCAL_EMBEDDING_WORKER_ERROR_CODES = {
|
||||
exited: "LOCAL_EMBEDDING_WORKER_EXITED",
|
||||
processError: "LOCAL_EMBEDDING_WORKER_PROCESS_ERROR",
|
||||
ipcError: "LOCAL_EMBEDDING_WORKER_IPC_ERROR",
|
||||
} as const;
|
||||
|
||||
/** Error code union for local embedding worker failures. */
|
||||
export type LocalEmbeddingWorkerFailureCode =
|
||||
(typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES)[keyof typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES];
|
||||
|
||||
/** Cause category for local embedding worker failures. */
|
||||
export type LocalEmbeddingWorkerFailureReason = "exit" | "signal" | "process-error" | "ipc";
|
||||
|
||||
/** Error shape used by callers that need retry/status decisions. */
|
||||
export type LocalEmbeddingWorkerFailureError = Error & {
|
||||
code: LocalEmbeddingWorkerFailureCode;
|
||||
reason: LocalEmbeddingWorkerFailureReason;
|
||||
exitCode?: number | null;
|
||||
signal?: NodeJS.Signals | null;
|
||||
};
|
||||
|
||||
/** Create a local embedding worker failure with stable metadata fields. */
|
||||
export function createLocalEmbeddingWorkerFailureError(params: {
|
||||
message: string;
|
||||
code: LocalEmbeddingWorkerFailureCode;
|
||||
reason: LocalEmbeddingWorkerFailureReason;
|
||||
exitCode?: number | null;
|
||||
signal?: NodeJS.Signals | null;
|
||||
cause?: unknown;
|
||||
}): LocalEmbeddingWorkerFailureError {
|
||||
return Object.assign(new Error(params.message), {
|
||||
code: params.code,
|
||||
reason: params.reason,
|
||||
...(params.exitCode !== undefined ? { exitCode: params.exitCode } : {}),
|
||||
...(params.signal !== undefined ? { signal: params.signal } : {}),
|
||||
...(params.cause !== undefined ? { cause: params.cause } : {}),
|
||||
});
|
||||
}
|
||||
28
packages/memory-host-sdk/src/host/embeddings-debug.ts
Normal file
28
packages/memory-host-sdk/src/host/embeddings-debug.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
// Memory Host SDK module implements embeddings debug behavior.
|
||||
import { normalizeLowercaseStringOrEmpty } from "./string-utils.js";
|
||||
|
||||
// Lightweight stderr debug logging for memory embedding internals.
|
||||
|
||||
const debugEmbeddings = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS);
|
||||
|
||||
/** Write embedding debug metadata when OPENCLAW_DEBUG_MEMORY_EMBEDDINGS is enabled. */
|
||||
export function debugEmbeddingsLog(message: string, meta?: Record<string, unknown>): void {
|
||||
if (!debugEmbeddings) {
|
||||
return;
|
||||
}
|
||||
const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
|
||||
process.stderr.write(`${message}${suffix}\n`);
|
||||
}
|
||||
|
||||
/** Parse common truthy env values for debug toggles. */
|
||||
function isTruthyEnvValue(value?: string): boolean {
|
||||
switch (normalizeLowercaseStringOrEmpty(value)) {
|
||||
case "1":
|
||||
case "on":
|
||||
case "true":
|
||||
case "yes":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Memory Host SDK tests cover embeddings model normalize behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeEmbeddingModelWithPrefixes } from "./embeddings-model-normalize.js";
|
||||
|
||||
describe("normalizeEmbeddingModelWithPrefixes", () => {
|
||||
it("returns default model when input is blank", () => {
|
||||
expect(
|
||||
normalizeEmbeddingModelWithPrefixes({
|
||||
model: " ",
|
||||
defaultModel: "fallback-model",
|
||||
prefixes: ["openai/"],
|
||||
}),
|
||||
).toBe("fallback-model");
|
||||
});
|
||||
|
||||
it("strips the first matching prefix", () => {
|
||||
expect(
|
||||
normalizeEmbeddingModelWithPrefixes({
|
||||
model: "openai/text-embedding-3-small",
|
||||
defaultModel: "fallback-model",
|
||||
prefixes: ["openai/"],
|
||||
}),
|
||||
).toBe("text-embedding-3-small");
|
||||
});
|
||||
|
||||
it("keeps explicit model names when no prefix matches", () => {
|
||||
expect(
|
||||
normalizeEmbeddingModelWithPrefixes({
|
||||
model: "voyage-4-large",
|
||||
defaultModel: "fallback-model",
|
||||
prefixes: ["voyage/"],
|
||||
}),
|
||||
).toBe("voyage-4-large");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
// Normalizes user-provided embedding model ids by removing accepted provider prefixes.
|
||||
|
||||
/** Trim a configured model id, fall back when empty, and strip known prefixes. */
|
||||
export function normalizeEmbeddingModelWithPrefixes(params: {
|
||||
model: string;
|
||||
defaultModel: string;
|
||||
prefixes: string[];
|
||||
}): string {
|
||||
const trimmed = params.model.trim();
|
||||
if (!trimmed) {
|
||||
return params.defaultModel;
|
||||
}
|
||||
for (const prefix of params.prefixes) {
|
||||
if (trimmed.startsWith(prefix)) {
|
||||
return trimmed.slice(prefix.length);
|
||||
}
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// Memory Host SDK tests cover embeddings remote client behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveRemoteEmbeddingBearerClient } from "./embeddings-remote-client.js";
|
||||
|
||||
describe("resolveRemoteEmbeddingBearerClient", () => {
|
||||
it("uses configured OpenAI provider baseUrl for memory embeddings", async () => {
|
||||
const client = await resolveRemoteEmbeddingBearerClient({
|
||||
provider: "openai",
|
||||
defaultBaseUrl: "https://api.openai.com/v1",
|
||||
options: {
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://proxy.example.test/openai/v1",
|
||||
},
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
model: "text-embedding-3-small",
|
||||
remote: {
|
||||
apiKey: "sk-test",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(client.baseUrl).toBe("https://proxy.example.test/openai/v1");
|
||||
});
|
||||
|
||||
it("adds OpenClaw attribution to native OpenAI embedding requests", async () => {
|
||||
vi.stubEnv("OPENCLAW_VERSION", "2026.3.22");
|
||||
const client = await resolveRemoteEmbeddingBearerClient({
|
||||
provider: "openai",
|
||||
defaultBaseUrl: "https://api.openai.com/v1",
|
||||
options: {
|
||||
config: { models: {} } as never,
|
||||
model: "text-embedding-3-large",
|
||||
remote: {
|
||||
apiKey: "sk-test",
|
||||
headers: {
|
||||
originator: "openclaw",
|
||||
"User-Agent": "openclaw",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(client.headers).toEqual({
|
||||
Authorization: "Bearer sk-test",
|
||||
"Content-Type": "application/json",
|
||||
originator: "openclaw",
|
||||
version: "2026.3.22",
|
||||
"User-Agent": "openclaw/2026.3.22",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
// Memory Host SDK module implements embeddings remote client behavior.
|
||||
import type { EmbeddingProviderOptions } from "./embeddings.types.js";
|
||||
import { requireApiKey, resolveApiKeyForProvider } from "./openclaw-runtime-auth.js";
|
||||
import { buildRemoteBaseUrlPolicy } from "./remote-http.js";
|
||||
import { resolveMemorySecretInputString } from "./secret-input.js";
|
||||
import type { SsrFPolicy } from "./ssrf-policy.js";
|
||||
import { normalizeOptionalString } from "./string-utils.js";
|
||||
|
||||
// Builds authenticated remote embedding HTTP clients from agent memory config.
|
||||
|
||||
/** Provider id used for remote embedding auth and config lookup. */
|
||||
export type RemoteEmbeddingProviderId = string;
|
||||
|
||||
/** Attribution headers for native OpenAI embedding calls. */
|
||||
function resolveOpenClawAttributionHeaders(): Record<string, string> {
|
||||
const version = typeof process !== "undefined" ? process.env.OPENCLAW_VERSION?.trim() : undefined;
|
||||
return {
|
||||
originator: "openclaw",
|
||||
...(version ? { version } : {}),
|
||||
"User-Agent": version ? `openclaw/${version}` : "openclaw",
|
||||
};
|
||||
}
|
||||
|
||||
/** Detect the native OpenAI embeddings API route that accepts attribution headers. */
|
||||
function isNativeOpenAIEmbeddingRoute(provider: string, baseUrl: string): boolean {
|
||||
if (provider !== "openai") {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return new URL(baseUrl).hostname.toLowerCase().replace(/\.+$/, "") === "api.openai.com";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve base URL, bearer headers, header overrides, and SSRF policy for remote embeddings. */
|
||||
export async function resolveRemoteEmbeddingBearerClient(params: {
|
||||
provider: RemoteEmbeddingProviderId;
|
||||
options: EmbeddingProviderOptions;
|
||||
defaultBaseUrl: string;
|
||||
}): Promise<{ baseUrl: string; headers: Record<string, string>; ssrfPolicy?: SsrFPolicy }> {
|
||||
const remote = params.options.remote;
|
||||
const remoteApiKey = resolveMemorySecretInputString({
|
||||
value: remote?.apiKey,
|
||||
path: "agents.*.memorySearch.remote.apiKey",
|
||||
});
|
||||
const remoteBaseUrl = normalizeOptionalString(remote?.baseUrl);
|
||||
const providerConfig = params.options.config.models?.providers?.[params.provider];
|
||||
const apiKey = remoteApiKey
|
||||
? remoteApiKey
|
||||
: requireApiKey(
|
||||
await resolveApiKeyForProvider({
|
||||
provider: params.provider,
|
||||
cfg: params.options.config,
|
||||
agentDir: params.options.agentDir,
|
||||
}),
|
||||
params.provider,
|
||||
);
|
||||
const baseUrl =
|
||||
remoteBaseUrl || normalizeOptionalString(providerConfig?.baseUrl) || params.defaultBaseUrl;
|
||||
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
...headerOverrides,
|
||||
};
|
||||
if (isNativeOpenAIEmbeddingRoute(params.provider, baseUrl)) {
|
||||
Object.assign(headers, resolveOpenClawAttributionHeaders());
|
||||
}
|
||||
return { baseUrl, headers, ssrfPolicy: buildRemoteBaseUrlPolicy(baseUrl) };
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Memory Host SDK tests cover embeddings remote fetch behavior.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { fetchRemoteEmbeddingVectors } from "./embeddings-remote-fetch.js";
|
||||
|
||||
const postJsonMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./post-json.js", () => ({
|
||||
postJson: postJsonMock,
|
||||
}));
|
||||
|
||||
function requirePostJsonParams(): {
|
||||
url?: unknown;
|
||||
headers?: unknown;
|
||||
signal?: unknown;
|
||||
body?: unknown;
|
||||
errorPrefix?: unknown;
|
||||
} {
|
||||
const [call] = postJsonMock.mock.calls;
|
||||
if (!call) {
|
||||
throw new Error("expected postJson call");
|
||||
}
|
||||
const [params] = call;
|
||||
if (typeof params !== "object" || params === null || Array.isArray(params)) {
|
||||
throw new Error("expected postJson params to be an object");
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
describe("fetchRemoteEmbeddingVectors", () => {
|
||||
beforeEach(() => {
|
||||
postJsonMock.mockReset();
|
||||
});
|
||||
|
||||
it("maps remote embedding response data to vectors", async () => {
|
||||
postJsonMock.mockImplementationOnce(async (params) => {
|
||||
return await params.parse({
|
||||
data: [{ embedding: [0.1, 0.2] }, { embedding: [0.4] }, { embedding: [0.3] }],
|
||||
});
|
||||
});
|
||||
|
||||
const vectors = await fetchRemoteEmbeddingVectors({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
body: { input: ["one", "two", "three"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
});
|
||||
|
||||
expect(vectors).toEqual([[0.1, 0.2], [0.4], [0.3]]);
|
||||
const postJsonParams = requirePostJsonParams();
|
||||
expect(postJsonParams.url).toBe("https://memory.example/v1/embeddings");
|
||||
expect(postJsonParams.headers).toEqual({ Authorization: "Bearer test" });
|
||||
expect(postJsonParams.body).toEqual({ input: ["one", "two", "three"] });
|
||||
expect(postJsonParams.errorPrefix).toBe("embedding fetch failed");
|
||||
});
|
||||
|
||||
it("passes abort signals to the JSON request", async () => {
|
||||
const controller = new AbortController();
|
||||
postJsonMock.mockImplementationOnce(async (params) => {
|
||||
return await params.parse({ data: [{ embedding: [0.1] }] });
|
||||
});
|
||||
|
||||
await fetchRemoteEmbeddingVectors({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
headers: {},
|
||||
signal: controller.signal,
|
||||
body: { input: ["one"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
});
|
||||
|
||||
expect(requirePostJsonParams().signal).toBe(controller.signal);
|
||||
});
|
||||
|
||||
it("throws a status-rich error on non-ok responses", async () => {
|
||||
postJsonMock.mockRejectedValueOnce(new Error("embedding fetch failed: 403 forbidden"));
|
||||
|
||||
await expect(
|
||||
fetchRemoteEmbeddingVectors({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
headers: {},
|
||||
body: { input: ["one"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
}),
|
||||
).rejects.toThrow("embedding fetch failed: 403 forbidden");
|
||||
});
|
||||
|
||||
it("rejects non-object embedding responses", async () => {
|
||||
postJsonMock.mockImplementationOnce(async (params) => await params.parse([]));
|
||||
|
||||
await expect(
|
||||
fetchRemoteEmbeddingVectors({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
headers: {},
|
||||
body: { input: ["one"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
}),
|
||||
).rejects.toThrow("embedding fetch failed: malformed JSON response");
|
||||
});
|
||||
|
||||
it("rejects missing embedding data arrays", async () => {
|
||||
postJsonMock.mockImplementationOnce(async (params) => await params.parse({}));
|
||||
|
||||
await expect(
|
||||
fetchRemoteEmbeddingVectors({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
headers: {},
|
||||
body: { input: ["one"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
}),
|
||||
).rejects.toThrow("embedding fetch failed: malformed JSON response");
|
||||
});
|
||||
|
||||
it("rejects embedding counts that do not match the submitted input batch", async () => {
|
||||
postJsonMock.mockImplementationOnce(async (params) => {
|
||||
return await params.parse({ data: [{ embedding: [0.1] }] });
|
||||
});
|
||||
|
||||
await expect(
|
||||
fetchRemoteEmbeddingVectors({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
headers: {},
|
||||
body: { input: ["one", "two"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
}),
|
||||
).rejects.toThrow("embedding fetch failed: malformed JSON response");
|
||||
});
|
||||
|
||||
it("rejects wrong nested embedding vector types", async () => {
|
||||
postJsonMock.mockImplementationOnce(async (params) => {
|
||||
return await params.parse({ data: [{ embedding: [0.1, "bad"] }] });
|
||||
});
|
||||
|
||||
await expect(
|
||||
fetchRemoteEmbeddingVectors({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
headers: {},
|
||||
body: { input: ["one"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
}),
|
||||
).rejects.toThrow("embedding fetch failed: malformed JSON response");
|
||||
});
|
||||
});
|
||||
74
packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts
Normal file
74
packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
// Memory Host SDK module implements embeddings remote fetch behavior.
|
||||
import { postJson } from "./post-json.js";
|
||||
import type { SsrFPolicy } from "./ssrf-policy.js";
|
||||
|
||||
// Fetches and validates OpenAI-compatible embedding responses.
|
||||
|
||||
/** Narrow unknown JSON payloads to plain objects. */
|
||||
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Build the common malformed embedding response error. */
|
||||
function malformedEmbeddingResponse(errorPrefix: string): Error {
|
||||
return new Error(`${errorPrefix}: malformed JSON response`);
|
||||
}
|
||||
|
||||
/** Validate and return one finite embedding vector. */
|
||||
function readEmbeddingVector(value: unknown, errorPrefix: string): number[] {
|
||||
if (!Array.isArray(value)) {
|
||||
throw malformedEmbeddingResponse(errorPrefix);
|
||||
}
|
||||
for (const entry of value) {
|
||||
if (typeof entry !== "number" || !Number.isFinite(entry)) {
|
||||
throw malformedEmbeddingResponse(errorPrefix);
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Resolve expected response count from the request body when input is an array. */
|
||||
function resolveExpectedEmbeddingCount(body: unknown): number | undefined {
|
||||
const input = asRecord(body)?.input;
|
||||
return Array.isArray(input) ? input.length : undefined;
|
||||
}
|
||||
|
||||
/** POST an embedding request and return validated vectors in provider response order. */
|
||||
export async function fetchRemoteEmbeddingVectors(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
signal?: AbortSignal;
|
||||
body: unknown;
|
||||
errorPrefix: string;
|
||||
}): Promise<number[][]> {
|
||||
return await postJson({
|
||||
url: params.url,
|
||||
headers: params.headers,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
fetchImpl: params.fetchImpl,
|
||||
signal: params.signal,
|
||||
body: params.body,
|
||||
errorPrefix: params.errorPrefix,
|
||||
parse: (payload) => {
|
||||
const root = asRecord(payload);
|
||||
if (!root || !Array.isArray(root.data)) {
|
||||
throw malformedEmbeddingResponse(params.errorPrefix);
|
||||
}
|
||||
const expectedCount = resolveExpectedEmbeddingCount(params.body);
|
||||
if (expectedCount !== undefined && root.data.length !== expectedCount) {
|
||||
throw malformedEmbeddingResponse(params.errorPrefix);
|
||||
}
|
||||
return root.data.map((entry) => {
|
||||
const record = asRecord(entry);
|
||||
if (!record) {
|
||||
throw malformedEmbeddingResponse(params.errorPrefix);
|
||||
}
|
||||
return readEmbeddingVector(record.embedding, params.errorPrefix);
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Memory Host SDK module implements embeddings remote provider behavior.
|
||||
import {
|
||||
resolveRemoteEmbeddingBearerClient,
|
||||
type RemoteEmbeddingProviderId,
|
||||
} from "./embeddings-remote-client.js";
|
||||
import { fetchRemoteEmbeddingVectors } from "./embeddings-remote-fetch.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js";
|
||||
import type { SsrFPolicy } from "./ssrf-policy.js";
|
||||
|
||||
// Remote embedding provider factory for OpenAI-compatible embeddings APIs.
|
||||
|
||||
/** HTTP client details required by a remote embedding provider. */
|
||||
export type RemoteEmbeddingClient = {
|
||||
baseUrl: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
model: string;
|
||||
};
|
||||
|
||||
/** Create an EmbeddingProvider backed by a remote embeddings endpoint. */
|
||||
export function createRemoteEmbeddingProvider(params: {
|
||||
id: string;
|
||||
client: RemoteEmbeddingClient;
|
||||
errorPrefix: string;
|
||||
maxInputTokens?: number;
|
||||
}): EmbeddingProvider {
|
||||
const { client } = params;
|
||||
const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
|
||||
|
||||
const embed = async (input: string[], signal?: AbortSignal): Promise<number[][]> => {
|
||||
if (input.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return await fetchRemoteEmbeddingVectors({
|
||||
url,
|
||||
headers: client.headers,
|
||||
ssrfPolicy: client.ssrfPolicy,
|
||||
fetchImpl: client.fetchImpl,
|
||||
signal,
|
||||
body: { model: client.model, input },
|
||||
errorPrefix: params.errorPrefix,
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
id: params.id,
|
||||
model: client.model,
|
||||
...(typeof params.maxInputTokens === "number" ? { maxInputTokens: params.maxInputTokens } : {}),
|
||||
embedQuery: async (text, options) => {
|
||||
const [vec] = await embed([text], options?.signal);
|
||||
return vec ?? [];
|
||||
},
|
||||
embedBatch: async (texts, options) => await embed(texts, options?.signal),
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolve a normalized remote embedding client from provider config and model options. */
|
||||
export async function resolveRemoteEmbeddingClient(params: {
|
||||
provider: RemoteEmbeddingProviderId;
|
||||
options: EmbeddingProviderOptions;
|
||||
defaultBaseUrl: string;
|
||||
normalizeModel: (model: string) => string;
|
||||
}): Promise<RemoteEmbeddingClient> {
|
||||
const { baseUrl, headers, ssrfPolicy } = await resolveRemoteEmbeddingBearerClient({
|
||||
provider: params.provider,
|
||||
options: params.options,
|
||||
defaultBaseUrl: params.defaultBaseUrl,
|
||||
});
|
||||
const model = params.normalizeModel(params.options.model);
|
||||
return { baseUrl, headers, ssrfPolicy, model };
|
||||
}
|
||||
120
packages/memory-host-sdk/src/host/embeddings-worker-child.ts
Normal file
120
packages/memory-host-sdk/src/host/embeddings-worker-child.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
// Memory Host SDK module implements embeddings worker child behavior.
|
||||
import { createLocalEmbeddingProviderInProcess } from "./embeddings.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js";
|
||||
|
||||
// Child process entrypoint for local embedding work.
|
||||
|
||||
/** Request payloads accepted from the parent worker client. */
|
||||
type LocalEmbeddingWorkerRequest =
|
||||
| {
|
||||
id: number;
|
||||
type: "initialize";
|
||||
options: EmbeddingProviderOptions;
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
type: "embedQuery";
|
||||
options: EmbeddingProviderOptions;
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
type: "embedBatch";
|
||||
options: EmbeddingProviderOptions;
|
||||
texts: string[];
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
type: "close";
|
||||
};
|
||||
|
||||
/** Serialized error shape returned over JSON IPC. */
|
||||
type LocalEmbeddingWorkerSerializedError = {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
|
||||
let provider: EmbeddingProvider | null = null;
|
||||
let providerOptionsKey: string | null = null;
|
||||
let requestQueue: Promise<void> = Promise.resolve();
|
||||
|
||||
/** Send one JSON IPC message when the child still has an IPC channel. */
|
||||
function send(message: unknown): void {
|
||||
if (typeof process.send === "function") {
|
||||
process.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reuse the current provider while options are unchanged, otherwise rebuild it. */
|
||||
async function getProvider(options: EmbeddingProviderOptions): Promise<EmbeddingProvider> {
|
||||
const key = JSON.stringify(options);
|
||||
if (provider && providerOptionsKey === key) {
|
||||
return provider;
|
||||
}
|
||||
await provider?.close?.();
|
||||
provider = await createLocalEmbeddingProviderInProcess(options);
|
||||
providerOptionsKey = key;
|
||||
return provider;
|
||||
}
|
||||
|
||||
/** Close and forget the active in-process provider. */
|
||||
async function closeProvider(): Promise<void> {
|
||||
const current = provider;
|
||||
provider = null;
|
||||
providerOptionsKey = null;
|
||||
await current?.close?.();
|
||||
}
|
||||
|
||||
/** Preserve error message and code across JSON IPC. */
|
||||
function serializeError(err: unknown): LocalEmbeddingWorkerSerializedError {
|
||||
if (!(err instanceof Error)) {
|
||||
return { message: String(err) };
|
||||
}
|
||||
const code = (err as Error & { code?: unknown }).code;
|
||||
return {
|
||||
message: err.message,
|
||||
...(typeof code === "string" ? { code } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Handle one parent request after queue serialization. */
|
||||
async function handleRequest(request: LocalEmbeddingWorkerRequest): Promise<void> {
|
||||
if (request.type === "close") {
|
||||
await closeProvider();
|
||||
send({ id: request.id, ok: true });
|
||||
return;
|
||||
}
|
||||
|
||||
const currentProvider = await getProvider(request.options);
|
||||
if (request.type === "initialize") {
|
||||
send({ id: request.id, ok: true });
|
||||
return;
|
||||
}
|
||||
if (request.type === "embedQuery") {
|
||||
const value = await currentProvider.embedQuery(request.text);
|
||||
send({ id: request.id, ok: true, value });
|
||||
return;
|
||||
}
|
||||
|
||||
const value = await currentProvider.embedBatch(request.texts);
|
||||
send({ id: request.id, ok: true, value });
|
||||
}
|
||||
|
||||
// Requests are serialized so node-llama-cpp context state is not used concurrently.
|
||||
process.on("message", (message) => {
|
||||
const request = message as LocalEmbeddingWorkerRequest;
|
||||
requestQueue = requestQueue.then(async () => {
|
||||
try {
|
||||
await handleRequest(request);
|
||||
} catch (err) {
|
||||
send({ id: request.id, ok: false, error: serializeError(err) });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Parent disconnect means the worker is orphaned; close provider resources before exiting.
|
||||
process.once("disconnect", () => {
|
||||
void closeProvider().finally(() => {
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
||||
413
packages/memory-host-sdk/src/host/embeddings-worker.ts
Normal file
413
packages/memory-host-sdk/src/host/embeddings-worker.ts
Normal file
@@ -0,0 +1,413 @@
|
||||
// Memory Host SDK module implements embeddings worker behavior.
|
||||
import { fork, type ChildProcess } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js";
|
||||
import {
|
||||
createLocalEmbeddingWorkerFailureError,
|
||||
LOCAL_EMBEDDING_WORKER_ERROR_CODES,
|
||||
} from "./embedding-worker-errors.js";
|
||||
import type { LocalEmbeddingProviderRuntimeOptions } from "./embeddings.js";
|
||||
import type {
|
||||
EmbeddingProvider,
|
||||
EmbeddingProviderCallOptions,
|
||||
EmbeddingProviderOptions,
|
||||
} from "./embeddings.types.js";
|
||||
import { normalizeOptionalString } from "./string-utils.js";
|
||||
|
||||
// Parent-side local embedding worker client for isolating node-llama-cpp state.
|
||||
|
||||
/** Request payloads sent from the parent process to the local embedding worker child. */
|
||||
type LocalEmbeddingWorkerRequestPayload =
|
||||
| {
|
||||
type: "initialize";
|
||||
options: EmbeddingProviderOptions;
|
||||
}
|
||||
| {
|
||||
type: "embedQuery";
|
||||
options: EmbeddingProviderOptions;
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "embedBatch";
|
||||
options: EmbeddingProviderOptions;
|
||||
texts: string[];
|
||||
}
|
||||
| {
|
||||
type: "close";
|
||||
};
|
||||
|
||||
type LocalEmbeddingWorkerRequest = LocalEmbeddingWorkerRequestPayload & { id: number };
|
||||
|
||||
/** Response payloads sent from the local embedding worker child back to the parent. */
|
||||
type LocalEmbeddingWorkerResponse =
|
||||
| {
|
||||
id: number;
|
||||
ok: true;
|
||||
value?: number[] | number[][];
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
ok: false;
|
||||
error:
|
||||
| string
|
||||
| {
|
||||
message?: string;
|
||||
code?: string;
|
||||
};
|
||||
};
|
||||
|
||||
/** Pending parent request plus abort cleanup. */
|
||||
type PendingRequest = {
|
||||
resolve: (value: number[] | number[][] | undefined) => void;
|
||||
reject: (err: unknown) => void;
|
||||
abort?: () => void;
|
||||
};
|
||||
|
||||
/** Resolve the worker child script for source, package, and bundled runtime layouts. */
|
||||
function resolveDefaultWorkerScriptPath(): string {
|
||||
const currentPath = fileURLToPath(import.meta.url);
|
||||
const extension = path.extname(currentPath);
|
||||
const currentName = path.basename(currentPath);
|
||||
const sibling =
|
||||
extension === ".ts"
|
||||
? "embeddings-worker-child.ts"
|
||||
: currentName.startsWith("embeddings-worker.")
|
||||
? "embeddings-worker-child.js"
|
||||
: "memory-core-local-embedding-worker.js";
|
||||
return path.join(path.dirname(currentPath), sibling);
|
||||
}
|
||||
|
||||
/** Keep only local embedding options that are safe and necessary to send over IPC. */
|
||||
function serializeLocalEmbeddingOptions(
|
||||
options: EmbeddingProviderOptions,
|
||||
runtimeOptions?: LocalEmbeddingProviderRuntimeOptions,
|
||||
): EmbeddingProviderOptions {
|
||||
return {
|
||||
config: {},
|
||||
provider: "local",
|
||||
model: options.model,
|
||||
fallback: "none",
|
||||
outputDimensionality: options.outputDimensionality,
|
||||
local: {
|
||||
...options.local,
|
||||
...(runtimeOptions?.nodeLlamaCppImportUrl
|
||||
? { nodeLlamaCppImportUrl: runtimeOptions.nodeLlamaCppImportUrl }
|
||||
: {}),
|
||||
} as EmbeddingProviderOptions["local"],
|
||||
};
|
||||
}
|
||||
|
||||
/** Create a typed failure for unexpected worker process exits. */
|
||||
function createWorkerExitError(code: number | null, signal: NodeJS.Signals | null): Error {
|
||||
const detail = signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`;
|
||||
return createLocalEmbeddingWorkerFailureError({
|
||||
message: `Local embedding worker exited unexpectedly (${detail})`,
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
reason: signal ? "signal" : "exit",
|
||||
exitCode: code,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
/** Convert worker response errors into Error objects while preserving worker error codes. */
|
||||
function createWorkerResponseError(error: LocalEmbeddingWorkerResponse & { ok: false }): Error {
|
||||
if (typeof error.error === "object" && error.error) {
|
||||
const message = error.error.message || "Local embedding worker failed";
|
||||
const workerError = new Error(message) as Error & { code?: string };
|
||||
if (error.error.code) {
|
||||
workerError.code = error.error.code;
|
||||
}
|
||||
return workerError;
|
||||
}
|
||||
return new Error(error.error || "Local embedding worker failed");
|
||||
}
|
||||
|
||||
const WORKER_UNSAFE_EXEC_ARGV_FLAGS = new Set(["--inspect", "--inspect-brk"]);
|
||||
|
||||
const WORKER_UNSAFE_EXEC_ARGV_FLAGS_WITH_VALUE = new Set([
|
||||
"--eval",
|
||||
"-e",
|
||||
"--print",
|
||||
"-p",
|
||||
"--input-type",
|
||||
"--inspect-port",
|
||||
]);
|
||||
|
||||
const WORKER_UNSAFE_EXEC_ARGV_OPTION_PREFIXES = [
|
||||
"--eval=",
|
||||
"--print=",
|
||||
"--input-type=",
|
||||
"--inspect=",
|
||||
"--inspect-brk=",
|
||||
"--inspect-port=",
|
||||
];
|
||||
|
||||
const WORKER_CLOSE_GRACE_MS = 250;
|
||||
|
||||
/** Drop execArgv flags that would make forked workers debug/eval stateful or unsafe. */
|
||||
function resolveWorkerExecArgv(): string[] {
|
||||
const args: string[] = [];
|
||||
let skipNext = false;
|
||||
for (const arg of process.execArgv) {
|
||||
if (skipNext) {
|
||||
skipNext = false;
|
||||
continue;
|
||||
}
|
||||
if (WORKER_UNSAFE_EXEC_ARGV_FLAGS.has(arg)) {
|
||||
continue;
|
||||
}
|
||||
if (WORKER_UNSAFE_EXEC_ARGV_FLAGS_WITH_VALUE.has(arg)) {
|
||||
skipNext = true;
|
||||
continue;
|
||||
}
|
||||
if (WORKER_UNSAFE_EXEC_ARGV_OPTION_PREFIXES.some((prefix) => arg.startsWith(prefix))) {
|
||||
continue;
|
||||
}
|
||||
args.push(arg);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
/** IPC client that serializes local embedding calls through one child process. */
|
||||
class LocalEmbeddingWorkerClient {
|
||||
private child: ChildProcess | null = null;
|
||||
private nextRequestId = 1;
|
||||
private pending = new Map<number, PendingRequest>();
|
||||
|
||||
constructor(private readonly scriptPath: string) {}
|
||||
|
||||
/** Start or reuse the child worker and initialize its provider. */
|
||||
async initialize(options: EmbeddingProviderOptions): Promise<void> {
|
||||
await this.send({ type: "initialize", options });
|
||||
}
|
||||
|
||||
/** Request one query embedding from the child worker. */
|
||||
async embedQuery(
|
||||
options: EmbeddingProviderOptions,
|
||||
text: string,
|
||||
callOptions?: EmbeddingProviderCallOptions,
|
||||
): Promise<number[]> {
|
||||
const result = await this.send({ type: "embedQuery", options, text }, callOptions);
|
||||
return Array.isArray(result) ? (result as number[]) : [];
|
||||
}
|
||||
|
||||
/** Request a batch of embeddings from the child worker. */
|
||||
async embedBatch(
|
||||
options: EmbeddingProviderOptions,
|
||||
texts: string[],
|
||||
callOptions?: EmbeddingProviderCallOptions,
|
||||
): Promise<number[][]> {
|
||||
const result = await this.send({ type: "embedBatch", options, texts }, callOptions);
|
||||
return Array.isArray(result) ? (result as number[][]) : [];
|
||||
}
|
||||
|
||||
/** Ask the child to close gracefully, then force shutdown after a short grace period. */
|
||||
async close(): Promise<void> {
|
||||
const child = this.child;
|
||||
if (!child) {
|
||||
return;
|
||||
}
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const closeRequest = this.send({ type: "close" }).then(() => "closed" as const);
|
||||
const closeTimeout = new Promise<"timeout">((resolve) => {
|
||||
timeout = setTimeout(() => resolve("timeout"), WORKER_CLOSE_GRACE_MS);
|
||||
timeout.unref?.();
|
||||
});
|
||||
try {
|
||||
const result = await Promise.race([closeRequest, closeTimeout]);
|
||||
if (result === "timeout") {
|
||||
closeRequest.catch(() => {});
|
||||
}
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
this.shutdownChild();
|
||||
}
|
||||
}
|
||||
|
||||
/** Ensure the child process exists and has lifecycle failure handlers installed. */
|
||||
private ensureChild(): ChildProcess {
|
||||
if (this.child?.connected) {
|
||||
return this.child;
|
||||
}
|
||||
|
||||
const child = fork(this.scriptPath, [], {
|
||||
execArgv: resolveWorkerExecArgv(),
|
||||
serialization: "json",
|
||||
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
||||
});
|
||||
child.on("message", (message) => this.handleMessage(message));
|
||||
child.on("exit", (code, signal) => {
|
||||
if (this.child === child) {
|
||||
this.child = null;
|
||||
}
|
||||
this.rejectPending(createWorkerExitError(code, signal));
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
if (this.child === child) {
|
||||
this.child = null;
|
||||
}
|
||||
this.rejectPending(
|
||||
createLocalEmbeddingWorkerFailureError({
|
||||
message: `Local embedding worker process failed: ${err.message}`,
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.processError,
|
||||
reason: "process-error",
|
||||
cause: err,
|
||||
}),
|
||||
);
|
||||
});
|
||||
this.child = child;
|
||||
return child;
|
||||
}
|
||||
|
||||
/** Send one request over IPC and bind its abort signal to child shutdown. */
|
||||
private async send(
|
||||
request: LocalEmbeddingWorkerRequestPayload,
|
||||
options?: EmbeddingProviderCallOptions,
|
||||
): Promise<number[] | number[][] | undefined> {
|
||||
options?.signal?.throwIfAborted();
|
||||
const child = this.ensureChild();
|
||||
const id = this.nextRequestId++;
|
||||
const payload = { ...request, id } as LocalEmbeddingWorkerRequest;
|
||||
return await new Promise((resolve, reject) => {
|
||||
const pending: PendingRequest = { resolve, reject };
|
||||
if (options?.signal) {
|
||||
const abort = () => {
|
||||
this.pending.delete(id);
|
||||
this.shutdownChild();
|
||||
reject(
|
||||
toLintErrorObject(
|
||||
options.signal?.reason ?? new Error("Local embedding request aborted"),
|
||||
"Non-Error rejection",
|
||||
),
|
||||
);
|
||||
};
|
||||
options.signal.addEventListener("abort", abort, { once: true });
|
||||
pending.abort = () => options.signal?.removeEventListener("abort", abort);
|
||||
}
|
||||
this.pending.set(id, pending);
|
||||
child.send(payload, (err) => {
|
||||
if (err) {
|
||||
this.pending.delete(id);
|
||||
pending.abort?.();
|
||||
reject(
|
||||
createLocalEmbeddingWorkerFailureError({
|
||||
message: `Local embedding worker IPC failed: ${err.message}`,
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.ipcError,
|
||||
reason: "ipc",
|
||||
cause: err,
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Route one worker response to the matching pending request. */
|
||||
private handleMessage(message: unknown): void {
|
||||
const response = message as Partial<LocalEmbeddingWorkerResponse>;
|
||||
if (typeof response.id !== "number") {
|
||||
return;
|
||||
}
|
||||
const pending = this.pending.get(response.id);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
this.pending.delete(response.id);
|
||||
pending.abort?.();
|
||||
if (response.ok) {
|
||||
pending.resolve(response.value);
|
||||
return;
|
||||
}
|
||||
pending.reject(
|
||||
createWorkerResponseError(response as LocalEmbeddingWorkerResponse & { ok: false }),
|
||||
);
|
||||
}
|
||||
|
||||
/** Disconnect and kill the current child process if it is still alive. */
|
||||
private shutdownChild(): void {
|
||||
const child = this.child;
|
||||
this.child = null;
|
||||
if (!child) {
|
||||
return;
|
||||
}
|
||||
if (child.connected) {
|
||||
child.disconnect();
|
||||
}
|
||||
if (!child.killed) {
|
||||
child.kill();
|
||||
}
|
||||
}
|
||||
|
||||
/** Reject all pending requests after child process failure. */
|
||||
private rejectPending(err: unknown): void {
|
||||
const pending = [...this.pending.values()];
|
||||
this.pending.clear();
|
||||
for (const entry of pending) {
|
||||
entry.abort?.();
|
||||
entry.reject(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Create the public local embedding provider backed by the child worker client. */
|
||||
export async function createLocalEmbeddingWorkerProvider(
|
||||
options: EmbeddingProviderOptions,
|
||||
runtimeOptions?: LocalEmbeddingProviderRuntimeOptions,
|
||||
): Promise<EmbeddingProvider> {
|
||||
const modelPath = normalizeOptionalString(options.local?.modelPath) || DEFAULT_LOCAL_MODEL;
|
||||
const workerOptions = serializeLocalEmbeddingOptions(options, runtimeOptions);
|
||||
const client = new LocalEmbeddingWorkerClient(
|
||||
runtimeOptions?.workerScriptPath ?? resolveDefaultWorkerScriptPath(),
|
||||
);
|
||||
try {
|
||||
await client.initialize(workerOptions);
|
||||
} catch (err) {
|
||||
await client.close().catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
let closed = false;
|
||||
|
||||
const throwIfClosed = () => {
|
||||
if (closed) {
|
||||
throw new Error("Local embedding provider has been closed");
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
id: "local",
|
||||
model: modelPath,
|
||||
embedQuery: async (text, callOptions) => {
|
||||
throwIfClosed();
|
||||
return await client.embedQuery(workerOptions, text, callOptions);
|
||||
},
|
||||
embedBatch: async (texts, callOptions) => {
|
||||
throwIfClosed();
|
||||
return await client.embedBatch(workerOptions, texts, callOptions);
|
||||
},
|
||||
close: async () => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
await client.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Convert abort reasons or arbitrary thrown values into lint-safe Error objects. */
|
||||
function toLintErrorObject(value: unknown, fallbackMessage: string): Error {
|
||||
if (value instanceof Error) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return new Error(value);
|
||||
}
|
||||
const error = new Error(fallbackMessage, { cause: value });
|
||||
if ((typeof value === "object" && value !== null) || typeof value === "function") {
|
||||
Object.assign(error, value);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
625
packages/memory-host-sdk/src/host/embeddings.test.ts
Normal file
625
packages/memory-host-sdk/src/host/embeddings.test.ts
Normal file
@@ -0,0 +1,625 @@
|
||||
// Memory Host SDK tests cover embeddings behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { LOCAL_EMBEDDING_WORKER_ERROR_CODES } from "./embedding-worker-errors.js";
|
||||
import { createLocalEmbeddingWorkerProvider } from "./embeddings-worker.js";
|
||||
import { createLocalEmbeddingProviderInProcess, DEFAULT_LOCAL_MODEL } from "./embeddings.js";
|
||||
|
||||
const nodeLlamaMock = vi.hoisted(() => ({
|
||||
importNodeLlamaCpp: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./node-llama.js", () => ({
|
||||
importNodeLlamaCpp: nodeLlamaMock.importNodeLlamaCpp,
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
function createDeferred<T>() {
|
||||
let resolve: ((value: T) => void) | undefined;
|
||||
let reject: ((reason?: unknown) => void) | undefined;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
if (!resolve || !reject) {
|
||||
throw new Error("Expected deferred callbacks to be initialized");
|
||||
}
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function mockLocalEmbeddingRuntime(
|
||||
vector: ArrayLike<number> = new Float32Array([2.35, 3.45, 0.63, 4.3]),
|
||||
) {
|
||||
const disposeContext = vi.fn();
|
||||
const disposeModel = vi.fn();
|
||||
const disposeLlama = vi.fn();
|
||||
const getEmbeddingFor = vi.fn().mockResolvedValue({ vector });
|
||||
const createEmbeddingContext = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ getEmbeddingFor, dispose: disposeContext });
|
||||
const loadModel = vi.fn().mockResolvedValue({ createEmbeddingContext, dispose: disposeModel });
|
||||
const getLlama = vi.fn(async () => ({ loadModel, dispose: disposeLlama }));
|
||||
const resolveModelFile = vi.fn(async (modelPath: string) => `/resolved/${modelPath}`);
|
||||
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama,
|
||||
resolveModelFile,
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
|
||||
return {
|
||||
createEmbeddingContext,
|
||||
disposeContext,
|
||||
disposeLlama,
|
||||
disposeModel,
|
||||
getLlama,
|
||||
getEmbeddingFor,
|
||||
loadModel,
|
||||
resolveModelFile,
|
||||
};
|
||||
}
|
||||
|
||||
describe("local embedding provider", () => {
|
||||
it("normalizes local embeddings and resolves the default local model", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedding = await provider.embedQuery("test query");
|
||||
const magnitude = Math.sqrt(embedding.reduce((sum, value) => sum + value * value, 0));
|
||||
|
||||
expect(DEFAULT_LOCAL_MODEL).toBe(
|
||||
"hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf",
|
||||
);
|
||||
expect(magnitude).toBeCloseTo(1, 5);
|
||||
expect(runtime.resolveModelFile).toHaveBeenCalledWith(
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
expect.objectContaining({ signal: expect.any(AbortSignal) }),
|
||||
);
|
||||
expect(runtime.loadModel).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
modelPath: `/resolved/${DEFAULT_LOCAL_MODEL}`,
|
||||
loadSignal: expect.any(AbortSignal),
|
||||
}),
|
||||
);
|
||||
expect(runtime.getEmbeddingFor).toHaveBeenCalledWith("test query");
|
||||
});
|
||||
|
||||
it("truncates local embeddings before normalizing them", async () => {
|
||||
mockLocalEmbeddingRuntime(new Float32Array([3, 4, 12]));
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
outputDimensionality: 2,
|
||||
});
|
||||
|
||||
await expect(provider.embedQuery("test query")).resolves.toEqual([0.6, 0.8]);
|
||||
await expect(provider.embedBatch(["test document"])).resolves.toEqual([[0.6, 0.8]]);
|
||||
});
|
||||
|
||||
it("does not read local embedding coordinates past outputDimensionality", async () => {
|
||||
mockLocalEmbeddingRuntime({
|
||||
length: 3,
|
||||
0: 3,
|
||||
1: 4,
|
||||
get 2(): number {
|
||||
throw new Error("tail coordinate should not be read");
|
||||
},
|
||||
});
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
outputDimensionality: 2,
|
||||
});
|
||||
|
||||
await expect(provider.embedQuery("test query")).resolves.toEqual([0.6, 0.8]);
|
||||
});
|
||||
|
||||
it("passes default contextSize (4096) to createEmbeddingContext when not configured", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size default test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ contextSize: 4096, createSignal: expect.any(AbortSignal) }),
|
||||
);
|
||||
});
|
||||
|
||||
it("imports node-llama-cpp from an explicit module URL when provided", async () => {
|
||||
mockLocalEmbeddingRuntime();
|
||||
|
||||
await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: {
|
||||
nodeLlamaCppImportUrl: "file:///plugins/llama-cpp/node-llama-cpp.js",
|
||||
} as never,
|
||||
});
|
||||
|
||||
expect(nodeLlamaMock.importNodeLlamaCpp).toHaveBeenCalledWith(
|
||||
"file:///plugins/llama-cpp/node-llama-cpp.js",
|
||||
);
|
||||
});
|
||||
|
||||
it("passes configured contextSize to createEmbeddingContext", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: { contextSize: 2048 },
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size custom test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ contextSize: 2048, createSignal: expect.any(AbortSignal) }),
|
||||
);
|
||||
});
|
||||
|
||||
it('passes "auto" contextSize to createEmbeddingContext when explicitly set', async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: { contextSize: "auto" },
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size auto test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ contextSize: "auto", createSignal: expect.any(AbortSignal) }),
|
||||
);
|
||||
});
|
||||
|
||||
it("runs local batch embeddings sequentially", async () => {
|
||||
const calls: string[] = [];
|
||||
const firstGate = createDeferred<{ vector: Float32Array }>();
|
||||
const secondGate = createDeferred<{ vector: Float32Array }>();
|
||||
const getEmbeddingFor = vi.fn((text: string) => {
|
||||
calls.push(text);
|
||||
return text === "first" ? firstGate.promise : secondGate.promise;
|
||||
});
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: vi.fn(async () => ({
|
||||
loadModel: vi.fn(async () => ({
|
||||
createEmbeddingContext: vi.fn(async () => ({ getEmbeddingFor })),
|
||||
})),
|
||||
})),
|
||||
resolveModelFile: vi.fn(async () => "/resolved/model.gguf"),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const batchPromise = provider.embedBatch(["first", "second"]);
|
||||
await expect.poll(() => calls.join(",")).toBe("first");
|
||||
firstGate.resolve({ vector: new Float32Array([1, 0]) });
|
||||
await expect.poll(() => calls.join(",")).toBe("first,second");
|
||||
secondGate.resolve({ vector: new Float32Array([0, 1]) });
|
||||
|
||||
await expect(batchPromise).resolves.toHaveLength(2);
|
||||
});
|
||||
|
||||
it("trims explicit local model paths and cache directories", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime(new Float32Array([1, 0]));
|
||||
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: {
|
||||
modelPath: " /models/embed.gguf ",
|
||||
modelCacheDir: " /cache/models ",
|
||||
},
|
||||
});
|
||||
|
||||
await provider.embedBatch(["a", "b"]);
|
||||
|
||||
expect(provider.model).toBe("/models/embed.gguf");
|
||||
expect(runtime.resolveModelFile).toHaveBeenCalledWith(
|
||||
"/models/embed.gguf",
|
||||
expect.objectContaining({
|
||||
directory: "/cache/models",
|
||||
signal: expect.any(AbortSignal),
|
||||
}),
|
||||
);
|
||||
expect(runtime.getEmbeddingFor).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("disposes cached local llama resources when closed", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
await provider.embedQuery("load local resources");
|
||||
await provider.close?.();
|
||||
await provider.close?.();
|
||||
|
||||
expect(runtime.disposeContext).toHaveBeenCalledTimes(1);
|
||||
expect(runtime.disposeModel).toHaveBeenCalledTimes(1);
|
||||
expect(runtime.disposeLlama).toHaveBeenCalledTimes(1);
|
||||
await expect(provider.embedQuery("after close")).rejects.toThrow(
|
||||
"Local embedding provider has been closed",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not wait for pending local llama initialization before close resolves", async () => {
|
||||
const disposeLlama = vi.fn();
|
||||
const getLlamaGate = createDeferred<unknown>();
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: async () => (await getLlamaGate.promise) as never,
|
||||
resolveModelFile: vi.fn(async (modelPath: string) => `/resolved/${modelPath}`),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedPromise = provider.embedQuery("pending init");
|
||||
await expect(provider.close?.()).resolves.toBeUndefined();
|
||||
|
||||
getLlamaGate.resolve({ loadModel: vi.fn(), dispose: disposeLlama });
|
||||
await expect(embedPromise).rejects.toThrow("Local embedding provider has been closed");
|
||||
expect(disposeLlama).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("aborts pending local llama model loads when closed", async () => {
|
||||
const loadModelStarted = createDeferred<void>();
|
||||
const loadModelGate = createDeferred<never>();
|
||||
const disposeLlama = vi.fn();
|
||||
let capturedResolveSignal: AbortSignal | undefined;
|
||||
let capturedLoadSignal: AbortSignal | undefined;
|
||||
const loadModel = vi.fn(
|
||||
(params: { modelPath: string; loadSignal?: AbortSignal }): Promise<never> => {
|
||||
capturedLoadSignal = params.loadSignal;
|
||||
loadModelStarted.resolve();
|
||||
return loadModelGate.promise;
|
||||
},
|
||||
);
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: async () => ({ loadModel, dispose: disposeLlama }),
|
||||
resolveModelFile: vi.fn(async (_modelPath: string, options?: { signal?: AbortSignal }) => {
|
||||
capturedResolveSignal = options?.signal;
|
||||
return "/resolved/model.gguf";
|
||||
}),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedPromise = provider.embedQuery("pending model load");
|
||||
await loadModelStarted.promise;
|
||||
await expect(provider.close?.()).resolves.toBeUndefined();
|
||||
|
||||
expect(capturedResolveSignal?.aborted).toBe(true);
|
||||
expect(capturedLoadSignal?.aborted).toBe(true);
|
||||
expect(disposeLlama).toHaveBeenCalledTimes(1);
|
||||
loadModelGate.reject(new Error("load aborted"));
|
||||
await expect(embedPromise).rejects.toThrow("load aborted");
|
||||
});
|
||||
|
||||
it("aborts pending local llama embedding context creation when closed", async () => {
|
||||
const createContextStarted = createDeferred<void>();
|
||||
const createContextGate = createDeferred<never>();
|
||||
const disposeLlama = vi.fn();
|
||||
const disposeModel = vi.fn();
|
||||
let capturedCreateSignal: AbortSignal | undefined;
|
||||
const createEmbeddingContext = vi.fn(
|
||||
(options?: { createSignal?: AbortSignal }): Promise<never> => {
|
||||
capturedCreateSignal = options?.createSignal;
|
||||
createContextStarted.resolve();
|
||||
return createContextGate.promise;
|
||||
},
|
||||
);
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: async () => ({
|
||||
loadModel: vi.fn(async () => ({ createEmbeddingContext, dispose: disposeModel })),
|
||||
dispose: disposeLlama,
|
||||
}),
|
||||
resolveModelFile: vi.fn(async () => "/resolved/model.gguf"),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedPromise = provider.embedQuery("pending context create");
|
||||
await createContextStarted.promise;
|
||||
await expect(provider.close?.()).resolves.toBeUndefined();
|
||||
|
||||
expect(capturedCreateSignal?.aborted).toBe(true);
|
||||
expect(disposeModel).toHaveBeenCalledTimes(1);
|
||||
expect(disposeLlama).toHaveBeenCalledTimes(1);
|
||||
createContextGate.reject(new Error("context create aborted"));
|
||||
await expect(embedPromise).rejects.toThrow("context create aborted");
|
||||
});
|
||||
|
||||
it("uses a worker process for the public local provider", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
if (message.type === "initialize") {
|
||||
if (message.options.local?.nodeLlamaCppImportUrl !== "file:///plugin/node-llama-cpp.js") {
|
||||
process.send({ id: message.id, ok: false, error: "missing nodeLlamaCppImportUrl" });
|
||||
return;
|
||||
}
|
||||
if (message.options.outputDimensionality !== 2) {
|
||||
process.send({ id: message.id, ok: false, error: "missing outputDimensionality" });
|
||||
return;
|
||||
}
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
if (message.type === "embedQuery") {
|
||||
process.send({ id: message.id, ok: true, value: [1, 0] });
|
||||
return;
|
||||
}
|
||||
if (message.type === "embedBatch") {
|
||||
process.send({ id: message.id, ok: true, value: message.texts.map(() => [0, 1]) });
|
||||
return;
|
||||
}
|
||||
process.send({ id: message.id, ok: true });
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
outputDimensionality: 2,
|
||||
},
|
||||
{
|
||||
workerScriptPath: workerScript,
|
||||
nodeLlamaCppImportUrl: "file:///plugin/node-llama-cpp.js",
|
||||
},
|
||||
);
|
||||
|
||||
await expect(provider.embedQuery("hello")).resolves.toEqual([1, 0]);
|
||||
await expect(provider.embedBatch(["a", "b"])).resolves.toEqual([
|
||||
[0, 1],
|
||||
[0, 1],
|
||||
]);
|
||||
await expect(provider.close?.()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("terminates the worker when close runs behind a pending request", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
const embedStartedPath = path.join(tempDir, "embed-started");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
const fs = require("node:fs");
|
||||
const embedStartedPath = ${JSON.stringify(embedStartedPath)};
|
||||
let busy = false;
|
||||
|
||||
process.on("message", (message) => {
|
||||
if (busy) {
|
||||
return;
|
||||
}
|
||||
if (message.type === "initialize") {
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
if (message.type === "embedQuery") {
|
||||
busy = true;
|
||||
fs.writeFileSync(embedStartedPath, "1");
|
||||
}
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
|
||||
const embedPromise = provider.embedQuery("stuck");
|
||||
const embedError = embedPromise.then(
|
||||
() => undefined,
|
||||
(err: unknown) => err,
|
||||
);
|
||||
await expect
|
||||
.poll(async () => {
|
||||
try {
|
||||
await fs.access(embedStartedPath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.toBe(true);
|
||||
|
||||
const closePromise = provider.close?.() ?? Promise.resolve();
|
||||
const closeResult = await Promise.race([
|
||||
closePromise.then(() => "closed" as const),
|
||||
new Promise<"timeout">((resolve) => {
|
||||
setTimeout(() => resolve("timeout"), 1_000);
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(closeResult).toBe("closed");
|
||||
await expect(embedError).resolves.toMatchObject({
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not pass inline-source or inspector exec args to the file-backed worker", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
if (message.type === "initialize" || message.type === "close") {
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
process.send({ id: message.id, ok: true, value: [process.execArgv.length] });
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const originalExecArgv = [...process.execArgv];
|
||||
let provider: Awaited<ReturnType<typeof createLocalEmbeddingWorkerProvider>> | undefined;
|
||||
try {
|
||||
process.execArgv.splice(
|
||||
0,
|
||||
process.execArgv.length,
|
||||
"--eval",
|
||||
"setInterval(() => {}, 1000)",
|
||||
"--print",
|
||||
"1 + 1",
|
||||
"--input-type=module",
|
||||
"--inspect-brk=127.0.0.1:0",
|
||||
"--inspect-port",
|
||||
"0",
|
||||
);
|
||||
provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
await expect(provider.embedQuery("hello")).resolves.toEqual([0]);
|
||||
} finally {
|
||||
process.execArgv.splice(0, process.execArgv.length, ...originalExecArgv);
|
||||
await provider?.close?.();
|
||||
}
|
||||
});
|
||||
|
||||
it("reports worker initialization failures during provider creation", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
process.send({
|
||||
id: message.id,
|
||||
ok: false,
|
||||
error: { message: "Cannot find package 'node-llama-cpp'", code: "ERR_MODULE_NOT_FOUND" },
|
||||
});
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
try {
|
||||
await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
throw new Error("expected local embedding provider creation to fail");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect((err as Error).message).toBe("Cannot find package 'node-llama-cpp'");
|
||||
expect((err as Error & { code?: string }).code).toBe("ERR_MODULE_NOT_FOUND");
|
||||
}
|
||||
});
|
||||
|
||||
it("reports worker exits with structured failure codes", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
if (message.type === "initialize") {
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
process.exit(134);
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
|
||||
await expect(provider.embedQuery("hello")).rejects.toMatchObject({
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
reason: "exit",
|
||||
exitCode: 134,
|
||||
});
|
||||
});
|
||||
});
|
||||
206
packages/memory-host-sdk/src/host/embeddings.ts
Normal file
206
packages/memory-host-sdk/src/host/embeddings.ts
Normal file
@@ -0,0 +1,206 @@
|
||||
import { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js";
|
||||
import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js";
|
||||
import { createLocalEmbeddingWorkerProvider } from "./embeddings-worker.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js";
|
||||
import {
|
||||
importNodeLlamaCpp,
|
||||
type Llama,
|
||||
type LlamaEmbeddingContext,
|
||||
type LlamaModel,
|
||||
} from "./node-llama.js";
|
||||
// Memory Host SDK module implements embeddings behavior.
|
||||
import { toLintErrorObject } from "./retry-utils.js";
|
||||
import { normalizeOptionalString } from "./string-utils.js";
|
||||
|
||||
type DisposableResource = {
|
||||
dispose?: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
export type {
|
||||
EmbeddingProvider,
|
||||
EmbeddingProviderFallback,
|
||||
EmbeddingProviderId,
|
||||
EmbeddingProviderOptions,
|
||||
EmbeddingProviderRequest,
|
||||
GeminiTaskType,
|
||||
} from "./embeddings.types.js";
|
||||
|
||||
export { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js";
|
||||
|
||||
export type LocalEmbeddingProviderRuntimeOptions = {
|
||||
workerScriptPath?: string;
|
||||
nodeLlamaCppImportUrl?: string;
|
||||
};
|
||||
|
||||
function copyEmbeddingVector(vector: ArrayLike<number>, maxLength?: number): number[] {
|
||||
const length = Math.min(maxLength ?? vector.length, vector.length);
|
||||
const values: number[] = [];
|
||||
for (let index = 0; index < length; index += 1) {
|
||||
values.push(vector[index]);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
async function disposeResources(
|
||||
resources: Array<DisposableResource | null | undefined>,
|
||||
): Promise<void> {
|
||||
let firstError: unknown;
|
||||
for (const resource of resources) {
|
||||
try {
|
||||
await resource?.dispose?.();
|
||||
} catch (err) {
|
||||
firstError ??= err;
|
||||
}
|
||||
}
|
||||
if (firstError) {
|
||||
throw toLintErrorObject(firstError, "Non-Error thrown");
|
||||
}
|
||||
}
|
||||
|
||||
export async function createLocalEmbeddingProvider(
|
||||
options: EmbeddingProviderOptions,
|
||||
runtimeOptions?: LocalEmbeddingProviderRuntimeOptions,
|
||||
): Promise<EmbeddingProvider> {
|
||||
return await createLocalEmbeddingWorkerProvider(options, runtimeOptions);
|
||||
}
|
||||
|
||||
export async function createLocalEmbeddingProviderInProcess(
|
||||
options: EmbeddingProviderOptions,
|
||||
): Promise<EmbeddingProvider> {
|
||||
const modelPath = normalizeOptionalString(options.local?.modelPath) || DEFAULT_LOCAL_MODEL;
|
||||
const modelCacheDir = normalizeOptionalString(options.local?.modelCacheDir);
|
||||
const nodeLlamaCppImportUrl = normalizeOptionalString(
|
||||
(options.local as EmbeddingProviderOptions["local"] & { nodeLlamaCppImportUrl?: string })
|
||||
?.nodeLlamaCppImportUrl,
|
||||
);
|
||||
const contextSize: number | "auto" = options.local?.contextSize ?? 4096;
|
||||
|
||||
// Lazy-load node-llama-cpp to keep startup light unless local is enabled.
|
||||
const { getLlama, resolveModelFile, LlamaLogLevel } =
|
||||
await importNodeLlamaCpp(nodeLlamaCppImportUrl);
|
||||
|
||||
let llama: Llama | null = null;
|
||||
let embeddingModel: LlamaModel | null = null;
|
||||
let embeddingContext: LlamaEmbeddingContext | null = null;
|
||||
let initPromise: Promise<LlamaEmbeddingContext> | null = null;
|
||||
let initAbortController: AbortController | null = null;
|
||||
let closePromise: Promise<void> | null = null;
|
||||
let closed = false;
|
||||
|
||||
const throwIfClosed = () => {
|
||||
if (closed) {
|
||||
throw new Error("Local embedding provider has been closed");
|
||||
}
|
||||
};
|
||||
const disposeAndThrowIfClosed = async <T extends DisposableResource>(resource: T): Promise<T> => {
|
||||
if (!closed) {
|
||||
return resource;
|
||||
}
|
||||
await disposeResources([resource]);
|
||||
throwIfClosed();
|
||||
return resource;
|
||||
};
|
||||
|
||||
const ensureContext = async (): Promise<LlamaEmbeddingContext> => {
|
||||
throwIfClosed();
|
||||
if (embeddingContext) {
|
||||
return embeddingContext;
|
||||
}
|
||||
if (initPromise) {
|
||||
return initPromise;
|
||||
}
|
||||
initPromise = (async () => {
|
||||
const abortController = new AbortController();
|
||||
initAbortController = abortController;
|
||||
try {
|
||||
if (!llama) {
|
||||
const nextLlama = await getLlama({
|
||||
logLevel: LlamaLogLevel.error,
|
||||
});
|
||||
llama = await disposeAndThrowIfClosed(nextLlama);
|
||||
}
|
||||
if (!embeddingModel) {
|
||||
const resolved = await resolveModelFile(modelPath, {
|
||||
...(modelCacheDir ? { directory: modelCacheDir } : {}),
|
||||
signal: abortController.signal,
|
||||
});
|
||||
throwIfClosed();
|
||||
const nextModel = await llama.loadModel({
|
||||
modelPath: resolved,
|
||||
loadSignal: abortController.signal,
|
||||
});
|
||||
embeddingModel = await disposeAndThrowIfClosed(nextModel);
|
||||
}
|
||||
if (!embeddingContext) {
|
||||
const nextContext = await embeddingModel.createEmbeddingContext({
|
||||
contextSize,
|
||||
createSignal: abortController.signal,
|
||||
});
|
||||
embeddingContext = await disposeAndThrowIfClosed(nextContext);
|
||||
}
|
||||
return embeddingContext;
|
||||
} catch (err) {
|
||||
initPromise = null;
|
||||
throw err;
|
||||
} finally {
|
||||
if (initAbortController === abortController) {
|
||||
initAbortController = null;
|
||||
}
|
||||
}
|
||||
})();
|
||||
return initPromise;
|
||||
};
|
||||
|
||||
const outputDimensionality =
|
||||
typeof options.outputDimensionality === "number" ? options.outputDimensionality : undefined;
|
||||
const normalize = (vector: ArrayLike<number>): number[] =>
|
||||
sanitizeAndNormalizeEmbedding(copyEmbeddingVector(vector, outputDimensionality));
|
||||
|
||||
return {
|
||||
id: "local",
|
||||
model: modelPath,
|
||||
embedQuery: async (text, optionsValue) => {
|
||||
throwIfClosed();
|
||||
optionsValue?.signal?.throwIfAborted();
|
||||
const ctx = await ensureContext();
|
||||
throwIfClosed();
|
||||
optionsValue?.signal?.throwIfAborted();
|
||||
const embedding = await ctx.getEmbeddingFor(text);
|
||||
return normalize(embedding.vector);
|
||||
},
|
||||
embedBatch: async (texts, optionsLocal) => {
|
||||
throwIfClosed();
|
||||
optionsLocal?.signal?.throwIfAborted();
|
||||
const ctx = await ensureContext();
|
||||
throwIfClosed();
|
||||
optionsLocal?.signal?.throwIfAborted();
|
||||
const embeddings: number[][] = [];
|
||||
for (const text of texts) {
|
||||
throwIfClosed();
|
||||
optionsLocal?.signal?.throwIfAborted();
|
||||
const embedding = await ctx.getEmbeddingFor(text);
|
||||
embeddings.push(normalize(embedding.vector));
|
||||
}
|
||||
return embeddings;
|
||||
},
|
||||
close: async () => {
|
||||
if (closePromise) {
|
||||
return closePromise;
|
||||
}
|
||||
closed = true;
|
||||
initAbortController?.abort();
|
||||
initAbortController = null;
|
||||
closePromise = (async () => {
|
||||
const context = embeddingContext;
|
||||
const model = embeddingModel;
|
||||
const runtime = llama;
|
||||
embeddingContext = null;
|
||||
embeddingModel = null;
|
||||
llama = null;
|
||||
initPromise = null;
|
||||
await disposeResources([context, model, runtime]);
|
||||
})();
|
||||
return closePromise;
|
||||
},
|
||||
};
|
||||
}
|
||||
65
packages/memory-host-sdk/src/host/embeddings.types.ts
Normal file
65
packages/memory-host-sdk/src/host/embeddings.types.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
// Memory Host SDK type module defines shared TypeScript contracts.
|
||||
import type { OpenClawConfig, SecretInput } from "../engine-foundation.js";
|
||||
import type { EmbeddingInput } from "./embedding-inputs.js";
|
||||
|
||||
export type EmbeddingProvider = {
|
||||
id: string;
|
||||
model: string;
|
||||
maxInputTokens?: number;
|
||||
embedQuery: (text: string, options?: EmbeddingProviderCallOptions) => Promise<number[]>;
|
||||
embedBatch: (texts: string[], options?: EmbeddingProviderCallOptions) => Promise<number[][]>;
|
||||
embedBatchInputs?: (
|
||||
inputs: EmbeddingInput[],
|
||||
options?: EmbeddingProviderCallOptions,
|
||||
) => Promise<number[][]>;
|
||||
close?: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
export type EmbeddingProviderCallOptions = {
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
export type EmbeddingProviderId = string;
|
||||
export type EmbeddingProviderRequest = string;
|
||||
export type EmbeddingProviderFallback = string;
|
||||
|
||||
export type GeminiTaskType =
|
||||
| "RETRIEVAL_QUERY"
|
||||
| "RETRIEVAL_DOCUMENT"
|
||||
| "SEMANTIC_SIMILARITY"
|
||||
| "CLASSIFICATION"
|
||||
| "CLUSTERING"
|
||||
| "QUESTION_ANSWERING"
|
||||
| "FACT_VERIFICATION";
|
||||
|
||||
export type EmbeddingProviderOptions = {
|
||||
config: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
provider?: EmbeddingProviderRequest;
|
||||
remote?: {
|
||||
baseUrl?: string;
|
||||
apiKey?: SecretInput;
|
||||
headers?: Record<string, string>;
|
||||
};
|
||||
model: string;
|
||||
inputType?: string;
|
||||
queryInputType?: string;
|
||||
documentInputType?: string;
|
||||
fallback?: EmbeddingProviderFallback;
|
||||
local?: {
|
||||
modelPath?: string;
|
||||
modelCacheDir?: string;
|
||||
/**
|
||||
* Context size passed to node-llama-cpp `createEmbeddingContext`.
|
||||
* Default: 4096, chosen to cover typical memory-search chunks (128–512 tokens)
|
||||
* while keeping non-weight VRAM bounded.
|
||||
* Set `"auto"` to let node-llama-cpp use the model's trained maximum — not
|
||||
* recommended for 8B+ models (e.g. Qwen3-Embedding-8B: up to 40 960 tokens → ~32 GB VRAM).
|
||||
*/
|
||||
contextSize?: number | "auto";
|
||||
};
|
||||
/** Provider-specific output vector dimensions for supported embedding families. */
|
||||
outputDimensionality?: number;
|
||||
/** Gemini: override the default task type sent with embedding requests. */
|
||||
taskType?: GeminiTaskType;
|
||||
};
|
||||
92
packages/memory-host-sdk/src/host/error-utils.ts
Normal file
92
packages/memory-host-sdk/src/host/error-utils.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
// Memory Host SDK helper module supports error utils behavior.
|
||||
const SECRET_PATTERNS: RegExp[] = [
|
||||
/\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD)\b\s*[=:]\s*(["']?)([^\s"'\\]+)\1/g,
|
||||
/[?&](?:access[-_]?token|auth[-_]?token|hook[-_]?token|refresh[-_]?token|api[-_]?key|client[-_]?secret|token|key|secret|password|pass|passwd|auth|signature)=([^&\s"'<>]+)/gi,
|
||||
/"(?:apiKey|token|secret|password|passwd|accessToken|refreshToken)"\s*:\s*"([^"]+)"/g,
|
||||
/--(?:api[-_]?key|hook[-_]?token|token|secret|password|passwd)\s+(["']?)([^\s"']+)\1/g,
|
||||
/Authorization\s*[:=]\s*Bearer\s+([A-Za-z0-9._\-+=]+)/g,
|
||||
/\bBearer\s+([A-Za-z0-9._\-+=]{18,})\b/g,
|
||||
/(^|[\s,;])(?:access_token|refresh_token|api[-_]?key|token|secret|password|passwd)=([^\s&#]+)/g,
|
||||
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----/g,
|
||||
/\b(sk-[A-Za-z0-9_-]{8,})\b/g,
|
||||
/\b(ghp_[A-Za-z0-9]{20,})\b/g,
|
||||
/\b(github_pat_[A-Za-z0-9_]{20,})\b/g,
|
||||
/\b(xox[baprs]-[A-Za-z0-9-]{10,})\b/g,
|
||||
/\b(xapp-[A-Za-z0-9-]{10,})\b/g,
|
||||
/\b(gsk_[A-Za-z0-9_-]{10,})\b/g,
|
||||
/\b(AIza[0-9A-Za-z\-_]{20,})\b/g,
|
||||
/\b(pplx-[A-Za-z0-9_-]{10,})\b/g,
|
||||
/\b(npm_[A-Za-z0-9]{10,})\b/g,
|
||||
/\bbot(\d{6,}:[A-Za-z0-9_-]{20,})\b/g,
|
||||
/\b(\d{6,}:[A-Za-z0-9_-]{20,})\b/g,
|
||||
];
|
||||
|
||||
// Redact common token/key shapes before errors leave memory host internals.
|
||||
function maskToken(token: string): string {
|
||||
if (token.length < 18) {
|
||||
return "***";
|
||||
}
|
||||
return `${token.slice(0, 6)}...${token.slice(-4)}`;
|
||||
}
|
||||
|
||||
function redactPemBlock(block: string): string {
|
||||
const lines = block.split(/\r?\n/).filter(Boolean);
|
||||
if (lines.length < 2) {
|
||||
return "***";
|
||||
}
|
||||
return `${lines[0]}\n...redacted...\n${lines[lines.length - 1]}`;
|
||||
}
|
||||
|
||||
function redactMatch(match: string, groups: string[]): string {
|
||||
if (match.includes("PRIVATE KEY-----")) {
|
||||
return redactPemBlock(match);
|
||||
}
|
||||
const token = groups.findLast((value) => typeof value === "string" && value.length > 0) ?? match;
|
||||
const masked = maskToken(token);
|
||||
return token === match ? masked : match.replace(token, masked);
|
||||
}
|
||||
|
||||
function redactSensitiveText(text: string): string {
|
||||
let next = text;
|
||||
for (const pattern of SECRET_PATTERNS) {
|
||||
next = next.replace(pattern, (...args: string[]) =>
|
||||
redactMatch(args[0] ?? "", args.slice(1, -2)),
|
||||
);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Format unknown errors with causes while redacting likely secrets. */
|
||||
export function formatErrorMessage(err: unknown): string {
|
||||
let formatted: string;
|
||||
if (err instanceof Error) {
|
||||
formatted = err.message || err.name || "Error";
|
||||
let cause: unknown = err.cause;
|
||||
const seen = new Set<unknown>([err]);
|
||||
while (cause && !seen.has(cause)) {
|
||||
seen.add(cause);
|
||||
if (cause instanceof Error) {
|
||||
if (cause.message) {
|
||||
formatted += ` | ${cause.message}`;
|
||||
}
|
||||
cause = cause.cause;
|
||||
} else if (typeof cause === "string") {
|
||||
formatted += ` | ${cause}`;
|
||||
break;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (typeof err === "string") {
|
||||
formatted = err;
|
||||
} else if (typeof err === "number" || typeof err === "boolean" || typeof err === "bigint") {
|
||||
formatted = String(err);
|
||||
} else {
|
||||
try {
|
||||
formatted = JSON.stringify(err);
|
||||
} catch {
|
||||
formatted = Object.prototype.toString.call(err);
|
||||
}
|
||||
}
|
||||
return redactSensitiveText(formatted);
|
||||
}
|
||||
34
packages/memory-host-sdk/src/host/fs-utils.ts
Normal file
34
packages/memory-host-sdk/src/host/fs-utils.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
// Memory Host SDK helper module supports fs utils behavior.
|
||||
import { configureFsSafePython } from "@openclaw/fs-safe/config";
|
||||
// fs-safe facade with Python validation disabled by default for this package's
|
||||
// host-side memory file operations.
|
||||
export { root } from "@openclaw/fs-safe/root";
|
||||
export { isPathInside, isPathInsideWithRealpath } from "@openclaw/fs-safe/path";
|
||||
export {
|
||||
assertNoSymlinkParents,
|
||||
readRegularFile,
|
||||
statRegularFile,
|
||||
type RegularFileStatResult,
|
||||
} from "@openclaw/fs-safe/advanced";
|
||||
export { walkDirectory, type WalkDirectoryEntry } from "@openclaw/fs-safe/walk";
|
||||
|
||||
const hasPythonModeOverride =
|
||||
process.env.FS_SAFE_PYTHON_MODE != null || process.env.OPENCLAW_FS_SAFE_PYTHON_MODE != null;
|
||||
|
||||
if (!hasPythonModeOverride) {
|
||||
configureFsSafePython({ mode: "off" });
|
||||
}
|
||||
|
||||
/** True for missing-file errors emitted by Node or fs-safe. */
|
||||
export function isFileMissingError(
|
||||
err: unknown,
|
||||
): err is NodeJS.ErrnoException & { code: "ENOENT" | "ENOTDIR" | "not-found" } {
|
||||
return Boolean(
|
||||
err &&
|
||||
typeof err === "object" &&
|
||||
"code" in err &&
|
||||
((err as Partial<NodeJS.ErrnoException>).code === "ENOENT" ||
|
||||
(err as Partial<NodeJS.ErrnoException>).code === "ENOTDIR" ||
|
||||
(err as { code?: unknown }).code === "not-found"),
|
||||
);
|
||||
}
|
||||
7
packages/memory-host-sdk/src/host/hash.ts
Normal file
7
packages/memory-host-sdk/src/host/hash.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
// Memory Host SDK module implements hash behavior.
|
||||
import crypto from "node:crypto";
|
||||
|
||||
/** SHA-256 hash helper for stable cache/content keys. */
|
||||
export function hashText(value: string): string {
|
||||
return crypto.createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
250
packages/memory-host-sdk/src/host/internal.test.ts
Normal file
250
packages/memory-host-sdk/src/host/internal.test.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
// Memory Host SDK tests cover internal behavior.
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildFileEntry,
|
||||
buildMultimodalChunkForIndexing,
|
||||
chunkMarkdown,
|
||||
ensureDir,
|
||||
isMemoryPath,
|
||||
listMemoryFiles,
|
||||
normalizeExtraMemoryPaths,
|
||||
remapChunkLines,
|
||||
} from "./internal.js";
|
||||
import {
|
||||
DEFAULT_MEMORY_MULTIMODAL_MAX_FILE_BYTES,
|
||||
type MemoryMultimodalSettings,
|
||||
} from "./multimodal.js";
|
||||
|
||||
type FileEntry = NonNullable<Awaited<ReturnType<typeof buildFileEntry>>>;
|
||||
type MultimodalIndexingChunk = NonNullable<
|
||||
Awaited<ReturnType<typeof buildMultimodalChunkForIndexing>>
|
||||
>;
|
||||
|
||||
let sharedTempRoot = "";
|
||||
let sharedTempId = 0;
|
||||
|
||||
beforeAll(() => {
|
||||
sharedTempRoot = fsSync.mkdtempSync(path.join(os.tmpdir(), "memory-host-sdk-package-tests-"));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (sharedTempRoot) {
|
||||
fsSync.rmSync(sharedTempRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function setupTempDirLifecycle(prefix: string): () => string {
|
||||
let tmpDir = "";
|
||||
beforeEach(() => {
|
||||
tmpDir = path.join(sharedTempRoot, `${prefix}${sharedTempId++}`);
|
||||
fsSync.mkdirSync(tmpDir, { recursive: true });
|
||||
});
|
||||
return () => tmpDir;
|
||||
}
|
||||
|
||||
function expectFileEntry(entry: Awaited<ReturnType<typeof buildFileEntry>>): FileEntry {
|
||||
if (!entry) {
|
||||
throw new Error("Expected file entry to be built");
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
function expectMultimodalIndexingChunk(
|
||||
built: Awaited<ReturnType<typeof buildMultimodalChunkForIndexing>>,
|
||||
): MultimodalIndexingChunk {
|
||||
if (!built) {
|
||||
throw new Error("Expected multimodal indexing chunk to be built");
|
||||
}
|
||||
return built;
|
||||
}
|
||||
|
||||
function expectEmbeddingInput(
|
||||
chunk: MultimodalIndexingChunk["chunk"],
|
||||
): NonNullable<MultimodalIndexingChunk["chunk"]["embeddingInput"]> {
|
||||
if (!chunk.embeddingInput) {
|
||||
throw new Error("Expected multimodal chunk embedding input");
|
||||
}
|
||||
return chunk.embeddingInput;
|
||||
}
|
||||
|
||||
const multimodal: MemoryMultimodalSettings = {
|
||||
enabled: true,
|
||||
modalities: ["image", "audio"],
|
||||
maxFileBytes: DEFAULT_MEMORY_MULTIMODAL_MAX_FILE_BYTES,
|
||||
};
|
||||
|
||||
describe("memory host SDK package internals", () => {
|
||||
const getTmpDir = setupTempDirLifecycle("memory-package-");
|
||||
|
||||
it("propagates directory creation failures", () => {
|
||||
const mkdirError = new Error("disk full");
|
||||
const targetDir = path.join(getTmpDir(), "blocked");
|
||||
const mkdirSync = vi.spyOn(fsSync, "mkdirSync").mockImplementation(() => {
|
||||
throw mkdirError;
|
||||
});
|
||||
|
||||
expect(() => ensureDir(targetDir)).toThrow(mkdirError);
|
||||
expect(mkdirSync).toHaveBeenCalledWith(targetDir, { recursive: true });
|
||||
});
|
||||
|
||||
it("normalizes additional memory paths", () => {
|
||||
const workspaceDir = path.join(os.tmpdir(), "memory-test-workspace");
|
||||
const absPath = path.resolve(path.sep, "shared-notes");
|
||||
expect(
|
||||
normalizeExtraMemoryPaths(workspaceDir, [
|
||||
" notes ",
|
||||
"./notes",
|
||||
absPath,
|
||||
absPath,
|
||||
"~/shared-notes",
|
||||
"~",
|
||||
"",
|
||||
]),
|
||||
).toEqual([
|
||||
path.resolve(workspaceDir, "notes"),
|
||||
absPath,
|
||||
path.join(os.homedir(), "shared-notes"),
|
||||
os.homedir(),
|
||||
]);
|
||||
});
|
||||
|
||||
it("lists canonical markdown and enabled multimodal files", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
fsSync.writeFileSync(path.join(tmpDir, "memory.md"), "# Legacy memory");
|
||||
const extraDir = path.join(tmpDir, "extra");
|
||||
fsSync.mkdirSync(extraDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(extraDir, "note.md"), "# Note");
|
||||
fsSync.writeFileSync(path.join(extraDir, "diagram.png"), Buffer.from("png"));
|
||||
fsSync.writeFileSync(path.join(extraDir, "ignore.txt"), "ignored");
|
||||
|
||||
const files = await listMemoryFiles(
|
||||
tmpDir,
|
||||
[path.join(tmpDir, "memory.md"), extraDir],
|
||||
multimodal,
|
||||
);
|
||||
|
||||
expect(files.map((file) => path.relative(tmpDir, file)).toSorted()).toEqual([
|
||||
"MEMORY.md",
|
||||
path.join("extra", "diagram.png"),
|
||||
path.join("extra", "note.md"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("allows top-level dreams path casing variants", () => {
|
||||
expect(isMemoryPath("dreams.md")).toBe(true);
|
||||
expect(isMemoryPath("DREAMS.md")).toBe(true);
|
||||
});
|
||||
|
||||
it("builds markdown and multimodal file entries", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const notePath = path.join(tmpDir, "note.md");
|
||||
const imagePath = path.join(tmpDir, "diagram.png");
|
||||
fsSync.writeFileSync(notePath, "hello", "utf-8");
|
||||
fsSync.writeFileSync(imagePath, Buffer.from("png"));
|
||||
|
||||
const note = await buildFileEntry(notePath, tmpDir);
|
||||
const image = await buildFileEntry(imagePath, tmpDir, multimodal);
|
||||
|
||||
const noteEntry = expectFileEntry(note);
|
||||
expect(noteEntry.path).toBe("note.md");
|
||||
expect(noteEntry.kind).toBe("markdown");
|
||||
const imageEntry = expectFileEntry(image);
|
||||
expect(imageEntry.path).toBe("diagram.png");
|
||||
expect(imageEntry.kind).toBe("multimodal");
|
||||
expect(imageEntry.modality).toBe("image");
|
||||
expect(imageEntry.mimeType).toBe("image/png");
|
||||
expect(imageEntry.contentText).toBe("Image file: diagram.png");
|
||||
});
|
||||
|
||||
it("retries transient markdown reads while building file entries", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const notePath = path.join(tmpDir, "note.md");
|
||||
fsSync.writeFileSync(notePath, "hello", "utf-8");
|
||||
|
||||
const realOpen = fs.open;
|
||||
let attempts = 0;
|
||||
const openSpy = vi
|
||||
.spyOn(fs, "open")
|
||||
.mockImplementation(async (...args: Parameters<typeof realOpen>) => {
|
||||
const [target, flags, mode] = args;
|
||||
if (typeof target === "string" && path.resolve(target) === notePath && attempts++ === 0) {
|
||||
const err = new Error(
|
||||
"Unknown system error -11: Unknown system error -11, open",
|
||||
) as NodeJS.ErrnoException;
|
||||
err.code = "UNKNOWN";
|
||||
err.errno = -11;
|
||||
throw err;
|
||||
}
|
||||
return await realOpen(target, flags, mode);
|
||||
});
|
||||
|
||||
try {
|
||||
const entry = expectFileEntry(await buildFileEntry(notePath, tmpDir));
|
||||
expect(entry.path).toBe("note.md");
|
||||
expect(entry.kind).toBe("markdown");
|
||||
expect(attempts).toBe(2);
|
||||
} finally {
|
||||
openSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("builds multimodal chunks lazily and rejects changed files", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const imagePath = path.join(tmpDir, "diagram.png");
|
||||
fsSync.writeFileSync(imagePath, Buffer.from("png"));
|
||||
|
||||
const entry = expectFileEntry(await buildFileEntry(imagePath, tmpDir, multimodal));
|
||||
const built = expectMultimodalIndexingChunk(await buildMultimodalChunkForIndexing(entry));
|
||||
const parts = expectEmbeddingInput(built.chunk).parts ?? [];
|
||||
expect(parts[0]).toEqual({ type: "text", text: "Image file: diagram.png" });
|
||||
const inlinePart = parts[1];
|
||||
if (inlinePart?.type !== "inline-data") {
|
||||
throw new Error("Expected multimodal inline-data embedding part");
|
||||
}
|
||||
expect(inlinePart.mimeType).toBe("image/png");
|
||||
|
||||
fsSync.writeFileSync(imagePath, Buffer.alloc(entry.size + 32, 1));
|
||||
await expect(buildMultimodalChunkForIndexing(entry)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it("chunks mixed text and preserves surrogate pairs", () => {
|
||||
const mixed = Array.from(
|
||||
{ length: 30 },
|
||||
(_, index) => `Line ${index}: 这是中英文混合的测试内容 with English`,
|
||||
).join("\n");
|
||||
const mixedChunks = chunkMarkdown(mixed, { tokens: 50, overlap: 0 });
|
||||
expect(mixedChunks.length).toBeGreaterThan(1);
|
||||
expect(mixedChunks.map((chunk) => chunk.text).join("\n")).toContain("Line 29");
|
||||
|
||||
const surrogateChar = "\u{20000}";
|
||||
const surrogateChunks = chunkMarkdown(surrogateChar.repeat(120), {
|
||||
tokens: 31,
|
||||
overlap: 0,
|
||||
});
|
||||
for (const chunk of surrogateChunks) {
|
||||
expect(chunk.text).not.toContain("\uFFFD");
|
||||
}
|
||||
});
|
||||
|
||||
it("remaps chunk lines using JSONL source line maps", () => {
|
||||
const lineMap = [4, 6, 7, 10, 13];
|
||||
const chunks = chunkMarkdown(
|
||||
"User: Hello\nAssistant: Hi\nUser: Question\nAssistant: Answer\nUser: Thanks",
|
||||
{ tokens: 400, overlap: 0 },
|
||||
);
|
||||
|
||||
remapChunkLines(chunks, lineMap);
|
||||
|
||||
expect(chunks[0].startLine).toBe(4);
|
||||
expect(chunks[chunks.length - 1].endLine).toBe(13);
|
||||
});
|
||||
});
|
||||
556
packages/memory-host-sdk/src/host/internal.ts
Normal file
556
packages/memory-host-sdk/src/host/internal.ts
Normal file
@@ -0,0 +1,556 @@
|
||||
// Memory Host SDK module implements internal behavior.
|
||||
import crypto from "node:crypto";
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import { homedir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { CANONICAL_ROOT_MEMORY_FILENAME } from "./config-utils.js";
|
||||
import { estimateStructuredEmbeddingInputBytes } from "./embedding-input-limits.js";
|
||||
import { buildTextEmbeddingInput, type EmbeddingInput } from "./embedding-inputs.js";
|
||||
import {
|
||||
isFileMissingError,
|
||||
readRegularFile,
|
||||
statRegularFile,
|
||||
walkDirectory,
|
||||
type WalkDirectoryEntry,
|
||||
} from "./fs-utils.js";
|
||||
import {
|
||||
buildMemoryMultimodalLabel,
|
||||
classifyMemoryMultimodalPath,
|
||||
type MemoryMultimodalModality,
|
||||
type MemoryMultimodalSettings,
|
||||
} from "./multimodal.js";
|
||||
import {
|
||||
CHARS_PER_TOKEN_ESTIMATE,
|
||||
detectMime,
|
||||
estimateStringChars,
|
||||
runTasksWithConcurrency,
|
||||
} from "./openclaw-runtime-io.js";
|
||||
import {
|
||||
resolveCanonicalRootMemoryFile,
|
||||
shouldSkipRootMemoryAuxiliaryPath,
|
||||
} from "./openclaw-runtime-memory.js";
|
||||
import { retryTransientMemoryRead } from "./read-retry.js";
|
||||
import { normalizeStringEntries, uniqueStrings } from "./string-utils.js";
|
||||
|
||||
export { hashText } from "./hash.js";
|
||||
import { hashText } from "./hash.js";
|
||||
|
||||
export type MemoryFileEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
hash: string;
|
||||
dataHash?: string;
|
||||
kind?: "markdown" | "multimodal";
|
||||
contentText?: string;
|
||||
modality?: MemoryMultimodalModality;
|
||||
mimeType?: string;
|
||||
};
|
||||
|
||||
export type MemoryChunk = {
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
text: string;
|
||||
hash: string;
|
||||
embeddingInput?: EmbeddingInput;
|
||||
};
|
||||
|
||||
export type MultimodalMemoryChunk = {
|
||||
chunk: MemoryChunk;
|
||||
structuredInputBytes: number;
|
||||
};
|
||||
|
||||
const DISABLED_MULTIMODAL_SETTINGS: MemoryMultimodalSettings = {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 0,
|
||||
};
|
||||
|
||||
export function ensureDir(dir: string): string {
|
||||
fsSync.mkdirSync(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
export function normalizeRelPath(value: string): string {
|
||||
const trimmed = value.trim().replace(/^[./]+/, "");
|
||||
return trimmed.replace(/\\/g, "/");
|
||||
}
|
||||
|
||||
function expandHomePath(value: string): string {
|
||||
if (value === "~") {
|
||||
return homedir();
|
||||
}
|
||||
if (value.startsWith("~/") || value.startsWith("~\\")) {
|
||||
return path.join(homedir(), value.slice(2));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeExtraMemoryPaths(workspaceDir: string, extraPaths?: string[]): string[] {
|
||||
if (!extraPaths?.length) {
|
||||
return [];
|
||||
}
|
||||
const resolved = normalizeStringEntries(extraPaths)
|
||||
.map((value) => expandHomePath(value))
|
||||
.map((value) =>
|
||||
path.isAbsolute(value) ? path.resolve(value) : path.resolve(workspaceDir, value),
|
||||
);
|
||||
return uniqueStrings(resolved);
|
||||
}
|
||||
|
||||
export function isMemoryPath(relPath: string): boolean {
|
||||
const normalized = normalizeRelPath(relPath);
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
if (normalized === CANONICAL_ROOT_MEMORY_FILENAME || normalized.toLowerCase() === "dreams.md") {
|
||||
return true;
|
||||
}
|
||||
return normalized.startsWith("memory/");
|
||||
}
|
||||
|
||||
function isAllowedMemoryFilePath(filePath: string, multimodal?: MemoryMultimodalSettings): boolean {
|
||||
if (filePath.endsWith(".md")) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
classifyMemoryMultimodalPath(filePath, multimodal ?? DISABLED_MULTIMODAL_SETTINGS) !== null
|
||||
);
|
||||
}
|
||||
|
||||
function shouldDescendMemoryEntry(
|
||||
entry: WalkDirectoryEntry,
|
||||
shouldSkipPath?: (absPath: string) => boolean,
|
||||
): boolean {
|
||||
if (shouldSkipPath?.(entry.path)) {
|
||||
return false;
|
||||
}
|
||||
return entry.kind === "directory" && entry.name !== ".openclaw-repair";
|
||||
}
|
||||
|
||||
async function collectMemoryFilesFromDir(
|
||||
dir: string,
|
||||
files: string[],
|
||||
multimodal?: MemoryMultimodalSettings,
|
||||
shouldSkipPath?: (absPath: string) => boolean,
|
||||
): Promise<void> {
|
||||
const scan = await walkDirectory(dir, {
|
||||
symlinks: "skip",
|
||||
descend: (entry) => shouldDescendMemoryEntry(entry, shouldSkipPath),
|
||||
include: (entry) =>
|
||||
!shouldSkipPath?.(entry.path) &&
|
||||
entry.kind === "file" &&
|
||||
isAllowedMemoryFilePath(entry.path, multimodal),
|
||||
});
|
||||
files.push(...scan.entries.map((entry) => entry.path));
|
||||
}
|
||||
|
||||
export async function listMemoryFiles(
|
||||
workspaceDir: string,
|
||||
extraPaths?: string[],
|
||||
multimodal?: MemoryMultimodalSettings,
|
||||
): Promise<string[]> {
|
||||
const result: string[] = [];
|
||||
const memoryDir = path.join(workspaceDir, "memory");
|
||||
|
||||
const shouldSkipWorkspaceMemoryPath = (absPath: string): boolean =>
|
||||
shouldSkipRootMemoryAuxiliaryPath({ workspaceDir, absPath });
|
||||
|
||||
const addMarkdownFile = async (absPath: string) => {
|
||||
try {
|
||||
const stat = await statRegularFile(absPath);
|
||||
if (stat.missing) {
|
||||
return;
|
||||
}
|
||||
if (!absPath.endsWith(".md")) {
|
||||
return;
|
||||
}
|
||||
result.push(absPath);
|
||||
} catch {}
|
||||
};
|
||||
|
||||
const memoryFile = await resolveCanonicalRootMemoryFile(workspaceDir);
|
||||
if (memoryFile) {
|
||||
await addMarkdownFile(memoryFile);
|
||||
}
|
||||
try {
|
||||
const dirStat = await fs.lstat(memoryDir);
|
||||
if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
|
||||
await collectMemoryFilesFromDir(memoryDir, result, multimodal, shouldSkipWorkspaceMemoryPath);
|
||||
}
|
||||
} catch {}
|
||||
|
||||
const normalizedExtraPaths = normalizeExtraMemoryPaths(workspaceDir, extraPaths);
|
||||
if (normalizedExtraPaths.length > 0) {
|
||||
for (const inputPath of normalizedExtraPaths) {
|
||||
if (shouldSkipWorkspaceMemoryPath(inputPath)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const stat = await fs.lstat(inputPath);
|
||||
if (stat.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
await collectMemoryFilesFromDir(
|
||||
inputPath,
|
||||
result,
|
||||
multimodal,
|
||||
shouldSkipWorkspaceMemoryPath,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile() && isAllowedMemoryFilePath(inputPath, multimodal)) {
|
||||
result.push(inputPath);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (result.length <= 1) {
|
||||
return result;
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const deduped: string[] = [];
|
||||
for (const entry of result) {
|
||||
let key = entry;
|
||||
try {
|
||||
key = await fs.realpath(entry);
|
||||
} catch {}
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
deduped.push(entry);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
export async function buildFileEntry(
|
||||
absPath: string,
|
||||
workspaceDir: string,
|
||||
multimodal?: MemoryMultimodalSettings,
|
||||
): Promise<MemoryFileEntry | null> {
|
||||
const regularFile = await statRegularFile(absPath);
|
||||
if (regularFile.missing) {
|
||||
return null;
|
||||
}
|
||||
const stat = regularFile.stat;
|
||||
const normalizedPath = path.relative(workspaceDir, absPath).replace(/\\/g, "/");
|
||||
const multimodalSettings = multimodal ?? DISABLED_MULTIMODAL_SETTINGS;
|
||||
const modality = classifyMemoryMultimodalPath(absPath, multimodalSettings);
|
||||
if (modality) {
|
||||
if (stat.size > multimodalSettings.maxFileBytes) {
|
||||
return null;
|
||||
}
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = (
|
||||
await retryTransientMemoryRead(
|
||||
() =>
|
||||
readRegularFile({
|
||||
filePath: absPath,
|
||||
maxBytes: multimodalSettings.maxFileBytes,
|
||||
}),
|
||||
`read multimodal memory file ${absPath}`,
|
||||
)
|
||||
).buffer;
|
||||
} catch (err) {
|
||||
if (isFileMissingError(err)) {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const mimeType = await detectMime({ buffer: buffer.subarray(0, 512), filePath: absPath });
|
||||
if (!mimeType || !mimeType.startsWith(`${modality}/`)) {
|
||||
return null;
|
||||
}
|
||||
const contentText = buildMemoryMultimodalLabel(modality, normalizedPath);
|
||||
const dataHash = crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
const chunkHash = hashText(
|
||||
JSON.stringify({
|
||||
path: normalizedPath,
|
||||
contentText,
|
||||
mimeType,
|
||||
dataHash,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
path: normalizedPath,
|
||||
absPath,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
hash: chunkHash,
|
||||
dataHash,
|
||||
kind: "multimodal",
|
||||
contentText,
|
||||
modality,
|
||||
mimeType,
|
||||
};
|
||||
}
|
||||
let content: string;
|
||||
try {
|
||||
content = (
|
||||
await retryTransientMemoryRead(
|
||||
() => readRegularFile({ filePath: absPath }),
|
||||
`read memory index file ${absPath}`,
|
||||
)
|
||||
).buffer.toString("utf-8");
|
||||
} catch (err) {
|
||||
if (isFileMissingError(err)) {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const hash = hashText(content);
|
||||
return {
|
||||
path: normalizedPath,
|
||||
absPath,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
hash,
|
||||
kind: "markdown",
|
||||
};
|
||||
}
|
||||
|
||||
async function loadMultimodalEmbeddingInput(
|
||||
entry: Pick<
|
||||
MemoryFileEntry,
|
||||
"absPath" | "contentText" | "mimeType" | "kind" | "size" | "dataHash"
|
||||
>,
|
||||
): Promise<EmbeddingInput | null> {
|
||||
if (entry.kind !== "multimodal" || !entry.contentText || !entry.mimeType) {
|
||||
return null;
|
||||
}
|
||||
const regularFile = await statRegularFile(entry.absPath);
|
||||
if (regularFile.missing) {
|
||||
return null;
|
||||
}
|
||||
const stat = regularFile.stat;
|
||||
if (stat.size !== entry.size) {
|
||||
return null;
|
||||
}
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = (
|
||||
await retryTransientMemoryRead(
|
||||
() => readRegularFile({ filePath: entry.absPath, maxBytes: entry.size }),
|
||||
`read multimodal indexing file ${entry.absPath}`,
|
||||
)
|
||||
).buffer;
|
||||
} catch (err) {
|
||||
if (isFileMissingError(err)) {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const dataHash = crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
if (entry.dataHash && entry.dataHash !== dataHash) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
text: entry.contentText,
|
||||
parts: [
|
||||
{ type: "text", text: entry.contentText },
|
||||
{
|
||||
type: "inline-data",
|
||||
mimeType: entry.mimeType,
|
||||
data: buffer.toString("base64"),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildMultimodalChunkForIndexing(
|
||||
entry: Pick<
|
||||
MemoryFileEntry,
|
||||
"absPath" | "contentText" | "mimeType" | "kind" | "hash" | "size" | "dataHash"
|
||||
>,
|
||||
): Promise<MultimodalMemoryChunk | null> {
|
||||
const embeddingInput = await loadMultimodalEmbeddingInput(entry);
|
||||
if (!embeddingInput) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
chunk: {
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
text: entry.contentText ?? embeddingInput.text,
|
||||
hash: entry.hash,
|
||||
embeddingInput,
|
||||
},
|
||||
structuredInputBytes: estimateStructuredEmbeddingInputBytes(embeddingInput),
|
||||
};
|
||||
}
|
||||
|
||||
export function chunkMarkdown(
|
||||
content: string,
|
||||
chunking: { tokens: number; overlap: number },
|
||||
): MemoryChunk[] {
|
||||
const lines = content.split("\n");
|
||||
if (lines.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const maxChars = Math.max(32, chunking.tokens * CHARS_PER_TOKEN_ESTIMATE);
|
||||
const overlapChars = Math.max(0, chunking.overlap * CHARS_PER_TOKEN_ESTIMATE);
|
||||
const chunks: MemoryChunk[] = [];
|
||||
|
||||
let current: Array<{ line: string; lineNo: number }> = [];
|
||||
let currentChars = 0;
|
||||
|
||||
const flush = () => {
|
||||
if (current.length === 0) {
|
||||
return;
|
||||
}
|
||||
const firstEntry = current[0];
|
||||
const lastEntry = current[current.length - 1];
|
||||
if (!firstEntry || !lastEntry) {
|
||||
return;
|
||||
}
|
||||
const text = current.map((entry) => entry.line).join("\n");
|
||||
const startLine = firstEntry.lineNo;
|
||||
const endLine = lastEntry.lineNo;
|
||||
chunks.push({
|
||||
startLine,
|
||||
endLine,
|
||||
text,
|
||||
hash: hashText(text),
|
||||
embeddingInput: buildTextEmbeddingInput(text),
|
||||
});
|
||||
};
|
||||
|
||||
const carryOverlap = () => {
|
||||
if (overlapChars <= 0 || current.length === 0) {
|
||||
current = [];
|
||||
currentChars = 0;
|
||||
return;
|
||||
}
|
||||
let acc = 0;
|
||||
const kept: Array<{ line: string; lineNo: number }> = [];
|
||||
for (let i = current.length - 1; i >= 0; i -= 1) {
|
||||
const entry = current[i];
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
acc += estimateStringChars(entry.line) + 1;
|
||||
kept.unshift(entry);
|
||||
if (acc >= overlapChars) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
current = kept;
|
||||
currentChars = acc;
|
||||
};
|
||||
|
||||
for (let i = 0; i < lines.length; i += 1) {
|
||||
const line = lines[i] ?? "";
|
||||
const lineNo = i + 1;
|
||||
const segments: string[] = [];
|
||||
if (line.length === 0) {
|
||||
segments.push("");
|
||||
} else {
|
||||
// First pass: slice at maxChars (preserves original behaviour for Latin).
|
||||
// Second pass: if a segment's *weighted* size still exceeds the budget
|
||||
// (happens for CJK-heavy text where 1 char ≈ 1 token), re-split it at
|
||||
// chunking.tokens so the chunk stays within the token budget.
|
||||
for (let start = 0; start < line.length; start += maxChars) {
|
||||
const coarse = line.slice(start, start + maxChars);
|
||||
if (estimateStringChars(coarse) > maxChars) {
|
||||
const fineStep = Math.max(1, chunking.tokens);
|
||||
for (let j = 0; j < coarse.length; ) {
|
||||
let end = Math.min(j + fineStep, coarse.length);
|
||||
// Avoid splitting inside a UTF-16 surrogate pair (CJK Extension B+).
|
||||
if (end < coarse.length) {
|
||||
const code = coarse.charCodeAt(end - 1);
|
||||
if (code >= 0xd800 && code <= 0xdbff) {
|
||||
end += 1; // include the low surrogate
|
||||
}
|
||||
}
|
||||
segments.push(coarse.slice(j, end));
|
||||
j = end; // advance cursor to the adjusted boundary
|
||||
}
|
||||
} else {
|
||||
segments.push(coarse);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const segment of segments) {
|
||||
const lineSize = estimateStringChars(segment) + 1;
|
||||
if (currentChars + lineSize > maxChars && current.length > 0) {
|
||||
flush();
|
||||
carryOverlap();
|
||||
}
|
||||
current.push({ line: segment, lineNo });
|
||||
currentChars += lineSize;
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remap chunk startLine/endLine from content-relative positions to original
|
||||
* source file positions using a lineMap. Each entry in lineMap gives the
|
||||
* 1-indexed source line for the corresponding 0-indexed content line.
|
||||
*
|
||||
* This is used for session JSONL files where buildSessionEntry() flattens
|
||||
* messages into a plain-text string before chunking. Without remapping the
|
||||
* stored line numbers would reference positions in the flattened text rather
|
||||
* than the original JSONL file.
|
||||
*/
|
||||
export function remapChunkLines(chunks: MemoryChunk[], lineMap: number[] | undefined): void {
|
||||
if (!lineMap || lineMap.length === 0) {
|
||||
return;
|
||||
}
|
||||
for (const chunk of chunks) {
|
||||
// startLine/endLine are 1-indexed; lineMap is 0-indexed by content line
|
||||
chunk.startLine = lineMap[chunk.startLine - 1] ?? chunk.startLine;
|
||||
chunk.endLine = lineMap[chunk.endLine - 1] ?? chunk.endLine;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseEmbedding(raw: string): number[] {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as number[];
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (a.length === 0 || b.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const len = Math.min(a.length, b.length);
|
||||
let dot = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < len; i += 1) {
|
||||
const av = a[i] ?? 0;
|
||||
const bv = b[i] ?? 0;
|
||||
dot += av * bv;
|
||||
normA += av * av;
|
||||
normB += bv * bv;
|
||||
}
|
||||
if (normA === 0 || normB === 0) {
|
||||
return 0;
|
||||
}
|
||||
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
||||
}
|
||||
|
||||
export async function runWithConcurrency<T>(
|
||||
tasks: Array<() => Promise<T>>,
|
||||
limit: number,
|
||||
): Promise<T[]> {
|
||||
const { results, firstError, hasError } = await runTasksWithConcurrency({
|
||||
tasks,
|
||||
limit,
|
||||
errorMode: "stop",
|
||||
});
|
||||
if (hasError) {
|
||||
throw firstError;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
340
packages/memory-host-sdk/src/host/memory-schema.test.ts
Normal file
340
packages/memory-host-sdk/src/host/memory-schema.test.ts
Normal file
@@ -0,0 +1,340 @@
|
||||
// Memory schema tests cover canonical table creation and shipped-name migration.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ensureMemoryIndexSchema } from "./memory-schema.js";
|
||||
|
||||
describe("memory index schema", () => {
|
||||
it("migrates shipped generic tables into canonical memory tables", () => {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
CREATE TABLE files (
|
||||
path TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
hash TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE chunks (
|
||||
id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE embedding_cache (
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
provider_key TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
dims INTEGER,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (provider, model, provider_key, hash)
|
||||
);
|
||||
CREATE VIRTUAL TABLE chunks_fts USING fts5(
|
||||
text, id UNINDEXED, path UNINDEXED, source UNINDEXED, model UNINDEXED,
|
||||
start_line UNINDEXED, end_line UNINDEXED
|
||||
);
|
||||
INSERT INTO meta VALUES ('memory_index_meta_v1', '{"vectorDims":3}');
|
||||
INSERT INTO files VALUES ('MEMORY.md', 'memory', 'file-hash', 10, 20);
|
||||
INSERT INTO chunks VALUES (
|
||||
'chunk-1', 'MEMORY.md', 'memory', 1, 2, 'chunk-hash', 'embed-model',
|
||||
'remember this', '[1,0,0]', 30
|
||||
);
|
||||
INSERT INTO embedding_cache VALUES (
|
||||
'openai', 'embed-model', 'key', 'chunk-hash', '[1,0,0]', 3, 40
|
||||
);
|
||||
INSERT INTO chunks_fts VALUES (
|
||||
'remember this', 'chunk-1', 'MEMORY.md', 'memory', 'embed-model', 1, 2
|
||||
);
|
||||
`);
|
||||
|
||||
const result = ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled: true,
|
||||
ftsEnabled: true,
|
||||
});
|
||||
|
||||
expect(result.ftsAvailable).toBe(true);
|
||||
expect(db.prepare("SELECT * FROM memory_index_sources").all()).toEqual([
|
||||
{ path: "MEMORY.md", source: "memory", hash: "file-hash", mtime: 10, size: 20 },
|
||||
]);
|
||||
expect(db.prepare("SELECT id, text FROM memory_index_chunks").all()).toEqual([
|
||||
{ id: "chunk-1", text: "remember this" },
|
||||
]);
|
||||
expect(db.prepare("SELECT id, text FROM memory_index_chunks_fts").all()).toEqual([
|
||||
{ id: "chunk-1", text: "remember this" },
|
||||
]);
|
||||
expect(db.prepare("SELECT provider, hash FROM memory_embedding_cache").all()).toEqual([
|
||||
{ provider: "openai", hash: "chunk-hash" },
|
||||
]);
|
||||
expect(
|
||||
db
|
||||
.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks', 'embedding_cache', 'chunks_fts')",
|
||||
)
|
||||
.all(),
|
||||
).toEqual([]);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not import a legacy sidecar memory database during schema startup", () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-sidecar-"));
|
||||
const legacyPath = path.join(rootDir, "memory", "main.sqlite");
|
||||
const agentPath = path.join(rootDir, "agents", "main", "agent", "openclaw-agent.sqlite");
|
||||
fs.mkdirSync(path.dirname(legacyPath), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(agentPath), { recursive: true });
|
||||
const legacyDb = new DatabaseSync(legacyPath);
|
||||
try {
|
||||
legacyDb.exec(`
|
||||
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
CREATE TABLE files (
|
||||
path TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
hash TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE chunks (
|
||||
id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE embedding_cache (
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
provider_key TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
dims INTEGER,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (provider, model, provider_key, hash)
|
||||
);
|
||||
INSERT INTO meta VALUES ('memory_index_meta_v1', '{"vectorDims":3}');
|
||||
INSERT INTO files VALUES ('MEMORY.md', 'memory', 'file-hash', 10, 20);
|
||||
INSERT INTO chunks VALUES (
|
||||
'chunk-1', 'MEMORY.md', 'memory', 1, 2, 'chunk-hash', 'embed-model',
|
||||
'remember this', '[1,0,0]', 30
|
||||
);
|
||||
INSERT INTO embedding_cache VALUES (
|
||||
'openai', 'embed-model', 'key', 'chunk-hash', '[1,0,0]', 3, 40
|
||||
);
|
||||
`);
|
||||
} finally {
|
||||
legacyDb.close();
|
||||
}
|
||||
|
||||
const db = new DatabaseSync(agentPath);
|
||||
try {
|
||||
const result = ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled: true,
|
||||
ftsEnabled: true,
|
||||
});
|
||||
|
||||
expect(result.ftsAvailable).toBe(true);
|
||||
expect(db.prepare("SELECT * FROM memory_index_sources").all()).toEqual([]);
|
||||
expect(db.prepare("SELECT id, text FROM memory_index_chunks").all()).toEqual([]);
|
||||
expect(db.prepare("SELECT id, text FROM memory_index_chunks_fts").all()).toEqual([]);
|
||||
expect(db.prepare("SELECT provider, hash FROM memory_embedding_cache").all()).toEqual([]);
|
||||
expect(fs.existsSync(legacyPath)).toBe(true);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(rootDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("stores source records with the same path in separate sources", () => {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
try {
|
||||
ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled: false,
|
||||
ftsEnabled: false,
|
||||
});
|
||||
|
||||
db.prepare(
|
||||
"INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)",
|
||||
).run("shared.md", "memory", "memory-hash", 10, 20);
|
||||
db.prepare(
|
||||
"INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)",
|
||||
).run("shared.md", "sessions", "session-hash", 30, 40);
|
||||
|
||||
expect(
|
||||
db.prepare("SELECT path, source, hash FROM memory_index_sources ORDER BY source").all(),
|
||||
).toEqual([
|
||||
{ path: "shared.md", source: "memory", hash: "memory-hash" },
|
||||
{ path: "shared.md", source: "sessions", hash: "session-hash" },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("honors shipped custom cache and FTS table names", () => {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
try {
|
||||
const result = ensureMemoryIndexSchema({
|
||||
db,
|
||||
embeddingCacheTable: "embedding_cache",
|
||||
cacheEnabled: true,
|
||||
ftsTable: "chunks_fts",
|
||||
ftsEnabled: true,
|
||||
});
|
||||
|
||||
expect(result.ftsAvailable).toBe(true);
|
||||
expect(
|
||||
db
|
||||
.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('embedding_cache', 'chunks_fts', 'memory_embedding_cache', 'memory_index_chunks_fts') ORDER BY name",
|
||||
)
|
||||
.all(),
|
||||
).toEqual([{ name: "chunks_fts" }, { name: "embedding_cache" }]);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("upgrades canonical source tables keyed only by path", () => {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE memory_index_sources (
|
||||
path TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
hash TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO memory_index_sources VALUES ('shared.md', 'memory', 'memory-hash', 10, 20);
|
||||
`);
|
||||
|
||||
ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled: false,
|
||||
ftsEnabled: false,
|
||||
});
|
||||
|
||||
db.prepare(
|
||||
"INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)",
|
||||
).run("shared.md", "sessions", "session-hash", 30, 40);
|
||||
|
||||
expect(
|
||||
db.prepare("SELECT path, source, hash FROM memory_index_sources ORDER BY source").all(),
|
||||
).toEqual([
|
||||
{ path: "shared.md", source: "memory", hash: "memory-hash" },
|
||||
{ path: "shared.md", source: "sessions", hash: "session-hash" },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves unrelated generic tables untouched", () => {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL, owner TEXT);
|
||||
CREATE TABLE files (
|
||||
path TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE chunks (
|
||||
id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
`);
|
||||
|
||||
ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled: false,
|
||||
ftsEnabled: false,
|
||||
});
|
||||
|
||||
expect(
|
||||
db
|
||||
.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name",
|
||||
)
|
||||
.all(),
|
||||
).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps legacy tables when canonical rows conflict", () => {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
CREATE TABLE files (
|
||||
path TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
hash TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE chunks (
|
||||
id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE memory_index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
INSERT INTO meta VALUES ('memory_index_meta_v1', 'legacy');
|
||||
INSERT INTO memory_index_meta VALUES ('memory_index_meta_v1', 'canonical');
|
||||
`);
|
||||
|
||||
expect(() =>
|
||||
ensureMemoryIndexSchema({
|
||||
db,
|
||||
cacheEnabled: false,
|
||||
ftsEnabled: false,
|
||||
}),
|
||||
).toThrow("legacy memory meta rows conflict");
|
||||
expect(db.prepare("SELECT value FROM meta").get()).toEqual({ value: "legacy" });
|
||||
expect(db.prepare("SELECT value FROM memory_index_meta").get()).toEqual({
|
||||
value: "canonical",
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
428
packages/memory-host-sdk/src/host/memory-schema.ts
Normal file
428
packages/memory-host-sdk/src/host/memory-schema.ts
Normal file
@@ -0,0 +1,428 @@
|
||||
// Memory Host SDK module implements memory schema behavior.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { formatErrorMessage } from "./error-utils.js";
|
||||
|
||||
// SQLite schema setup for builtin memory index, embedding cache, and FTS.
|
||||
|
||||
export const MEMORY_INDEX_META_TABLE = "memory_index_meta";
|
||||
export const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources";
|
||||
export const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks";
|
||||
export const MEMORY_EMBEDDING_CACHE_TABLE = "memory_embedding_cache";
|
||||
export const MEMORY_INDEX_STATE_TABLE = "memory_index_state";
|
||||
export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts";
|
||||
export const MEMORY_INDEX_VECTOR_TABLE = "memory_index_chunks_vec";
|
||||
|
||||
const LEGACY_MEMORY_INDEX_TRIGGERS = [
|
||||
"memory_files_revision_after_insert",
|
||||
"memory_files_revision_after_update",
|
||||
"memory_files_revision_after_delete",
|
||||
"memory_chunks_revision_after_insert",
|
||||
"memory_chunks_revision_after_update",
|
||||
"memory_chunks_revision_after_delete",
|
||||
] as const;
|
||||
|
||||
const MEMORY_INDEX_SOURCE_COLUMNS = ["path", "source", "hash", "mtime", "size"] as const;
|
||||
|
||||
function tableColumns(db: DatabaseSync, tableName: string, schema = "main"): Set<string> {
|
||||
const rows = db.prepare(`PRAGMA ${schema}.table_info(${tableName})`).all() as Array<{
|
||||
name?: unknown;
|
||||
}>;
|
||||
return new Set(rows.flatMap((row) => (typeof row.name === "string" ? [row.name] : [])));
|
||||
}
|
||||
|
||||
function tableHasExactColumns(
|
||||
db: DatabaseSync,
|
||||
tableName: string,
|
||||
expected: readonly string[],
|
||||
schema = "main",
|
||||
): boolean {
|
||||
const columns = tableColumns(db, tableName, schema);
|
||||
return columns.size === expected.length && expected.every((column) => columns.has(column));
|
||||
}
|
||||
|
||||
function tablePrimaryKeyColumns(db: DatabaseSync, tableName: string): string[] {
|
||||
const rows = db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{
|
||||
name?: unknown;
|
||||
pk?: unknown;
|
||||
}>;
|
||||
return rows
|
||||
.flatMap((row) =>
|
||||
typeof row.name === "string" && typeof row.pk === "number" && row.pk > 0
|
||||
? [{ name: row.name, pk: row.pk }]
|
||||
: [],
|
||||
)
|
||||
.toSorted((left, right) => left.pk - right.pk)
|
||||
.map((row) => row.name);
|
||||
}
|
||||
|
||||
function tableHasPrimaryKey(
|
||||
db: DatabaseSync,
|
||||
tableName: string,
|
||||
expectedColumns: readonly string[],
|
||||
): boolean {
|
||||
const columns = tablePrimaryKeyColumns(db, tableName);
|
||||
return (
|
||||
columns.length === expectedColumns.length &&
|
||||
columns.every((column, index) => column === expectedColumns[index])
|
||||
);
|
||||
}
|
||||
|
||||
function assertLegacyRowsCopied(db: DatabaseSync, query: string, tableName: string): void {
|
||||
const row = db.prepare(query).get() as { missing?: unknown } | undefined;
|
||||
if (Number(row?.missing ?? 0) > 0) {
|
||||
throw new Error(`legacy memory ${tableName} rows conflict with canonical memory index rows`);
|
||||
}
|
||||
}
|
||||
|
||||
function migrateCanonicalMemoryIndexSourcesPrimaryKey(db: DatabaseSync): void {
|
||||
if (
|
||||
!tableHasExactColumns(db, MEMORY_INDEX_SOURCES_TABLE, MEMORY_INDEX_SOURCE_COLUMNS) ||
|
||||
tableHasPrimaryKey(db, MEMORY_INDEX_SOURCES_TABLE, ["path", "source"])
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (!tableHasPrimaryKey(db, MEMORY_INDEX_SOURCES_TABLE, ["path"])) {
|
||||
return;
|
||||
}
|
||||
|
||||
db.exec("SAVEPOINT migrate_memory_index_sources_primary_key");
|
||||
try {
|
||||
db.exec(`
|
||||
DROP TRIGGER IF EXISTS memory_index_sources_revision_after_insert;
|
||||
DROP TRIGGER IF EXISTS memory_index_sources_revision_after_update;
|
||||
DROP TRIGGER IF EXISTS memory_index_sources_revision_after_delete;
|
||||
|
||||
ALTER TABLE ${MEMORY_INDEX_SOURCES_TABLE}
|
||||
RENAME TO memory_index_sources_path_pk_migration;
|
||||
CREATE TABLE ${MEMORY_INDEX_SOURCES_TABLE} (
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
hash TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
PRIMARY KEY (path, source)
|
||||
);
|
||||
INSERT INTO ${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size)
|
||||
SELECT path, source, hash, mtime, size FROM memory_index_sources_path_pk_migration;
|
||||
DROP TABLE memory_index_sources_path_pk_migration;
|
||||
RELEASE migrate_memory_index_sources_primary_key;
|
||||
`);
|
||||
} catch (err) {
|
||||
db.exec("ROLLBACK TO migrate_memory_index_sources_primary_key");
|
||||
db.exec("RELEASE migrate_memory_index_sources_primary_key");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function hasLegacyMemoryIndexTables(db: DatabaseSync, schema = "main"): boolean {
|
||||
return (
|
||||
tableHasExactColumns(db, "meta", ["key", "value"], schema) &&
|
||||
tableHasExactColumns(db, "files", ["path", "source", "hash", "mtime", "size"], schema) &&
|
||||
tableHasExactColumns(
|
||||
db,
|
||||
"chunks",
|
||||
[
|
||||
"id",
|
||||
"path",
|
||||
"source",
|
||||
"start_line",
|
||||
"end_line",
|
||||
"hash",
|
||||
"model",
|
||||
"text",
|
||||
"embedding",
|
||||
"updated_at",
|
||||
],
|
||||
schema,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function hasLegacyEmbeddingCacheTable(db: DatabaseSync, schema = "main"): boolean {
|
||||
return tableHasExactColumns(
|
||||
db,
|
||||
"embedding_cache",
|
||||
["provider", "model", "provider_key", "hash", "embedding", "dims", "updated_at"],
|
||||
schema,
|
||||
);
|
||||
}
|
||||
|
||||
function copyLegacyMemoryIndexRows(
|
||||
db: DatabaseSync,
|
||||
schema: string,
|
||||
preservedEmbeddingCacheTable?: string,
|
||||
): void {
|
||||
db.exec(`
|
||||
INSERT OR IGNORE INTO main.${MEMORY_INDEX_META_TABLE} (key, value)
|
||||
SELECT key, value FROM ${schema}.meta;
|
||||
|
||||
INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size)
|
||||
SELECT path, source, hash, mtime, size FROM ${schema}.files;
|
||||
|
||||
INSERT OR IGNORE INTO main.${MEMORY_INDEX_CHUNKS_TABLE} (
|
||||
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
)
|
||||
SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
|
||||
FROM ${schema}.chunks;
|
||||
`);
|
||||
assertLegacyRowsCopied(
|
||||
db,
|
||||
`SELECT COUNT(*) AS missing
|
||||
FROM ${schema}.meta AS legacy
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM main.${MEMORY_INDEX_META_TABLE} AS canonical
|
||||
WHERE canonical.key = legacy.key AND canonical.value IS legacy.value
|
||||
)`,
|
||||
"meta",
|
||||
);
|
||||
assertLegacyRowsCopied(
|
||||
db,
|
||||
`SELECT COUNT(*) AS missing
|
||||
FROM ${schema}.files AS legacy
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical
|
||||
WHERE canonical.path = legacy.path
|
||||
AND canonical.source IS legacy.source
|
||||
AND canonical.hash IS legacy.hash
|
||||
AND canonical.mtime IS legacy.mtime
|
||||
AND canonical.size IS legacy.size
|
||||
)`,
|
||||
"files",
|
||||
);
|
||||
assertLegacyRowsCopied(
|
||||
db,
|
||||
`SELECT COUNT(*) AS missing
|
||||
FROM ${schema}.chunks AS legacy
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical
|
||||
WHERE canonical.id = legacy.id
|
||||
AND canonical.path IS legacy.path
|
||||
AND canonical.source IS legacy.source
|
||||
AND canonical.start_line IS legacy.start_line
|
||||
AND canonical.end_line IS legacy.end_line
|
||||
AND canonical.hash IS legacy.hash
|
||||
AND canonical.model IS legacy.model
|
||||
AND canonical.text IS legacy.text
|
||||
AND canonical.embedding IS legacy.embedding
|
||||
AND canonical.updated_at IS legacy.updated_at
|
||||
)`,
|
||||
"chunks",
|
||||
);
|
||||
if (
|
||||
preservedEmbeddingCacheTable !== "embedding_cache" &&
|
||||
hasLegacyEmbeddingCacheTable(db, schema)
|
||||
) {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS main.${MEMORY_EMBEDDING_CACHE_TABLE} (
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
provider_key TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
dims INTEGER,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (provider, model, provider_key, hash)
|
||||
);
|
||||
INSERT OR IGNORE INTO main.${MEMORY_EMBEDDING_CACHE_TABLE} (
|
||||
provider, model, provider_key, hash, embedding, dims, updated_at
|
||||
)
|
||||
SELECT provider, model, provider_key, hash, embedding, dims, updated_at
|
||||
FROM ${schema}.embedding_cache;
|
||||
`);
|
||||
assertLegacyRowsCopied(
|
||||
db,
|
||||
`SELECT COUNT(*) AS missing
|
||||
FROM ${schema}.embedding_cache AS legacy
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM main.${MEMORY_EMBEDDING_CACHE_TABLE} AS canonical
|
||||
WHERE canonical.provider = legacy.provider
|
||||
AND canonical.model = legacy.model
|
||||
AND canonical.provider_key = legacy.provider_key
|
||||
AND canonical.hash = legacy.hash
|
||||
AND canonical.embedding IS legacy.embedding
|
||||
AND canonical.dims IS legacy.dims
|
||||
AND canonical.updated_at IS legacy.updated_at
|
||||
)`,
|
||||
"embedding_cache",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function migrateLegacyMemoryIndexTables(
|
||||
db: DatabaseSync,
|
||||
preservedEmbeddingCacheTable?: string,
|
||||
): void {
|
||||
if (!hasLegacyMemoryIndexTables(db)) {
|
||||
return;
|
||||
}
|
||||
|
||||
db.exec("SAVEPOINT migrate_legacy_memory_index_tables");
|
||||
try {
|
||||
copyLegacyMemoryIndexRows(db, "main", preservedEmbeddingCacheTable);
|
||||
if (preservedEmbeddingCacheTable !== "embedding_cache" && hasLegacyEmbeddingCacheTable(db)) {
|
||||
db.exec("DROP TABLE embedding_cache");
|
||||
}
|
||||
for (const trigger of LEGACY_MEMORY_INDEX_TRIGGERS) {
|
||||
db.exec(`DROP TRIGGER IF EXISTS ${trigger}`);
|
||||
}
|
||||
db.exec(`
|
||||
DROP TABLE IF EXISTS chunks_fts;
|
||||
DROP TABLE chunks;
|
||||
DROP TABLE files;
|
||||
DROP TABLE meta;
|
||||
RELEASE migrate_legacy_memory_index_tables;
|
||||
`);
|
||||
} catch (err) {
|
||||
db.exec("ROLLBACK TO migrate_legacy_memory_index_tables");
|
||||
db.exec("RELEASE migrate_legacy_memory_index_tables");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Ensure canonical memory index tables and the optional FTS table exist. */
|
||||
export function ensureMemoryIndexSchema(params: {
|
||||
db: DatabaseSync;
|
||||
/** @deprecated Omit to use the canonical memory cache table. */
|
||||
embeddingCacheTable?: string;
|
||||
cacheEnabled: boolean;
|
||||
/** @deprecated Omit to use the canonical memory FTS table. */
|
||||
ftsTable?: string;
|
||||
ftsEnabled: boolean;
|
||||
ftsTokenizer?: "unicode61" | "trigram";
|
||||
}): { ftsAvailable: boolean; ftsError?: string } {
|
||||
const embeddingCacheTable = params.embeddingCacheTable ?? MEMORY_EMBEDDING_CACHE_TABLE;
|
||||
const ftsTable = params.ftsTable ?? MEMORY_INDEX_FTS_TABLE;
|
||||
params.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_META_TABLE} (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_SOURCES_TABLE} (
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
hash TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
PRIMARY KEY (path, source)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNKS_TABLE} (
|
||||
id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'memory',
|
||||
start_line INTEGER NOT NULL,
|
||||
end_line INTEGER NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_STATE_TABLE} (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
revision INTEGER NOT NULL
|
||||
);
|
||||
INSERT OR IGNORE INTO ${MEMORY_INDEX_STATE_TABLE} (id, revision) VALUES (1, 0);
|
||||
`);
|
||||
migrateCanonicalMemoryIndexSourcesPrimaryKey(params.db);
|
||||
params.db.exec(`
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_insert
|
||||
AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE}
|
||||
BEGIN
|
||||
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_update
|
||||
AFTER UPDATE ON ${MEMORY_INDEX_SOURCES_TABLE}
|
||||
BEGIN
|
||||
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_delete
|
||||
AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE}
|
||||
BEGIN
|
||||
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_insert
|
||||
AFTER INSERT ON ${MEMORY_INDEX_CHUNKS_TABLE}
|
||||
BEGIN
|
||||
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_update
|
||||
AFTER UPDATE ON ${MEMORY_INDEX_CHUNKS_TABLE}
|
||||
BEGIN
|
||||
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_delete
|
||||
AFTER DELETE ON ${MEMORY_INDEX_CHUNKS_TABLE}
|
||||
BEGIN
|
||||
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_index_sources_source
|
||||
ON ${MEMORY_INDEX_SOURCES_TABLE}(source);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path_source
|
||||
ON ${MEMORY_INDEX_CHUNKS_TABLE}(path, source);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path
|
||||
ON ${MEMORY_INDEX_CHUNKS_TABLE}(path);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source
|
||||
ON ${MEMORY_INDEX_CHUNKS_TABLE}(source);
|
||||
`);
|
||||
migrateLegacyMemoryIndexTables(params.db, params.embeddingCacheTable);
|
||||
if (params.cacheEnabled) {
|
||||
const updatedAtIndex =
|
||||
embeddingCacheTable === MEMORY_EMBEDDING_CACHE_TABLE
|
||||
? "idx_memory_embedding_cache_updated_at"
|
||||
: "idx_embedding_cache_updated_at";
|
||||
params.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ${embeddingCacheTable} (
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
provider_key TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
embedding TEXT NOT NULL,
|
||||
dims INTEGER,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (provider, model, provider_key, hash)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ${updatedAtIndex}
|
||||
ON ${embeddingCacheTable}(updated_at);
|
||||
`);
|
||||
}
|
||||
|
||||
let ftsAvailable = false;
|
||||
let ftsError: string | undefined;
|
||||
if (params.ftsEnabled) {
|
||||
try {
|
||||
const tokenizer = params.ftsTokenizer ?? "unicode61";
|
||||
const tokenizeClause = tokenizer === "trigram" ? `, tokenize='trigram case_sensitive 0'` : "";
|
||||
params.db.exec(
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS ${ftsTable} USING fts5(\n` +
|
||||
` text,\n` +
|
||||
` id UNINDEXED,\n` +
|
||||
` path UNINDEXED,\n` +
|
||||
` source UNINDEXED,\n` +
|
||||
` model UNINDEXED,\n` +
|
||||
` start_line UNINDEXED,\n` +
|
||||
` end_line UNINDEXED\n` +
|
||||
`${tokenizeClause});`,
|
||||
);
|
||||
// The shipped generic-table migration and a later FTS enablement both
|
||||
// create an empty derived table beside already-canonical chunk rows.
|
||||
params.db.exec(`
|
||||
INSERT INTO ${ftsTable} (
|
||||
text, id, path, source, model, start_line, end_line
|
||||
)
|
||||
SELECT text, id, path, source, model, start_line, end_line
|
||||
FROM ${MEMORY_INDEX_CHUNKS_TABLE}
|
||||
WHERE NOT EXISTS (SELECT 1 FROM ${ftsTable} LIMIT 1);
|
||||
`);
|
||||
ftsAvailable = true;
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
ftsAvailable = false;
|
||||
ftsError = message;
|
||||
}
|
||||
}
|
||||
|
||||
return { ftsAvailable, ...(ftsError ? { ftsError } : {}) };
|
||||
}
|
||||
117
packages/memory-host-sdk/src/host/multimodal.ts
Normal file
117
packages/memory-host-sdk/src/host/multimodal.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
// Memory Host SDK module implements multimodal behavior.
|
||||
import { normalizeLowercaseStringOrEmpty } from "./string-utils.js";
|
||||
|
||||
// Multimodal memory settings and file classification helpers.
|
||||
|
||||
const MEMORY_MULTIMODAL_SPECS = {
|
||||
image: {
|
||||
labelPrefix: "Image file",
|
||||
extensions: [".jpg", ".jpeg", ".png", ".webp", ".gif", ".heic", ".heif"],
|
||||
},
|
||||
audio: {
|
||||
labelPrefix: "Audio file",
|
||||
extensions: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".aac", ".flac"],
|
||||
},
|
||||
} as const;
|
||||
|
||||
/** Supported multimodal memory modality. */
|
||||
export type MemoryMultimodalModality = keyof typeof MEMORY_MULTIMODAL_SPECS;
|
||||
/** All supported multimodal memory modalities in stable config order. */
|
||||
export const MEMORY_MULTIMODAL_MODALITIES = Object.keys(
|
||||
MEMORY_MULTIMODAL_SPECS,
|
||||
) as MemoryMultimodalModality[];
|
||||
/** User selection for one modality or all modalities. */
|
||||
export type MemoryMultimodalSelection = MemoryMultimodalModality | "all";
|
||||
|
||||
/** Normalized multimodal memory ingestion settings. */
|
||||
export type MemoryMultimodalSettings = {
|
||||
enabled: boolean;
|
||||
modalities: MemoryMultimodalModality[];
|
||||
maxFileBytes: number;
|
||||
};
|
||||
|
||||
/** Default max bytes for one multimodal memory file. */
|
||||
export const DEFAULT_MEMORY_MULTIMODAL_MAX_FILE_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/** Normalize user modality selections to supported modalities. */
|
||||
export function normalizeMemoryMultimodalModalities(
|
||||
raw: MemoryMultimodalSelection[] | undefined,
|
||||
): MemoryMultimodalModality[] {
|
||||
if (raw === undefined || raw.includes("all")) {
|
||||
return [...MEMORY_MULTIMODAL_MODALITIES];
|
||||
}
|
||||
const normalized = new Set<MemoryMultimodalModality>();
|
||||
for (const value of raw) {
|
||||
if (value === "image" || value === "audio") {
|
||||
normalized.add(value);
|
||||
}
|
||||
}
|
||||
return Array.from(normalized);
|
||||
}
|
||||
|
||||
/** Normalize user multimodal settings, including disabled-state empty modality list. */
|
||||
export function normalizeMemoryMultimodalSettings(raw: {
|
||||
enabled?: boolean;
|
||||
modalities?: MemoryMultimodalSelection[];
|
||||
maxFileBytes?: number;
|
||||
}): MemoryMultimodalSettings {
|
||||
const enabled = raw.enabled === true;
|
||||
const maxFileBytes =
|
||||
typeof raw.maxFileBytes === "number" && Number.isFinite(raw.maxFileBytes)
|
||||
? Math.max(1, Math.floor(raw.maxFileBytes))
|
||||
: DEFAULT_MEMORY_MULTIMODAL_MAX_FILE_BYTES;
|
||||
return {
|
||||
enabled,
|
||||
modalities: enabled ? normalizeMemoryMultimodalModalities(raw.modalities) : [],
|
||||
maxFileBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/** Return true when multimodal memory ingestion has at least one enabled modality. */
|
||||
export function isMemoryMultimodalEnabled(settings: MemoryMultimodalSettings): boolean {
|
||||
return settings.enabled && settings.modalities.length > 0;
|
||||
}
|
||||
|
||||
/** Return accepted file extensions for a modality. */
|
||||
export function getMemoryMultimodalExtensions(
|
||||
modality: MemoryMultimodalModality,
|
||||
): readonly string[] {
|
||||
return MEMORY_MULTIMODAL_SPECS[modality].extensions;
|
||||
}
|
||||
|
||||
/** Build the text label that accompanies embedded multimodal file content. */
|
||||
export function buildMemoryMultimodalLabel(
|
||||
modality: MemoryMultimodalModality,
|
||||
normalizedPath: string,
|
||||
): string {
|
||||
return `${MEMORY_MULTIMODAL_SPECS[modality].labelPrefix}: ${normalizedPath}`;
|
||||
}
|
||||
|
||||
/** Build a glob that matches an extension case-insensitively for QMD sources. */
|
||||
export function buildCaseInsensitiveExtensionGlob(extension: string): string {
|
||||
const normalized = normalizeLowercaseStringOrEmpty(extension).replace(/^\./, "");
|
||||
if (!normalized) {
|
||||
return "*";
|
||||
}
|
||||
const parts = Array.from(normalized, (char) => `[${char.toLowerCase()}${char.toUpperCase()}]`);
|
||||
return `*.${parts.join("")}`;
|
||||
}
|
||||
|
||||
/** Classify a file path into a supported multimodal modality under current settings. */
|
||||
export function classifyMemoryMultimodalPath(
|
||||
filePath: string,
|
||||
settings: MemoryMultimodalSettings,
|
||||
): MemoryMultimodalModality | null {
|
||||
if (!isMemoryMultimodalEnabled(settings)) {
|
||||
return null;
|
||||
}
|
||||
const lower = normalizeLowercaseStringOrEmpty(filePath);
|
||||
for (const modality of settings.modalities) {
|
||||
for (const extension of getMemoryMultimodalExtensions(modality)) {
|
||||
if (lower.endsWith(extension)) {
|
||||
return modality;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
52
packages/memory-host-sdk/src/host/node-llama.ts
Normal file
52
packages/memory-host-sdk/src/host/node-llama.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
// Minimal node-llama-cpp type facade used by the local embedding provider.
|
||||
|
||||
/** Embedding vector returned by node-llama-cpp. */
|
||||
export type LlamaEmbedding = {
|
||||
vector: Float32Array | number[];
|
||||
};
|
||||
|
||||
/** Embedding context created from a loaded llama model. */
|
||||
export type LlamaEmbeddingContext = {
|
||||
getEmbeddingFor: (text: string) => Promise<LlamaEmbedding>;
|
||||
dispose?: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
/** Loaded llama model capable of creating embedding contexts. */
|
||||
export type LlamaModel = {
|
||||
createEmbeddingContext: (options?: {
|
||||
contextSize?: number | "auto";
|
||||
createSignal?: AbortSignal;
|
||||
}) => Promise<LlamaEmbeddingContext>;
|
||||
dispose?: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
/** Options accepted by node-llama-cpp model file resolution. */
|
||||
export type ResolveModelFileOptions = {
|
||||
directory?: string;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
/** Root llama runtime object exposed by node-llama-cpp. */
|
||||
export type Llama = {
|
||||
loadModel: (params: { modelPath: string; loadSignal?: AbortSignal }) => Promise<LlamaModel>;
|
||||
dispose?: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
/** Imported node-llama-cpp module shape used by local embeddings. */
|
||||
export type NodeLlamaCppModule = {
|
||||
LlamaLogLevel: {
|
||||
error: number;
|
||||
};
|
||||
getLlama: (params: { logLevel: number }) => Promise<Llama>;
|
||||
resolveModelFile: (
|
||||
modelPath: string,
|
||||
optionsOrDirectory?: string | ResolveModelFileOptions,
|
||||
) => Promise<string>;
|
||||
};
|
||||
|
||||
const NODE_LLAMA_CPP_MODULE = "node-llama-cpp";
|
||||
|
||||
/** Dynamically import node-llama-cpp so the optional dependency is loaded only when needed. */
|
||||
export async function importNodeLlamaCpp(moduleSpecifier = NODE_LLAMA_CPP_MODULE) {
|
||||
return import(moduleSpecifier) as Promise<NodeLlamaCppModule>;
|
||||
}
|
||||
23
packages/memory-host-sdk/src/host/openclaw-runtime-agent.ts
Normal file
23
packages/memory-host-sdk/src/host/openclaw-runtime-agent.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
// Agent-facing runtime facade for memory host packages.
|
||||
// Keep exports here limited to config/state helpers that memory plugins may reuse.
|
||||
export {
|
||||
DEFAULT_AGENT_COMPACTION_RESERVE_TOKENS_FLOOR,
|
||||
asToolParamsRecord,
|
||||
jsonResult,
|
||||
parseAgentSessionKey,
|
||||
readNumberParam,
|
||||
readStringParam,
|
||||
resolveAgentContextLimits,
|
||||
resolveAgentDir,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveCronStyleNow,
|
||||
resolveDefaultAgentId,
|
||||
resolveMemorySearchConfig,
|
||||
resolveMemorySearchSyncConfig,
|
||||
resolveSessionAgentId,
|
||||
} from "./openclaw-runtime.js";
|
||||
export type {
|
||||
AnyAgentTool,
|
||||
ResolvedMemorySearchConfig,
|
||||
ResolvedMemorySearchSyncConfig,
|
||||
} from "./openclaw-runtime.js";
|
||||
13
packages/memory-host-sdk/src/host/openclaw-runtime-auth.ts
Normal file
13
packages/memory-host-sdk/src/host/openclaw-runtime-auth.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
// Memory Host SDK module implements openclaw runtime auth behavior.
|
||||
import { requireApiKey } from "../../../../src/agents/model-auth-runtime-shared.js";
|
||||
import type { resolveApiKeyForProvider as ResolveApiKeyForProvider } from "../../../../src/agents/model-auth.js";
|
||||
|
||||
// Lazy auth facade so memory host helpers avoid eager model-auth module loading.
|
||||
|
||||
export { requireApiKey };
|
||||
|
||||
/** Resolve a provider API key through the core model-auth runtime. */
|
||||
export const resolveApiKeyForProvider: typeof ResolveApiKeyForProvider = async (...args) => {
|
||||
const auth = await import("../../../../src/agents/model-auth.js");
|
||||
return auth.resolveApiKeyForProvider(...args);
|
||||
};
|
||||
19
packages/memory-host-sdk/src/host/openclaw-runtime-cli.ts
Normal file
19
packages/memory-host-sdk/src/host/openclaw-runtime-cli.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
// Narrow CLI/runtime facade re-exported for memory host helpers.
|
||||
|
||||
export {
|
||||
colorize,
|
||||
defaultRuntime,
|
||||
formatDocsLink,
|
||||
formatErrorMessage,
|
||||
formatHelpExamples,
|
||||
isRich,
|
||||
isVerbose,
|
||||
resolveCommandSecretRefsViaGateway,
|
||||
setVerbose,
|
||||
shortenHomeInString,
|
||||
shortenHomePath,
|
||||
theme,
|
||||
withManager,
|
||||
withProgress,
|
||||
withProgressTotals,
|
||||
} from "./openclaw-runtime.js";
|
||||
24
packages/memory-host-sdk/src/host/openclaw-runtime-config.ts
Normal file
24
packages/memory-host-sdk/src/host/openclaw-runtime-config.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
// Config-facing runtime facade for memory host packages.
|
||||
// This keeps memory plugins off broader core config modules and their private helpers.
|
||||
export {
|
||||
getRuntimeConfig,
|
||||
hasConfiguredSecretInput,
|
||||
loadConfig,
|
||||
normalizeResolvedSecretInputString,
|
||||
parseDurationMs,
|
||||
parseNonNegativeByteSize,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
resolveStateDir,
|
||||
} from "./openclaw-runtime.js";
|
||||
export type {
|
||||
MemoryBackend,
|
||||
MemoryCitationsMode,
|
||||
MemoryQmdConfig,
|
||||
MemoryQmdIndexPath,
|
||||
MemoryQmdMcporterConfig,
|
||||
MemoryQmdSearchMode,
|
||||
MemorySearchConfig,
|
||||
OpenClawConfig,
|
||||
SecretInput,
|
||||
SessionSendPolicyConfig,
|
||||
} from "./openclaw-runtime.js";
|
||||
43
packages/memory-host-sdk/src/host/openclaw-runtime-io.ts
Normal file
43
packages/memory-host-sdk/src/host/openclaw-runtime-io.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
// Narrow IO/runtime facade re-exported for memory host helpers.
|
||||
|
||||
export {
|
||||
CHARS_PER_TOKEN_ESTIMATE,
|
||||
DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES,
|
||||
DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS,
|
||||
DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS,
|
||||
applyWindowsSpawnProgramPolicy,
|
||||
configureSqliteConnectionPragmas,
|
||||
configureSqliteWalMaintenance,
|
||||
root,
|
||||
createSubsystemLogger,
|
||||
detectMime,
|
||||
estimateStringChars,
|
||||
installProcessWarningFilter,
|
||||
materializeWindowsSpawnProgram,
|
||||
redactSensitiveText,
|
||||
resolveGlobalSingleton,
|
||||
resolveUserPath,
|
||||
resolveWindowsExecutablePath,
|
||||
resolveWindowsSpawnProgram,
|
||||
resolveWindowsSpawnProgramCandidate,
|
||||
runTasksWithConcurrency,
|
||||
shortenHomeInString,
|
||||
shortenHomePath,
|
||||
shouldIgnoreWarning,
|
||||
splitShellArgs,
|
||||
truncateUtf16Safe,
|
||||
} from "./openclaw-runtime.js";
|
||||
|
||||
export type {
|
||||
ProcessWarning,
|
||||
ResolveWindowsSpawnProgramCandidateParams,
|
||||
ResolveWindowsSpawnProgramParams,
|
||||
SqliteConnectionPragmaOptions,
|
||||
SqliteWalMaintenance,
|
||||
SqliteWalMaintenanceOptions,
|
||||
WindowsSpawnCandidateResolution,
|
||||
WindowsSpawnInvocation,
|
||||
WindowsSpawnProgram,
|
||||
WindowsSpawnProgramCandidate,
|
||||
WindowsSpawnResolution,
|
||||
} from "./openclaw-runtime.js";
|
||||
32
packages/memory-host-sdk/src/host/openclaw-runtime-memory.ts
Normal file
32
packages/memory-host-sdk/src/host/openclaw-runtime-memory.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
// Memory-facing runtime facade for plugin registration, embeddings, and prompt artifacts.
|
||||
// Re-export only stable host seams; plugin implementations should not import core internals.
|
||||
export {
|
||||
buildActiveMemoryPromptSection,
|
||||
emptyPluginConfigSchema,
|
||||
getMemoryCapabilityRegistration,
|
||||
getMemoryEmbeddingProvider,
|
||||
listActiveMemoryPublicArtifacts,
|
||||
listMemoryEmbeddingProviders,
|
||||
listRegisteredMemoryEmbeddingProviderAdapters,
|
||||
listRegisteredMemoryEmbeddingProviders,
|
||||
resolveCanonicalRootMemoryFile,
|
||||
shouldSkipRootMemoryAuxiliaryPath,
|
||||
} from "./openclaw-runtime.js";
|
||||
export type {
|
||||
MemoryEmbeddingBatchChunk,
|
||||
MemoryEmbeddingBatchOptions,
|
||||
MemoryEmbeddingProvider,
|
||||
MemoryEmbeddingProviderAdapter,
|
||||
MemoryEmbeddingProviderCallOptions,
|
||||
MemoryEmbeddingProviderCreateOptions,
|
||||
MemoryEmbeddingProviderCreateResult,
|
||||
MemoryEmbeddingProviderRuntime,
|
||||
MemoryFlushPlan,
|
||||
MemoryFlushPlanResolver,
|
||||
MemoryPluginCapability,
|
||||
MemoryPluginPublicArtifact,
|
||||
MemoryPluginPublicArtifactsProvider,
|
||||
MemoryPluginRuntime,
|
||||
MemoryPromptSectionBuilder,
|
||||
OpenClawPluginApi,
|
||||
} from "./openclaw-runtime.js";
|
||||
@@ -0,0 +1,5 @@
|
||||
// Narrow network/runtime facade re-exported for memory remote HTTP helpers.
|
||||
|
||||
export { fetchWithSsrFGuard } from "../../../../src/infra/net/fetch-guard.js";
|
||||
export { shouldUseEnvHttpProxyForUrl } from "../../../../src/infra/net/proxy-env.js";
|
||||
export { ssrfPolicyFromHttpBaseUrlAllowedHostname } from "../../../../src/infra/net/ssrf.js";
|
||||
@@ -0,0 +1,62 @@
|
||||
// Narrow session/runtime facade re-exported for memory transcript helpers.
|
||||
import path from "node:path";
|
||||
|
||||
export {
|
||||
canonicalizeMainSessionAlias,
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
getRuntimeConfig,
|
||||
HEARTBEAT_PROMPT,
|
||||
HEARTBEAT_TOKEN,
|
||||
SILENT_REPLY_TOKEN,
|
||||
hasInterSessionUserProvenance,
|
||||
isCompactionCheckpointTranscriptFileName,
|
||||
isCronRunSessionKey,
|
||||
isExecCompletionEvent,
|
||||
isHeartbeatUserMessage,
|
||||
isSessionArchiveArtifactName,
|
||||
isSilentReplyPayloadText,
|
||||
isUsageCountedSessionTranscriptFileName,
|
||||
listSessionEntries,
|
||||
onSessionTranscriptUpdate,
|
||||
parseUsageCountedSessionIdFromFileName,
|
||||
resolveSessionFilePath,
|
||||
resolveStorePath,
|
||||
resolveSessionAgentId,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
stripInboundMetadata,
|
||||
stripInternalRuntimeContext,
|
||||
type SessionEntry,
|
||||
} from "./openclaw-runtime.js";
|
||||
|
||||
/** Extracts the agent id from a canonical `agents/<id>/sessions` directory path. */
|
||||
export function extractAgentIdFromSessionsDir(sessionsDir: string): string | null {
|
||||
const parts = path.normalize(path.resolve(sessionsDir)).split(path.sep).filter(Boolean);
|
||||
const sessionsIndex = parts.length - 1;
|
||||
if (
|
||||
parts[sessionsIndex] !== "sessions" ||
|
||||
sessionsIndex < 2 ||
|
||||
parts[sessionsIndex - 2] !== "agents"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return parts[sessionsIndex - 1] || null;
|
||||
}
|
||||
|
||||
/** Session-key prefix marking transcripts generated by memory dreaming runs. */
|
||||
export const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-";
|
||||
|
||||
/** True when a session-store key belongs to a dreaming narrative run. */
|
||||
export function isDreamingNarrativeSessionStoreKey(sessionKey: string): boolean {
|
||||
const trimmed = sessionKey.trim();
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
const firstSeparator = trimmed.indexOf(":");
|
||||
if (firstSeparator < 0) {
|
||||
return trimmed.startsWith(DREAMING_NARRATIVE_RUN_PREFIX);
|
||||
}
|
||||
const secondSeparator = trimmed.indexOf(":", firstSeparator + 1);
|
||||
const sessionSegment = secondSeparator < 0 ? trimmed : trimmed.slice(secondSeparator + 1);
|
||||
return sessionSegment.startsWith(DREAMING_NARRATIVE_RUN_PREFIX);
|
||||
}
|
||||
182
packages/memory-host-sdk/src/host/openclaw-runtime.ts
Normal file
182
packages/memory-host-sdk/src/host/openclaw-runtime.ts
Normal file
@@ -0,0 +1,182 @@
|
||||
// Agent/runtime helpers.
|
||||
export { resolveCronStyleNow } from "../../../../src/agents/current-time.js";
|
||||
export {
|
||||
resolveAgentContextLimits,
|
||||
resolveAgentDir,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveDefaultAgentId,
|
||||
resolveSessionAgentId,
|
||||
} from "../../../../src/agents/agent-scope.js";
|
||||
export { requireApiKey, resolveApiKeyForProvider } from "../../../../src/agents/model-auth.js";
|
||||
export { stripInternalRuntimeContext } from "../../../../src/agents/internal-runtime-context.js";
|
||||
export { DEFAULT_AGENT_COMPACTION_RESERVE_TOKENS_FLOOR } from "../../../../src/agents/agent-settings.js";
|
||||
export {
|
||||
asToolParamsRecord,
|
||||
jsonResult,
|
||||
readNumberParam,
|
||||
readStringParam,
|
||||
} from "../../../../src/agents/tools/common.js";
|
||||
export type { AnyAgentTool } from "../../../../src/agents/tools/common.js";
|
||||
export {
|
||||
resolveMemorySearchConfig,
|
||||
resolveMemorySearchSyncConfig,
|
||||
type ResolvedMemorySearchConfig,
|
||||
type ResolvedMemorySearchSyncConfig,
|
||||
} from "../../../../src/agents/memory-search.js";
|
||||
|
||||
// Session and reply helpers.
|
||||
export { isHeartbeatUserMessage } from "../../../../src/auto-reply/heartbeat-filter.js";
|
||||
export { HEARTBEAT_PROMPT } from "../../../../src/auto-reply/heartbeat.js";
|
||||
export { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inbound-meta.js";
|
||||
export {
|
||||
HEARTBEAT_TOKEN,
|
||||
SILENT_REPLY_TOKEN,
|
||||
isSilentReplyPayloadText,
|
||||
} from "../../../../src/auto-reply/tokens.js";
|
||||
|
||||
// CLI/runtime/config helpers.
|
||||
export { formatErrorMessage, withManager } from "../../../../src/cli/cli-utils.js";
|
||||
export { resolveCommandSecretRefsViaGateway } from "../../../../src/cli/command-secret-gateway.js";
|
||||
export { formatHelpExamples } from "../../../../src/cli/help-format.js";
|
||||
export { parseDurationMs } from "../../../../src/cli/parse-duration.js";
|
||||
export { withProgress, withProgressTotals } from "../../../../src/cli/progress.js";
|
||||
export { parseNonNegativeByteSize } from "../../../../src/config/byte-size.js";
|
||||
export {
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
getRuntimeConfig,
|
||||
/** @deprecated Use getRuntimeConfig(), or pass the already loaded config through the call path. */
|
||||
loadConfig,
|
||||
} from "../../../../src/config/config.js";
|
||||
export type { OpenClawConfig } from "../../../../src/config/config.js";
|
||||
export { resolveStateDir } from "../../../../src/config/paths.js";
|
||||
export {
|
||||
isCompactionCheckpointTranscriptFileName,
|
||||
isSessionArchiveArtifactName,
|
||||
isUsageCountedSessionTranscriptFileName,
|
||||
parseUsageCountedSessionIdFromFileName,
|
||||
} from "../../../../src/config/sessions/artifacts.js";
|
||||
export { canonicalizeMainSessionAlias } from "../../../../src/config/sessions/main-session.js";
|
||||
export { resolveSessionTranscriptsDirForAgent } from "../../../../src/config/sessions/paths.js";
|
||||
export {
|
||||
listSessionEntries,
|
||||
resolveSessionFilePath,
|
||||
resolveStorePath,
|
||||
} from "../../../../src/plugin-sdk/session-store-runtime.js";
|
||||
export type { SessionEntry } from "../../../../src/config/sessions/types.js";
|
||||
export type { SessionSendPolicyConfig } from "../../../../src/config/types.base.js";
|
||||
export type {
|
||||
MemoryBackend,
|
||||
MemoryCitationsMode,
|
||||
MemoryQmdConfig,
|
||||
MemoryQmdIndexPath,
|
||||
MemoryQmdMcporterConfig,
|
||||
MemoryQmdSearchMode,
|
||||
} from "../../../../src/config/types.memory.js";
|
||||
export {
|
||||
hasConfiguredSecretInput,
|
||||
normalizeResolvedSecretInputString,
|
||||
} from "../../../../src/config/types.secrets.js";
|
||||
export type { SecretInput } from "../../../../src/config/types.secrets.js";
|
||||
export type { MemorySearchConfig } from "../../../../src/config/types.tools.js";
|
||||
export { isVerbose, setVerbose } from "../../../../src/globals.js";
|
||||
|
||||
// IO, network, and logging helpers.
|
||||
export { isExecCompletionEvent } from "../../../../src/infra/heartbeat-events-filter.js";
|
||||
export { root } from "../../../../src/infra/fs-safe.js";
|
||||
export { fetchWithSsrFGuard } from "../../../../src/infra/net/fetch-guard.js";
|
||||
export { shouldUseEnvHttpProxyForUrl } from "../../../../src/infra/net/proxy-env.js";
|
||||
export { ssrfPolicyFromHttpBaseUrlAllowedHostname } from "../../../../src/infra/net/ssrf.js";
|
||||
export {
|
||||
DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES,
|
||||
DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS,
|
||||
DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS,
|
||||
configureSqliteConnectionPragmas,
|
||||
configureSqliteWalMaintenance,
|
||||
} from "../../../../src/infra/sqlite-wal.js";
|
||||
export type {
|
||||
SqliteConnectionPragmaOptions,
|
||||
SqliteWalMaintenance,
|
||||
SqliteWalMaintenanceOptions,
|
||||
} from "../../../../src/infra/sqlite-wal.js";
|
||||
export {
|
||||
installProcessWarningFilter,
|
||||
shouldIgnoreWarning,
|
||||
} from "../../../../src/infra/warning-filter.js";
|
||||
export type { ProcessWarning } from "../../../../src/infra/warning-filter.js";
|
||||
export { redactSensitiveText } from "../../../../src/logging/redact.js";
|
||||
export { createSubsystemLogger } from "../../../../src/logging/subsystem.js";
|
||||
export { detectMime } from "@openclaw/media-core/mime";
|
||||
|
||||
// Memory plugin helpers.
|
||||
export {
|
||||
resolveCanonicalRootMemoryFile,
|
||||
shouldSkipRootMemoryAuxiliaryPath,
|
||||
} from "../../../../src/memory/root-memory-files.js";
|
||||
export {
|
||||
getMemoryEmbeddingProvider,
|
||||
listMemoryEmbeddingProviders,
|
||||
listRegisteredMemoryEmbeddingProviderAdapters,
|
||||
listRegisteredMemoryEmbeddingProviders,
|
||||
} from "../../../../src/plugins/memory-embedding-provider-runtime.js";
|
||||
export type {
|
||||
MemoryEmbeddingBatchChunk,
|
||||
MemoryEmbeddingBatchOptions,
|
||||
MemoryEmbeddingProvider,
|
||||
MemoryEmbeddingProviderAdapter,
|
||||
MemoryEmbeddingProviderCallOptions,
|
||||
MemoryEmbeddingProviderCreateOptions,
|
||||
MemoryEmbeddingProviderCreateResult,
|
||||
MemoryEmbeddingProviderRuntime,
|
||||
} from "../../../../src/plugins/memory-embedding-providers.js";
|
||||
export { emptyPluginConfigSchema } from "../../../../src/plugins/config-schema.js";
|
||||
export {
|
||||
buildMemoryPromptSection as buildActiveMemoryPromptSection,
|
||||
getMemoryCapabilityRegistration,
|
||||
listActiveMemoryPublicArtifacts,
|
||||
} from "../../../../src/plugins/memory-state.js";
|
||||
export type {
|
||||
MemoryFlushPlan,
|
||||
MemoryFlushPlanResolver,
|
||||
MemoryPluginCapability,
|
||||
MemoryPluginPublicArtifact,
|
||||
MemoryPluginPublicArtifactsProvider,
|
||||
MemoryPluginRuntime,
|
||||
MemoryPromptSectionBuilder,
|
||||
} from "../../../../src/plugins/memory-state.js";
|
||||
export type { OpenClawPluginApi } from "../../../../src/plugins/types.js";
|
||||
|
||||
// Shared session/text utilities.
|
||||
export { defaultRuntime } from "../../../../src/runtime.js";
|
||||
export { parseAgentSessionKey } from "../../../../src/routing/session-key.js";
|
||||
export { hasInterSessionUserProvenance } from "../../../../src/sessions/input-provenance.js";
|
||||
export { isCronRunSessionKey } from "../../../../src/sessions/session-key-utils.js";
|
||||
export { onSessionTranscriptUpdate } from "../../../../src/sessions/transcript-events.js";
|
||||
export { formatDocsLink } from "../../../terminal-core/src/links.js";
|
||||
export { colorize, isRich, theme } from "../../../terminal-core/src/theme.js";
|
||||
export { CHARS_PER_TOKEN_ESTIMATE, estimateStringChars } from "../../../../src/utils/cjk-chars.js";
|
||||
export { runTasksWithConcurrency } from "../../../../src/utils/run-with-concurrency.js";
|
||||
export { splitShellArgs } from "../../../../src/utils/shell-argv.js";
|
||||
export {
|
||||
resolveUserPath,
|
||||
shortenHomeInString,
|
||||
shortenHomePath,
|
||||
truncateUtf16Safe,
|
||||
} from "../../../../src/utils.js";
|
||||
export {
|
||||
applyWindowsSpawnProgramPolicy,
|
||||
materializeWindowsSpawnProgram,
|
||||
resolveWindowsExecutablePath,
|
||||
resolveWindowsSpawnProgram,
|
||||
resolveWindowsSpawnProgramCandidate,
|
||||
} from "../../../../src/plugin-sdk/windows-spawn.js";
|
||||
export type {
|
||||
ResolveWindowsSpawnProgramCandidateParams,
|
||||
ResolveWindowsSpawnProgramParams,
|
||||
WindowsSpawnCandidateResolution,
|
||||
WindowsSpawnInvocation,
|
||||
WindowsSpawnProgram,
|
||||
WindowsSpawnProgramCandidate,
|
||||
WindowsSpawnResolution,
|
||||
} from "../../../../src/plugin-sdk/windows-spawn.js";
|
||||
export { resolveGlobalSingleton } from "../../../../src/shared/global-singleton.js";
|
||||
242
packages/memory-host-sdk/src/host/post-json.test.ts
Normal file
242
packages/memory-host-sdk/src/host/post-json.test.ts
Normal file
@@ -0,0 +1,242 @@
|
||||
// Memory Host SDK tests cover post json behavior.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { postJson } from "./post-json.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
vi.mock("./remote-http.js", () => ({
|
||||
withRemoteHttpResponse: vi.fn(),
|
||||
}));
|
||||
|
||||
const remoteHttpMock = vi.mocked(withRemoteHttpResponse);
|
||||
|
||||
function jsonResponse(payload: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(payload), { status });
|
||||
}
|
||||
|
||||
function textResponse(body: string, status: number): Response {
|
||||
return new Response(body, { status });
|
||||
}
|
||||
|
||||
function streamingTextResponse(params: {
|
||||
body: string;
|
||||
status: number;
|
||||
headers?: HeadersInit;
|
||||
onCancel: () => void;
|
||||
}): Response {
|
||||
const encoded = new TextEncoder().encode(params.body);
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoded);
|
||||
},
|
||||
cancel() {
|
||||
params.onCancel();
|
||||
},
|
||||
});
|
||||
return new Response(stream, { status: params.status, headers: params.headers });
|
||||
}
|
||||
|
||||
function stallingSuccessResponse(onCancel: () => void): Response {
|
||||
const reader = {
|
||||
read: () => new Promise<ReadableStreamReadResult<Uint8Array>>(() => {}),
|
||||
cancel: async () => {
|
||||
onCancel();
|
||||
},
|
||||
releaseLock: () => undefined,
|
||||
} as ReadableStreamDefaultReader<Uint8Array>;
|
||||
|
||||
return {
|
||||
body: { getReader: () => reader },
|
||||
headers: new Headers(),
|
||||
ok: true,
|
||||
status: 200,
|
||||
} as Response;
|
||||
}
|
||||
|
||||
describe("postJson", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("parses JSON payload on successful response", async () => {
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(jsonResponse({ data: [{ embedding: [1, 2] }] }));
|
||||
});
|
||||
|
||||
const result = await postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
body: { input: ["x"] },
|
||||
errorPrefix: "post failed",
|
||||
parse: (payload) => payload,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ data: [{ embedding: [1, 2] }] });
|
||||
});
|
||||
|
||||
it("forwards abort signals to the remote HTTP request", async () => {
|
||||
const controller = new AbortController();
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
expect(params.signal).toBe(controller.signal);
|
||||
return await params.onResponse(jsonResponse({ ok: true }));
|
||||
});
|
||||
|
||||
await postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: {},
|
||||
body: {},
|
||||
signal: controller.signal,
|
||||
errorPrefix: "post failed",
|
||||
parse: (payload) => payload,
|
||||
});
|
||||
});
|
||||
|
||||
it("applies abort signals while reading successful response bodies", async () => {
|
||||
let canceled = false;
|
||||
const controller = new AbortController();
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
stallingSuccessResponse(() => {
|
||||
canceled = true;
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
const read = postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: {},
|
||||
body: {},
|
||||
signal: controller.signal,
|
||||
errorPrefix: "post failed",
|
||||
parse: () => ({}),
|
||||
});
|
||||
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 0);
|
||||
});
|
||||
controller.abort(new Error("body aborted"));
|
||||
|
||||
await expect(read).rejects.toThrow("body aborted");
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
|
||||
it("attaches status to thrown error when requested", async () => {
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(textResponse("bad gateway", 502));
|
||||
});
|
||||
|
||||
let error: unknown;
|
||||
try {
|
||||
await postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: {},
|
||||
body: {},
|
||||
errorPrefix: "post failed",
|
||||
attachStatus: true,
|
||||
parse: () => ({}),
|
||||
});
|
||||
} catch (caught) {
|
||||
error = caught;
|
||||
}
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as Error).message).toBe("post failed: 502 bad gateway");
|
||||
expect((error as { status?: unknown }).status).toBe(502);
|
||||
});
|
||||
|
||||
it("bounds non-ok response bodies before formatting the error", async () => {
|
||||
let canceled = false;
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
streamingTextResponse({
|
||||
body: "x".repeat(12_000),
|
||||
status: 502,
|
||||
onCancel: () => {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
await expect(
|
||||
postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: {},
|
||||
body: {},
|
||||
errorPrefix: "post failed",
|
||||
parse: () => ({}),
|
||||
}),
|
||||
).rejects.toThrow(`post failed: 502 ${"x".repeat(1_000)}... [truncated]`);
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
|
||||
it("wraps malformed success JSON with the request error prefix", async () => {
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(textResponse("{ nope", 200));
|
||||
});
|
||||
|
||||
await expect(
|
||||
postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: {},
|
||||
body: {},
|
||||
errorPrefix: "post failed",
|
||||
parse: () => ({}),
|
||||
}),
|
||||
).rejects.toThrow("post failed: malformed JSON response");
|
||||
});
|
||||
|
||||
it("rejects successful JSON responses with oversized content-length", async () => {
|
||||
let canceled = false;
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
streamingTextResponse({
|
||||
body: "{}",
|
||||
status: 200,
|
||||
headers: { "content-length": "32" },
|
||||
onCancel: () => {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
await expect(
|
||||
postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: {},
|
||||
body: {},
|
||||
errorPrefix: "post failed",
|
||||
maxResponseBytes: 8,
|
||||
parse: () => ({}),
|
||||
}),
|
||||
).rejects.toThrow("post failed: response body too large: 32 bytes (limit: 8 bytes)");
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
|
||||
it("cancels successful JSON responses that exceed the streaming byte cap", async () => {
|
||||
let canceled = false;
|
||||
remoteHttpMock.mockImplementationOnce(async (params) => {
|
||||
return await params.onResponse(
|
||||
streamingTextResponse({
|
||||
body: `{"data":"${"x".repeat(32)}"}`,
|
||||
status: 200,
|
||||
onCancel: () => {
|
||||
canceled = true;
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
await expect(
|
||||
postJson({
|
||||
url: "https://memory.example/v1/post",
|
||||
headers: {},
|
||||
body: {},
|
||||
errorPrefix: "post failed",
|
||||
maxResponseBytes: 16,
|
||||
parse: () => ({}),
|
||||
}),
|
||||
).rejects.toThrow("post failed: response body too large");
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
});
|
||||
50
packages/memory-host-sdk/src/host/post-json.ts
Normal file
50
packages/memory-host-sdk/src/host/post-json.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
// Memory Host SDK module implements post json behavior.
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
import { readResponseJsonWithLimit, readResponseTextSnippet } from "./response-snippet.js";
|
||||
import type { SsrFPolicy } from "./ssrf-policy.js";
|
||||
|
||||
// Shared JSON POST helper for guarded remote memory provider calls.
|
||||
|
||||
/** POST JSON, parse bounded response JSON, and attach status metadata when requested. */
|
||||
export async function postJson<T>(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
signal?: AbortSignal;
|
||||
body: unknown;
|
||||
errorPrefix: string;
|
||||
attachStatus?: boolean;
|
||||
maxResponseBytes?: number;
|
||||
parse: (payload: unknown) => T | Promise<T>;
|
||||
}): Promise<T> {
|
||||
return await withRemoteHttpResponse({
|
||||
url: params.url,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
fetchImpl: params.fetchImpl,
|
||||
signal: params.signal,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: params.headers,
|
||||
body: JSON.stringify(params.body),
|
||||
},
|
||||
onResponse: async (res) => {
|
||||
if (!res.ok) {
|
||||
const text = await readResponseTextSnippet(res, { signal: params.signal });
|
||||
const err = new Error(`${params.errorPrefix}: ${res.status} ${text}`) as Error & {
|
||||
status?: number;
|
||||
};
|
||||
if (params.attachStatus) {
|
||||
err.status = res.status;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const payload = await readResponseJsonWithLimit(res, {
|
||||
errorPrefix: params.errorPrefix,
|
||||
maxBytes: params.maxResponseBytes,
|
||||
signal: params.signal,
|
||||
});
|
||||
return await params.parse(payload);
|
||||
},
|
||||
});
|
||||
}
|
||||
143
packages/memory-host-sdk/src/host/qmd-process.real.test.ts
Normal file
143
packages/memory-host-sdk/src/host/qmd-process.real.test.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
// Memory Host SDK real-process tests cover QMD process-tree cleanup.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runCliCommand } from "./qmd-process.js";
|
||||
|
||||
type ProcessTreePids = {
|
||||
parent: number;
|
||||
grandchild: number;
|
||||
};
|
||||
|
||||
function isProcessRunning(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && "code" in error && error.code === "ESRCH") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function waitUntil(params: {
|
||||
condition: () => boolean | Promise<boolean>;
|
||||
description: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<void> {
|
||||
const deadline = Date.now() + (params.timeoutMs ?? 5_000);
|
||||
while (!(await params.condition())) {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(`timed out waiting for ${params.description}`);
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, 20);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function readProcessTreePids(pidFile: string): Promise<ProcessTreePids> {
|
||||
let pids: ProcessTreePids | undefined;
|
||||
await waitUntil({
|
||||
description: "the process-tree PID file",
|
||||
condition: async () => {
|
||||
try {
|
||||
pids = JSON.parse(await fs.readFile(pidFile, "utf8")) as ProcessTreePids;
|
||||
return Number.isInteger(pids.parent) && Number.isInteger(pids.grandchild);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
if (!pids) {
|
||||
throw new Error("process-tree PID file was not populated");
|
||||
}
|
||||
return pids;
|
||||
}
|
||||
|
||||
function killProcessTree(parentPid: number): void {
|
||||
if (process.platform === "win32") {
|
||||
const systemRoot = process.env.SystemRoot ?? process.env.WINDIR ?? "C:\\Windows";
|
||||
spawnSync(
|
||||
path.win32.join(systemRoot, "System32", "taskkill.exe"),
|
||||
["/PID", String(parentPid), "/T", "/F"],
|
||||
{ stdio: "ignore", windowsHide: true },
|
||||
);
|
||||
return;
|
||||
}
|
||||
// The production abort path already force-kills the group. Cleanup after a
|
||||
// failed assertion starts gracefully so it cannot kill a reused group id.
|
||||
process.kill(-parentPid, "SIGTERM");
|
||||
}
|
||||
|
||||
describe("runCliCommand real process lifecycle", () => {
|
||||
it("kills the command and its descendant when the caller aborts", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qmd-abort-"));
|
||||
const pidFile = path.join(tempDir, "pids.json");
|
||||
const controller = new AbortController();
|
||||
const abortError = new Error("memory_search timed out after 15s");
|
||||
let pending: ReturnType<typeof runCliCommand> | undefined;
|
||||
let pids: ProcessTreePids | undefined;
|
||||
|
||||
const childScript = `
|
||||
const { spawn } = require("node:child_process");
|
||||
const { renameSync, writeFileSync } = require("node:fs");
|
||||
const grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
const pidFile = process.argv[1];
|
||||
const temporaryPidFile = pidFile + ".tmp";
|
||||
writeFileSync(temporaryPidFile, JSON.stringify({
|
||||
parent: process.pid,
|
||||
grandchild: grandchild.pid,
|
||||
}));
|
||||
renameSync(temporaryPidFile, pidFile);
|
||||
setInterval(() => {}, 1000);
|
||||
`;
|
||||
|
||||
try {
|
||||
pending = runCliCommand({
|
||||
commandSummary: "real qmd process-tree fixture",
|
||||
spawnInvocation: { command: process.execPath, argv: ["-e", childScript, pidFile] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
timeoutMs: 60_000,
|
||||
maxOutputChars: 10_000,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
pids = await readProcessTreePids(pidFile);
|
||||
expect(isProcessRunning(pids.parent)).toBe(true);
|
||||
expect(isProcessRunning(pids.grandchild)).toBe(true);
|
||||
|
||||
controller.abort(abortError);
|
||||
await expect(pending).rejects.toBe(abortError);
|
||||
await waitUntil({
|
||||
description: "the detached process tree to exit",
|
||||
condition: () =>
|
||||
pids !== undefined &&
|
||||
!isProcessRunning(pids.parent) &&
|
||||
!isProcessRunning(pids.grandchild),
|
||||
});
|
||||
} finally {
|
||||
if (!controller.signal.aborted) {
|
||||
controller.abort(abortError);
|
||||
}
|
||||
await pending?.catch(() => undefined);
|
||||
if (pids?.parent && isProcessRunning(pids.parent)) {
|
||||
try {
|
||||
killProcessTree(pids.parent);
|
||||
} catch {
|
||||
// Best-effort cleanup when an assertion failed after the child exited.
|
||||
}
|
||||
}
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 15_000);
|
||||
});
|
||||
527
packages/memory-host-sdk/src/host/qmd-process.test.ts
Normal file
527
packages/memory-host-sdk/src/host/qmd-process.test.ts
Normal file
@@ -0,0 +1,527 @@
|
||||
// Memory Host SDK tests cover qmd process behavior.
|
||||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
type MockInstance,
|
||||
} from "vitest";
|
||||
import { MAX_SAFE_TIMEOUT_DELAY_MS } from "../../../gateway-client/src/timeouts.js";
|
||||
|
||||
const spawnMock = vi.hoisted(() => vi.fn());
|
||||
const spawnSyncMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("node:child_process", async () => {
|
||||
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
return {
|
||||
...actual,
|
||||
spawn: spawnMock,
|
||||
spawnSync: spawnSyncMock,
|
||||
};
|
||||
});
|
||||
|
||||
import {
|
||||
checkQmdBinaryAvailability,
|
||||
resolveCliSpawnInvocation,
|
||||
resolveQmdBinaryUnavailableReason,
|
||||
runCliCommand,
|
||||
type QmdBinaryAvailability,
|
||||
} from "./qmd-process.js";
|
||||
|
||||
function createMockChild(params: { pid?: number } = {}) {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
pid?: number;
|
||||
stdout: EventEmitter;
|
||||
stderr: EventEmitter;
|
||||
kill: ReturnType<typeof vi.fn>;
|
||||
closeWith: (code?: number | null, signal?: NodeJS.Signals | null) => void;
|
||||
};
|
||||
child.pid = params.pid;
|
||||
child.stdout = new EventEmitter();
|
||||
child.stderr = new EventEmitter();
|
||||
child.kill = vi.fn();
|
||||
child.closeWith = (code: number | null = 0, signal: NodeJS.Signals | null = null) => {
|
||||
child.emit("close", code, signal);
|
||||
};
|
||||
return child;
|
||||
}
|
||||
|
||||
let fixtureRoot = "";
|
||||
let tempDir = "";
|
||||
let platformSpy: MockInstance<() => NodeJS.Platform> | null = null;
|
||||
let fixtureId = 0;
|
||||
const originalPath = process.env.PATH;
|
||||
const originalPathExt = process.env.PATHEXT;
|
||||
const originalSystemRoot = process.env.SystemRoot;
|
||||
const originalWindir = process.env.WINDIR;
|
||||
const taskkillPath = path.win32.join("C:\\Windows", "System32", "taskkill.exe");
|
||||
|
||||
function restoreEnvValue(key: string, value: string | undefined): void {
|
||||
if (value === undefined) {
|
||||
delete process.env[key];
|
||||
return;
|
||||
}
|
||||
process.env[key] = value;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qmd-win-spawn-"));
|
||||
platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
platformSpy?.mockRestore();
|
||||
platformSpy = null;
|
||||
if (fixtureRoot) {
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = path.join(fixtureRoot, `case-${fixtureId++}`);
|
||||
await fs.mkdir(tempDir, { recursive: true });
|
||||
process.env.SystemRoot = "C:\\Windows";
|
||||
delete process.env.WINDIR;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
process.env.PATH = originalPath;
|
||||
process.env.PATHEXT = originalPathExt;
|
||||
restoreEnvValue("SystemRoot", originalSystemRoot);
|
||||
restoreEnvValue("WINDIR", originalWindir);
|
||||
platformSpy?.mockReturnValue("win32");
|
||||
spawnMock.mockReset();
|
||||
spawnSyncMock.mockReset();
|
||||
spawnSyncMock.mockReturnValue({ status: 0 });
|
||||
tempDir = "";
|
||||
});
|
||||
|
||||
describe("resolveCliSpawnInvocation", () => {
|
||||
it("unwraps npm cmd shims to a direct node entrypoint", async () => {
|
||||
const binDir = path.join(tempDir, "node_modules", ".bin");
|
||||
const packageDir = path.join(tempDir, "node_modules", "qmd");
|
||||
const scriptPath = path.join(packageDir, "dist", "cli.js");
|
||||
await fs.mkdir(path.dirname(scriptPath), { recursive: true });
|
||||
await fs.mkdir(binDir, { recursive: true });
|
||||
await fs.writeFile(path.join(binDir, "qmd.cmd"), "@echo off\r\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(packageDir, "package.json"),
|
||||
JSON.stringify({ name: "qmd", version: "0.0.0", bin: { qmd: "dist/cli.js" } }),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(scriptPath, "module.exports = {};\n", "utf8");
|
||||
|
||||
process.env.PATH = `${binDir};${originalPath ?? ""}`;
|
||||
process.env.PATHEXT = ".CMD;.EXE";
|
||||
|
||||
const invocation = resolveCliSpawnInvocation({
|
||||
command: "qmd",
|
||||
args: ["query", "hello"],
|
||||
env: process.env,
|
||||
packageName: "qmd",
|
||||
});
|
||||
|
||||
expect(invocation.command).toBe(process.execPath);
|
||||
expect(invocation.argv).toEqual([scriptPath, "query", "hello"]);
|
||||
expect(invocation.shell).not.toBe(true);
|
||||
expect(invocation.windowsHide).toBe(true);
|
||||
});
|
||||
|
||||
it("fails closed when a Windows cmd shim cannot be resolved without shell execution", async () => {
|
||||
const binDir = path.join(tempDir, "bad-bin");
|
||||
await fs.mkdir(binDir, { recursive: true });
|
||||
await fs.writeFile(path.join(binDir, "qmd.cmd"), "@echo off\r\nREM no entrypoint\r\n", "utf8");
|
||||
|
||||
process.env.PATH = `${binDir};${originalPath ?? ""}`;
|
||||
process.env.PATHEXT = ".CMD;.EXE";
|
||||
|
||||
expect(() =>
|
||||
resolveCliSpawnInvocation({
|
||||
command: "qmd",
|
||||
args: ["query", "hello"],
|
||||
env: process.env,
|
||||
packageName: "qmd",
|
||||
}),
|
||||
).toThrow(/without shell execution/);
|
||||
});
|
||||
|
||||
it("keeps bare commands bare when no Windows wrapper exists on PATH", () => {
|
||||
process.env.PATH = originalPath ?? "";
|
||||
process.env.PATHEXT = ".CMD;.EXE";
|
||||
|
||||
const invocation = resolveCliSpawnInvocation({
|
||||
command: "qmd",
|
||||
args: ["query", "hello"],
|
||||
env: process.env,
|
||||
packageName: "qmd",
|
||||
});
|
||||
|
||||
expect(invocation.command).toBe("qmd");
|
||||
expect(invocation.argv).toEqual(["query", "hello"]);
|
||||
expect(invocation.shell).not.toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("checkQmdBinaryAvailability", () => {
|
||||
it("keeps legacy unavailable probe results source-compatible", () => {
|
||||
const legacyUnavailable: QmdBinaryAvailability = {
|
||||
available: false,
|
||||
error: "spawn qmd ENOENT",
|
||||
};
|
||||
|
||||
expect(resolveQmdBinaryUnavailableReason(legacyUnavailable)).toBe("binary");
|
||||
});
|
||||
|
||||
it("returns available when the qmd process spawns successfully", async () => {
|
||||
const child = createMockChild({ pid: 12344 });
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => child.emit("spawn"));
|
||||
return child;
|
||||
});
|
||||
|
||||
await expect(
|
||||
checkQmdBinaryAvailability({ command: "qmd", env: process.env, cwd: tempDir }),
|
||||
).resolves.toEqual({ available: true });
|
||||
expect(spawnSyncMock).toHaveBeenCalledWith(taskkillPath, ["/PID", String(child.pid), "/T"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
expect(child.kill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("force-kills Windows availability probes when graceful taskkill fails", async () => {
|
||||
const child = createMockChild({ pid: 12345 });
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => child.emit("spawn"));
|
||||
return child;
|
||||
});
|
||||
spawnSyncMock.mockReset();
|
||||
spawnSyncMock.mockReturnValueOnce({ status: 1 }).mockReturnValueOnce({ status: 0 });
|
||||
|
||||
await expect(
|
||||
checkQmdBinaryAvailability({ command: "qmd", env: process.env, cwd: tempDir }),
|
||||
).resolves.toEqual({ available: true });
|
||||
|
||||
expect(spawnSyncMock).toHaveBeenNthCalledWith(1, taskkillPath, ["/PID", "12345", "/T"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
expect(spawnSyncMock).toHaveBeenNthCalledWith(2, taskkillPath, ["/PID", "12345", "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
expect(child.kill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns unavailable when the qmd process cannot be spawned", async () => {
|
||||
const child = createMockChild();
|
||||
const err = Object.assign(new Error("spawn qmd ENOENT"), { code: "ENOENT" });
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => child.emit("error", err));
|
||||
return child;
|
||||
});
|
||||
|
||||
await expect(
|
||||
checkQmdBinaryAvailability({ command: "qmd", env: process.env, cwd: tempDir }),
|
||||
).resolves.toEqual({ available: false, reason: "binary", error: "spawn qmd ENOENT" });
|
||||
});
|
||||
|
||||
it("returns an explicit workspace error when cwd is missing", async () => {
|
||||
const missingDir = path.join(tempDir, "missing-workspace");
|
||||
|
||||
await expect(
|
||||
checkQmdBinaryAvailability({ command: "qmd", env: process.env, cwd: missingDir }),
|
||||
).resolves.toEqual({
|
||||
available: false,
|
||||
reason: "workspace-cwd",
|
||||
error: `workspace directory missing: ${missingDir}`,
|
||||
});
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not treat close-before-spawn as a successful availability probe", async () => {
|
||||
const child = createMockChild();
|
||||
const err = Object.assign(new Error("spawn qmd ENOENT"), { code: "ENOENT" });
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => child.emit("close"));
|
||||
queueMicrotask(() => child.emit("error", err));
|
||||
return child;
|
||||
});
|
||||
|
||||
await expect(
|
||||
checkQmdBinaryAvailability({ command: "qmd", env: process.env, cwd: tempDir }),
|
||||
).resolves.toEqual({ available: false, reason: "binary", error: "spawn qmd ENOENT" });
|
||||
});
|
||||
|
||||
it("caps oversized availability probe timeouts before scheduling", async () => {
|
||||
vi.useFakeTimers();
|
||||
const child = createMockChild();
|
||||
const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
spawnMock.mockReturnValueOnce(child);
|
||||
|
||||
void checkQmdBinaryAvailability({
|
||||
command: "qmd",
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
timeoutMs: Number.MAX_SAFE_INTEGER,
|
||||
});
|
||||
|
||||
expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_SAFE_TIMEOUT_DELAY_MS);
|
||||
});
|
||||
it("kills timed-out availability probes by process group on POSIX", async () => {
|
||||
platformSpy?.mockReturnValue("linux");
|
||||
const killProcess = vi.spyOn(process, "kill").mockImplementation(() => true);
|
||||
const child = createMockChild({ pid: 4321 });
|
||||
spawnMock.mockReturnValueOnce(child);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
checkQmdBinaryAvailability({
|
||||
command: "qmd",
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
timeoutMs: 1,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
available: false,
|
||||
reason: "binary",
|
||||
error: "spawn qmd timed out after 1ms",
|
||||
});
|
||||
|
||||
expect(spawnMock.mock.calls[0]?.[2]).toMatchObject({ detached: true });
|
||||
expect(killProcess).toHaveBeenCalledWith(-4321, "SIGKILL");
|
||||
expect(child.kill).not.toHaveBeenCalledWith("SIGKILL");
|
||||
} finally {
|
||||
killProcess.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("runCliCommand", () => {
|
||||
it("keeps stdout and stderr on non-zero exits", async () => {
|
||||
const child = createMockChild();
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
child.stdout.emit("data", '[{"docid":"abc","score":0.93}]');
|
||||
child.stderr.emit("data", "ggml-metal-device.m:612");
|
||||
child.closeWith(134);
|
||||
});
|
||||
return child;
|
||||
});
|
||||
|
||||
try {
|
||||
await runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 10_000,
|
||||
});
|
||||
throw new Error("expected runCliCommand to reject");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
if (!(err instanceof Error)) {
|
||||
throw err;
|
||||
}
|
||||
expect(err.name).toBe("CliCommandError");
|
||||
expect(err).toMatchObject({
|
||||
code: 134,
|
||||
signal: null,
|
||||
stdout: '[{"docid":"abc","score":0.93}]',
|
||||
stderr: "ggml-metal-device.m:612",
|
||||
});
|
||||
expect(err.message).toContain("qmd query test failed (code 134)");
|
||||
}
|
||||
});
|
||||
|
||||
it("records signal-only command failures", async () => {
|
||||
const child = createMockChild();
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
child.stdout.emit("data", "[]");
|
||||
child.closeWith(null, "SIGABRT");
|
||||
});
|
||||
return child;
|
||||
});
|
||||
|
||||
await expect(
|
||||
runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 10_000,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
code: null,
|
||||
signal: "SIGABRT",
|
||||
stdout: "[]",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not expose truncated output as a recoverable command failure", async () => {
|
||||
const child = createMockChild();
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
child.stdout.emit("data", "too much output");
|
||||
child.closeWith(1);
|
||||
});
|
||||
return child;
|
||||
});
|
||||
|
||||
await expect(
|
||||
runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 4,
|
||||
}),
|
||||
).rejects.toThrow(/produced too much output/);
|
||||
});
|
||||
|
||||
it("counts surrogate pairs as one character when capping failed command output", async () => {
|
||||
const child = createMockChild();
|
||||
spawnMock.mockImplementationOnce(() => {
|
||||
queueMicrotask(() => {
|
||||
child.stderr.emit("data", "a🙂");
|
||||
child.closeWith(1);
|
||||
});
|
||||
return child;
|
||||
});
|
||||
|
||||
await expect(
|
||||
runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 2,
|
||||
}),
|
||||
).rejects.toThrow(/🙂/);
|
||||
});
|
||||
|
||||
it("caps oversized command timeouts before scheduling", async () => {
|
||||
vi.useFakeTimers();
|
||||
const child = createMockChild();
|
||||
const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
spawnMock.mockReturnValueOnce(child);
|
||||
|
||||
void runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 10_000,
|
||||
timeoutMs: Number.MAX_SAFE_INTEGER,
|
||||
}).catch(() => undefined);
|
||||
|
||||
expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_SAFE_TIMEOUT_DELAY_MS);
|
||||
});
|
||||
|
||||
it("kills aborted cli command process groups on POSIX and rejects with the abort reason", async () => {
|
||||
platformSpy?.mockReturnValue("linux");
|
||||
const killProcess = vi.spyOn(process, "kill").mockImplementation(() => true);
|
||||
const child = createMockChild({ pid: 7654 });
|
||||
spawnMock.mockReturnValueOnce(child);
|
||||
const controller = new AbortController();
|
||||
|
||||
try {
|
||||
const pending = runCliCommand({
|
||||
commandSummary: "qmd query slow",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "slow", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 10_000,
|
||||
timeoutMs: 60_000,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
controller.abort(new Error("memory_search timed out after 15s"));
|
||||
|
||||
await expect(pending).rejects.toThrow("memory_search timed out after 15s");
|
||||
expect(spawnMock.mock.calls[0]?.[2]).toMatchObject({ detached: true });
|
||||
expect(killProcess).toHaveBeenCalledWith(-7654, "SIGKILL");
|
||||
expect(child.kill).not.toHaveBeenCalledWith("SIGKILL");
|
||||
} finally {
|
||||
killProcess.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects immediately without spawning when the signal is already aborted", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort(new Error("memory_search timed out after 15s"));
|
||||
|
||||
await expect(
|
||||
runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 10_000,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow("memory_search timed out after 15s");
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("kills timed-out cli command process groups on POSIX", async () => {
|
||||
platformSpy?.mockReturnValue("linux");
|
||||
const killProcess = vi.spyOn(process, "kill").mockImplementation(() => true);
|
||||
const child = createMockChild({ pid: 8765 });
|
||||
spawnMock.mockReturnValueOnce(child);
|
||||
|
||||
try {
|
||||
const pending = runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 10_000,
|
||||
timeoutMs: 1,
|
||||
});
|
||||
const timeoutAssertion = expect(pending).rejects.toThrow(
|
||||
"qmd query test timed out after 1ms",
|
||||
);
|
||||
|
||||
await timeoutAssertion;
|
||||
|
||||
expect(spawnMock.mock.calls[0]?.[2]).toMatchObject({ detached: true });
|
||||
expect(killProcess).toHaveBeenCalledWith(-8765, "SIGKILL");
|
||||
expect(child.kill).not.toHaveBeenCalledWith("SIGKILL");
|
||||
} finally {
|
||||
killProcess.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("force-kills timed-out Windows cli commands with taskkill", async () => {
|
||||
const child = createMockChild({ pid: 12346 });
|
||||
spawnMock.mockReturnValueOnce(child);
|
||||
|
||||
const pending = runCliCommand({
|
||||
commandSummary: "qmd query test",
|
||||
spawnInvocation: { command: "qmd", argv: ["query", "test", "--json"] },
|
||||
env: process.env,
|
||||
cwd: tempDir,
|
||||
maxOutputChars: 10_000,
|
||||
timeoutMs: 1,
|
||||
});
|
||||
|
||||
await expect(pending).rejects.toThrow("qmd query test timed out after 1ms");
|
||||
|
||||
expect(spawnSyncMock).toHaveBeenCalledWith(taskkillPath, ["/PID", "12346", "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
expect(child.kill).not.toHaveBeenCalledWith("SIGKILL");
|
||||
});
|
||||
});
|
||||
425
packages/memory-host-sdk/src/host/qmd-process.ts
Normal file
425
packages/memory-host-sdk/src/host/qmd-process.ts
Normal file
@@ -0,0 +1,425 @@
|
||||
// Memory Host SDK module implements qmd process behavior.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolveSafeTimeoutDelayMs } from "../../../gateway-client/src/timeouts.js";
|
||||
import { materializeWindowsSpawnProgram, resolveWindowsSpawnProgram } from "./windows-spawn.js";
|
||||
|
||||
export type CliSpawnInvocation = {
|
||||
command: string;
|
||||
argv: string[];
|
||||
shell?: boolean;
|
||||
windowsHide?: boolean;
|
||||
};
|
||||
|
||||
type QmdChildProcess = {
|
||||
pid?: number;
|
||||
kill: (signal?: NodeJS.Signals) => boolean;
|
||||
};
|
||||
|
||||
const DEFAULT_WINDOWS_SYSTEM_ROOT = "C:\\Windows";
|
||||
|
||||
export type QmdBinaryUnavailableReason = "binary" | "workspace-cwd";
|
||||
|
||||
export type QmdBinaryUnavailable = {
|
||||
available: false;
|
||||
/**
|
||||
* Optional for source compatibility with older plugin SDK callers that
|
||||
* returned only `{ available: false, error }`.
|
||||
*/
|
||||
reason?: QmdBinaryUnavailableReason;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export type QmdBinaryAvailability = { available: true } | QmdBinaryUnavailable;
|
||||
|
||||
export function resolveQmdBinaryUnavailableReason(
|
||||
result: QmdBinaryUnavailable,
|
||||
): QmdBinaryUnavailableReason {
|
||||
return result.reason ?? "binary";
|
||||
}
|
||||
|
||||
export function resolveCliSpawnInvocation(params: {
|
||||
command: string;
|
||||
args: string[];
|
||||
env: NodeJS.ProcessEnv;
|
||||
packageName: string;
|
||||
}): CliSpawnInvocation {
|
||||
const program = resolveWindowsSpawnProgram({
|
||||
command: params.command,
|
||||
platform: process.platform,
|
||||
env: params.env,
|
||||
execPath: process.execPath,
|
||||
packageName: params.packageName,
|
||||
allowShellFallback: false,
|
||||
});
|
||||
return materializeWindowsSpawnProgram(program, params.args);
|
||||
}
|
||||
|
||||
export async function checkQmdBinaryAvailability(params: {
|
||||
command: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
cwd?: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<QmdBinaryAvailability> {
|
||||
let spawnInvocation: CliSpawnInvocation;
|
||||
try {
|
||||
spawnInvocation = resolveCliSpawnInvocation({
|
||||
command: params.command,
|
||||
args: [],
|
||||
env: params.env,
|
||||
packageName: "qmd",
|
||||
});
|
||||
} catch (err) {
|
||||
return { available: false, reason: "binary", error: formatQmdAvailabilityError(err) };
|
||||
}
|
||||
|
||||
const cwd = params.cwd ?? process.cwd();
|
||||
const cwdError = validateQmdProbeCwd(cwd);
|
||||
if (cwdError) {
|
||||
return cwdError;
|
||||
}
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
let settled = false;
|
||||
let didSpawn = false;
|
||||
const finish = (result: QmdBinaryAvailability) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
const child = spawn(spawnInvocation.command, spawnInvocation.argv, {
|
||||
env: params.env,
|
||||
cwd,
|
||||
shell: spawnInvocation.shell,
|
||||
windowsHide: spawnInvocation.windowsHide,
|
||||
stdio: "ignore",
|
||||
detached: shouldUseQmdProcessGroup(),
|
||||
});
|
||||
const timeoutMs = resolveSafeTimeoutDelayMs(params.timeoutMs ?? 2_000, { minMs: 0 });
|
||||
const timer = setTimeout(() => {
|
||||
signalQmdProcessTree(child, "SIGKILL");
|
||||
finish({
|
||||
available: false,
|
||||
reason: "binary",
|
||||
error: `spawn ${params.command} timed out after ${timeoutMs}ms`,
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
||||
child.once("error", (err) => {
|
||||
finish({ available: false, reason: "binary", error: formatQmdAvailabilityError(err) });
|
||||
});
|
||||
child.once("spawn", () => {
|
||||
didSpawn = true;
|
||||
signalQmdProcessTree(child);
|
||||
finish({ available: true });
|
||||
});
|
||||
child.once("close", () => {
|
||||
if (!didSpawn) {
|
||||
return;
|
||||
}
|
||||
finish({ available: true });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function validateQmdProbeCwd(cwd: string): QmdBinaryAvailability | null {
|
||||
try {
|
||||
const stat = statSync(cwd);
|
||||
if (!stat.isDirectory()) {
|
||||
return {
|
||||
available: false,
|
||||
reason: "workspace-cwd",
|
||||
error: `workspace directory is not a directory: ${cwd}`,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
} catch (err) {
|
||||
if (typeof err === "object" && err && "code" in err && err.code === "ENOENT") {
|
||||
return {
|
||||
available: false,
|
||||
reason: "workspace-cwd",
|
||||
error: `workspace directory missing: ${cwd}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
available: false,
|
||||
reason: "workspace-cwd",
|
||||
error: `workspace directory unavailable: ${cwd} (${formatQmdAvailabilityError(err)})`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize an aborted signal into the error used to reject a killed command.
|
||||
* Prefers the caller-supplied abort reason (so a deadline message survives) and
|
||||
* falls back to a stable per-command abort error.
|
||||
*/
|
||||
function abortReason(signal: AbortSignal | undefined, commandSummary: string): Error {
|
||||
const reason = signal?.reason;
|
||||
if (reason instanceof Error) {
|
||||
return reason;
|
||||
}
|
||||
if (typeof reason === "string" && reason.length > 0) {
|
||||
return new Error(reason);
|
||||
}
|
||||
return new Error(`${commandSummary} aborted`);
|
||||
}
|
||||
|
||||
export async function runCliCommand(params: {
|
||||
commandSummary: string;
|
||||
spawnInvocation: CliSpawnInvocation;
|
||||
env: NodeJS.ProcessEnv;
|
||||
cwd: string;
|
||||
timeoutMs?: number;
|
||||
maxOutputChars: number;
|
||||
discardStdout?: boolean;
|
||||
/**
|
||||
* Caller-owned cancellation. When the signal aborts, the spawned child is
|
||||
* killed immediately and the call rejects, so a caller that already stopped
|
||||
* waiting (for example after its own deadline) does not leave an orphaned
|
||||
* process running for the full command timeout.
|
||||
*/
|
||||
signal?: AbortSignal;
|
||||
}): Promise<{ stdout: string; stderr: string }> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const { signal } = params;
|
||||
if (signal?.aborted) {
|
||||
reject(abortReason(signal, params.commandSummary));
|
||||
return;
|
||||
}
|
||||
const child = spawn(params.spawnInvocation.command, params.spawnInvocation.argv, {
|
||||
env: params.env,
|
||||
cwd: params.cwd,
|
||||
shell: params.spawnInvocation.shell,
|
||||
windowsHide: params.spawnInvocation.windowsHide,
|
||||
detached: shouldUseQmdProcessGroup(),
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let stdoutTruncated = false;
|
||||
let stderrTruncated = false;
|
||||
let settled = false;
|
||||
const discardStdout = params.discardStdout === true;
|
||||
const timeoutMs =
|
||||
params.timeoutMs === undefined ? undefined : resolveSafeTimeoutDelayMs(params.timeoutMs);
|
||||
const timer = timeoutMs
|
||||
? setTimeout(() => {
|
||||
signalQmdProcessTree(child, "SIGKILL");
|
||||
settle(() =>
|
||||
reject(new Error(`${params.commandSummary} timed out after ${timeoutMs}ms`)),
|
||||
);
|
||||
}, timeoutMs)
|
||||
: null;
|
||||
const onAbort = () => {
|
||||
signalQmdProcessTree(child, "SIGKILL");
|
||||
settle(() => reject(abortReason(signal, params.commandSummary)));
|
||||
};
|
||||
function settle(run: () => void): void {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
run();
|
||||
}
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
child.stdout.on("data", (data) => {
|
||||
if (discardStdout) {
|
||||
return;
|
||||
}
|
||||
const next = appendOutputWithCap(stdout, data.toString("utf8"), params.maxOutputChars);
|
||||
stdout = next.text;
|
||||
stdoutTruncated = stdoutTruncated || next.truncated;
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
const next = appendOutputWithCap(stderr, data.toString("utf8"), params.maxOutputChars);
|
||||
stderr = next.text;
|
||||
stderrTruncated = stderrTruncated || next.truncated;
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
settle(() => reject(err));
|
||||
});
|
||||
child.on("close", (code, closeSignal) => {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
settle(() => {
|
||||
if (!discardStdout && (stdoutTruncated || stderrTruncated)) {
|
||||
reject(
|
||||
new Error(
|
||||
`${params.commandSummary} produced too much output (limit ${params.maxOutputChars} chars)`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (code === 0) {
|
||||
resolve({ stdout, stderr });
|
||||
} else {
|
||||
reject(
|
||||
new CliCommandError({
|
||||
commandSummary: params.commandSummary,
|
||||
code,
|
||||
signal: closeSignal ?? null,
|
||||
stdout,
|
||||
stderr,
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function shouldUseQmdProcessGroup(): boolean {
|
||||
return process.platform !== "win32";
|
||||
}
|
||||
|
||||
function getEnvValueCaseInsensitive(
|
||||
env: Record<string, string | undefined>,
|
||||
expectedKey: string,
|
||||
): string | undefined {
|
||||
const direct = env[expectedKey];
|
||||
if (direct !== undefined) {
|
||||
return direct;
|
||||
}
|
||||
const expected = expectedKey.toUpperCase();
|
||||
const actualKey = Object.keys(env).find((key) => key.toUpperCase() === expected);
|
||||
return actualKey ? env[actualKey] : undefined;
|
||||
}
|
||||
|
||||
function normalizeWindowsSystemRoot(raw: string | undefined): string | null {
|
||||
const trimmed = raw?.trim();
|
||||
if (
|
||||
!trimmed ||
|
||||
trimmed.includes("\0") ||
|
||||
trimmed.includes("\r") ||
|
||||
trimmed.includes("\n") ||
|
||||
trimmed.includes(";")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const normalized = path.win32.normalize(trimmed);
|
||||
if (!path.win32.isAbsolute(normalized) || normalized.startsWith("\\\\")) {
|
||||
return null;
|
||||
}
|
||||
const parsed = path.win32.parse(normalized);
|
||||
if (!/^[A-Za-z]:\\$/.test(parsed.root) || normalized.length <= parsed.root.length) {
|
||||
return null;
|
||||
}
|
||||
return normalized.replace(/[\\/]+$/, "");
|
||||
}
|
||||
|
||||
function resolveWindowsTaskkillPath(env: Record<string, string | undefined> = process.env): string {
|
||||
const systemRoot =
|
||||
normalizeWindowsSystemRoot(getEnvValueCaseInsensitive(env, "SystemRoot")) ??
|
||||
normalizeWindowsSystemRoot(getEnvValueCaseInsensitive(env, "WINDIR")) ??
|
||||
DEFAULT_WINDOWS_SYSTEM_ROOT;
|
||||
return path.win32.join(systemRoot, "System32", "taskkill.exe");
|
||||
}
|
||||
|
||||
function signalQmdProcessTree(child: QmdChildProcess, signal?: NodeJS.Signals): void {
|
||||
if (shouldUseQmdProcessGroup() && typeof child.pid === "number") {
|
||||
try {
|
||||
if (signal === undefined) {
|
||||
process.kill(-child.pid);
|
||||
} else {
|
||||
process.kill(-child.pid, signal);
|
||||
}
|
||||
return;
|
||||
} catch {
|
||||
// Fall back to the direct child if the process group already disappeared.
|
||||
}
|
||||
}
|
||||
if (!shouldUseQmdProcessGroup() && typeof child.pid === "number") {
|
||||
const taskkillPath = resolveWindowsTaskkillPath();
|
||||
const args = ["/PID", String(child.pid), "/T"];
|
||||
if (signal === "SIGKILL") {
|
||||
args.push("/F");
|
||||
}
|
||||
const result = spawnSync(taskkillPath, args, { stdio: "ignore", windowsHide: true });
|
||||
if (!result.error && result.status === 0) {
|
||||
return;
|
||||
}
|
||||
if (signal !== "SIGKILL") {
|
||||
const forceResult = spawnSync(taskkillPath, [...args, "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
if (!forceResult.error && forceResult.status === 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (signal === undefined) {
|
||||
child.kill();
|
||||
} else {
|
||||
child.kill(signal);
|
||||
}
|
||||
}
|
||||
|
||||
class CliCommandError extends Error {
|
||||
readonly code: number | null;
|
||||
readonly signal: NodeJS.Signals | null;
|
||||
readonly stdout: string;
|
||||
readonly stderr: string;
|
||||
|
||||
constructor(params: {
|
||||
commandSummary: string;
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}) {
|
||||
super(formatCliCommandFailureMessage(params));
|
||||
this.name = "CliCommandError";
|
||||
this.code = params.code;
|
||||
this.signal = params.signal;
|
||||
this.stdout = params.stdout;
|
||||
this.stderr = params.stderr;
|
||||
}
|
||||
}
|
||||
|
||||
function formatCliCommandFailureMessage(params: {
|
||||
commandSummary: string;
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}): string {
|
||||
const exit =
|
||||
params.code === null ? `signal ${params.signal ?? "unknown"}` : `code ${String(params.code)}`;
|
||||
return `${params.commandSummary} failed (${exit}): ${params.stderr || params.stdout}`;
|
||||
}
|
||||
|
||||
function appendOutputWithCap(
|
||||
current: string,
|
||||
chunk: string,
|
||||
maxChars: number,
|
||||
): { text: string; truncated: boolean } {
|
||||
const appended = current + chunk;
|
||||
const chars = Array.from(appended);
|
||||
if (chars.length <= maxChars) {
|
||||
return { text: appended, truncated: false };
|
||||
}
|
||||
return { text: chars.slice(-maxChars).join(""), truncated: true };
|
||||
}
|
||||
|
||||
function formatQmdAvailabilityError(err: unknown): string {
|
||||
if (err instanceof Error && err.message) {
|
||||
return err.message;
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
74
packages/memory-host-sdk/src/host/qmd-query-parser.test.ts
Normal file
74
packages/memory-host-sdk/src/host/qmd-query-parser.test.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
// Memory Host SDK tests cover qmd query parser behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseQmdQueryJson } from "./qmd-query-parser.js";
|
||||
|
||||
describe("parseQmdQueryJson", () => {
|
||||
it("parses clean qmd JSON output", () => {
|
||||
const results = parseQmdQueryJson('[{"docid":"abc","score":1,"snippet":"@@ -1,1\\none"}]', "");
|
||||
expect(results).toEqual([
|
||||
{
|
||||
docid: "abc",
|
||||
score: 1,
|
||||
snippet: "@@ -1,1\none",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts embedded result arrays from noisy stdout", () => {
|
||||
const results = parseQmdQueryJson(
|
||||
`initializing
|
||||
{"payload":"ok"}
|
||||
[{"docid":"abc","score":0.5}]
|
||||
complete`,
|
||||
"",
|
||||
);
|
||||
expect(results).toEqual([{ docid: "abc", score: 0.5 }]);
|
||||
});
|
||||
|
||||
it("preserves explicit qmd line metadata when present", () => {
|
||||
const results = parseQmdQueryJson(
|
||||
'[{"docid":"abc","score":0.5,"start_line":4,"end_line":6,"snippet":"@@ -10,1\\nignored"}]',
|
||||
"",
|
||||
);
|
||||
expect(results).toEqual([
|
||||
{
|
||||
docid: "abc",
|
||||
score: 0.5,
|
||||
snippet: "@@ -10,1\nignored",
|
||||
startLine: 4,
|
||||
endLine: 6,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("drops non-integer qmd line metadata", () => {
|
||||
const results = parseQmdQueryJson(
|
||||
`[{"docid":"abc","start_line":4.5,"end_line":${Number.MAX_SAFE_INTEGER + 1}}]`,
|
||||
"",
|
||||
);
|
||||
|
||||
expect(results).toEqual([{ docid: "abc" }]);
|
||||
});
|
||||
|
||||
it("treats plain-text no-results from stderr as an empty result set", () => {
|
||||
const results = parseQmdQueryJson("", "No results found\n");
|
||||
expect(results).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("treats prefixed no-results marker output as an empty result set", () => {
|
||||
expect(parseQmdQueryJson("warning: no results found", "")).toStrictEqual([]);
|
||||
expect(parseQmdQueryJson("", "[qmd] warning: no results found\n")).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("does not treat arbitrary non-marker text as no-results output", () => {
|
||||
expect(() =>
|
||||
parseQmdQueryJson("warning: search completed; no results found for this query", ""),
|
||||
).toThrow(/qmd query returned invalid JSON/i);
|
||||
});
|
||||
|
||||
it("throws when stdout cannot be interpreted as qmd JSON", () => {
|
||||
expect(() => parseQmdQueryJson("this is not json", "")).toThrow(
|
||||
/qmd query returned invalid JSON/i,
|
||||
);
|
||||
});
|
||||
});
|
||||
169
packages/memory-host-sdk/src/host/qmd-query-parser.ts
Normal file
169
packages/memory-host-sdk/src/host/qmd-query-parser.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
// Memory Host SDK module implements qmd query parser behavior.
|
||||
import { formatErrorMessage } from "./error-utils.js";
|
||||
import { normalizeLowercaseStringOrEmpty } from "./string-utils.js";
|
||||
|
||||
// Parser for qmd query JSON output, including noisy CLI wrapper output.
|
||||
|
||||
/** Normalized qmd query result consumed by memory search. */
|
||||
export type QmdQueryResult = {
|
||||
docid?: string;
|
||||
score?: number;
|
||||
collection?: string;
|
||||
file?: string;
|
||||
snippet?: string;
|
||||
body?: string;
|
||||
startLine?: number;
|
||||
endLine?: number;
|
||||
};
|
||||
|
||||
/** Parse qmd stdout/stderr into normalized results, accepting known no-result markers. */
|
||||
export function parseQmdQueryJson(stdout: string, stderr: string): QmdQueryResult[] {
|
||||
const trimmedStdout = stdout.trim();
|
||||
const trimmedStderr = stderr.trim();
|
||||
const stdoutIsMarker = trimmedStdout.length > 0 && isQmdNoResultsOutput(trimmedStdout);
|
||||
const stderrIsMarker = trimmedStderr.length > 0 && isQmdNoResultsOutput(trimmedStderr);
|
||||
if (stdoutIsMarker || (!trimmedStdout && stderrIsMarker)) {
|
||||
return [];
|
||||
}
|
||||
if (!trimmedStdout) {
|
||||
const context = trimmedStderr ? ` (stderr: ${summarizeQmdStderr(trimmedStderr)})` : "";
|
||||
const message = `stdout empty${context}`;
|
||||
warnQmdQueryParseError(message);
|
||||
throw new Error(`qmd query returned invalid JSON: ${message}`);
|
||||
}
|
||||
try {
|
||||
const parsed = parseQmdQueryResultArray(trimmedStdout);
|
||||
if (parsed !== null) {
|
||||
return parsed;
|
||||
}
|
||||
const noisyPayload = extractFirstJsonArray(trimmedStdout);
|
||||
if (!noisyPayload) {
|
||||
throw new Error("qmd query JSON response was not an array");
|
||||
}
|
||||
const fallback = parseQmdQueryResultArray(noisyPayload);
|
||||
if (fallback !== null) {
|
||||
return fallback;
|
||||
}
|
||||
throw new Error("qmd query JSON response was not an array");
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
warnQmdQueryParseError(message);
|
||||
throw new Error(`qmd query returned invalid JSON: ${message}`, { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
/** Emit parse warnings outside tests so broken qmd output is visible to operators. */
|
||||
function warnQmdQueryParseError(message: string): void {
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
return;
|
||||
}
|
||||
process.stderr.write(`qmd query returned invalid JSON: ${message}\n`);
|
||||
}
|
||||
|
||||
/** Detect qmd no-result marker output on stdout or stderr. */
|
||||
function isQmdNoResultsOutput(raw: string): boolean {
|
||||
const lines = raw
|
||||
.split(/\r?\n/)
|
||||
.map((line) => normalizeLowercaseStringOrEmpty(line).replace(/\s+/g, " "))
|
||||
.filter((line) => line.length > 0);
|
||||
return lines.some((line) => isQmdNoResultsLine(line));
|
||||
}
|
||||
|
||||
/** Match qmd no-result lines with optional warning/info prefixes. */
|
||||
function isQmdNoResultsLine(line: string): boolean {
|
||||
if (line === "no results found" || line === "no results found.") {
|
||||
return true;
|
||||
}
|
||||
return /^(?:\[[^\]]+\]\s*)?(?:(?:warn(?:ing)?|info|error|qmd)\s*:\s*)+no results found\.?$/.test(
|
||||
line,
|
||||
);
|
||||
}
|
||||
|
||||
/** Bound stderr context included in parse errors. */
|
||||
function summarizeQmdStderr(raw: string): string {
|
||||
return raw.length <= 120 ? raw : `${raw.slice(0, 117)}...`;
|
||||
}
|
||||
|
||||
/** Parse and normalize a strict qmd JSON array payload. */
|
||||
function parseQmdQueryResultArray(raw: string): QmdQueryResult[] | null {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!Array.isArray(parsed)) {
|
||||
return null;
|
||||
}
|
||||
return parsed.map((item) => {
|
||||
if (typeof item !== "object" || item === null) {
|
||||
return item as QmdQueryResult;
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const docid = typeof record.docid === "string" ? record.docid : undefined;
|
||||
const score =
|
||||
typeof record.score === "number" && Number.isFinite(record.score)
|
||||
? record.score
|
||||
: undefined;
|
||||
const collection = typeof record.collection === "string" ? record.collection : undefined;
|
||||
const file = typeof record.file === "string" ? record.file : undefined;
|
||||
const snippet = typeof record.snippet === "string" ? record.snippet : undefined;
|
||||
const body = typeof record.body === "string" ? record.body : undefined;
|
||||
return {
|
||||
docid,
|
||||
score,
|
||||
collection,
|
||||
file,
|
||||
snippet,
|
||||
body,
|
||||
startLine: parseQmdLineNumber(record.start_line ?? record.startLine),
|
||||
endLine: parseQmdLineNumber(record.end_line ?? record.endLine),
|
||||
} as QmdQueryResult;
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Normalize qmd line numbers, rejecting zero, negative, and non-integer values. */
|
||||
function parseQmdLineNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
/** Extract the first complete JSON array from noisy stdout. */
|
||||
function extractFirstJsonArray(raw: string): string | null {
|
||||
const start = raw.indexOf("[");
|
||||
if (start < 0) {
|
||||
return null;
|
||||
}
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
for (let i = start; i < raw.length; i += 1) {
|
||||
const char = raw[i];
|
||||
if (char === undefined) {
|
||||
break;
|
||||
}
|
||||
if (inString) {
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
if (char === "\\") {
|
||||
escaped = true;
|
||||
} else if (char === '"') {
|
||||
inString = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (char === '"') {
|
||||
inString = true;
|
||||
continue;
|
||||
}
|
||||
if (char === "[") {
|
||||
depth += 1;
|
||||
} else if (char === "]") {
|
||||
depth -= 1;
|
||||
if (depth === 0) {
|
||||
return raw.slice(start, i + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
55
packages/memory-host-sdk/src/host/qmd-scope.test.ts
Normal file
55
packages/memory-host-sdk/src/host/qmd-scope.test.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
// Memory Host SDK tests cover qmd scope behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ResolvedQmdConfig } from "./backend-config.js";
|
||||
import { deriveQmdScopeChannel, deriveQmdScopeChatType, isQmdScopeAllowed } from "./qmd-scope.js";
|
||||
|
||||
describe("qmd scope", () => {
|
||||
const allowDirect: ResolvedQmdConfig["scope"] = {
|
||||
default: "deny",
|
||||
rules: [{ action: "allow", match: { chatType: "direct" } }],
|
||||
};
|
||||
|
||||
it("derives channel and chat type from canonical keys once", () => {
|
||||
expect(deriveQmdScopeChannel("Workspace:group:123")).toBe("workspace");
|
||||
expect(deriveQmdScopeChatType("Workspace:group:123")).toBe("group");
|
||||
});
|
||||
|
||||
it("derives channel and chat type from stored key suffixes", () => {
|
||||
expect(deriveQmdScopeChannel("agent:agent-1:workspace:channel:chan-123")).toBe("workspace");
|
||||
expect(deriveQmdScopeChatType("agent:agent-1:workspace:channel:chan-123")).toBe("channel");
|
||||
});
|
||||
|
||||
it("treats parsed keys with no chat prefix as direct", () => {
|
||||
expect(deriveQmdScopeChannel("agent:agent-1:peer-direct")).toBeUndefined();
|
||||
expect(deriveQmdScopeChatType("agent:agent-1:peer-direct")).toBe("direct");
|
||||
expect(isQmdScopeAllowed(allowDirect, "agent:agent-1:peer-direct")).toBe(true);
|
||||
expect(isQmdScopeAllowed(allowDirect, "agent:agent-1:peer:group:abc")).toBe(false);
|
||||
});
|
||||
|
||||
it("applies scoped key-prefix checks against normalized key", () => {
|
||||
const scope: ResolvedQmdConfig["scope"] = {
|
||||
default: "deny",
|
||||
rules: [{ action: "allow", match: { keyPrefix: "workspace:" } }],
|
||||
};
|
||||
expect(isQmdScopeAllowed(scope, "agent:agent-1:workspace:group:123")).toBe(true);
|
||||
expect(isQmdScopeAllowed(scope, "agent:agent-1:other:group:123")).toBe(false);
|
||||
});
|
||||
|
||||
it("supports rawKeyPrefix matches for agent-prefixed keys", () => {
|
||||
const scope: ResolvedQmdConfig["scope"] = {
|
||||
default: "allow",
|
||||
rules: [{ action: "deny", match: { rawKeyPrefix: "agent:main:guildchat:" } }],
|
||||
};
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:guildchat:channel:c123")).toBe(false);
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:workspace:channel:c123")).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps legacy agent-prefixed keyPrefix rules working", () => {
|
||||
const scope: ResolvedQmdConfig["scope"] = {
|
||||
default: "allow",
|
||||
rules: [{ action: "deny", match: { keyPrefix: "agent:main:guildchat:" } }],
|
||||
};
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:guildchat:channel:c123")).toBe(false);
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:workspace:channel:c123")).toBe(true);
|
||||
});
|
||||
});
|
||||
123
packages/memory-host-sdk/src/host/qmd-scope.ts
Normal file
123
packages/memory-host-sdk/src/host/qmd-scope.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
// Memory Host SDK module implements qmd scope behavior.
|
||||
import type { ResolvedQmdConfig } from "./backend-config.js";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalLowercaseString,
|
||||
} from "./string-utils.js";
|
||||
|
||||
type ParsedQmdSessionScope = {
|
||||
channel?: string;
|
||||
chatType?: "channel" | "group" | "direct";
|
||||
normalizedKey?: string;
|
||||
};
|
||||
|
||||
export function isQmdScopeAllowed(scope: ResolvedQmdConfig["scope"], sessionKey?: string): boolean {
|
||||
if (!scope) {
|
||||
return true;
|
||||
}
|
||||
const parsed = parseQmdSessionScope(sessionKey);
|
||||
const channel = parsed.channel;
|
||||
const chatType = parsed.chatType;
|
||||
const normalizedKey = parsed.normalizedKey ?? "";
|
||||
const rawKey = normalizeLowercaseStringOrEmpty(sessionKey ?? "");
|
||||
for (const rule of scope.rules ?? []) {
|
||||
if (!rule) {
|
||||
continue;
|
||||
}
|
||||
const match = rule.match ?? {};
|
||||
if (match.channel && match.channel !== channel) {
|
||||
continue;
|
||||
}
|
||||
if (match.chatType && match.chatType !== chatType) {
|
||||
continue;
|
||||
}
|
||||
const normalizedPrefix = normalizeOptionalLowercaseString(match.keyPrefix) || undefined;
|
||||
const rawPrefix = normalizeOptionalLowercaseString(match.rawKeyPrefix) || undefined;
|
||||
|
||||
if (rawPrefix && !rawKey.startsWith(rawPrefix)) {
|
||||
continue;
|
||||
}
|
||||
if (normalizedPrefix) {
|
||||
// Backward compat: older configs used `keyPrefix: "agent:<id>:..."` to match raw keys.
|
||||
const isLegacyRaw = normalizedPrefix.startsWith("agent:");
|
||||
if (isLegacyRaw) {
|
||||
if (!rawKey.startsWith(normalizedPrefix)) {
|
||||
continue;
|
||||
}
|
||||
} else if (!normalizedKey.startsWith(normalizedPrefix)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return rule.action === "allow";
|
||||
}
|
||||
const fallback = scope.default ?? "allow";
|
||||
return fallback === "allow";
|
||||
}
|
||||
|
||||
export function deriveQmdScopeChannel(key?: string): string | undefined {
|
||||
return parseQmdSessionScope(key).channel;
|
||||
}
|
||||
|
||||
export function deriveQmdScopeChatType(key?: string): "channel" | "group" | "direct" | undefined {
|
||||
return parseQmdSessionScope(key).chatType;
|
||||
}
|
||||
|
||||
function parseQmdSessionScope(key?: string): ParsedQmdSessionScope {
|
||||
const normalized = normalizeQmdSessionKey(key);
|
||||
if (!normalized) {
|
||||
return {};
|
||||
}
|
||||
const parts = normalized.split(":").filter(Boolean);
|
||||
let chatType: ParsedQmdSessionScope["chatType"];
|
||||
if (
|
||||
parts.length >= 2 &&
|
||||
(parts[1] === "group" || parts[1] === "channel" || parts[1] === "direct" || parts[1] === "dm")
|
||||
) {
|
||||
if (parts.includes("group")) {
|
||||
chatType = "group";
|
||||
} else if (parts.includes("channel")) {
|
||||
chatType = "channel";
|
||||
}
|
||||
return {
|
||||
normalizedKey: normalized,
|
||||
channel: normalizeOptionalLowercaseString(parts[0]),
|
||||
chatType: chatType ?? "direct",
|
||||
};
|
||||
}
|
||||
if (normalized.includes(":group:")) {
|
||||
return { normalizedKey: normalized, chatType: "group" };
|
||||
}
|
||||
if (normalized.includes(":channel:")) {
|
||||
return { normalizedKey: normalized, chatType: "channel" };
|
||||
}
|
||||
return { normalizedKey: normalized, chatType: "direct" };
|
||||
}
|
||||
|
||||
function normalizeQmdSessionKey(key?: string): string | undefined {
|
||||
if (!key) {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = key.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = parseAgentSessionKey(trimmed);
|
||||
const normalized = normalizeLowercaseStringOrEmpty(parsed?.rest ?? trimmed);
|
||||
if (normalized.startsWith("subagent:")) {
|
||||
return undefined;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseAgentSessionKey(sessionKey: string | undefined | null): { rest: string } | null {
|
||||
const raw = normalizeOptionalLowercaseString(sessionKey);
|
||||
if (!raw) {
|
||||
return null;
|
||||
}
|
||||
const parts = raw.split(":").filter(Boolean);
|
||||
if (parts.length < 3 || parts[0] !== "agent") {
|
||||
return null;
|
||||
}
|
||||
const rest = parts.slice(2).join(":");
|
||||
return rest ? { rest } : null;
|
||||
}
|
||||
184
packages/memory-host-sdk/src/host/query-expansion.test.ts
Normal file
184
packages/memory-host-sdk/src/host/query-expansion.test.ts
Normal file
@@ -0,0 +1,184 @@
|
||||
// Memory Host SDK tests cover query keyword extraction behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { extractKeywords } from "./query-expansion.js";
|
||||
|
||||
describe("extractKeywords", () => {
|
||||
it("extracts keywords from English conversational query", () => {
|
||||
const keywords = extractKeywords("that thing we discussed about the API");
|
||||
expect(keywords).toStrictEqual(["discussed", "api"]);
|
||||
});
|
||||
|
||||
it("extracts keywords from Chinese conversational query", () => {
|
||||
const keywords = extractKeywords("之前讨论的那个方案");
|
||||
expect(keywords).toStrictEqual([
|
||||
"之",
|
||||
"讨",
|
||||
"论",
|
||||
"个",
|
||||
"方",
|
||||
"案",
|
||||
"前讨",
|
||||
"讨论",
|
||||
"论的",
|
||||
"的那",
|
||||
"个方",
|
||||
"方案",
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts keywords from mixed language query", () => {
|
||||
const keywords = extractKeywords("昨天讨论的 API design");
|
||||
expect(keywords).toStrictEqual([
|
||||
"昨",
|
||||
"天",
|
||||
"讨",
|
||||
"论",
|
||||
"天讨",
|
||||
"讨论",
|
||||
"论的",
|
||||
"api",
|
||||
"design",
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns specific technical terms", () => {
|
||||
const keywords = extractKeywords("what was the solution for the CFR bug");
|
||||
expect(keywords).toStrictEqual(["solution", "cfr", "bug"]);
|
||||
});
|
||||
|
||||
it("extracts keywords from Korean conversational query", () => {
|
||||
const keywords = extractKeywords("어제 논의한 배포 전략");
|
||||
expect(keywords).toStrictEqual(["논의한", "배포", "전략"]);
|
||||
});
|
||||
|
||||
it("strips Korean particles to extract stems", () => {
|
||||
const keywords = extractKeywords("서버에서 발생한 에러를 확인");
|
||||
expect(keywords).toStrictEqual(["서버에서", "서버", "발생한", "에러를", "에러", "확인"]);
|
||||
});
|
||||
|
||||
it("filters Korean stop words including inflected forms", () => {
|
||||
const keywords = extractKeywords("나는 그리고 그래서");
|
||||
expect(keywords).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("filters inflected Korean stop words not explicitly listed", () => {
|
||||
const keywords = extractKeywords("그녀는 우리는");
|
||||
expect(keywords).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("does not produce bogus single-char stems from particle stripping", () => {
|
||||
const keywords = extractKeywords("논의");
|
||||
expect(keywords).toStrictEqual(["논의"]);
|
||||
});
|
||||
|
||||
it("strips longest Korean trailing particles first", () => {
|
||||
const keywords = extractKeywords("기능으로 설명");
|
||||
expect(keywords).toStrictEqual(["기능으로", "기능", "설명"]);
|
||||
});
|
||||
|
||||
it("keeps stripped ASCII stems for mixed Korean tokens", () => {
|
||||
const keywords = extractKeywords("API를 배포했다");
|
||||
expect(keywords).toStrictEqual(["api를", "api", "배포했다"]);
|
||||
});
|
||||
|
||||
it("handles mixed Korean and English query", () => {
|
||||
const keywords = extractKeywords("API 배포에 대한 논의");
|
||||
expect(keywords).toStrictEqual(["api", "배포에", "배포", "대한", "논의"]);
|
||||
});
|
||||
|
||||
it("extracts keywords from Japanese conversational query", () => {
|
||||
const keywords = extractKeywords("昨日話したデプロイ戦略");
|
||||
expect(keywords).toStrictEqual(["昨日話", "日話", "デプロイ", "戦略"]);
|
||||
});
|
||||
|
||||
it("handles mixed Japanese and English query", () => {
|
||||
const keywords = extractKeywords("昨日話したAPIのバグ");
|
||||
expect(keywords).toStrictEqual(["昨日話", "日話", "api", "バグ"]);
|
||||
});
|
||||
|
||||
it("filters Japanese stop words", () => {
|
||||
const keywords = extractKeywords("これ それ そして どう");
|
||||
expect(keywords).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("extracts keywords from Spanish conversational query", () => {
|
||||
const keywords = extractKeywords("ayer hablamos sobre la estrategia de despliegue");
|
||||
expect(keywords).toStrictEqual(["hablamos", "estrategia", "despliegue"]);
|
||||
});
|
||||
|
||||
it("extracts keywords from Portuguese conversational query", () => {
|
||||
const keywords = extractKeywords("ontem falamos sobre a estratégia de implantação");
|
||||
expect(keywords).toStrictEqual(["falamos", "estratégia", "implantação"]);
|
||||
});
|
||||
|
||||
it("filters Spanish and Portuguese question stop words", () => {
|
||||
const keywords = extractKeywords("cómo cuando donde porquê quando onde");
|
||||
expect(keywords).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("extracts keywords from Arabic conversational query", () => {
|
||||
const keywords = extractKeywords("بالأمس ناقشنا استراتيجية النشر");
|
||||
expect(keywords).toStrictEqual(["ناقشنا", "استراتيجية", "النشر"]);
|
||||
});
|
||||
|
||||
it("filters Arabic question stop words", () => {
|
||||
const keywords = extractKeywords("كيف متى أين ماذا");
|
||||
expect(keywords).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("handles empty query", () => {
|
||||
expect(extractKeywords("")).toStrictEqual([]);
|
||||
expect(extractKeywords(" ")).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("handles query with only stop words", () => {
|
||||
const keywords = extractKeywords("the a an is are");
|
||||
expect(keywords).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("removes duplicate keywords", () => {
|
||||
const keywords = extractKeywords("test test testing");
|
||||
expect(keywords).toStrictEqual(["test", "testing"]);
|
||||
});
|
||||
|
||||
describe("with trigram tokenizer", () => {
|
||||
const trigramOpts = { ftsTokenizer: "trigram" as const };
|
||||
|
||||
it("emits whole CJK block instead of unigrams in trigram mode", () => {
|
||||
const defaultKeywords = extractKeywords("之前讨论的那个方案");
|
||||
const trigramKeywords = extractKeywords("之前讨论的那个方案", trigramOpts);
|
||||
expect(defaultKeywords).toStrictEqual([
|
||||
"之",
|
||||
"讨",
|
||||
"论",
|
||||
"个",
|
||||
"方",
|
||||
"案",
|
||||
"前讨",
|
||||
"讨论",
|
||||
"论的",
|
||||
"的那",
|
||||
"个方",
|
||||
"方案",
|
||||
]);
|
||||
expect(trigramKeywords).toStrictEqual(["之前讨论的那个方案"]);
|
||||
});
|
||||
|
||||
it("skips Japanese kanji bigrams in trigram mode", () => {
|
||||
const defaultKeywords = extractKeywords("経済政策について");
|
||||
const trigramKeywords = extractKeywords("経済政策について", trigramOpts);
|
||||
expect(defaultKeywords).toStrictEqual(["経済政策", "経済", "済政", "政策", "について"]);
|
||||
expect(trigramKeywords).toStrictEqual(["経済政策", "について"]);
|
||||
});
|
||||
|
||||
it("still filters stop words in trigram mode", () => {
|
||||
const keywords = extractKeywords("これ それ そして どう", trigramOpts);
|
||||
expect(keywords).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("does not affect English keyword extraction", () => {
|
||||
const keywords = extractKeywords("that thing we discussed about the API", trigramOpts);
|
||||
expect(keywords).toStrictEqual(["discussed", "api"]);
|
||||
});
|
||||
});
|
||||
});
|
||||
776
packages/memory-host-sdk/src/host/query-expansion.ts
Normal file
776
packages/memory-host-sdk/src/host/query-expansion.ts
Normal file
@@ -0,0 +1,776 @@
|
||||
// Memory Host SDK module implements query expansion behavior.
|
||||
import { normalizeLowercaseStringOrEmpty } from "./string-utils.js";
|
||||
|
||||
/**
|
||||
* Query expansion for FTS-only search mode.
|
||||
*
|
||||
* When no embedding provider is available, we fall back to FTS (full-text search).
|
||||
* FTS works best with specific keywords, but users often ask conversational queries
|
||||
* like "that thing we discussed yesterday" or "之前讨论的那个方案".
|
||||
*
|
||||
* This module extracts meaningful keywords from such queries to improve FTS results.
|
||||
*/
|
||||
|
||||
// Common stop words that don't add search value
|
||||
const STOP_WORDS_EN = new Set([
|
||||
// Articles and determiners
|
||||
"a",
|
||||
"an",
|
||||
"the",
|
||||
"this",
|
||||
"that",
|
||||
"these",
|
||||
"those",
|
||||
// Pronouns
|
||||
"i",
|
||||
"me",
|
||||
"my",
|
||||
"we",
|
||||
"our",
|
||||
"you",
|
||||
"your",
|
||||
"he",
|
||||
"she",
|
||||
"it",
|
||||
"they",
|
||||
"them",
|
||||
// Common verbs
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"being",
|
||||
"have",
|
||||
"has",
|
||||
"had",
|
||||
"do",
|
||||
"does",
|
||||
"did",
|
||||
"will",
|
||||
"would",
|
||||
"could",
|
||||
"should",
|
||||
"can",
|
||||
"may",
|
||||
"might",
|
||||
// Prepositions
|
||||
"in",
|
||||
"on",
|
||||
"at",
|
||||
"to",
|
||||
"for",
|
||||
"of",
|
||||
"with",
|
||||
"by",
|
||||
"from",
|
||||
"about",
|
||||
"into",
|
||||
"through",
|
||||
"during",
|
||||
"before",
|
||||
"after",
|
||||
"above",
|
||||
"below",
|
||||
"between",
|
||||
"under",
|
||||
"over",
|
||||
// Conjunctions
|
||||
"and",
|
||||
"or",
|
||||
"but",
|
||||
"if",
|
||||
"then",
|
||||
"because",
|
||||
"as",
|
||||
"while",
|
||||
"when",
|
||||
"where",
|
||||
"what",
|
||||
"which",
|
||||
"who",
|
||||
"how",
|
||||
"why",
|
||||
// Time references (vague, not useful for FTS)
|
||||
"yesterday",
|
||||
"today",
|
||||
"tomorrow",
|
||||
"earlier",
|
||||
"later",
|
||||
"recently",
|
||||
"before",
|
||||
"ago",
|
||||
"just",
|
||||
"now",
|
||||
// Vague references
|
||||
"thing",
|
||||
"things",
|
||||
"stuff",
|
||||
"something",
|
||||
"anything",
|
||||
"everything",
|
||||
"nothing",
|
||||
// Question words
|
||||
"please",
|
||||
"help",
|
||||
"find",
|
||||
"show",
|
||||
"get",
|
||||
"tell",
|
||||
"give",
|
||||
]);
|
||||
|
||||
const STOP_WORDS_ES = new Set([
|
||||
// Articles and determiners
|
||||
"el",
|
||||
"la",
|
||||
"los",
|
||||
"las",
|
||||
"un",
|
||||
"una",
|
||||
"unos",
|
||||
"unas",
|
||||
"este",
|
||||
"esta",
|
||||
"ese",
|
||||
"esa",
|
||||
// Pronouns
|
||||
"yo",
|
||||
"me",
|
||||
"mi",
|
||||
"nosotros",
|
||||
"nosotras",
|
||||
"tu",
|
||||
"tus",
|
||||
"usted",
|
||||
"ustedes",
|
||||
"ellos",
|
||||
"ellas",
|
||||
// Prepositions and conjunctions
|
||||
"de",
|
||||
"del",
|
||||
"a",
|
||||
"en",
|
||||
"con",
|
||||
"por",
|
||||
"para",
|
||||
"sobre",
|
||||
"entre",
|
||||
"y",
|
||||
"o",
|
||||
"pero",
|
||||
"si",
|
||||
"porque",
|
||||
"como",
|
||||
// Common verbs / auxiliaries
|
||||
"es",
|
||||
"son",
|
||||
"fue",
|
||||
"fueron",
|
||||
"ser",
|
||||
"estar",
|
||||
"haber",
|
||||
"tener",
|
||||
"hacer",
|
||||
// Time references (vague)
|
||||
"ayer",
|
||||
"hoy",
|
||||
"mañana",
|
||||
"antes",
|
||||
"despues",
|
||||
"después",
|
||||
"ahora",
|
||||
"recientemente",
|
||||
// Question/request words
|
||||
"que",
|
||||
"qué",
|
||||
"cómo",
|
||||
"cuando",
|
||||
"cuándo",
|
||||
"donde",
|
||||
"dónde",
|
||||
"porqué",
|
||||
"favor",
|
||||
"ayuda",
|
||||
]);
|
||||
|
||||
const STOP_WORDS_PT = new Set([
|
||||
// Articles and determiners
|
||||
"o",
|
||||
"a",
|
||||
"os",
|
||||
"as",
|
||||
"um",
|
||||
"uma",
|
||||
"uns",
|
||||
"umas",
|
||||
"este",
|
||||
"esta",
|
||||
"esse",
|
||||
"essa",
|
||||
// Pronouns
|
||||
"eu",
|
||||
"me",
|
||||
"meu",
|
||||
"minha",
|
||||
"nos",
|
||||
"nós",
|
||||
"você",
|
||||
"vocês",
|
||||
"ele",
|
||||
"ela",
|
||||
"eles",
|
||||
"elas",
|
||||
// Prepositions and conjunctions
|
||||
"de",
|
||||
"do",
|
||||
"da",
|
||||
"em",
|
||||
"com",
|
||||
"por",
|
||||
"para",
|
||||
"sobre",
|
||||
"entre",
|
||||
"e",
|
||||
"ou",
|
||||
"mas",
|
||||
"se",
|
||||
"porque",
|
||||
"como",
|
||||
// Common verbs / auxiliaries
|
||||
"é",
|
||||
"são",
|
||||
"foi",
|
||||
"foram",
|
||||
"ser",
|
||||
"estar",
|
||||
"ter",
|
||||
"fazer",
|
||||
// Time references (vague)
|
||||
"ontem",
|
||||
"hoje",
|
||||
"amanhã",
|
||||
"antes",
|
||||
"depois",
|
||||
"agora",
|
||||
"recentemente",
|
||||
// Question/request words
|
||||
"que",
|
||||
"quê",
|
||||
"quando",
|
||||
"onde",
|
||||
"porquê",
|
||||
"favor",
|
||||
"ajuda",
|
||||
]);
|
||||
|
||||
const STOP_WORDS_AR = new Set([
|
||||
// Articles and connectors
|
||||
"ال",
|
||||
"و",
|
||||
"أو",
|
||||
"لكن",
|
||||
"ثم",
|
||||
"بل",
|
||||
// Pronouns / references
|
||||
"أنا",
|
||||
"نحن",
|
||||
"هو",
|
||||
"هي",
|
||||
"هم",
|
||||
"هذا",
|
||||
"هذه",
|
||||
"ذلك",
|
||||
"تلك",
|
||||
"هنا",
|
||||
"هناك",
|
||||
// Common prepositions
|
||||
"من",
|
||||
"إلى",
|
||||
"الى",
|
||||
"في",
|
||||
"على",
|
||||
"عن",
|
||||
"مع",
|
||||
"بين",
|
||||
"ل",
|
||||
"ب",
|
||||
"ك",
|
||||
// Common auxiliaries / vague verbs
|
||||
"كان",
|
||||
"كانت",
|
||||
"يكون",
|
||||
"تكون",
|
||||
"صار",
|
||||
"أصبح",
|
||||
"يمكن",
|
||||
"ممكن",
|
||||
// Time references (vague)
|
||||
"بالأمس",
|
||||
"امس",
|
||||
"اليوم",
|
||||
"غدا",
|
||||
"الآن",
|
||||
"قبل",
|
||||
"بعد",
|
||||
"مؤخرا",
|
||||
// Question/request words
|
||||
"لماذا",
|
||||
"كيف",
|
||||
"ماذا",
|
||||
"متى",
|
||||
"أين",
|
||||
"هل",
|
||||
"من فضلك",
|
||||
"فضلا",
|
||||
"ساعد",
|
||||
]);
|
||||
|
||||
const STOP_WORDS_KO = new Set([
|
||||
// Particles (조사)
|
||||
"은",
|
||||
"는",
|
||||
"이",
|
||||
"가",
|
||||
"을",
|
||||
"를",
|
||||
"의",
|
||||
"에",
|
||||
"에서",
|
||||
"로",
|
||||
"으로",
|
||||
"와",
|
||||
"과",
|
||||
"도",
|
||||
"만",
|
||||
"까지",
|
||||
"부터",
|
||||
"한테",
|
||||
"에게",
|
||||
"께",
|
||||
"처럼",
|
||||
"같이",
|
||||
"보다",
|
||||
"마다",
|
||||
"밖에",
|
||||
"대로",
|
||||
// Pronouns (대명사)
|
||||
"나",
|
||||
"나는",
|
||||
"내가",
|
||||
"나를",
|
||||
"너",
|
||||
"우리",
|
||||
"저",
|
||||
"저희",
|
||||
"그",
|
||||
"그녀",
|
||||
"그들",
|
||||
"이것",
|
||||
"저것",
|
||||
"그것",
|
||||
"여기",
|
||||
"저기",
|
||||
"거기",
|
||||
// Common verbs / auxiliaries (일반 동사/보조 동사)
|
||||
"있다",
|
||||
"없다",
|
||||
"하다",
|
||||
"되다",
|
||||
"이다",
|
||||
"아니다",
|
||||
"보다",
|
||||
"주다",
|
||||
"오다",
|
||||
"가다",
|
||||
// Nouns (의존 명사 / vague)
|
||||
"것",
|
||||
"거",
|
||||
"등",
|
||||
"수",
|
||||
"때",
|
||||
"곳",
|
||||
"중",
|
||||
"분",
|
||||
// Adverbs
|
||||
"잘",
|
||||
"더",
|
||||
"또",
|
||||
"매우",
|
||||
"정말",
|
||||
"아주",
|
||||
"많이",
|
||||
"너무",
|
||||
"좀",
|
||||
// Conjunctions
|
||||
"그리고",
|
||||
"하지만",
|
||||
"그래서",
|
||||
"그런데",
|
||||
"그러나",
|
||||
"또는",
|
||||
"그러면",
|
||||
// Question words
|
||||
"왜",
|
||||
"어떻게",
|
||||
"뭐",
|
||||
"언제",
|
||||
"어디",
|
||||
"누구",
|
||||
"무엇",
|
||||
"어떤",
|
||||
// Time (vague)
|
||||
"어제",
|
||||
"오늘",
|
||||
"내일",
|
||||
"최근",
|
||||
"지금",
|
||||
"아까",
|
||||
"나중",
|
||||
"전에",
|
||||
// Request words
|
||||
"제발",
|
||||
"부탁",
|
||||
]);
|
||||
|
||||
// Common Korean trailing particles to strip from words for tokenization
|
||||
// Sorted by descending length so longest-match-first is guaranteed.
|
||||
const KO_TRAILING_PARTICLES = [
|
||||
"에서",
|
||||
"으로",
|
||||
"에게",
|
||||
"한테",
|
||||
"처럼",
|
||||
"같이",
|
||||
"보다",
|
||||
"까지",
|
||||
"부터",
|
||||
"마다",
|
||||
"밖에",
|
||||
"대로",
|
||||
"은",
|
||||
"는",
|
||||
"이",
|
||||
"가",
|
||||
"을",
|
||||
"를",
|
||||
"의",
|
||||
"에",
|
||||
"로",
|
||||
"와",
|
||||
"과",
|
||||
"도",
|
||||
"만",
|
||||
].toSorted((a, b) => b.length - a.length);
|
||||
|
||||
function stripKoreanTrailingParticle(token: string): string | null {
|
||||
for (const particle of KO_TRAILING_PARTICLES) {
|
||||
if (token.length > particle.length && token.endsWith(particle)) {
|
||||
return token.slice(0, -particle.length);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isUsefulKoreanStem(stem: string): boolean {
|
||||
// Prevent bogus one-syllable stems from words like "논의" -> "논".
|
||||
if (/[\uac00-\ud7af]/.test(stem)) {
|
||||
return stem.length >= 2;
|
||||
}
|
||||
// Keep stripped ASCII stems for mixed tokens like "API를" -> "api".
|
||||
return /^[a-z0-9_]+$/i.test(stem);
|
||||
}
|
||||
|
||||
const STOP_WORDS_JA = new Set([
|
||||
// Pronouns and references
|
||||
"これ",
|
||||
"それ",
|
||||
"あれ",
|
||||
"この",
|
||||
"その",
|
||||
"あの",
|
||||
"ここ",
|
||||
"そこ",
|
||||
"あそこ",
|
||||
// Common auxiliaries / vague verbs
|
||||
"する",
|
||||
"した",
|
||||
"して",
|
||||
"です",
|
||||
"ます",
|
||||
"いる",
|
||||
"ある",
|
||||
"なる",
|
||||
"できる",
|
||||
// Particles / connectors
|
||||
"の",
|
||||
"こと",
|
||||
"もの",
|
||||
"ため",
|
||||
"そして",
|
||||
"しかし",
|
||||
"また",
|
||||
"でも",
|
||||
"から",
|
||||
"まで",
|
||||
"より",
|
||||
"だけ",
|
||||
// Question words
|
||||
"なぜ",
|
||||
"どう",
|
||||
"何",
|
||||
"いつ",
|
||||
"どこ",
|
||||
"誰",
|
||||
"どれ",
|
||||
// Time (vague)
|
||||
"昨日",
|
||||
"今日",
|
||||
"明日",
|
||||
"最近",
|
||||
"今",
|
||||
"さっき",
|
||||
"前",
|
||||
"後",
|
||||
]);
|
||||
|
||||
const STOP_WORDS_ZH = new Set([
|
||||
// Pronouns
|
||||
"我",
|
||||
"我们",
|
||||
"你",
|
||||
"你们",
|
||||
"他",
|
||||
"她",
|
||||
"它",
|
||||
"他们",
|
||||
"这",
|
||||
"那",
|
||||
"这个",
|
||||
"那个",
|
||||
"这些",
|
||||
"那些",
|
||||
// Auxiliary words
|
||||
"的",
|
||||
"了",
|
||||
"着",
|
||||
"过",
|
||||
"得",
|
||||
"地",
|
||||
"吗",
|
||||
"呢",
|
||||
"吧",
|
||||
"啊",
|
||||
"呀",
|
||||
"嘛",
|
||||
"啦",
|
||||
// Verbs (common, vague)
|
||||
"是",
|
||||
"有",
|
||||
"在",
|
||||
"被",
|
||||
"把",
|
||||
"给",
|
||||
"让",
|
||||
"用",
|
||||
"到",
|
||||
"去",
|
||||
"来",
|
||||
"做",
|
||||
"说",
|
||||
"看",
|
||||
"找",
|
||||
"想",
|
||||
"要",
|
||||
"能",
|
||||
"会",
|
||||
"可以",
|
||||
// Prepositions and conjunctions
|
||||
"和",
|
||||
"与",
|
||||
"或",
|
||||
"但",
|
||||
"但是",
|
||||
"因为",
|
||||
"所以",
|
||||
"如果",
|
||||
"虽然",
|
||||
"而",
|
||||
"也",
|
||||
"都",
|
||||
"就",
|
||||
"还",
|
||||
"又",
|
||||
"再",
|
||||
"才",
|
||||
"只",
|
||||
// Time (vague)
|
||||
"之前",
|
||||
"以前",
|
||||
"之后",
|
||||
"以后",
|
||||
"刚才",
|
||||
"现在",
|
||||
"昨天",
|
||||
"今天",
|
||||
"明天",
|
||||
"最近",
|
||||
// Vague references
|
||||
"东西",
|
||||
"事情",
|
||||
"事",
|
||||
"什么",
|
||||
"哪个",
|
||||
"哪些",
|
||||
"怎么",
|
||||
"为什么",
|
||||
"多少",
|
||||
// Question/request words
|
||||
"请",
|
||||
"帮",
|
||||
"帮忙",
|
||||
"告诉",
|
||||
]);
|
||||
|
||||
/** Returns true for low-value conversational tokens that should not drive FTS matching. */
|
||||
export function isQueryStopWordToken(token: string): boolean {
|
||||
return (
|
||||
STOP_WORDS_EN.has(token) ||
|
||||
STOP_WORDS_ES.has(token) ||
|
||||
STOP_WORDS_PT.has(token) ||
|
||||
STOP_WORDS_AR.has(token) ||
|
||||
STOP_WORDS_ZH.has(token) ||
|
||||
STOP_WORDS_KO.has(token) ||
|
||||
STOP_WORDS_JA.has(token)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a token looks like a meaningful keyword.
|
||||
* Returns false for short tokens, numbers-only, etc.
|
||||
*/
|
||||
function isValidKeyword(token: string): boolean {
|
||||
if (!token || token.length === 0) {
|
||||
return false;
|
||||
}
|
||||
// Skip very short English words (likely stop words or fragments)
|
||||
if (/^[a-zA-Z]+$/.test(token) && token.length < 3) {
|
||||
return false;
|
||||
}
|
||||
// Skip pure numbers (not useful for semantic search)
|
||||
if (/^\d+$/.test(token)) {
|
||||
return false;
|
||||
}
|
||||
// Skip tokens that are all punctuation
|
||||
if (/^[\p{P}\p{S}]+$/u.test(token)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple tokenizer that handles English, Chinese, Korean, and Japanese text.
|
||||
* For Chinese, we do character-based splitting since we don't have a proper segmenter.
|
||||
* For English, we split on whitespace and punctuation.
|
||||
*/
|
||||
function tokenize(text: string, opts?: { ftsTokenizer?: "unicode61" | "trigram" }): string[] {
|
||||
const useTrigram = opts?.ftsTokenizer === "trigram";
|
||||
const tokens: string[] = [];
|
||||
const normalized = normalizeLowercaseStringOrEmpty(text);
|
||||
|
||||
// Split into segments (English words, Chinese character sequences, etc.)
|
||||
const segments = normalized.split(/[\s\p{P}]+/u).filter(Boolean);
|
||||
|
||||
for (const segment of segments) {
|
||||
// Japanese text often mixes scripts (kanji/kana/ASCII) without spaces.
|
||||
// Extract script-specific chunks so technical terms like "API" / "バグ" are retained.
|
||||
if (/[\u3040-\u30ff]/.test(segment)) {
|
||||
const jpParts =
|
||||
segment.match(/[a-z0-9_]+|[\u30a0-\u30ffー]+|[\u4e00-\u9fff]+|[\u3040-\u309f]{2,}/g) ?? [];
|
||||
for (const part of jpParts) {
|
||||
if (/^[\u4e00-\u9fff]+$/.test(part)) {
|
||||
tokens.push(part);
|
||||
if (!useTrigram) {
|
||||
for (let i = 0; i < part.length - 1; i++) {
|
||||
tokens.push(part[i] + part[i + 1]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tokens.push(part);
|
||||
}
|
||||
}
|
||||
} else if (/[\u4e00-\u9fff]/.test(segment)) {
|
||||
// Check if segment contains CJK characters (Chinese)
|
||||
const chars = Array.from(segment).filter((c) => /[\u4e00-\u9fff]/.test(c));
|
||||
if (useTrigram) {
|
||||
// In trigram mode, push the whole contiguous CJK block (mirroring the
|
||||
// Japanese kanji path). SQLite's trigram FTS requires at least 3 characters
|
||||
// per query term — individual characters silently return no results.
|
||||
const block = chars.join("");
|
||||
if (block.length > 0) {
|
||||
tokens.push(block);
|
||||
}
|
||||
} else {
|
||||
// Default mode: unigrams + bigrams for phrase matching
|
||||
tokens.push(...chars);
|
||||
for (let i = 0; i < chars.length - 1; i++) {
|
||||
tokens.push(chars[i] + chars[i + 1]);
|
||||
}
|
||||
}
|
||||
} else if (/[\uac00-\ud7af\u3131-\u3163]/.test(segment)) {
|
||||
// For Korean (Hangul syllables and jamo), keep the word as-is unless it is
|
||||
// effectively a stop word once trailing particles are removed.
|
||||
const stem = stripKoreanTrailingParticle(segment);
|
||||
const stemIsStopWord = stem !== null && STOP_WORDS_KO.has(stem);
|
||||
if (!STOP_WORDS_KO.has(segment) && !stemIsStopWord) {
|
||||
tokens.push(segment);
|
||||
}
|
||||
// Also emit particle-stripped stems when they are useful keywords.
|
||||
if (stem && !STOP_WORDS_KO.has(stem) && isUsefulKoreanStem(stem)) {
|
||||
tokens.push(stem);
|
||||
}
|
||||
} else {
|
||||
// For non-CJK, keep as single token
|
||||
tokens.push(segment);
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract keywords from a conversational query for FTS search.
|
||||
*
|
||||
* Examples:
|
||||
* - "that thing we discussed about the API" → ["discussed", "API"]
|
||||
* - "之前讨论的那个方案" → ["讨论", "方案"]
|
||||
* - "what was the solution for the bug" → ["solution", "bug"]
|
||||
*/
|
||||
export function extractKeywords(
|
||||
query: string,
|
||||
opts?: { ftsTokenizer?: "unicode61" | "trigram" },
|
||||
): string[] {
|
||||
const tokens = tokenize(query, opts);
|
||||
const keywords: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const token of tokens) {
|
||||
// Skip stop words
|
||||
if (isQueryStopWordToken(token)) {
|
||||
continue;
|
||||
}
|
||||
// Skip invalid keywords
|
||||
if (!isValidKeyword(token)) {
|
||||
continue;
|
||||
}
|
||||
// Skip duplicates
|
||||
if (seen.has(token)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(token);
|
||||
keywords.push(token);
|
||||
}
|
||||
|
||||
return keywords;
|
||||
}
|
||||
37
packages/memory-host-sdk/src/host/read-file-shared.test.ts
Normal file
37
packages/memory-host-sdk/src/host/read-file-shared.test.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
// Memory Host SDK tests cover read file shared behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildMemoryReadResult, buildMemoryReadResultFromSlice } from "./read-file-shared.js";
|
||||
|
||||
describe("memory read result slicing", () => {
|
||||
it("uses default line windows for non-finite from and lines values", () => {
|
||||
expect(
|
||||
buildMemoryReadResult({
|
||||
content: "one\ntwo\nthree",
|
||||
relPath: "memory/test.md",
|
||||
from: Number.NaN,
|
||||
lines: Number.NaN,
|
||||
}),
|
||||
).toEqual({
|
||||
text: "one\ntwo\nthree",
|
||||
path: "memory/test.md",
|
||||
from: 1,
|
||||
lines: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the default character budget for non-finite maxChars values", () => {
|
||||
expect(
|
||||
buildMemoryReadResultFromSlice({
|
||||
selectedLines: ["one", "two"],
|
||||
relPath: "memory/test.md",
|
||||
startLine: Number.POSITIVE_INFINITY,
|
||||
maxChars: Number.NaN,
|
||||
}),
|
||||
).toEqual({
|
||||
text: "one\ntwo",
|
||||
path: "memory/test.md",
|
||||
from: 1,
|
||||
lines: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
130
packages/memory-host-sdk/src/host/read-file-shared.ts
Normal file
130
packages/memory-host-sdk/src/host/read-file-shared.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
// Memory Host SDK module implements read file shared behavior.
|
||||
import type { MemoryReadResult } from "./types.js";
|
||||
|
||||
// Shared memory-file read result shaping and truncation notices.
|
||||
|
||||
/** Default number of lines returned by memory read helpers. */
|
||||
export const DEFAULT_MEMORY_READ_LINES = 120;
|
||||
/** Default max character budget for memory read helper output. */
|
||||
export const DEFAULT_MEMORY_READ_MAX_CHARS = 12_000;
|
||||
|
||||
export type { MemoryReadResult } from "./types.js";
|
||||
|
||||
/** Build the continuation notice appended to truncated memory excerpts. */
|
||||
function buildContinuationNotice(params: {
|
||||
nextFrom: number | undefined;
|
||||
suggestReadFallback?: boolean;
|
||||
}): string {
|
||||
const base =
|
||||
typeof params.nextFrom === "number"
|
||||
? `[More content available. Use from=${params.nextFrom} to continue.]`
|
||||
: "[More content available. Requested excerpt exceeded the default maxChars budget.]";
|
||||
const fallback = params.suggestReadFallback
|
||||
? " If you need the full raw line, use read on the source file."
|
||||
: "";
|
||||
return `\n\n${base.slice(0, -1)}${fallback}]`;
|
||||
}
|
||||
|
||||
/** Fit line slices to the response character budget while preserving line boundaries. */
|
||||
function fitLinesToCharBudget(params: { lines: string[]; maxChars: number }): {
|
||||
text: string;
|
||||
includedLines: number;
|
||||
hardTruncatedSingleLine: boolean;
|
||||
} {
|
||||
const { lines, maxChars } = params;
|
||||
if (lines.length === 0) {
|
||||
return { text: "", includedLines: 0, hardTruncatedSingleLine: false };
|
||||
}
|
||||
|
||||
let includedLines = lines.length;
|
||||
let text = lines.join("\n");
|
||||
while (includedLines > 1 && text.length > maxChars) {
|
||||
includedLines -= 1;
|
||||
text = lines.slice(0, includedLines).join("\n");
|
||||
}
|
||||
|
||||
if (text.length <= maxChars) {
|
||||
return { text, includedLines, hardTruncatedSingleLine: false };
|
||||
}
|
||||
|
||||
return {
|
||||
text: text.slice(0, maxChars),
|
||||
includedLines: 1,
|
||||
hardTruncatedSingleLine: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Normalize optional numeric config to a positive integer fallback. */
|
||||
function normalizePositiveInteger(value: number | undefined, fallback: number): number {
|
||||
return typeof value === "number" && Number.isFinite(value)
|
||||
? Math.max(1, Math.floor(value))
|
||||
: fallback;
|
||||
}
|
||||
|
||||
/** Build a memory read result from an already-selected line slice. */
|
||||
export function buildMemoryReadResultFromSlice(params: {
|
||||
selectedLines: string[];
|
||||
relPath: string;
|
||||
startLine: number;
|
||||
moreSourceLinesRemain?: boolean;
|
||||
maxChars?: number;
|
||||
suggestReadFallback?: boolean;
|
||||
}): MemoryReadResult {
|
||||
const start = normalizePositiveInteger(params.startLine, 1);
|
||||
const fitted = fitLinesToCharBudget({
|
||||
lines: params.selectedLines,
|
||||
maxChars: normalizePositiveInteger(params.maxChars, DEFAULT_MEMORY_READ_MAX_CHARS),
|
||||
});
|
||||
const moreSourceLinesRemain = params.moreSourceLinesRemain ?? false;
|
||||
const charCapTruncated =
|
||||
fitted.hardTruncatedSingleLine || fitted.includedLines < params.selectedLines.length;
|
||||
const nextFrom =
|
||||
!fitted.hardTruncatedSingleLine &&
|
||||
(moreSourceLinesRemain || fitted.includedLines < params.selectedLines.length)
|
||||
? start + fitted.includedLines
|
||||
: undefined;
|
||||
const truncated = charCapTruncated || moreSourceLinesRemain;
|
||||
const text =
|
||||
truncated && fitted.text
|
||||
? `${fitted.text}${buildContinuationNotice({
|
||||
nextFrom,
|
||||
suggestReadFallback: fitted.hardTruncatedSingleLine && params.suggestReadFallback,
|
||||
})}`
|
||||
: fitted.text;
|
||||
return {
|
||||
text,
|
||||
path: params.relPath,
|
||||
from: start,
|
||||
lines: fitted.includedLines,
|
||||
...(truncated ? { truncated: true } : {}),
|
||||
...(typeof nextFrom === "number" ? { nextFrom } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Build a memory read result from raw file content and caller range options. */
|
||||
export function buildMemoryReadResult(params: {
|
||||
content: string;
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
defaultLines?: number;
|
||||
maxChars?: number;
|
||||
suggestReadFallback?: boolean;
|
||||
}): MemoryReadResult {
|
||||
const fileLines = params.content.split("\n");
|
||||
const start = normalizePositiveInteger(params.from, 1);
|
||||
const requestedCount = normalizePositiveInteger(
|
||||
params.lines ?? params.defaultLines,
|
||||
DEFAULT_MEMORY_READ_LINES,
|
||||
);
|
||||
const selectedLines = fileLines.slice(start - 1, start - 1 + requestedCount);
|
||||
const moreSourceLinesRemain = start - 1 + selectedLines.length < fileLines.length;
|
||||
return buildMemoryReadResultFromSlice({
|
||||
selectedLines,
|
||||
relPath: params.relPath,
|
||||
startLine: start,
|
||||
moreSourceLinesRemain,
|
||||
maxChars: params.maxChars,
|
||||
suggestReadFallback: params.suggestReadFallback,
|
||||
});
|
||||
}
|
||||
161
packages/memory-host-sdk/src/host/read-file.test.ts
Normal file
161
packages/memory-host-sdk/src/host/read-file.test.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
// Memory Host SDK tests cover read file behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { readMemoryFile } from "./read-file.js";
|
||||
|
||||
async function createDirectorySymlink(target: string, linkPath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.symlink(target, linkPath, "dir");
|
||||
return true;
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "EPERM" || code === "EACCES") {
|
||||
return false;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
describe("readMemoryFile", () => {
|
||||
it("returns empty text for missing files under extra path directories", async () => {
|
||||
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-read-file-"));
|
||||
try {
|
||||
const workspaceDir = path.join(tmpRoot, "workspace");
|
||||
const extraDir = path.join(tmpRoot, "extra");
|
||||
const missingPath = path.join(extraDir, "missing.md");
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: missingPath,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
text: "",
|
||||
path: path.relative(workspaceDir, missingPath).replace(/\\/g, "/"),
|
||||
});
|
||||
|
||||
const nonDirectoryParentPath = path.join(extraDir, "note.md", "child.md");
|
||||
await fs.writeFile(path.join(extraDir, "note.md"), "note", "utf-8");
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: nonDirectoryParentPath,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
text: "",
|
||||
path: path.relative(workspaceDir, nonDirectoryParentPath).replace(/\\/g, "/"),
|
||||
});
|
||||
} finally {
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects extra path reads through symlinked directory components", async () => {
|
||||
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-read-file-"));
|
||||
try {
|
||||
const workspaceDir = path.join(tmpRoot, "workspace");
|
||||
const extraDir = path.join(tmpRoot, "extra");
|
||||
const outsideDir = path.join(tmpRoot, "outside");
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.mkdir(outsideDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "inside.md"), "inside", "utf-8");
|
||||
await fs.writeFile(path.join(outsideDir, "private.md"), "private", "utf-8");
|
||||
|
||||
const inside = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: path.join(extraDir, "inside.md"),
|
||||
});
|
||||
expect(inside.text).toBe("inside");
|
||||
|
||||
const insideLinkPath = path.join(extraDir, "inside-link");
|
||||
if (!(await createDirectorySymlink(extraDir, insideLinkPath))) {
|
||||
return;
|
||||
}
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: path.join(insideLinkPath, "inside.md"),
|
||||
}),
|
||||
).rejects.toThrow("path required");
|
||||
|
||||
const outsideLinkPath = path.join(extraDir, "link");
|
||||
if (!(await createDirectorySymlink(outsideDir, outsideLinkPath))) {
|
||||
return;
|
||||
}
|
||||
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: path.join(outsideLinkPath, "private.md"),
|
||||
}),
|
||||
).rejects.toThrow("path required");
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: path.join(outsideLinkPath, "missing.md"),
|
||||
}),
|
||||
).rejects.toThrow("path required");
|
||||
} finally {
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("retries transient read errors for workspace memory files", async () => {
|
||||
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-read-file-"));
|
||||
try {
|
||||
const workspaceDir = path.join(tmpRoot, "workspace");
|
||||
const relPath = "memory/retry.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, "alpha\nbeta", "utf-8");
|
||||
|
||||
const realOpen = fs.open;
|
||||
let attempts = 0;
|
||||
const openSpy = vi
|
||||
.spyOn(fs, "open")
|
||||
.mockImplementation(async (...args: Parameters<typeof realOpen>) => {
|
||||
const [target, flags, mode] = args;
|
||||
if (typeof target === "string" && path.resolve(target) === absPath && attempts++ === 0) {
|
||||
const err = new Error(
|
||||
"Unknown system error -11: Unknown system error -11, open",
|
||||
) as NodeJS.ErrnoException;
|
||||
err.code = "UNKNOWN";
|
||||
err.errno = -11;
|
||||
throw err;
|
||||
}
|
||||
return await realOpen(target, flags, mode);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
text: "alpha\nbeta",
|
||||
path: relPath,
|
||||
from: 1,
|
||||
lines: 2,
|
||||
});
|
||||
expect(attempts).toBe(2);
|
||||
} finally {
|
||||
openSpy.mockRestore();
|
||||
}
|
||||
} finally {
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
182
packages/memory-host-sdk/src/host/read-file.ts
Normal file
182
packages/memory-host-sdk/src/host/read-file.ts
Normal file
@@ -0,0 +1,182 @@
|
||||
// Memory Host SDK module implements read file behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
resolveAgentContextLimits,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveMemorySearchConfig,
|
||||
type OpenClawConfig,
|
||||
} from "./config-utils.js";
|
||||
import {
|
||||
assertNoSymlinkParents,
|
||||
isFileMissingError,
|
||||
isPathInside,
|
||||
isPathInsideWithRealpath,
|
||||
readRegularFile,
|
||||
root,
|
||||
statRegularFile,
|
||||
} from "./fs-utils.js";
|
||||
import { isMemoryPath, normalizeExtraMemoryPaths } from "./internal.js";
|
||||
import {
|
||||
buildMemoryReadResult,
|
||||
DEFAULT_MEMORY_READ_LINES,
|
||||
type MemoryReadResult,
|
||||
} from "./read-file-shared.js";
|
||||
import { retryTransientMemoryRead } from "./read-retry.js";
|
||||
|
||||
// Secure markdown memory-file reader for workspace and configured extra paths.
|
||||
|
||||
/** Check that an absolute path stays inside an allowed extra directory without symlink escapes. */
|
||||
async function isAllowedAdditionalDirectoryPath(
|
||||
additionalPath: string,
|
||||
absPath: string,
|
||||
): Promise<boolean> {
|
||||
if (!isPathInside(additionalPath, absPath)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await assertNoSymlinkParents({ rootDir: additionalPath, targetPath: absPath });
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (!isPathInsideWithRealpath(additionalPath, absPath)) {
|
||||
try {
|
||||
await fs.lstat(absPath);
|
||||
} catch (err) {
|
||||
return isFileMissingError(err);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Return true when a file vanished after path validation but before content read. */
|
||||
function isFileDisappearedDuringReadError(err: unknown): boolean {
|
||||
return (
|
||||
isFileMissingError(err) ||
|
||||
Boolean(
|
||||
err &&
|
||||
typeof err === "object" &&
|
||||
"code" in err &&
|
||||
(err as { code?: unknown }).code === "path-mismatch",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/** Read a validated memory markdown file from workspace or configured extra paths. */
|
||||
export async function readMemoryFile(params: {
|
||||
workspaceDir: string;
|
||||
extraPaths?: string[];
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
defaultLines?: number;
|
||||
maxChars?: number;
|
||||
}): Promise<MemoryReadResult> {
|
||||
const rawPath = params.relPath.trim();
|
||||
if (!rawPath) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const absPath = path.isAbsolute(rawPath)
|
||||
? path.resolve(rawPath)
|
||||
: path.resolve(params.workspaceDir, rawPath);
|
||||
const relPath = path.relative(params.workspaceDir, absPath).replace(/\\/g, "/");
|
||||
const inWorkspace = relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
|
||||
const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
|
||||
let allowedAdditional = false;
|
||||
if (!allowedWorkspace && (params.extraPaths?.length ?? 0) > 0) {
|
||||
const additionalPaths = normalizeExtraMemoryPaths(params.workspaceDir, params.extraPaths);
|
||||
for (const additionalPath of additionalPaths) {
|
||||
try {
|
||||
const stat = await fs.lstat(additionalPath);
|
||||
if (stat.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
if (await isAllowedAdditionalDirectoryPath(additionalPath, absPath)) {
|
||||
const candidateStat = await fs.lstat(absPath).catch(() => null);
|
||||
if (candidateStat?.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile() && absPath === additionalPath && absPath.endsWith(".md")) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (!allowedWorkspace && !allowedAdditional) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
if (!absPath.endsWith(".md")) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
if (allowedWorkspace) {
|
||||
try {
|
||||
// Workspace reads use the safe fs root so symlink escapes are rejected before file IO.
|
||||
const workspaceRoot = await root(params.workspaceDir);
|
||||
await workspaceRoot.resolve(relPath);
|
||||
} catch (err) {
|
||||
if (isFileMissingError(err)) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
const statResult = await statRegularFile(absPath);
|
||||
if (statResult.missing) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
let content: string;
|
||||
try {
|
||||
content = (
|
||||
await retryTransientMemoryRead(
|
||||
() => readRegularFile({ filePath: absPath }),
|
||||
`read memory file ${absPath}`,
|
||||
)
|
||||
).buffer.toString("utf-8");
|
||||
} catch (err) {
|
||||
if (isFileDisappearedDuringReadError(err)) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return buildMemoryReadResult({
|
||||
content,
|
||||
relPath,
|
||||
from: params.from,
|
||||
lines: params.lines,
|
||||
defaultLines: params.defaultLines ?? DEFAULT_MEMORY_READ_LINES,
|
||||
maxChars: params.maxChars,
|
||||
suggestReadFallback: allowedWorkspace,
|
||||
});
|
||||
}
|
||||
|
||||
/** Resolve agent memory config and read one memory file for that agent. */
|
||||
export async function readAgentMemoryFile(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<MemoryReadResult> {
|
||||
const settings = resolveMemorySearchConfig(params.cfg, params.agentId);
|
||||
if (!settings) {
|
||||
throw new Error("memory search disabled");
|
||||
}
|
||||
const contextLimits = resolveAgentContextLimits(params.cfg, params.agentId);
|
||||
return await readMemoryFile({
|
||||
workspaceDir: resolveAgentWorkspaceDir(params.cfg, params.agentId),
|
||||
extraPaths: settings.extraPaths,
|
||||
relPath: params.relPath,
|
||||
from: params.from,
|
||||
lines: params.lines,
|
||||
defaultLines: contextLimits?.memoryGetDefaultLines,
|
||||
maxChars: contextLimits?.memoryGetMaxChars,
|
||||
});
|
||||
}
|
||||
26
packages/memory-host-sdk/src/host/read-retry.test.ts
Normal file
26
packages/memory-host-sdk/src/host/read-retry.test.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
// Memory Host SDK tests cover read retry behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { retryTransientMemoryRead } from "./read-retry.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("retryTransientMemoryRead", () => {
|
||||
it("uses a short two-retry budget for transient file read errors", async () => {
|
||||
const err = new Error("Unknown system error -11: Unknown system error -11, read");
|
||||
const run = vi.fn<() => Promise<string>>().mockRejectedValue(err);
|
||||
const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation((callback) => {
|
||||
if (typeof callback === "function") {
|
||||
callback();
|
||||
}
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
});
|
||||
|
||||
await expect(retryTransientMemoryRead(run)).rejects.toThrow("Unknown system error -11");
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(3);
|
||||
expect(timeoutSpy).toHaveBeenNthCalledWith(1, expect.any(Function), 25);
|
||||
expect(timeoutSpy).toHaveBeenNthCalledWith(2, expect.any(Function), 50);
|
||||
});
|
||||
});
|
||||
51
packages/memory-host-sdk/src/host/read-retry.ts
Normal file
51
packages/memory-host-sdk/src/host/read-retry.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
// Memory Host SDK module implements read retry behavior.
|
||||
import { retryAsync } from "./retry-utils.js";
|
||||
|
||||
// Retry helper for transient filesystem reads observed on memory stores.
|
||||
|
||||
const TRANSIENT_MEMORY_READ_ERRNO = -11;
|
||||
const TRANSIENT_MEMORY_READ_CODES = new Set(["EAGAIN", "EWOULDBLOCK", "EDEADLK"]);
|
||||
const TRANSIENT_MEMORY_READ_MESSAGE = /Unknown system error -11\b/i;
|
||||
|
||||
/** Extract errno from Node filesystem-style errors. */
|
||||
function getErrno(error: unknown): number | undefined {
|
||||
return typeof (error as NodeJS.ErrnoException | undefined)?.errno === "number"
|
||||
? (error as NodeJS.ErrnoException).errno
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Extract code from Node filesystem-style errors. */
|
||||
function getCode(error: unknown): string | undefined {
|
||||
return typeof (error as NodeJS.ErrnoException | undefined)?.code === "string"
|
||||
? (error as NodeJS.ErrnoException).code
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Return true for transient memory read failures that should be retried. */
|
||||
export function isTransientMemoryReadError(error: unknown): boolean {
|
||||
const code = getCode(error);
|
||||
if (code && TRANSIENT_MEMORY_READ_CODES.has(code)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const errno = getErrno(error);
|
||||
if (errno === TRANSIENT_MEMORY_READ_ERRNO) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return error instanceof Error && TRANSIENT_MEMORY_READ_MESSAGE.test(error.message);
|
||||
}
|
||||
|
||||
/** Retry a memory read with the narrow transient error predicate. */
|
||||
export async function retryTransientMemoryRead<T>(
|
||||
read: () => Promise<T>,
|
||||
label = "memory read",
|
||||
): Promise<T> {
|
||||
return await retryAsync(read, {
|
||||
attempts: 3,
|
||||
minDelayMs: 25,
|
||||
maxDelayMs: 50,
|
||||
label,
|
||||
shouldRetry: (error) => isTransientMemoryReadError(error),
|
||||
});
|
||||
}
|
||||
61
packages/memory-host-sdk/src/host/remote-http.test.ts
Normal file
61
packages/memory-host-sdk/src/host/remote-http.test.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
// Memory Host SDK tests cover remote http behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE, withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
describe("package withRemoteHttpResponse", () => {
|
||||
function makeFetchDeps({ useEnvProxy = false }: { useEnvProxy?: boolean } = {}) {
|
||||
const calls: unknown[] = [];
|
||||
return {
|
||||
calls,
|
||||
fetchWithSsrFGuardImpl: async (params: unknown) => {
|
||||
calls.push(params);
|
||||
return {
|
||||
response: new Response("ok", { status: 200 }),
|
||||
finalUrl: "https://memory.example/v1",
|
||||
release: async () => {},
|
||||
};
|
||||
},
|
||||
shouldUseEnvHttpProxyForUrlImpl: () => useEnvProxy,
|
||||
};
|
||||
}
|
||||
|
||||
it("uses trusted env proxy mode when the target will use EnvHttpProxyAgent", async () => {
|
||||
const deps = makeFetchDeps({ useEnvProxy: true });
|
||||
|
||||
await withRemoteHttpResponse({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
onResponse: async () => undefined,
|
||||
...deps,
|
||||
});
|
||||
|
||||
expect(deps.calls[0]).toHaveProperty("url", "https://memory.example/v1/embeddings");
|
||||
expect(deps.calls[0]).toHaveProperty("mode", MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE);
|
||||
});
|
||||
|
||||
it("keeps strict guarded fetch mode when proxy env would not proxy the target", async () => {
|
||||
const deps = makeFetchDeps();
|
||||
|
||||
await withRemoteHttpResponse({
|
||||
url: "https://internal.corp.example/v1/embeddings",
|
||||
onResponse: async () => undefined,
|
||||
...deps,
|
||||
});
|
||||
|
||||
expect(deps.calls).toHaveLength(1);
|
||||
expect(deps.calls[0]).not.toHaveProperty("mode");
|
||||
});
|
||||
|
||||
it("passes abort signals to the guarded fetch", async () => {
|
||||
const deps = makeFetchDeps();
|
||||
const controller = new AbortController();
|
||||
|
||||
await withRemoteHttpResponse({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
signal: controller.signal,
|
||||
onResponse: async () => undefined,
|
||||
...deps,
|
||||
});
|
||||
|
||||
expect(deps.calls[0]).toHaveProperty("signal", controller.signal);
|
||||
});
|
||||
});
|
||||
46
packages/memory-host-sdk/src/host/remote-http.ts
Normal file
46
packages/memory-host-sdk/src/host/remote-http.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
// Memory Host SDK module implements remote http behavior.
|
||||
import {
|
||||
fetchWithSsrFGuard,
|
||||
shouldUseEnvHttpProxyForUrl,
|
||||
ssrfPolicyFromHttpBaseUrlAllowedHostname,
|
||||
} from "./openclaw-runtime-network.js";
|
||||
import type { SsrFPolicy } from "./ssrf-policy.js";
|
||||
|
||||
// Remote memory HTTP wrapper that applies SSRF policy and releases guarded sockets.
|
||||
|
||||
/** Proxy mode used only for URLs that the runtime classified as env-proxy safe. */
|
||||
export const MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE = "trusted_env_proxy";
|
||||
|
||||
/** Build an SSRF allow policy from a configured remote base URL. */
|
||||
export const buildRemoteBaseUrlPolicy: (baseUrl: string) => SsrFPolicy | undefined =
|
||||
ssrfPolicyFromHttpBaseUrlAllowedHostname;
|
||||
|
||||
/** Execute a remote HTTP request under SSRF guard and always release the response handle. */
|
||||
export async function withRemoteHttpResponse<T>(params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
signal?: AbortSignal;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
fetchWithSsrFGuardImpl?: typeof fetchWithSsrFGuard;
|
||||
shouldUseEnvHttpProxyForUrlImpl?: typeof shouldUseEnvHttpProxyForUrl;
|
||||
auditContext?: string;
|
||||
onResponse: (response: Response) => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const guardedFetch = params.fetchWithSsrFGuardImpl ?? fetchWithSsrFGuard;
|
||||
const shouldUseEnvProxy = params.shouldUseEnvHttpProxyForUrlImpl ?? shouldUseEnvHttpProxyForUrl;
|
||||
const { response, release } = await guardedFetch({
|
||||
url: params.url,
|
||||
fetchImpl: params.fetchImpl,
|
||||
init: params.init,
|
||||
signal: params.signal,
|
||||
policy: params.ssrfPolicy,
|
||||
auditContext: params.auditContext ?? "memory-remote",
|
||||
...(shouldUseEnvProxy(params.url) ? { mode: MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE } : {}),
|
||||
});
|
||||
try {
|
||||
return await params.onResponse(response);
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
77
packages/memory-host-sdk/src/host/response-snippet.test.ts
Normal file
77
packages/memory-host-sdk/src/host/response-snippet.test.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
// Memory Host SDK tests cover response snippet behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readResponseJsonWithLimit, readResponseTextSnippet } from "./response-snippet.js";
|
||||
|
||||
describe("readResponseTextSnippet", () => {
|
||||
function stallingResponse(onCancel: () => void): Response {
|
||||
const reader = {
|
||||
read: () => new Promise<ReadableStreamReadResult<Uint8Array>>(() => {}),
|
||||
cancel: async () => {
|
||||
onCancel();
|
||||
},
|
||||
releaseLock: () => undefined,
|
||||
} as ReadableStreamDefaultReader<Uint8Array>;
|
||||
|
||||
return {
|
||||
body: { getReader: () => reader },
|
||||
headers: new Headers(),
|
||||
} as Response;
|
||||
}
|
||||
|
||||
it("does not wait for another chunk after reading the byte cap exactly", async () => {
|
||||
let canceled = false;
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("abcd"));
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
readResponseTextSnippet(new Response(stream), { maxBytes: 4, maxChars: 100 }),
|
||||
).resolves.toBe("abcd... [truncated]");
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
|
||||
it("cancels snippet body reads when the caller signal aborts", async () => {
|
||||
let canceled = false;
|
||||
const response = stallingResponse(() => {
|
||||
canceled = true;
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const read = readResponseTextSnippet(response, {
|
||||
maxBytes: 1024,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 0);
|
||||
});
|
||||
controller.abort(new Error("snippet aborted"));
|
||||
|
||||
await expect(read).rejects.toThrow("snippet aborted");
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
|
||||
it("cancels JSON body reads when the caller signal aborts", async () => {
|
||||
let canceled = false;
|
||||
const response = stallingResponse(() => {
|
||||
canceled = true;
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const read = readResponseJsonWithLimit(response, {
|
||||
errorPrefix: "remote memory",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 0);
|
||||
});
|
||||
controller.abort(new Error("json aborted"));
|
||||
|
||||
await expect(read).rejects.toThrow("json aborted");
|
||||
expect(canceled).toBe(true);
|
||||
});
|
||||
});
|
||||
247
packages/memory-host-sdk/src/host/response-snippet.ts
Normal file
247
packages/memory-host-sdk/src/host/response-snippet.ts
Normal file
@@ -0,0 +1,247 @@
|
||||
// Memory Host SDK module implements response snippet behavior.
|
||||
const DEFAULT_ERROR_BODY_MAX_BYTES = 8 * 1024;
|
||||
const DEFAULT_ERROR_BODY_MAX_CHARS = 1_000;
|
||||
const DEFAULT_JSON_BODY_MAX_BYTES = 64 * 1024 * 1024;
|
||||
const TRUNCATED_SUFFIX = "... [truncated]";
|
||||
|
||||
// Bounded response readers for provider/remote HTTP errors and JSON bodies.
|
||||
type ResponseTextSnippetOptions = {
|
||||
maxBytes?: number;
|
||||
maxChars?: number;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
type ResponseJsonOptions = {
|
||||
maxBytes?: number;
|
||||
errorPrefix: string;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
type ResponsePrefix = {
|
||||
bytes: Uint8Array[];
|
||||
length: number;
|
||||
truncated: boolean;
|
||||
};
|
||||
|
||||
/** Read a small collapsed text snippet from a response body. */
|
||||
export async function readResponseTextSnippet(
|
||||
res: Response,
|
||||
options: ResponseTextSnippetOptions = {},
|
||||
): Promise<string> {
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_ERROR_BODY_MAX_BYTES;
|
||||
const maxChars = options.maxChars ?? DEFAULT_ERROR_BODY_MAX_CHARS;
|
||||
const prefix = await readResponsePrefix(res, maxBytes, options.signal);
|
||||
if (prefix.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const text = new TextDecoder().decode(joinChunks(prefix.bytes, prefix.length));
|
||||
const collapsed = text.replace(/\s+/g, " ").trim();
|
||||
if (!collapsed) {
|
||||
return "";
|
||||
}
|
||||
if (prefix.truncated || collapsed.length > maxChars) {
|
||||
return `${collapsed.slice(0, maxChars)}${TRUNCATED_SUFFIX}`;
|
||||
}
|
||||
return collapsed;
|
||||
}
|
||||
|
||||
/** Read and parse JSON while enforcing a hard byte limit. */
|
||||
export async function readResponseJsonWithLimit(
|
||||
res: Response,
|
||||
options: ResponseJsonOptions,
|
||||
): Promise<unknown> {
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_JSON_BODY_MAX_BYTES;
|
||||
const contentLength = parseContentLength(res.headers.get("content-length"), options.errorPrefix);
|
||||
if (typeof contentLength === "number" && contentLength > maxBytes) {
|
||||
await cancelResponseBody(res);
|
||||
throw responseTooLarge(options.errorPrefix, contentLength, maxBytes);
|
||||
}
|
||||
|
||||
const text = await readResponseTextWithLimit(res, maxBytes, options.errorPrefix, options.signal);
|
||||
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch (cause) {
|
||||
throw new Error(`${options.errorPrefix}: malformed JSON response`, { cause });
|
||||
}
|
||||
}
|
||||
|
||||
function toAbortError(signal: AbortSignal, fallbackMessage: string): Error {
|
||||
return signal.reason instanceof Error ? signal.reason : new Error(fallbackMessage);
|
||||
}
|
||||
|
||||
async function readChunkWithAbort(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
signal: AbortSignal | undefined,
|
||||
fallbackMessage: string,
|
||||
): Promise<ReadableStreamReadResult<Uint8Array>> {
|
||||
if (!signal) {
|
||||
return await reader.read();
|
||||
}
|
||||
if (signal.aborted) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
throw toAbortError(signal, fallbackMessage);
|
||||
}
|
||||
|
||||
let removeAbortListener: (() => void) | undefined;
|
||||
const abortPromise = new Promise<ReadableStreamReadResult<Uint8Array>>((_resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
void reader.cancel().catch(() => undefined);
|
||||
reject(toAbortError(signal, fallbackMessage));
|
||||
};
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
removeAbortListener = () => signal.removeEventListener("abort", onAbort);
|
||||
});
|
||||
|
||||
try {
|
||||
return await Promise.race([reader.read(), abortPromise]);
|
||||
} finally {
|
||||
removeAbortListener?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function readResponsePrefix(
|
||||
res: Response,
|
||||
maxBytes: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ResponsePrefix> {
|
||||
const body = res.body;
|
||||
if (!body || typeof body.getReader !== "function") {
|
||||
return { bytes: [], length: 0, truncated: false };
|
||||
}
|
||||
|
||||
const reader = body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let length = 0;
|
||||
let truncated = false;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await readChunkWithAbort(
|
||||
reader,
|
||||
signal,
|
||||
"Response snippet body read aborted",
|
||||
);
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
if (!value?.length) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const remaining = maxBytes - length;
|
||||
if (value.length >= remaining) {
|
||||
// Keep only the configured prefix and cancel the body so callers do not
|
||||
// accidentally buffer large provider error responses.
|
||||
if (remaining > 0) {
|
||||
chunks.push(value.subarray(0, remaining));
|
||||
length += remaining;
|
||||
}
|
||||
truncated = true;
|
||||
await reader.cancel().catch(() => undefined);
|
||||
break;
|
||||
}
|
||||
|
||||
chunks.push(value);
|
||||
length += value.length;
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
reader.releaseLock();
|
||||
} catch {}
|
||||
}
|
||||
|
||||
return { bytes: chunks, length, truncated };
|
||||
}
|
||||
|
||||
async function readResponseTextWithLimit(
|
||||
res: Response,
|
||||
maxBytes: number,
|
||||
errorPrefix: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string> {
|
||||
const body = res.body;
|
||||
if (!body || typeof body.getReader !== "function") {
|
||||
return "";
|
||||
}
|
||||
|
||||
const reader = body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let length = 0;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await readChunkWithAbort(
|
||||
reader,
|
||||
signal,
|
||||
`${errorPrefix}: response body read aborted`,
|
||||
);
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
if (!value?.length) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextLength = length + value.length;
|
||||
if (nextLength > maxBytes) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
throw responseTooLarge(errorPrefix, nextLength, maxBytes);
|
||||
}
|
||||
|
||||
chunks.push(value);
|
||||
length = nextLength;
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
reader.releaseLock();
|
||||
} catch {}
|
||||
}
|
||||
|
||||
return new TextDecoder().decode(joinChunks(chunks, length));
|
||||
}
|
||||
|
||||
async function cancelResponseBody(res: Response): Promise<void> {
|
||||
const body = res.body;
|
||||
if (!body || typeof body.cancel !== "function") {
|
||||
return;
|
||||
}
|
||||
await body.cancel().catch(() => undefined);
|
||||
}
|
||||
|
||||
function parseContentLength(raw: string | null, errorPrefix: string): number | undefined {
|
||||
const trimmed = raw?.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
if (!/^(0|[1-9]\d*)$/.test(trimmed)) {
|
||||
throw new Error(`${errorPrefix}: invalid content-length header: ${raw}`);
|
||||
}
|
||||
const value = Number(trimmed);
|
||||
if (!Number.isSafeInteger(value)) {
|
||||
throw new Error(`${errorPrefix}: invalid content-length header: ${raw}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function responseTooLarge(errorPrefix: string, size: number, maxBytes: number): Error {
|
||||
return new Error(responseTooLargeMessage(errorPrefix, size, maxBytes));
|
||||
}
|
||||
|
||||
function responseTooLargeMessage(errorPrefix: string, size: number, maxBytes: number): string {
|
||||
return `${errorPrefix}: response body too large: ${size} bytes (limit: ${maxBytes} bytes)`;
|
||||
}
|
||||
|
||||
function joinChunks(chunks: Uint8Array[], length: number): Uint8Array {
|
||||
if (chunks.length === 1 && chunks[0]?.length === length) {
|
||||
return chunks[0];
|
||||
}
|
||||
const joined = new Uint8Array(length);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
joined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return joined;
|
||||
}
|
||||
86
packages/memory-host-sdk/src/host/retry-utils.test.ts
Normal file
86
packages/memory-host-sdk/src/host/retry-utils.test.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
// Memory Host SDK tests cover retry utils behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { MAX_SAFE_TIMEOUT_DELAY_MS } from "../../../gateway-client/src/timeouts.js";
|
||||
import { resolveRetryConfig, retryAsync } from "./retry-utils.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("resolveRetryConfig", () => {
|
||||
const defaults = {
|
||||
attempts: 4,
|
||||
minDelayMs: 0,
|
||||
maxDelayMs: 0,
|
||||
jitter: 0,
|
||||
};
|
||||
|
||||
it("does not round malformed attempt counts", () => {
|
||||
expect(resolveRetryConfig(defaults, { attempts: 1.5 }).attempts).toBe(4);
|
||||
expect(resolveRetryConfig(defaults, { attempts: Number.POSITIVE_INFINITY }).attempts).toBe(4);
|
||||
expect(resolveRetryConfig(defaults, { attempts: Number.NaN }).attempts).toBe(4);
|
||||
});
|
||||
|
||||
it("caps oversized retry delays at the timer-safe ceiling", () => {
|
||||
const config = resolveRetryConfig(defaults, {
|
||||
minDelayMs: Number.MAX_SAFE_INTEGER,
|
||||
maxDelayMs: Number.MAX_SAFE_INTEGER,
|
||||
});
|
||||
|
||||
expect(config.minDelayMs).toBe(MAX_SAFE_TIMEOUT_DELAY_MS);
|
||||
expect(config.maxDelayMs).toBe(MAX_SAFE_TIMEOUT_DELAY_MS);
|
||||
});
|
||||
});
|
||||
|
||||
describe("retryAsync", () => {
|
||||
it("falls back to the default attempt count for malformed numeric counts", async () => {
|
||||
const run = vi.fn(async () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
|
||||
await expect(retryAsync(run, Number.NaN, 0)).rejects.toThrow("boom");
|
||||
|
||||
expect(run).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("caps legacy numeric retry sleeps at the timer-safe ceiling", async () => {
|
||||
const run = vi
|
||||
.fn<() => Promise<string>>()
|
||||
.mockRejectedValueOnce(new Error("boom"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation((callback) => {
|
||||
if (typeof callback === "function") {
|
||||
callback();
|
||||
}
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
});
|
||||
|
||||
await expect(retryAsync(run, 2, Number.MAX_SAFE_INTEGER)).resolves.toBe("ok");
|
||||
|
||||
expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_SAFE_TIMEOUT_DELAY_MS);
|
||||
});
|
||||
|
||||
it("caps retryAfterMs sleeps at the timer-safe ceiling", async () => {
|
||||
const run = vi
|
||||
.fn<() => Promise<string>>()
|
||||
.mockRejectedValueOnce(new Error("boom"))
|
||||
.mockResolvedValueOnce("ok");
|
||||
const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation((callback) => {
|
||||
if (typeof callback === "function") {
|
||||
callback();
|
||||
}
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
});
|
||||
|
||||
await expect(
|
||||
retryAsync(run, {
|
||||
attempts: 2,
|
||||
minDelayMs: 0,
|
||||
maxDelayMs: Number.MAX_SAFE_INTEGER,
|
||||
retryAfterMs: () => Number.MAX_SAFE_INTEGER,
|
||||
}),
|
||||
).resolves.toBe("ok");
|
||||
|
||||
expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_SAFE_TIMEOUT_DELAY_MS);
|
||||
});
|
||||
});
|
||||
175
packages/memory-host-sdk/src/host/retry-utils.ts
Normal file
175
packages/memory-host-sdk/src/host/retry-utils.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
// Memory Host SDK helper module supports retry utils behavior.
|
||||
import { resolveSafeTimeoutDelayMs } from "../../../gateway-client/src/timeouts.js";
|
||||
|
||||
/** Retry timing configuration with optional jitter. */
|
||||
export type RetryConfig = {
|
||||
attempts?: number;
|
||||
minDelayMs?: number;
|
||||
maxDelayMs?: number;
|
||||
jitter?: number;
|
||||
};
|
||||
|
||||
/** Retry callback payload. */
|
||||
export type RetryInfo = {
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
delayMs: number;
|
||||
err: unknown;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
/** Retry options for retryAsync. */
|
||||
export type RetryOptions = RetryConfig & {
|
||||
label?: string;
|
||||
shouldRetry?: (err: unknown, attempt: number) => boolean;
|
||||
retryAfterMs?: (err: unknown) => number | undefined;
|
||||
onRetry?: (info: RetryInfo) => void;
|
||||
};
|
||||
|
||||
const DEFAULT_RETRY_CONFIG = {
|
||||
attempts: 3,
|
||||
minDelayMs: 300,
|
||||
maxDelayMs: 30_000,
|
||||
jitter: 0,
|
||||
};
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function asFiniteNumber(value: unknown): number | undefined {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
return undefined;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function clampNumber(value: unknown, fallback: number, min?: number, max?: number): number {
|
||||
const next = asFiniteNumber(value);
|
||||
if (next === undefined) {
|
||||
return fallback;
|
||||
}
|
||||
const floor = typeof min === "number" ? min : Number.NEGATIVE_INFINITY;
|
||||
const ceiling = typeof max === "number" ? max : Number.POSITIVE_INFINITY;
|
||||
return Math.min(Math.max(next, floor), ceiling);
|
||||
}
|
||||
|
||||
function resolveAttempts(value: unknown, fallback: number): number {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value)) {
|
||||
return fallback;
|
||||
}
|
||||
return Math.max(1, value);
|
||||
}
|
||||
|
||||
/** Resolve retry settings with clamped positive timeout values. */
|
||||
export function resolveRetryConfig(
|
||||
defaults: Required<RetryConfig> = DEFAULT_RETRY_CONFIG,
|
||||
overrides?: RetryConfig,
|
||||
): Required<RetryConfig> {
|
||||
const attempts = resolveAttempts(overrides?.attempts, defaults.attempts);
|
||||
const minDelayMs = resolveSafeTimeoutDelayMs(
|
||||
Math.round(clampNumber(overrides?.minDelayMs, defaults.minDelayMs, 0)),
|
||||
{ minMs: 0 },
|
||||
);
|
||||
const maxDelayMs = Math.max(
|
||||
minDelayMs,
|
||||
resolveSafeTimeoutDelayMs(
|
||||
Math.round(clampNumber(overrides?.maxDelayMs, defaults.maxDelayMs, 0)),
|
||||
{ minMs: 0 },
|
||||
),
|
||||
);
|
||||
const jitter = clampNumber(overrides?.jitter, defaults.jitter, 0, 1);
|
||||
return { attempts, minDelayMs, maxDelayMs, jitter };
|
||||
}
|
||||
|
||||
function applyJitter(delayMs: number, jitter: number): number {
|
||||
if (jitter <= 0) {
|
||||
return delayMs;
|
||||
}
|
||||
const offset = (Math.random() * 2 - 1) * jitter;
|
||||
return Math.max(0, Math.round(delayMs * (1 + offset)));
|
||||
}
|
||||
|
||||
/** Run an async operation with exponential backoff retry handling. */
|
||||
export async function retryAsync<T>(
|
||||
fn: () => Promise<T>,
|
||||
attemptsOrOptions: number | RetryOptions = 3,
|
||||
initialDelayMs = 300,
|
||||
): Promise<T> {
|
||||
if (typeof attemptsOrOptions === "number") {
|
||||
const attempts = resolveAttempts(attemptsOrOptions, DEFAULT_RETRY_CONFIG.attempts);
|
||||
let lastErr: unknown;
|
||||
for (let i = 0; i < attempts; i += 1) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (i === attempts - 1) {
|
||||
break;
|
||||
}
|
||||
await sleep(resolveSafeTimeoutDelayMs(initialDelayMs * 2 ** i, { minMs: 0 }));
|
||||
}
|
||||
}
|
||||
throw toLintErrorObject(lastErr ?? new Error("Retry failed"), "Non-Error thrown");
|
||||
}
|
||||
|
||||
const options = attemptsOrOptions;
|
||||
const resolved = resolveRetryConfig(DEFAULT_RETRY_CONFIG, options);
|
||||
const maxAttempts = resolved.attempts;
|
||||
const minDelayMs = resolved.minDelayMs;
|
||||
const maxDelayMs =
|
||||
Number.isFinite(resolved.maxDelayMs) && resolved.maxDelayMs > 0
|
||||
? resolved.maxDelayMs
|
||||
: Number.POSITIVE_INFINITY;
|
||||
const shouldRetry = options.shouldRetry ?? (() => true);
|
||||
let lastErr: unknown;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (attempt >= maxAttempts || !shouldRetry(err, attempt)) {
|
||||
break;
|
||||
}
|
||||
|
||||
const retryAfterMs = options.retryAfterMs?.(err);
|
||||
const hasRetryAfter = typeof retryAfterMs === "number" && Number.isFinite(retryAfterMs);
|
||||
const baseDelay = hasRetryAfter
|
||||
? Math.max(resolveSafeTimeoutDelayMs(retryAfterMs, { minMs: 0 }), minDelayMs)
|
||||
: resolveSafeTimeoutDelayMs(minDelayMs * 2 ** (attempt - 1), { minMs: 0 });
|
||||
let delay = Math.min(baseDelay, maxDelayMs);
|
||||
delay = applyJitter(delay, resolved.jitter);
|
||||
delay = Math.min(Math.max(delay, minDelayMs), maxDelayMs);
|
||||
|
||||
options.onRetry?.({
|
||||
attempt,
|
||||
maxAttempts,
|
||||
delayMs: delay,
|
||||
err,
|
||||
label: options.label,
|
||||
});
|
||||
if (delay > 0) {
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw toLintErrorObject(lastErr ?? new Error("Retry failed"), "Non-Error thrown");
|
||||
}
|
||||
|
||||
export function toLintErrorObject(value: unknown, fallbackMessage: string): Error {
|
||||
if (value instanceof Error) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return new Error(value);
|
||||
}
|
||||
const error = new Error(fallbackMessage, { cause: value });
|
||||
if ((typeof value === "object" && value !== null) || typeof value === "function") {
|
||||
Object.assign(error, value);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
164
packages/memory-host-sdk/src/host/secret-input-utils.ts
Normal file
164
packages/memory-host-sdk/src/host/secret-input-utils.ts
Normal file
@@ -0,0 +1,164 @@
|
||||
// Secret input parsing shared by memory provider config and gateway-resolved snapshots.
|
||||
|
||||
/** Supported secret reference backing stores. */
|
||||
export type SecretRefSource = "env" | "file" | "exec";
|
||||
|
||||
/** Canonical secret reference shape used after gateway resolution. */
|
||||
export type SecretRef = {
|
||||
source: SecretRefSource;
|
||||
provider: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
const DEFAULT_SECRET_PROVIDER_ALIAS = "default";
|
||||
const ENV_SECRET_REF_ID_RE = /^[A-Z][A-Z0-9_]{0,127}$/;
|
||||
const LEGACY_SECRETREF_ENV_MARKER_PREFIX = "secretref-env:";
|
||||
const ENV_SECRET_TEMPLATE_RE = /^\$\{([A-Z][A-Z0-9_]{0,127})\}$/;
|
||||
const SECRET_REF_SOURCES = new Set<SecretRefSource>(["env", "file", "exec"]);
|
||||
|
||||
/** Narrow unknown JSON config values to plain records. */
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** Normalize literal secret strings and reject empty placeholders. */
|
||||
function normalizeSecretInputString(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
/** Narrow a string to a supported SecretRef source. */
|
||||
function hasSecretRefSource(value: unknown): value is SecretRefSource {
|
||||
return typeof value === "string" && SECRET_REF_SOURCES.has(value as SecretRefSource);
|
||||
}
|
||||
|
||||
/** Narrow unknown values to non-empty strings. */
|
||||
function hasNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
/** Detect canonical three-field SecretRef objects. */
|
||||
function isSecretRef(value: unknown): value is SecretRef {
|
||||
if (!isRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
const keys = Object.keys(value);
|
||||
return (
|
||||
keys.length === 3 &&
|
||||
hasSecretRefSource(value.source) &&
|
||||
hasNonEmptyString(value.provider) &&
|
||||
hasNonEmptyString(value.id)
|
||||
);
|
||||
}
|
||||
|
||||
/** Detect legacy refs that predate explicit provider names. */
|
||||
function isLegacySecretRefWithoutProvider(
|
||||
value: unknown,
|
||||
): value is { source: SecretRefSource; id: string } {
|
||||
if (!isRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
hasSecretRefSource(value.source) && hasNonEmptyString(value.id) && value.provider === undefined
|
||||
);
|
||||
}
|
||||
|
||||
/** Parse env template shorthand such as "${OPENAI_API_KEY}". */
|
||||
function parseEnvTemplateSecretRef(value: unknown): SecretRef | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const match = ENV_SECRET_TEMPLATE_RE.exec(value.trim());
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
source: "env",
|
||||
provider: DEFAULT_SECRET_PROVIDER_ALIAS,
|
||||
id: match[1] ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse legacy secretref-env markers from older config snapshots. */
|
||||
function parseLegacySecretRefEnvMarker(value: unknown): SecretRef | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed.startsWith(LEGACY_SECRETREF_ENV_MARKER_PREFIX)) {
|
||||
return null;
|
||||
}
|
||||
const id = trimmed.slice(LEGACY_SECRETREF_ENV_MARKER_PREFIX.length);
|
||||
if (!ENV_SECRET_REF_ID_RE.test(id)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
source: "env",
|
||||
provider: DEFAULT_SECRET_PROVIDER_ALIAS,
|
||||
id,
|
||||
};
|
||||
}
|
||||
|
||||
/** Coerce all accepted shipped secret reference shapes to canonical SecretRef. */
|
||||
function coerceSecretRef(value: unknown): SecretRef | null {
|
||||
if (isSecretRef(value)) {
|
||||
return value;
|
||||
}
|
||||
if (isLegacySecretRefWithoutProvider(value)) {
|
||||
return {
|
||||
source: value.source,
|
||||
provider: DEFAULT_SECRET_PROVIDER_ALIAS,
|
||||
id: value.id,
|
||||
};
|
||||
}
|
||||
return parseEnvTemplateSecretRef(value) ?? parseLegacySecretRefEnvMarker(value);
|
||||
}
|
||||
|
||||
/** Return true when a secret input has either a literal value or resolvable reference shape. */
|
||||
export function hasConfiguredSecretInput(value: unknown): boolean {
|
||||
if (normalizeSecretInputString(value)) {
|
||||
return true;
|
||||
}
|
||||
return coerceSecretRef(value) !== null;
|
||||
}
|
||||
|
||||
/** Format a ref label without revealing a resolved secret value. */
|
||||
function formatSecretRefLabel(ref: SecretRef): string {
|
||||
return `${ref.source}:${ref.provider}:${ref.id}`;
|
||||
}
|
||||
|
||||
/** Build the unresolved-ref error used when callers bypass gateway secret resolution. */
|
||||
function createUnresolvedSecretInputError(params: { path: string; ref: SecretRef }): Error {
|
||||
return new Error(
|
||||
`${params.path}: unresolved SecretRef "${formatSecretRefLabel(params.ref)}". Resolve this command against an active gateway runtime snapshot before reading it.`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Return a canonical SecretRef when the input is a supported reference shape. */
|
||||
export function resolveSecretInputRef(value: unknown): SecretRef | null {
|
||||
return coerceSecretRef(value);
|
||||
}
|
||||
|
||||
/** Normalize literal secrets, or throw for refs that still require gateway resolution. */
|
||||
export function normalizeResolvedSecretInputString(params: {
|
||||
value: unknown;
|
||||
path: string;
|
||||
}): string | undefined {
|
||||
const normalized = normalizeSecretInputString(params.value);
|
||||
if (normalized) {
|
||||
return normalized;
|
||||
}
|
||||
const ref = resolveSecretInputRef(params.value);
|
||||
if (!ref) {
|
||||
return undefined;
|
||||
}
|
||||
throw createUnresolvedSecretInputError({ path: params.path, ref });
|
||||
}
|
||||
|
||||
/** Normalize env-provided secret values before use. */
|
||||
export function normalizeEnvSecretInputString(value: unknown): string | undefined {
|
||||
return normalizeSecretInputString(value);
|
||||
}
|
||||
32
packages/memory-host-sdk/src/host/secret-input.ts
Normal file
32
packages/memory-host-sdk/src/host/secret-input.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
// Memory Host SDK module implements secret input behavior.
|
||||
import {
|
||||
hasConfiguredSecretInput,
|
||||
normalizeEnvSecretInputString,
|
||||
normalizeResolvedSecretInputString,
|
||||
resolveSecretInputRef,
|
||||
} from "./secret-input-utils.js";
|
||||
|
||||
// Memory-specific facade for resolving provider secret input from config.
|
||||
|
||||
/** Return true when a configured memory secret contains a literal value or reference. */
|
||||
export function hasConfiguredMemorySecretInput(value: unknown): boolean {
|
||||
return hasConfiguredSecretInput(value);
|
||||
}
|
||||
|
||||
/** Resolve memory secret input, reading env refs directly when available. */
|
||||
export function resolveMemorySecretInputString(params: {
|
||||
value: unknown;
|
||||
path: string;
|
||||
}): string | undefined {
|
||||
const ref = resolveSecretInputRef(params.value);
|
||||
if (ref?.source === "env") {
|
||||
const envValue = normalizeEnvSecretInputString(process.env[ref.id]);
|
||||
if (envValue) {
|
||||
return envValue;
|
||||
}
|
||||
}
|
||||
return normalizeResolvedSecretInputString({
|
||||
value: params.value,
|
||||
path: params.path,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Memory Host SDK tests cover session files yield behavior.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { fileState } = vi.hoisted(() => ({
|
||||
fileState: { raw: "" },
|
||||
}));
|
||||
|
||||
vi.mock("./fs-utils.js", () => ({
|
||||
readRegularFile: vi.fn(async () => ({
|
||||
buffer: Buffer.from(fileState.raw, "utf-8"),
|
||||
})),
|
||||
statRegularFile: vi.fn(async () => ({
|
||||
missing: false,
|
||||
stat: {
|
||||
mtimeMs: 1,
|
||||
size: Buffer.byteLength(fileState.raw, "utf-8"),
|
||||
},
|
||||
})),
|
||||
}));
|
||||
|
||||
import { buildSessionEntry } from "./session-files.js";
|
||||
|
||||
describe("buildSessionEntry responsiveness", () => {
|
||||
afterEach(() => {
|
||||
fileState.raw = "";
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("yields while parsing a single large transcript", async () => {
|
||||
fileState.raw = Array.from({ length: 25 }, (_value, index) =>
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: `message ${index}` },
|
||||
}),
|
||||
).join("\n");
|
||||
let immediateRan = false;
|
||||
const immediate = new Promise<void>((resolve) => {
|
||||
setImmediate(() => {
|
||||
immediateRan = true;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
const entry = await buildSessionEntry("/tmp/session.jsonl", {
|
||||
generatedByCronRun: false,
|
||||
generatedByDreamingNarrative: false,
|
||||
parseYieldEveryLines: 10,
|
||||
});
|
||||
|
||||
expect(entry?.lineMap).toHaveLength(25);
|
||||
expect(immediateRan).toBe(true);
|
||||
await immediate;
|
||||
});
|
||||
});
|
||||
915
packages/memory-host-sdk/src/host/session-files.test.ts
Normal file
915
packages/memory-host-sdk/src/host/session-files.test.ts
Normal file
@@ -0,0 +1,915 @@
|
||||
// Memory Host SDK tests cover session files behavior.
|
||||
import fsSync from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { clearConfigCache, clearRuntimeConfigSnapshot } from "./openclaw-runtime-session.js";
|
||||
import {
|
||||
buildSessionEntry,
|
||||
listSessionFilesForAgent,
|
||||
listSessionTranscriptCorpusEntriesForAgent,
|
||||
loadSessionTranscriptClassificationForAgent,
|
||||
parseCanonicalSessionSyncTargetFromPath,
|
||||
resolveSessionIdentityForTranscriptFile,
|
||||
resolveSessionFileForSyncTarget,
|
||||
sessionPathForFile,
|
||||
type SessionFileEntry,
|
||||
} from "./session-files.js";
|
||||
|
||||
function captureStateDirEnv() {
|
||||
const stateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
const configPath = process.env.OPENCLAW_CONFIG_PATH;
|
||||
return {
|
||||
restore() {
|
||||
if (stateDir === undefined) {
|
||||
Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR");
|
||||
} else {
|
||||
Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir);
|
||||
}
|
||||
if (configPath === undefined) {
|
||||
Reflect.deleteProperty(process.env, "OPENCLAW_CONFIG_PATH");
|
||||
} else {
|
||||
Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let fixtureRoot: string;
|
||||
let tmpDir: string;
|
||||
let envSnapshot: ReturnType<typeof captureStateDirEnv> | undefined;
|
||||
let fixtureId = 0;
|
||||
|
||||
beforeAll(() => {
|
||||
fixtureRoot = fsSync.mkdtempSync(path.join(os.tmpdir(), "session-entry-test-"));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fsSync.rmSync(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = path.join(fixtureRoot, `case-${fixtureId++}`);
|
||||
fsSync.mkdirSync(tmpDir, { recursive: true });
|
||||
envSnapshot = captureStateDirEnv();
|
||||
Reflect.set(process.env, "OPENCLAW_STATE_DIR", tmpDir);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
envSnapshot?.restore();
|
||||
envSnapshot = undefined;
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
});
|
||||
|
||||
function requireSessionEntry(entry: SessionFileEntry | null): SessionFileEntry {
|
||||
if (!entry) {
|
||||
throw new Error("expected session entry");
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
describe("listSessionFilesForAgent", () => {
|
||||
it("includes reset and deleted transcripts in session file listing", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(path.join(sessionsDir, "archive"), { recursive: true });
|
||||
|
||||
const included = [
|
||||
"active.jsonl",
|
||||
"active.jsonl.reset.2026-02-16T22-26-33.000Z",
|
||||
"active.jsonl.deleted.2026-02-16T22-27-33.000Z",
|
||||
];
|
||||
const excluded = ["active.jsonl.bak.2026-02-16T22-28-33.000Z", "sessions.json", "notes.md"];
|
||||
excluded.push("active.checkpoint.11111111-1111-4111-8111-111111111111.jsonl");
|
||||
|
||||
for (const fileName of [...included, ...excluded]) {
|
||||
fsSync.writeFileSync(path.join(sessionsDir, fileName), "");
|
||||
}
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "archive", "nested.jsonl.deleted.2026-02-16T22-29-33.000Z"),
|
||||
"",
|
||||
);
|
||||
|
||||
const files = await listSessionFilesForAgent("main");
|
||||
|
||||
expect(files.map((filePath) => path.basename(filePath)).toSorted()).toEqual(
|
||||
included.toSorted(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("listSessionTranscriptCorpusEntriesForAgent", () => {
|
||||
it("lists active session entries with accessor-backed identity and classification", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(sessionsDir, "narrative.jsonl"), "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:dreaming-narrative-run-1": {
|
||||
sessionFile: "narrative.jsonl",
|
||||
sessionId: "narrative",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([
|
||||
{
|
||||
agentId: "main",
|
||||
artifactKind: "active-session",
|
||||
generatedByDreamingNarrative: true,
|
||||
sessionFile: path.join(sessionsDir, "narrative.jsonl"),
|
||||
sessionId: "narrative",
|
||||
sessionKey: "agent:main:dreaming-narrative-run-1",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps archive artifacts in the corpus and inherits active session classification", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
const activePath = path.join(sessionsDir, "cron-run.jsonl");
|
||||
const archivePath = path.join(sessionsDir, "cron-run.jsonl.deleted.2026-02-16T22-27-33.000Z");
|
||||
fsSync.writeFileSync(activePath, "");
|
||||
fsSync.writeFileSync(archivePath, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:cron:job-1:run:run-1": {
|
||||
sessionFile: "cron-run.jsonl",
|
||||
sessionId: "cron-run",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const classification = loadSessionTranscriptClassificationForAgent("main");
|
||||
|
||||
expect(classification.cronRunTranscriptPaths).toEqual(
|
||||
new Set([activePath, archivePath].map((filePath) => path.resolve(filePath))),
|
||||
);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toContainEqual({
|
||||
agentId: "main",
|
||||
artifactKind: "archive-artifact",
|
||||
generatedByCronRun: true,
|
||||
sessionFile: archivePath,
|
||||
sessionId: "cron-run",
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies active entries through cron parentage chains", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
const cronPath = path.join(sessionsDir, "cron-run.jsonl");
|
||||
const spawnedChildPath = path.join(sessionsDir, "spawned-child.jsonl");
|
||||
const keyedChildPath = path.join(sessionsDir, "keyed-child.jsonl");
|
||||
const orphanChildPath = path.join(sessionsDir, "orphan-child.jsonl");
|
||||
const normalPath = path.join(sessionsDir, "normal-child.jsonl");
|
||||
for (const filePath of [
|
||||
cronPath,
|
||||
spawnedChildPath,
|
||||
keyedChildPath,
|
||||
orphanChildPath,
|
||||
normalPath,
|
||||
]) {
|
||||
fsSync.writeFileSync(filePath, "");
|
||||
}
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:cron:job-1:run:run-1": {
|
||||
sessionFile: "cron-run.jsonl",
|
||||
sessionId: "cron-run",
|
||||
},
|
||||
"agent:main:subagent:spawned-child": {
|
||||
sessionFile: "spawned-child.jsonl",
|
||||
sessionId: "spawned-child",
|
||||
spawnedBy: "agent:main:cron:job-1:run:run-1",
|
||||
},
|
||||
"agent:main:subagent:keyed-child": {
|
||||
parentSessionKey: "agent:main:subagent:spawned-child",
|
||||
sessionFile: "keyed-child.jsonl",
|
||||
sessionId: "keyed-child",
|
||||
},
|
||||
"agent:main:subagent:orphan-child": {
|
||||
sessionFile: "orphan-child.jsonl",
|
||||
sessionId: "orphan-child",
|
||||
spawnedBy: "agent:main:cron:job-1:run:missing",
|
||||
},
|
||||
"agent:main:subagent:normal-child": {
|
||||
sessionFile: "normal-child.jsonl",
|
||||
sessionId: "normal-child",
|
||||
spawnedBy: "agent:main:chat:manual",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const classification = loadSessionTranscriptClassificationForAgent("main");
|
||||
|
||||
expect(classification.cronRunTranscriptPaths).toEqual(
|
||||
new Set(
|
||||
[cronPath, spawnedChildPath, keyedChildPath, orphanChildPath].map((filePath) =>
|
||||
path.resolve(filePath),
|
||||
),
|
||||
),
|
||||
);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
generatedByCronRun: true,
|
||||
sessionFile: spawnedChildPath,
|
||||
sessionKey: "agent:main:subagent:spawned-child",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
generatedByCronRun: true,
|
||||
sessionFile: keyedChildPath,
|
||||
sessionKey: "agent:main:subagent:keyed-child",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
generatedByCronRun: true,
|
||||
sessionFile: orphanChildPath,
|
||||
sessionKey: "agent:main:subagent:orphan-child",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
sessionFile: normalPath,
|
||||
sessionKey: "agent:main:subagent:normal-child",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
const entries = await listSessionTranscriptCorpusEntriesForAgent("main");
|
||||
expect(entries.find((entry) => entry.sessionFile === normalPath)?.generatedByCronRun).toBe(
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps archive classification when the active transcript is missing", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
const archivePath = path.join(sessionsDir, "cron-run.jsonl.reset.2026-02-16T22-26-33.000Z");
|
||||
fsSync.writeFileSync(archivePath, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:cron:job-1:run:run-1": {
|
||||
sessionFile: "cron-run.jsonl",
|
||||
sessionId: "cron-run",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const expectedArchivePath = archivePath;
|
||||
const classification = loadSessionTranscriptClassificationForAgent("main");
|
||||
|
||||
expect(classification.cronRunTranscriptPaths).toEqual(new Set([expectedArchivePath]));
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([expectedArchivePath]);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([
|
||||
{
|
||||
agentId: "main",
|
||||
artifactKind: "archive-artifact",
|
||||
generatedByCronRun: true,
|
||||
sessionFile: expectedArchivePath,
|
||||
sessionId: "cron-run",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits active session entries whose transcript files are missing", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:missing": {
|
||||
sessionFile: "missing.jsonl",
|
||||
sessionId: "missing",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([]);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("omits active session entries whose transcript path is a symlink", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
const targetPath = path.join(tmpDir, "external.jsonl");
|
||||
const symlinkPath = path.join(sessionsDir, "linked.jsonl");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(targetPath, "");
|
||||
fsSync.symlinkSync(targetPath, symlinkPath);
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:linked": {
|
||||
sessionFile: "linked.jsonl",
|
||||
sessionId: "linked",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([]);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects session ids that would escape the sessions directory", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(tmpDir, "secret.jsonl"), "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:escape": {
|
||||
sessionId: "../secret",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([]);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("does not classify a fallback transcript when explicit sessionFile is invalid", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
const sessionFile = path.join(sessionsDir, "active.jsonl");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(sessionFile, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:cron:job-1:run:run-1": {
|
||||
sessionFile: "../old.jsonl",
|
||||
sessionId: "active",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([
|
||||
{
|
||||
agentId: "main",
|
||||
artifactKind: "orphan-file-artifact",
|
||||
sessionFile,
|
||||
sessionId: "active",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects relative sessionFile values that escape through nested segments", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
const secretPath = path.join(tmpDir, "agents", "main", "secret.jsonl");
|
||||
fsSync.mkdirSync(path.join(sessionsDir, "sub"), { recursive: true });
|
||||
fsSync.writeFileSync(secretPath, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:escape-file": {
|
||||
sessionFile: "sub/../../secret.jsonl",
|
||||
sessionId: "secret",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects absolute transcript paths owned by another agent", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
const otherSessionsDir = path.join(tmpDir, "agents", "ops", "sessions");
|
||||
const otherSessionFile = path.join(otherSessionsDir, "private.jsonl");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.mkdirSync(otherSessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(otherSessionFile, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:cross-agent": {
|
||||
sessionFile: otherSessionFile,
|
||||
sessionId: "private",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("falls back to transcript filename identity when an active row lacks sessionId", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
const sessionFile = path.join(sessionsDir, "active-thread-456.jsonl");
|
||||
fsSync.writeFileSync(sessionFile, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:thread-456": {
|
||||
sessionFile: "active-thread-456.jsonl",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([
|
||||
{
|
||||
agentId: "main",
|
||||
artifactKind: "active-session",
|
||||
sessionFile,
|
||||
sessionId: "active-thread-456",
|
||||
sessionKey: "agent:main:chat:thread-456",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("lists only the requested agent's active transcripts from a shared custom store", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "custom-sessions");
|
||||
const sessionFile = path.join(sessionsDir, "custom-thread.jsonl");
|
||||
const otherSessionFile = path.join(sessionsDir, "ops-thread.jsonl");
|
||||
const storePath = path.join(sessionsDir, "sessions.json");
|
||||
const configPath = path.join(tmpDir, "openclaw.json");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(sessionFile, "");
|
||||
fsSync.writeFileSync(otherSessionFile, "");
|
||||
fsSync.writeFileSync(
|
||||
storePath,
|
||||
JSON.stringify({
|
||||
"agent:main:chat:custom": {
|
||||
sessionFile: "custom-thread.jsonl",
|
||||
sessionId: "custom-thread",
|
||||
},
|
||||
"agent:ops:chat:custom": {
|
||||
sessionFile: "ops-thread.jsonl",
|
||||
sessionId: "ops-thread",
|
||||
},
|
||||
}),
|
||||
);
|
||||
fsSync.writeFileSync(configPath, JSON.stringify({ session: { store: storePath } }));
|
||||
Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([sessionFile]);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([
|
||||
{
|
||||
agentId: "main",
|
||||
artifactKind: "active-session",
|
||||
sessionFile,
|
||||
sessionId: "custom-thread",
|
||||
sessionKey: "agent:main:chat:custom",
|
||||
},
|
||||
]);
|
||||
await expect(listSessionFilesForAgent("ops")).resolves.toEqual([otherSessionFile]);
|
||||
});
|
||||
|
||||
it("keeps unowned archives from an agent-owned fixed session store", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
const archivePath = path.join(sessionsDir, "retained.jsonl.deleted.2026-02-16T22-27-33.000Z");
|
||||
const configPath = path.join(tmpDir, "openclaw.json");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(archivePath, "");
|
||||
fsSync.writeFileSync(path.join(sessionsDir, "sessions.json"), "{}");
|
||||
fsSync.writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify({ session: { store: path.join(sessionsDir, "sessions.json") } }),
|
||||
);
|
||||
Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([archivePath]);
|
||||
await expect(listSessionTranscriptCorpusEntriesForAgent("main")).resolves.toEqual([
|
||||
{
|
||||
agentId: "main",
|
||||
artifactKind: "archive-artifact",
|
||||
sessionFile: archivePath,
|
||||
sessionId: "retained",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("resolves absolute transcript paths from a fixed custom store", async () => {
|
||||
const storeDir = path.join(tmpDir, "custom-sessions");
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
const sessionFile = path.join(sessionsDir, "absolute-thread.jsonl");
|
||||
const archivePath = path.join(
|
||||
sessionsDir,
|
||||
"absolute-thread.jsonl.deleted.2026-02-16T22-27-33.000Z",
|
||||
);
|
||||
const storePath = path.join(storeDir, "sessions.json");
|
||||
const configPath = path.join(tmpDir, "openclaw.json");
|
||||
fsSync.mkdirSync(storeDir, { recursive: true });
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(sessionFile, "");
|
||||
fsSync.writeFileSync(archivePath, "");
|
||||
fsSync.writeFileSync(
|
||||
storePath,
|
||||
JSON.stringify({
|
||||
"agent:main:chat:absolute": {
|
||||
sessionFile,
|
||||
sessionId: "absolute-thread",
|
||||
},
|
||||
}),
|
||||
);
|
||||
fsSync.writeFileSync(configPath, JSON.stringify({ session: { store: storePath } }));
|
||||
Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([sessionFile, archivePath]);
|
||||
});
|
||||
|
||||
it("keeps legacy session keys in non-main per-agent stores", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "ops", "sessions");
|
||||
const sessionFile = path.join(sessionsDir, "legacy-thread.jsonl");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(sessionFile, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"slack:workspace:thread": {
|
||||
sessionFile: "legacy-thread.jsonl",
|
||||
sessionId: "legacy-thread",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(listSessionFilesForAgent("ops")).resolves.toEqual([sessionFile]);
|
||||
await expect(listSessionFilesForAgent("main")).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps legacy main aliases in a renamed default agent store", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "ops", "sessions");
|
||||
const sessionFile = path.join(sessionsDir, "legacy-main.jsonl");
|
||||
const configPath = path.join(tmpDir, "openclaw.json");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(sessionFile, "");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:main": {
|
||||
sessionFile: "legacy-main.jsonl",
|
||||
sessionId: "legacy-main",
|
||||
},
|
||||
}),
|
||||
);
|
||||
fsSync.writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify({ agents: { list: [{ id: "ops", default: true }] } }),
|
||||
);
|
||||
Reflect.set(process.env, "OPENCLAW_CONFIG_PATH", configPath);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
|
||||
await expect(listSessionFilesForAgent("ops")).resolves.toEqual([sessionFile]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sessionPathForFile", () => {
|
||||
it("includes the owning agent id when the transcript lives under an agent sessions dir", () => {
|
||||
const absPath = path.join(
|
||||
tmpDir,
|
||||
"agents",
|
||||
"main",
|
||||
"sessions",
|
||||
"deleted-session.jsonl.deleted.2026-02-16T22-27-33.000Z",
|
||||
);
|
||||
|
||||
expect(sessionPathForFile(absPath)).toBe(
|
||||
"sessions/main/deleted-session.jsonl.deleted.2026-02-16T22-27-33.000Z",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the legacy basename-only path when the agent owner cannot be derived", () => {
|
||||
expect(sessionPathForFile(path.join(tmpDir, "loose-session.jsonl"))).toBe(
|
||||
"sessions/loose-session.jsonl",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory session sync targets", () => {
|
||||
it("parses deprecated canonical OpenClaw transcript paths into sync identity", () => {
|
||||
const sessionFile = path.join(tmpDir, "agents", "main", "sessions", "active.jsonl");
|
||||
fsSync.mkdirSync(path.dirname(sessionFile), { recursive: true });
|
||||
|
||||
expect(parseCanonicalSessionSyncTargetFromPath(sessionFile)).toEqual({
|
||||
agentId: "main",
|
||||
sessionId: "active",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects arbitrary deprecated transcript path hints", () => {
|
||||
expect(parseCanonicalSessionSyncTargetFromPath(path.join(tmpDir, "active.jsonl"))).toBeNull();
|
||||
expect(
|
||||
parseCanonicalSessionSyncTargetFromPath(
|
||||
path.join(tmpDir, "agents", "main", "sessions", "active.trajectory.jsonl"),
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("resolves identity sync targets to the current file-backed transcript", () => {
|
||||
expect(resolveSessionFileForSyncTarget({ sessionId: "active" }, "main")).toEqual({
|
||||
agentId: "main",
|
||||
sessionId: "active",
|
||||
sessionFile: path.join(tmpDir, "agents", "main", "sessions", "active.jsonl"),
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes agent ids before resolving identity sync targets", () => {
|
||||
expect(resolveSessionFileForSyncTarget({ agentId: "MAIN", sessionId: "active" })).toEqual({
|
||||
agentId: "main",
|
||||
sessionId: "active",
|
||||
sessionFile: path.join(tmpDir, "agents", "main", "sessions", "active.jsonl"),
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects identity sync targets that would escape the sessions directory", () => {
|
||||
expect(resolveSessionFileForSyncTarget({ sessionId: "../outside" }, "main")).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects identity sync targets that normalize to another transcript", () => {
|
||||
expect(resolveSessionFileForSyncTarget({ sessionId: "foo/../active" }, "main")).toBeNull();
|
||||
});
|
||||
|
||||
it("resolves identity sync targets through persisted session keys", () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:thread-456": {
|
||||
sessionFile: "active-thread-456.jsonl",
|
||||
sessionId: "active",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
resolveSessionFileForSyncTarget({
|
||||
agentId: "main",
|
||||
sessionId: "active",
|
||||
sessionKey: "agent:main:chat:thread-456",
|
||||
}),
|
||||
).toEqual({
|
||||
agentId: "main",
|
||||
sessionId: "active",
|
||||
sessionFile: path.join(sessionsDir, "active-thread-456.jsonl"),
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves identity sync targets through persisted session ids", () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:thread-456": {
|
||||
sessionFile: "active-thread-456.jsonl",
|
||||
sessionId: "active",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(resolveSessionFileForSyncTarget({ agentId: "main", sessionId: "active" })).toEqual({
|
||||
agentId: "main",
|
||||
sessionId: "active",
|
||||
sessionFile: path.join(sessionsDir, "active-thread-456.jsonl"),
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves transcript file identities through persisted session keys", () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
const sessionFile = path.join(sessionsDir, "active-thread-456.jsonl");
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:chat:thread-456": {
|
||||
sessionFile: "active-thread-456.jsonl",
|
||||
sessionId: "active",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(resolveSessionIdentityForTranscriptFile(sessionFile)).toEqual({
|
||||
agentId: "main",
|
||||
sessionId: "active",
|
||||
sessionKey: "agent:main:chat:thread-456",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildSessionEntry", () => {
|
||||
it("returns lineMap tracking original JSONL line numbers", async () => {
|
||||
// Simulate a real session JSONL file with metadata records interspersed
|
||||
// Lines 1-3: non-message metadata records
|
||||
// Line 4: user message
|
||||
// Line 5: metadata
|
||||
// Line 6: assistant message
|
||||
// Line 7: user message
|
||||
const jsonlLines = [
|
||||
JSON.stringify({ type: "custom", customType: "model-snapshot", data: {} }),
|
||||
JSON.stringify({ type: "custom", customType: "openclaw.cache-ttl", data: {} }),
|
||||
JSON.stringify({ type: "session-meta", agentId: "test" }),
|
||||
JSON.stringify({ type: "message", message: { role: "user", content: "Hello world" } }),
|
||||
JSON.stringify({ type: "custom", customType: "tool-result", data: {} }),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "Hi there, how can I help?" },
|
||||
}),
|
||||
JSON.stringify({ type: "message", message: { role: "user", content: "Tell me a joke" } }),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(filePath));
|
||||
expect(entry.content).toBe(
|
||||
"User: Hello world\nAssistant: Hi there, how can I help?\nUser: Tell me a joke",
|
||||
);
|
||||
|
||||
// lineMap should map each content line to its original JSONL line (1-indexed)
|
||||
// Content line 0 → JSONL line 4 (the first user message)
|
||||
// Content line 1 → JSONL line 6 (the assistant message)
|
||||
// Content line 2 → JSONL line 7 (the second user message)
|
||||
expect(entry.lineMap).toStrictEqual([4, 6, 7]);
|
||||
});
|
||||
|
||||
it("returns empty lineMap when no messages are found", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({ type: "custom", customType: "model-snapshot", data: {} }),
|
||||
JSON.stringify({ type: "session-meta", agentId: "test" }),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "empty-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(filePath));
|
||||
expect(entry.content).toBe("");
|
||||
expect(entry.lineMap).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("indexes usage-counted reset/deleted archives but still skips bak and checkpoint artifacts", async () => {
|
||||
const resetPath = path.join(tmpDir, "ordinary.jsonl.reset.2026-02-16T22-26-33.000Z");
|
||||
const deletedPath = path.join(tmpDir, "ordinary.jsonl.deleted.2026-02-16T22-27-33.000Z");
|
||||
const bakPath = path.join(tmpDir, "ordinary.jsonl.bak.2026-02-16T22-28-33.000Z");
|
||||
const checkpointPath = path.join(
|
||||
tmpDir,
|
||||
"ordinary.checkpoint.11111111-1111-4111-8111-111111111111.jsonl",
|
||||
);
|
||||
const content = JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "Archived hello" },
|
||||
});
|
||||
fsSync.writeFileSync(resetPath, content);
|
||||
fsSync.writeFileSync(deletedPath, content);
|
||||
fsSync.writeFileSync(bakPath, content);
|
||||
fsSync.writeFileSync(checkpointPath, content);
|
||||
|
||||
const resetEntry = requireSessionEntry(await buildSessionEntry(resetPath));
|
||||
const deletedEntry = requireSessionEntry(await buildSessionEntry(deletedPath));
|
||||
const bakEntry = requireSessionEntry(await buildSessionEntry(bakPath));
|
||||
const checkpointEntry = requireSessionEntry(await buildSessionEntry(checkpointPath));
|
||||
|
||||
// Usage-counted archives (reset, deleted) must surface real content so
|
||||
// post-reset memory_search can recover prior session history.
|
||||
expect(resetEntry.content).toBe("User: Archived hello");
|
||||
expect(resetEntry.lineMap).toStrictEqual([1]);
|
||||
expect(deletedEntry.content).toBe("User: Archived hello");
|
||||
expect(deletedEntry.lineMap).toStrictEqual([1]);
|
||||
|
||||
// .bak and compaction checkpoints remain opaque pre-archive / snapshot
|
||||
// artifacts and stay empty so they do not get double-indexed.
|
||||
expect(bakEntry.content).toBe("");
|
||||
expect(bakEntry.lineMap).toStrictEqual([]);
|
||||
expect(checkpointEntry.content).toBe("");
|
||||
expect(checkpointEntry.lineMap).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("keeps cron-run deleted archives opaque when the live session store entry is gone", async () => {
|
||||
const archivePath = path.join(tmpDir, "cron-run.jsonl.deleted.2026-02-16T22-27-33.000Z");
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: "[cron:job-1 Codex Sessions Sync] Run internal sync.",
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "Internal cron output that must stay out." },
|
||||
}),
|
||||
];
|
||||
fsSync.writeFileSync(archivePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(archivePath));
|
||||
|
||||
expect(entry.content).toBe("");
|
||||
expect(entry.lineMap).toStrictEqual([]);
|
||||
expect(entry.generatedByCronRun).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps cron-run reset archives opaque when session metadata preserves the cron key", async () => {
|
||||
const archivePath = path.join(tmpDir, "cron-run.jsonl.reset.2026-02-16T22-26-33.000Z");
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "session-meta",
|
||||
data: { sessionKey: "agent:main:cron:job-1:run:run-1" },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "Internal cron output that must stay out." },
|
||||
}),
|
||||
];
|
||||
fsSync.writeFileSync(archivePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(archivePath));
|
||||
|
||||
expect(entry.content).toBe("");
|
||||
expect(entry.lineMap).toStrictEqual([]);
|
||||
expect(entry.generatedByCronRun).toBe(true);
|
||||
});
|
||||
|
||||
it("skips blank lines and invalid JSON without breaking lineMap", async () => {
|
||||
const jsonlLines = [
|
||||
"",
|
||||
"not valid json",
|
||||
JSON.stringify({ type: "message", message: { role: "user", content: "First" } }),
|
||||
"",
|
||||
JSON.stringify({ type: "message", message: { role: "assistant", content: "Second" } }),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "gaps.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(filePath));
|
||||
expect(entry.lineMap).toStrictEqual([3, 5]);
|
||||
});
|
||||
|
||||
it("strips inbound metadata when a user envelope is split across text blocks", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Conversation info (untrusted metadata):" },
|
||||
{ type: "text", text: "```json" },
|
||||
{ type: "text", text: '{"message_id":"msg-100","chat_id":"-100123"}' },
|
||||
{ type: "text", text: "```" },
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: "Sender (untrusted metadata):" },
|
||||
{ type: "text", text: "```json" },
|
||||
{ type: "text", text: '{"label":"Chris","id":"42"}' },
|
||||
{ type: "text", text: "```" },
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: "Actual user text" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "enveloped-session-array.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(filePath));
|
||||
expect(entry.content).toBe("User: Actual user text");
|
||||
});
|
||||
|
||||
it("skips inter-session user messages", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: "A background task completed. Internal relay text.",
|
||||
provenance: { kind: "inter_session", sourceTool: "subagent_announce" },
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "User-facing summary." },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "Actual user follow-up." },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "inter-session-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(filePath));
|
||||
expect(entry.content).toBe("Assistant: User-facing summary.\nUser: Actual user follow-up.");
|
||||
expect(entry.lineMap).toStrictEqual([2, 3]);
|
||||
});
|
||||
|
||||
it("drops Date-invalid numeric message timestamps", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: "Hello",
|
||||
timestamp: 8_640_000_000_000_001,
|
||||
},
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "invalid-timestamp-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = requireSessionEntry(await buildSessionEntry(filePath));
|
||||
expect(entry.messageTimestampsMs).toStrictEqual([0]);
|
||||
});
|
||||
});
|
||||
870
packages/memory-host-sdk/src/host/session-files.ts
Normal file
870
packages/memory-host-sdk/src/host/session-files.ts
Normal file
@@ -0,0 +1,870 @@
|
||||
// Memory Host SDK module implements session files behavior.
|
||||
import fsSync from "node:fs";
|
||||
import path from "node:path";
|
||||
import { normalizeAgentId } from "./config-utils.js";
|
||||
import { readRegularFile, statRegularFile } from "./fs-utils.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import { createSubsystemLogger, redactSensitiveText } from "./openclaw-runtime-io.js";
|
||||
import {
|
||||
DREAMING_NARRATIVE_RUN_PREFIX,
|
||||
isDreamingNarrativeSessionStoreKey,
|
||||
extractAgentIdFromSessionsDir,
|
||||
HEARTBEAT_PROMPT,
|
||||
HEARTBEAT_TOKEN,
|
||||
hasInterSessionUserProvenance,
|
||||
isCompactionCheckpointTranscriptFileName,
|
||||
isCronRunSessionKey,
|
||||
isExecCompletionEvent,
|
||||
isHeartbeatUserMessage,
|
||||
isSessionArchiveArtifactName,
|
||||
isSilentReplyPayloadText,
|
||||
isUsageCountedSessionTranscriptFileName,
|
||||
parseUsageCountedSessionIdFromFileName,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
stripInboundMetadata,
|
||||
stripInternalRuntimeContext,
|
||||
} from "./openclaw-runtime-session.js";
|
||||
import { retryTransientMemoryRead } from "./read-retry.js";
|
||||
import {
|
||||
listSessionTranscriptCorpusEntriesForAgent,
|
||||
listSessionTranscriptCorpusEntriesForAgentSync,
|
||||
type SessionTranscriptCorpusEntry,
|
||||
} from "./session-transcript-corpus.js";
|
||||
import type { MemorySessionSyncTarget } from "./types.js";
|
||||
|
||||
export {
|
||||
listSessionTranscriptCorpusEntriesForAgent,
|
||||
type SessionTranscriptCorpusArtifactKind,
|
||||
type SessionTranscriptCorpusEntry,
|
||||
} from "./session-transcript-corpus.js";
|
||||
|
||||
// Keep the historical one-line-per-message export shape for normal turns, but
|
||||
// wrap pathological long messages so downstream indexers never ingest a single
|
||||
// toxic line. Wrapped continuation lines still map back to the same JSONL line.
|
||||
// This limit applies to content only; the role label adds up to 11 chars.
|
||||
const SESSION_EXPORT_CONTENT_WRAP_CHARS = 800;
|
||||
const SESSION_ENTRY_PARSE_YIELD_LINES = 250;
|
||||
const MAX_DATE_TIMESTAMP_MS = 8_640_000_000_000_000;
|
||||
const DIRECT_CRON_PROMPT_RE = /^\[cron:[^\]]+\]\s*/;
|
||||
|
||||
export type SessionFileEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
mtimeMs: number;
|
||||
size: number;
|
||||
hash: string;
|
||||
content: string;
|
||||
/** Maps each content line (0-indexed) to its 1-indexed JSONL source line. */
|
||||
lineMap: number[];
|
||||
/** Maps each content line (0-indexed) to epoch ms; 0 means unknown timestamp. */
|
||||
messageTimestampsMs: number[];
|
||||
/** True when this transcript belongs to an internal dreaming narrative run. */
|
||||
generatedByDreamingNarrative?: boolean;
|
||||
/** True when this transcript belongs to an isolated cron run session. */
|
||||
generatedByCronRun?: boolean;
|
||||
};
|
||||
|
||||
export type BuildSessionEntryOptions = {
|
||||
/** Optional preclassification from a caller-managed dreaming transcript lookup. */
|
||||
generatedByDreamingNarrative?: boolean;
|
||||
/** Optional preclassification from a caller-managed cron transcript lookup. */
|
||||
generatedByCronRun?: boolean;
|
||||
/** Override for tests or specialized callers that need a tighter parse yield cadence. */
|
||||
parseYieldEveryLines?: number;
|
||||
};
|
||||
|
||||
export type SessionTranscriptClassification = {
|
||||
dreamingNarrativeTranscriptPaths: ReadonlySet<string>;
|
||||
cronRunTranscriptPaths: ReadonlySet<string>;
|
||||
};
|
||||
|
||||
export type ResolvedMemorySessionSyncTarget = {
|
||||
agentId: string;
|
||||
sessionFile: string;
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
export type ResolvedSessionTranscriptIdentity = {
|
||||
agentId: string;
|
||||
sessionId: string;
|
||||
sessionKey?: string;
|
||||
};
|
||||
|
||||
type SessionTranscriptStoreEntry = {
|
||||
sessionFile?: unknown;
|
||||
sessionId?: unknown;
|
||||
};
|
||||
|
||||
function shouldSkipTranscriptFileForDreaming(absPath: string): boolean {
|
||||
const fileName = path.basename(absPath);
|
||||
// Compaction checkpoints are always skipped: they are derived snapshots of an
|
||||
// active session and would double-index the same content.
|
||||
if (isCompactionCheckpointTranscriptFileName(fileName)) {
|
||||
return true;
|
||||
}
|
||||
// Legacy backups and `.jsonl.bak.<iso>` rotations are opaque pre-archive
|
||||
// copies, not a user-facing session artifact; skip them too.
|
||||
if (
|
||||
isSessionArchiveArtifactName(fileName) &&
|
||||
!isUsageCountedSessionTranscriptFileName(fileName)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
// Usage-counted archives (`.jsonl.reset.<iso>` / `.jsonl.deleted.<iso>`) are
|
||||
// the rotated-but-retained copies of real sessions and must stay indexed so
|
||||
// `memory_search` can surface hits on post-reset / post-delete history.
|
||||
return false;
|
||||
}
|
||||
|
||||
function isUsageCountedSessionArchiveTranscriptPath(absPath: string): boolean {
|
||||
const fileName = path.basename(absPath);
|
||||
return (
|
||||
isUsageCountedSessionTranscriptFileName(fileName) &&
|
||||
isSessionArchiveArtifactName(fileName) &&
|
||||
parseUsageCountedSessionIdFromFileName(fileName) !== null
|
||||
);
|
||||
}
|
||||
|
||||
function isDreamingNarrativeBootstrapRecord(record: unknown): boolean {
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) {
|
||||
return false;
|
||||
}
|
||||
const candidate = record as {
|
||||
type?: unknown;
|
||||
customType?: unknown;
|
||||
data?: unknown;
|
||||
};
|
||||
if (
|
||||
candidate.type !== "custom" ||
|
||||
candidate.customType !== "openclaw:bootstrap-context:full" ||
|
||||
!candidate.data ||
|
||||
typeof candidate.data !== "object" ||
|
||||
Array.isArray(candidate.data)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const runId = (candidate.data as { runId?: unknown }).runId;
|
||||
return typeof runId === "string" && runId.startsWith(DREAMING_NARRATIVE_RUN_PREFIX);
|
||||
}
|
||||
|
||||
function hasDreamingNarrativeRunId(value: unknown): boolean {
|
||||
return typeof value === "string" && value.startsWith(DREAMING_NARRATIVE_RUN_PREFIX);
|
||||
}
|
||||
|
||||
function isDreamingNarrativeGeneratedRecord(record: unknown): boolean {
|
||||
if (isDreamingNarrativeBootstrapRecord(record)) {
|
||||
return true;
|
||||
}
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) {
|
||||
return false;
|
||||
}
|
||||
const candidate = record as {
|
||||
runId?: unknown;
|
||||
sessionKey?: unknown;
|
||||
data?: unknown;
|
||||
};
|
||||
if (
|
||||
hasDreamingNarrativeRunId(candidate.runId) ||
|
||||
hasDreamingNarrativeRunId(candidate.sessionKey)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (!candidate.data || typeof candidate.data !== "object" || Array.isArray(candidate.data)) {
|
||||
return false;
|
||||
}
|
||||
const nested = candidate.data as {
|
||||
runId?: unknown;
|
||||
sessionKey?: unknown;
|
||||
};
|
||||
return hasDreamingNarrativeRunId(nested.runId) || hasDreamingNarrativeRunId(nested.sessionKey);
|
||||
}
|
||||
|
||||
function hasCronRunSessionKey(value: unknown): boolean {
|
||||
return typeof value === "string" && isCronRunSessionKey(value);
|
||||
}
|
||||
|
||||
function isCronRunGeneratedRecord(record: unknown): boolean {
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) {
|
||||
return false;
|
||||
}
|
||||
const candidate = record as {
|
||||
sessionKey?: unknown;
|
||||
data?: unknown;
|
||||
};
|
||||
if (hasCronRunSessionKey(candidate.sessionKey)) {
|
||||
return true;
|
||||
}
|
||||
if (!candidate.data || typeof candidate.data !== "object" || Array.isArray(candidate.data)) {
|
||||
return false;
|
||||
}
|
||||
const nested = candidate.data as {
|
||||
sessionKey?: unknown;
|
||||
};
|
||||
return hasCronRunSessionKey(nested.sessionKey);
|
||||
}
|
||||
|
||||
function normalizeComparablePath(pathname: string): string {
|
||||
const resolved = path.resolve(pathname);
|
||||
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
||||
}
|
||||
|
||||
export function normalizeSessionTranscriptPathForComparison(pathname: string): string {
|
||||
return normalizeComparablePath(pathname);
|
||||
}
|
||||
|
||||
function resolveSessionStoreTranscriptPath(
|
||||
sessionsDir: string,
|
||||
entry: { sessionFile?: unknown; sessionId?: unknown } | undefined,
|
||||
): string | null {
|
||||
const resolved = resolveSessionStoreTranscriptResolvedPath(sessionsDir, entry);
|
||||
return resolved ? normalizeComparablePath(resolved) : null;
|
||||
}
|
||||
|
||||
function resolveSessionStoreTranscriptResolvedPath(
|
||||
sessionsDir: string,
|
||||
entry: { sessionFile?: unknown; sessionId?: unknown } | undefined,
|
||||
): string | null {
|
||||
if (typeof entry?.sessionFile === "string" && entry.sessionFile.trim().length > 0) {
|
||||
const sessionFile = entry.sessionFile.trim();
|
||||
return path.isAbsolute(sessionFile) ? sessionFile : path.resolve(sessionsDir, sessionFile);
|
||||
}
|
||||
if (typeof entry?.sessionId === "string" && entry.sessionId.trim().length > 0) {
|
||||
return path.join(sessionsDir, `${entry.sessionId.trim()}.jsonl`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isCanonicalSessionsDirForAgent(sessionsDir: string, agentId: string): boolean {
|
||||
return (
|
||||
normalizeComparablePath(sessionsDir) ===
|
||||
normalizeComparablePath(resolveSessionTranscriptsDirForAgent(agentId))
|
||||
);
|
||||
}
|
||||
|
||||
export function loadSessionTranscriptClassificationForSessionsDir(
|
||||
sessionsDir: string,
|
||||
): SessionTranscriptClassification {
|
||||
const agentId = extractAgentIdFromSessionsDir(sessionsDir);
|
||||
if (agentId && isCanonicalSessionsDirForAgent(sessionsDir, agentId)) {
|
||||
return classifySessionTranscriptCorpusEntries(
|
||||
listSessionTranscriptCorpusEntriesForAgentSync(agentId),
|
||||
);
|
||||
}
|
||||
const storePath = path.join(sessionsDir, "sessions.json");
|
||||
const store = readSessionTranscriptClassificationStore(storePath);
|
||||
const dreamingTranscriptPaths = new Set<string>();
|
||||
const cronRunTranscriptPaths = new Set<string>();
|
||||
for (const [sessionKey, entry] of Object.entries(store)) {
|
||||
const transcriptPath = resolveSessionStoreTranscriptPath(sessionsDir, entry);
|
||||
if (!transcriptPath) {
|
||||
continue;
|
||||
}
|
||||
if (isDreamingNarrativeSessionStoreKey(sessionKey)) {
|
||||
dreamingTranscriptPaths.add(transcriptPath);
|
||||
}
|
||||
if (isCronRunSessionKey(sessionKey)) {
|
||||
cronRunTranscriptPaths.add(transcriptPath);
|
||||
}
|
||||
}
|
||||
return {
|
||||
dreamingNarrativeTranscriptPaths: dreamingTranscriptPaths,
|
||||
cronRunTranscriptPaths,
|
||||
};
|
||||
}
|
||||
|
||||
function readSessionTranscriptClassificationStore(
|
||||
storePath: string,
|
||||
): Record<string, SessionTranscriptStoreEntry> {
|
||||
try {
|
||||
const parsed = JSON.parse(fsSync.readFileSync(storePath, "utf-8")) as unknown;
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
return {};
|
||||
}
|
||||
return parsed as Record<string, SessionTranscriptStoreEntry>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function classifySessionTranscriptCorpusEntries(
|
||||
corpusEntries: readonly SessionTranscriptCorpusEntry[],
|
||||
): SessionTranscriptClassification {
|
||||
const dreamingTranscriptPaths = new Set<string>();
|
||||
const cronRunTranscriptPaths = new Set<string>();
|
||||
for (const entry of corpusEntries) {
|
||||
const normalizedPath = normalizeComparablePath(entry.sessionFile);
|
||||
if (entry.generatedByDreamingNarrative) {
|
||||
dreamingTranscriptPaths.add(normalizedPath);
|
||||
}
|
||||
if (entry.generatedByCronRun) {
|
||||
cronRunTranscriptPaths.add(normalizedPath);
|
||||
}
|
||||
}
|
||||
return {
|
||||
dreamingNarrativeTranscriptPaths: dreamingTranscriptPaths,
|
||||
cronRunTranscriptPaths,
|
||||
};
|
||||
}
|
||||
|
||||
function findSessionTranscriptStoreEntryBySessionId(
|
||||
store: Record<string, SessionTranscriptStoreEntry>,
|
||||
sessionId: string,
|
||||
): SessionTranscriptStoreEntry | undefined {
|
||||
return Object.values(store).find((entry) => {
|
||||
return typeof entry.sessionId === "string" && entry.sessionId.trim() === sessionId;
|
||||
});
|
||||
}
|
||||
|
||||
export function loadDreamingNarrativeTranscriptPathSetForAgent(
|
||||
agentId: string,
|
||||
): ReadonlySet<string> {
|
||||
return loadSessionTranscriptClassificationForAgent(agentId).dreamingNarrativeTranscriptPaths;
|
||||
}
|
||||
|
||||
export function loadSessionTranscriptClassificationForAgent(
|
||||
agentId: string,
|
||||
): SessionTranscriptClassification {
|
||||
return classifySessionTranscriptCorpusEntries(
|
||||
listSessionTranscriptCorpusEntriesForAgentSync(agentId),
|
||||
);
|
||||
}
|
||||
|
||||
function classifySessionTranscriptFromSessionStore(absPath: string): {
|
||||
generatedByDreamingNarrative: boolean;
|
||||
generatedByCronRun: boolean;
|
||||
} {
|
||||
const sessionsDir = path.dirname(absPath);
|
||||
const normalizedAbsPath = normalizeComparablePath(absPath);
|
||||
const primarySessionId = parseUsageCountedSessionIdFromFileName(path.basename(absPath));
|
||||
const normalizedPrimaryPath =
|
||||
primarySessionId && isSessionArchiveArtifactName(path.basename(absPath))
|
||||
? normalizeComparablePath(path.join(sessionsDir, `${primarySessionId}.jsonl`))
|
||||
: null;
|
||||
const classification = loadSessionTranscriptClassificationForSessionsDir(sessionsDir);
|
||||
const hasClassifiedPath = (paths: ReadonlySet<string>) =>
|
||||
paths.has(normalizedAbsPath) ||
|
||||
(normalizedPrimaryPath !== null && paths.has(normalizedPrimaryPath));
|
||||
return {
|
||||
generatedByDreamingNarrative: hasClassifiedPath(
|
||||
classification.dreamingNarrativeTranscriptPaths,
|
||||
),
|
||||
generatedByCronRun: hasClassifiedPath(classification.cronRunTranscriptPaths),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listSessionFilesForAgent(agentId: string): Promise<string[]> {
|
||||
return (await listSessionTranscriptCorpusEntriesForAgent(agentId)).map(
|
||||
(entry) => entry.sessionFile,
|
||||
);
|
||||
}
|
||||
|
||||
function extractAgentIdFromSessionPath(absPath: string): string | null {
|
||||
const parts = path.normalize(path.resolve(absPath)).split(path.sep).filter(Boolean);
|
||||
const sessionsIndex = parts.lastIndexOf("sessions");
|
||||
if (sessionsIndex < 2 || parts[sessionsIndex - 2] !== "agents") {
|
||||
return null;
|
||||
}
|
||||
return parts[sessionsIndex - 1] || null;
|
||||
}
|
||||
|
||||
export function sessionPathForFile(absPath: string): string {
|
||||
const agentId = extractAgentIdFromSessionPath(absPath);
|
||||
return path
|
||||
.join("sessions", ...(agentId ? [agentId] : []), path.basename(absPath))
|
||||
.replace(/\\/g, "/");
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a deprecated path-shaped memory sync hint only when it points at an
|
||||
* OpenClaw-owned usage-counted transcript in the canonical agent sessions dir.
|
||||
*/
|
||||
export function parseCanonicalSessionSyncTargetFromPath(
|
||||
sessionFile: string,
|
||||
): MemorySessionSyncTarget | null {
|
||||
const trimmed = sessionFile.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
const resolved = path.resolve(trimmed);
|
||||
const fileName = path.basename(resolved);
|
||||
const sessionId = parseUsageCountedSessionIdFromFileName(fileName);
|
||||
if (!sessionId || !isUsageCountedSessionTranscriptFileName(fileName)) {
|
||||
return null;
|
||||
}
|
||||
const agentId = extractAgentIdFromSessionPath(resolved);
|
||||
if (!agentId) {
|
||||
return null;
|
||||
}
|
||||
const canonicalSessionsDir = normalizeComparablePath(
|
||||
resolveSessionTranscriptsDirForAgent(agentId),
|
||||
);
|
||||
if (normalizeComparablePath(path.dirname(resolved)) !== canonicalSessionsDir) {
|
||||
return null;
|
||||
}
|
||||
return { agentId, sessionId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a current transcript path back to the canonical session-store
|
||||
* identity when available, falling back to the usage-counted file identity.
|
||||
*/
|
||||
export function resolveSessionIdentityForTranscriptFile(
|
||||
sessionFile: string,
|
||||
): ResolvedSessionTranscriptIdentity | null {
|
||||
const parsed = parseCanonicalSessionSyncTargetFromPath(sessionFile);
|
||||
if (!parsed?.agentId) {
|
||||
return null;
|
||||
}
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent(parsed.agentId);
|
||||
const normalizedSessionFile = normalizeComparablePath(sessionFile);
|
||||
const store = readSessionTranscriptClassificationStore(path.join(sessionsDir, "sessions.json"));
|
||||
for (const [sessionKey, entry] of Object.entries(store)) {
|
||||
const transcriptPath = resolveSessionStoreTranscriptPath(sessionsDir, entry);
|
||||
if (transcriptPath !== normalizedSessionFile) {
|
||||
continue;
|
||||
}
|
||||
const sessionId = typeof entry.sessionId === "string" ? entry.sessionId.trim() : "";
|
||||
if (!sessionId) {
|
||||
continue;
|
||||
}
|
||||
return {
|
||||
agentId: parsed.agentId,
|
||||
sessionId,
|
||||
...(sessionKey.trim() ? { sessionKey } : {}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
agentId: parsed.agentId,
|
||||
sessionId: parsed.sessionId,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a storage-neutral memory sync target to the current file-backed
|
||||
* transcript. The SQLite adapter implements this identity contract without
|
||||
* deriving a path.
|
||||
*/
|
||||
export function resolveSessionFileForSyncTarget(
|
||||
target: MemorySessionSyncTarget,
|
||||
defaultAgentId?: string,
|
||||
): ResolvedMemorySessionSyncTarget | null {
|
||||
const sessionId = target.sessionId.trim();
|
||||
const rawAgentId = (target.agentId ?? defaultAgentId ?? "").trim();
|
||||
if (!rawAgentId || !sessionId) {
|
||||
return null;
|
||||
}
|
||||
const agentId = normalizeAgentId(rawAgentId);
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent(agentId);
|
||||
const sessionKey = target.sessionKey?.trim();
|
||||
let store: Record<string, SessionTranscriptStoreEntry> | null = null;
|
||||
if (sessionKey) {
|
||||
store = readSessionTranscriptClassificationStore(path.join(sessionsDir, "sessions.json"));
|
||||
const persistedPath = resolveSessionStoreTranscriptResolvedPath(sessionsDir, store[sessionKey]);
|
||||
const canonicalPath = resolveCanonicalSessionSyncFilePath(agentId, persistedPath);
|
||||
if (canonicalPath) {
|
||||
return {
|
||||
agentId,
|
||||
sessionId,
|
||||
sessionFile: canonicalPath,
|
||||
};
|
||||
}
|
||||
}
|
||||
store ??= readSessionTranscriptClassificationStore(path.join(sessionsDir, "sessions.json"));
|
||||
const persistedPath = resolveSessionStoreTranscriptResolvedPath(
|
||||
sessionsDir,
|
||||
findSessionTranscriptStoreEntryBySessionId(store, sessionId),
|
||||
);
|
||||
const canonicalPath = resolveCanonicalSessionSyncFilePath(agentId, persistedPath);
|
||||
if (canonicalPath) {
|
||||
return {
|
||||
agentId,
|
||||
sessionId,
|
||||
sessionFile: canonicalPath,
|
||||
};
|
||||
}
|
||||
const sessionFile = resolveCanonicalSessionSyncFilePath(
|
||||
agentId,
|
||||
path.join(sessionsDir, `${sessionId}.jsonl`),
|
||||
sessionId,
|
||||
);
|
||||
if (!sessionFile) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
agentId,
|
||||
sessionId,
|
||||
sessionFile,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCanonicalSessionSyncFilePath(
|
||||
agentId: string,
|
||||
sessionFile?: string | null,
|
||||
expectedSessionId?: string,
|
||||
): string | null {
|
||||
if (!sessionFile) {
|
||||
return null;
|
||||
}
|
||||
const resolved = path.resolve(sessionFile);
|
||||
const parsed = parseCanonicalSessionSyncTargetFromPath(resolved);
|
||||
if (parsed?.agentId !== agentId) {
|
||||
return null;
|
||||
}
|
||||
if (expectedSessionId !== undefined && parsed.sessionId !== expectedSessionId) {
|
||||
return null;
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function logSessionFileReadFailure(absPath: string, err: unknown): Promise<void> {
|
||||
createSubsystemLogger("memory").debug(`Failed reading session file ${absPath}: ${String(err)}`);
|
||||
}
|
||||
|
||||
function normalizeSessionText(value: string): string {
|
||||
return value
|
||||
.replace(/\s*\n+\s*/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function collectRawSessionText(content: unknown): string | null {
|
||||
if (typeof content === "string") {
|
||||
return content;
|
||||
}
|
||||
if (!Array.isArray(content)) {
|
||||
return null;
|
||||
}
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (!block || typeof block !== "object") {
|
||||
continue;
|
||||
}
|
||||
const record = block as { type?: unknown; text?: unknown };
|
||||
if (record.type === "text" && typeof record.text === "string") {
|
||||
parts.push(record.text);
|
||||
}
|
||||
}
|
||||
return parts.length > 0 ? parts.join("\n") : null;
|
||||
}
|
||||
|
||||
function isHighSurrogate(code: number): boolean {
|
||||
return code >= 0xd800 && code <= 0xdbff;
|
||||
}
|
||||
|
||||
function isLowSurrogate(code: number): boolean {
|
||||
return code >= 0xdc00 && code <= 0xdfff;
|
||||
}
|
||||
|
||||
function splitLongSessionLine(
|
||||
text: string,
|
||||
maxChars: number = SESSION_EXPORT_CONTENT_WRAP_CHARS,
|
||||
): string[] {
|
||||
const normalized = text.trim();
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
if (normalized.length <= maxChars) {
|
||||
return [normalized];
|
||||
}
|
||||
|
||||
const segments: string[] = [];
|
||||
let cursor = 0;
|
||||
while (cursor < normalized.length) {
|
||||
const remaining = normalized.length - cursor;
|
||||
if (remaining <= maxChars) {
|
||||
segments.push(normalized.slice(cursor).trim());
|
||||
break;
|
||||
}
|
||||
|
||||
const limit = cursor + maxChars;
|
||||
let splitAt = limit;
|
||||
for (let index = limit; index > cursor; index -= 1) {
|
||||
if (normalized[index] === " ") {
|
||||
splitAt = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (
|
||||
splitAt < normalized.length &&
|
||||
splitAt > cursor &&
|
||||
isHighSurrogate(normalized.charCodeAt(splitAt - 1)) &&
|
||||
isLowSurrogate(normalized.charCodeAt(splitAt))
|
||||
) {
|
||||
splitAt -= 1;
|
||||
}
|
||||
segments.push(normalized.slice(cursor, splitAt).trim());
|
||||
cursor = splitAt;
|
||||
while (cursor < normalized.length && normalized[cursor] === " ") {
|
||||
cursor += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return segments.filter(Boolean);
|
||||
}
|
||||
|
||||
function renderSessionExportLines(label: string, text: string): string[] {
|
||||
return splitLongSessionLine(text).map((segment) => `${label}: ${segment}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip OpenClaw-injected inbound metadata envelopes from a raw text block.
|
||||
*
|
||||
* User-role messages arriving from external channels (Telegram, Discord,
|
||||
* Slack, …) are stored with a multi-line prefix containing Conversation info,
|
||||
* Sender info, and other AI-facing metadata blocks. These envelopes must be
|
||||
* removed BEFORE normalization, because `stripInboundMetadata` relies on
|
||||
* newline structure and fenced `json` code fences to locate sentinels; once
|
||||
* `normalizeSessionText` collapses newlines into spaces, stripping is
|
||||
* impossible.
|
||||
*
|
||||
* See: https://github.com/openclaw/openclaw/issues/63921
|
||||
*/
|
||||
function stripInboundMetadataForUserRole(text: string, role: "user" | "assistant"): string {
|
||||
if (role !== "user") {
|
||||
return text;
|
||||
}
|
||||
return stripInboundMetadata(text);
|
||||
}
|
||||
|
||||
const GENERATED_SYSTEM_MESSAGE_RE = /^System(?: \(untrusted\))?: \[[^\]]+\]\s*/;
|
||||
|
||||
function isGeneratedSystemWrapperMessage(text: string, role: "user" | "assistant"): boolean {
|
||||
if (role !== "user") {
|
||||
return false;
|
||||
}
|
||||
return GENERATED_SYSTEM_MESSAGE_RE.test(text);
|
||||
}
|
||||
|
||||
function isGeneratedCronPromptMessage(text: string, role: "user" | "assistant"): boolean {
|
||||
if (role !== "user") {
|
||||
return false;
|
||||
}
|
||||
return DIRECT_CRON_PROMPT_RE.test(text);
|
||||
}
|
||||
|
||||
function isGeneratedHeartbeatPromptMessage(text: string, role: "user" | "assistant"): boolean {
|
||||
return role === "user" && isHeartbeatUserMessage({ role, content: text }, HEARTBEAT_PROMPT);
|
||||
}
|
||||
|
||||
function sanitizeSessionText(text: string, role: "user" | "assistant"): string | null {
|
||||
const strippedInbound = stripInboundMetadataForUserRole(text, role);
|
||||
const strippedInternal = stripInternalRuntimeContext(strippedInbound);
|
||||
const normalized = normalizeSessionText(strippedInternal);
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (isGeneratedSystemWrapperMessage(normalized, role)) {
|
||||
return null;
|
||||
}
|
||||
if (isGeneratedCronPromptMessage(normalized, role)) {
|
||||
return null;
|
||||
}
|
||||
if (isGeneratedHeartbeatPromptMessage(normalized, role)) {
|
||||
return null;
|
||||
}
|
||||
if (isSilentReplyPayloadText(normalized)) {
|
||||
return null;
|
||||
}
|
||||
// Assistant-side machinery acks: HEARTBEAT_OK is the canonical "all clear,
|
||||
// nothing to do" reply to a heartbeat tick. Drop on the assistant side
|
||||
// directly so we do not have to rely on cross-message coupling with the
|
||||
// preceding user message (which a real user could spoof).
|
||||
if (role === "assistant" && normalized === HEARTBEAT_TOKEN) {
|
||||
return null;
|
||||
}
|
||||
const withoutSystemEnvelope = normalized.replace(GENERATED_SYSTEM_MESSAGE_RE, "").trim();
|
||||
if (isExecCompletionEvent(withoutSystemEnvelope)) {
|
||||
return null;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseSessionTimestampMs(
|
||||
record: { timestamp?: unknown },
|
||||
message: { timestamp?: unknown },
|
||||
): number {
|
||||
const candidates = [message.timestamp, record.timestamp];
|
||||
for (const value of candidates) {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const ms = value > 0 && value < 1e11 ? value * 1000 : value;
|
||||
if (Number.isFinite(ms) && ms > 0 && ms <= MAX_DATE_TIMESTAMP_MS) {
|
||||
return ms;
|
||||
}
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
const parsed = Date.parse(value);
|
||||
if (Number.isFinite(parsed) && parsed > 0) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function resolveSessionEntryParseYieldLines(opts: BuildSessionEntryOptions): number {
|
||||
const configured = opts.parseYieldEveryLines;
|
||||
if (typeof configured === "number" && Number.isFinite(configured)) {
|
||||
return Math.max(1, Math.floor(configured));
|
||||
}
|
||||
return SESSION_ENTRY_PARSE_YIELD_LINES;
|
||||
}
|
||||
|
||||
async function yieldSessionEntryParseIfNeeded(
|
||||
lineIndex: number,
|
||||
everyLines: number,
|
||||
): Promise<void> {
|
||||
if (lineIndex > 0 && lineIndex % everyLines === 0) {
|
||||
await new Promise<void>((resolve) => {
|
||||
setImmediate(resolve);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildSessionEntry(
|
||||
absPath: string,
|
||||
opts: BuildSessionEntryOptions = {},
|
||||
): Promise<SessionFileEntry | null> {
|
||||
try {
|
||||
const regularFile = await statRegularFile(absPath);
|
||||
if (regularFile.missing) {
|
||||
return null;
|
||||
}
|
||||
const stat = regularFile.stat;
|
||||
if (shouldSkipTranscriptFileForDreaming(absPath)) {
|
||||
return {
|
||||
path: sessionPathForFile(absPath),
|
||||
absPath,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
hash: hashText("\n\n"),
|
||||
content: "",
|
||||
lineMap: [],
|
||||
messageTimestampsMs: [],
|
||||
};
|
||||
}
|
||||
const raw = (
|
||||
await retryTransientMemoryRead(
|
||||
() => readRegularFile({ filePath: absPath }),
|
||||
`read session transcript ${absPath}`,
|
||||
)
|
||||
).buffer.toString("utf-8");
|
||||
const collected: string[] = [];
|
||||
const lineMap: number[] = [];
|
||||
const messageTimestampsMs: number[] = [];
|
||||
const parseYieldEveryLines = resolveSessionEntryParseYieldLines(opts);
|
||||
const sessionStoreClassification =
|
||||
opts.generatedByDreamingNarrative === undefined || opts.generatedByCronRun === undefined
|
||||
? classifySessionTranscriptFromSessionStore(absPath)
|
||||
: null;
|
||||
let generatedByDreamingNarrative =
|
||||
opts.generatedByDreamingNarrative ??
|
||||
sessionStoreClassification?.generatedByDreamingNarrative ??
|
||||
false;
|
||||
let generatedByCronRun =
|
||||
opts.generatedByCronRun ?? sessionStoreClassification?.generatedByCronRun ?? false;
|
||||
const allowArchiveContentCronClassification =
|
||||
isUsageCountedSessionArchiveTranscriptPath(absPath);
|
||||
for (let jsonlIdx = 0, lineStart = 0; lineStart <= raw.length; jsonlIdx++) {
|
||||
await yieldSessionEntryParseIfNeeded(jsonlIdx, parseYieldEveryLines);
|
||||
const newlineIndex = raw.indexOf("\n", lineStart);
|
||||
const lineEnd = newlineIndex === -1 ? raw.length : newlineIndex;
|
||||
const line = raw.slice(lineStart, lineEnd);
|
||||
lineStart = newlineIndex === -1 ? raw.length + 1 : newlineIndex + 1;
|
||||
if (!line.trim()) {
|
||||
continue;
|
||||
}
|
||||
let record: unknown;
|
||||
try {
|
||||
record = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!generatedByDreamingNarrative && isDreamingNarrativeGeneratedRecord(record)) {
|
||||
generatedByDreamingNarrative = true;
|
||||
}
|
||||
if (
|
||||
!generatedByCronRun &&
|
||||
allowArchiveContentCronClassification &&
|
||||
isCronRunGeneratedRecord(record)
|
||||
) {
|
||||
generatedByCronRun = true;
|
||||
collected.length = 0;
|
||||
lineMap.length = 0;
|
||||
messageTimestampsMs.length = 0;
|
||||
}
|
||||
if (
|
||||
!record ||
|
||||
typeof record !== "object" ||
|
||||
(record as { type?: unknown }).type !== "message"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const message = (record as { message?: unknown }).message as
|
||||
| { role?: unknown; content?: unknown; provenance?: unknown }
|
||||
| undefined;
|
||||
if (!message || typeof message.role !== "string") {
|
||||
continue;
|
||||
}
|
||||
if (message.role !== "user" && message.role !== "assistant") {
|
||||
continue;
|
||||
}
|
||||
if (message.role === "user" && hasInterSessionUserProvenance(message)) {
|
||||
continue;
|
||||
}
|
||||
const rawText = collectRawSessionText(message.content);
|
||||
if (rawText === null) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
!generatedByCronRun &&
|
||||
allowArchiveContentCronClassification &&
|
||||
isGeneratedCronPromptMessage(normalizeSessionText(rawText), message.role)
|
||||
) {
|
||||
generatedByCronRun = true;
|
||||
collected.length = 0;
|
||||
lineMap.length = 0;
|
||||
messageTimestampsMs.length = 0;
|
||||
}
|
||||
const text = sanitizeSessionText(rawText, message.role);
|
||||
if (!text) {
|
||||
// Assistant-side machinery (silent replies, system wrappers) is already
|
||||
// dropped by sanitizeSessionText. We deliberately do NOT use the prior
|
||||
// user message's pattern-match to drop the next assistant message:
|
||||
// user-typed text can match those same patterns (`[cron:...]`,
|
||||
// `System (untrusted): ...`) and a cross-message drop would let users
|
||||
// exfiltrate real assistant replies from the dreaming corpus by
|
||||
// prefixing their own prompt. See PR #70737 review (aisle-research-bot).
|
||||
continue;
|
||||
}
|
||||
if (generatedByDreamingNarrative || generatedByCronRun) {
|
||||
continue;
|
||||
}
|
||||
const safe = redactSensitiveText(text, { mode: "tools" });
|
||||
const label = message.role === "user" ? "User" : "Assistant";
|
||||
const renderedLines = renderSessionExportLines(label, safe);
|
||||
const timestampMs = parseSessionTimestampMs(
|
||||
record as { timestamp?: unknown },
|
||||
message as { timestamp?: unknown },
|
||||
);
|
||||
collected.push(...renderedLines);
|
||||
lineMap.push(...renderedLines.map(() => jsonlIdx + 1));
|
||||
messageTimestampsMs.push(...renderedLines.map(() => timestampMs));
|
||||
}
|
||||
const content = collected.join("\n");
|
||||
return {
|
||||
path: sessionPathForFile(absPath),
|
||||
absPath,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
hash: hashText(content + "\n" + lineMap.join(",") + "\n" + messageTimestampsMs.join(",")),
|
||||
content,
|
||||
lineMap,
|
||||
messageTimestampsMs,
|
||||
...(generatedByDreamingNarrative ? { generatedByDreamingNarrative: true } : {}),
|
||||
...(generatedByCronRun ? { generatedByCronRun: true } : {}),
|
||||
};
|
||||
} catch (err) {
|
||||
void logSessionFileReadFailure(absPath, err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
425
packages/memory-host-sdk/src/host/session-transcript-corpus.ts
Normal file
425
packages/memory-host-sdk/src/host/session-transcript-corpus.ts
Normal file
@@ -0,0 +1,425 @@
|
||||
// Accessor-backed transcript corpus discovery for memory/QMD session indexing.
|
||||
import fsSync from "node:fs";
|
||||
import path from "node:path";
|
||||
import { normalizeAgentId } from "./config-utils.js";
|
||||
import {
|
||||
isDreamingNarrativeSessionStoreKey,
|
||||
extractAgentIdFromSessionsDir,
|
||||
canonicalizeMainSessionAlias,
|
||||
getRuntimeConfig,
|
||||
isCronRunSessionKey,
|
||||
isSessionArchiveArtifactName,
|
||||
isUsageCountedSessionTranscriptFileName,
|
||||
listSessionEntries,
|
||||
parseUsageCountedSessionIdFromFileName,
|
||||
resolveSessionAgentId,
|
||||
resolveSessionFilePath,
|
||||
resolveStorePath,
|
||||
type SessionEntry,
|
||||
} from "./openclaw-runtime-session.js";
|
||||
|
||||
export type SessionTranscriptCorpusArtifactKind =
|
||||
| "active-session"
|
||||
| "archive-artifact"
|
||||
| "orphan-file-artifact";
|
||||
|
||||
export type SessionTranscriptCorpusEntry = {
|
||||
agentId: string;
|
||||
sessionFile: string;
|
||||
sessionId: string;
|
||||
artifactKind: SessionTranscriptCorpusArtifactKind;
|
||||
sessionKey?: string;
|
||||
/** True when this transcript belongs to an internal dreaming narrative run. */
|
||||
generatedByDreamingNarrative?: boolean;
|
||||
/** True when this transcript belongs to an isolated cron run session. */
|
||||
generatedByCronRun?: boolean;
|
||||
};
|
||||
|
||||
type SessionEntrySummary = {
|
||||
sessionKey: string;
|
||||
entry: SessionEntry;
|
||||
};
|
||||
|
||||
function isDreamingNarrativeSessionKeyLike(value: unknown): boolean {
|
||||
return typeof value === "string" && isDreamingNarrativeSessionStoreKey(value);
|
||||
}
|
||||
|
||||
function normalizeComparablePath(pathname: string): string {
|
||||
const resolved = path.resolve(pathname);
|
||||
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
||||
}
|
||||
|
||||
function normalizeRealComparablePath(pathname: string): string {
|
||||
try {
|
||||
return normalizeComparablePath(fsSync.realpathSync(pathname));
|
||||
} catch {
|
||||
try {
|
||||
return normalizeComparablePath(
|
||||
path.join(fsSync.realpathSync(path.dirname(pathname)), path.basename(pathname)),
|
||||
);
|
||||
} catch {
|
||||
return normalizeComparablePath(pathname);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function rememberArtifactDir(dirs: Map<string, string>, dir: string): void {
|
||||
dirs.set(normalizeRealComparablePath(dir), dir);
|
||||
}
|
||||
|
||||
function extractAgentIdFromSessionPath(absPath: string): string | null {
|
||||
const parts = path.normalize(path.resolve(absPath)).split(path.sep).filter(Boolean);
|
||||
const sessionsIndex = parts.lastIndexOf("sessions");
|
||||
if (sessionsIndex < 2 || parts[sessionsIndex - 2] !== "agents") {
|
||||
return null;
|
||||
}
|
||||
return parts[sessionsIndex - 1] || null;
|
||||
}
|
||||
|
||||
function resolveSessionStoreTranscriptCorpusPath(
|
||||
agentId: string,
|
||||
sessionsDir: string,
|
||||
entry: { sessionFile?: unknown; sessionId?: unknown } | undefined,
|
||||
): string | null {
|
||||
const sessionFile =
|
||||
typeof entry?.sessionFile === "string" && entry.sessionFile.trim().length > 0
|
||||
? entry.sessionFile.trim()
|
||||
: undefined;
|
||||
const sessionId =
|
||||
typeof entry?.sessionId === "string" && entry.sessionId.trim().length > 0
|
||||
? entry.sessionId.trim()
|
||||
: sessionFile
|
||||
? parseUsageCountedSessionIdFromFileName(path.basename(sessionFile))
|
||||
: null;
|
||||
if (!sessionId) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
if (!sessionFile) {
|
||||
return resolveSessionFilePath(sessionId, undefined, { agentId, sessionsDir });
|
||||
}
|
||||
const resolved = resolveSessionFilePath(
|
||||
sessionId,
|
||||
{ sessionFile },
|
||||
{
|
||||
agentId,
|
||||
sessionsDir,
|
||||
},
|
||||
);
|
||||
if (!path.isAbsolute(sessionFile)) {
|
||||
const candidate = path.resolve(sessionsDir, sessionFile);
|
||||
if (
|
||||
normalizeComparablePath(path.dirname(candidate)) !== normalizeComparablePath(sessionsDir)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return normalizeRealComparablePath(resolved) === normalizeRealComparablePath(candidate)
|
||||
? candidate
|
||||
: null;
|
||||
}
|
||||
const pathAgentId = extractAgentIdFromSessionPath(sessionFile);
|
||||
if (pathAgentId && normalizeAgentId(pathAgentId) !== normalizeAgentId(agentId)) {
|
||||
return null;
|
||||
}
|
||||
return normalizeRealComparablePath(resolved) === normalizeRealComparablePath(sessionFile)
|
||||
? sessionFile
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function classifySessionEntry(
|
||||
sessionKey: string,
|
||||
entry: SessionEntry,
|
||||
cronGeneratedSessionKeys: ReadonlySet<string>,
|
||||
): {
|
||||
generatedByDreamingNarrative: boolean;
|
||||
generatedByCronRun: boolean;
|
||||
} {
|
||||
return {
|
||||
generatedByDreamingNarrative:
|
||||
isDreamingNarrativeSessionStoreKey(sessionKey) ||
|
||||
isDreamingNarrativeSessionKeyLike(entry.spawnedBy),
|
||||
generatedByCronRun: cronGeneratedSessionKeys.has(sessionKey),
|
||||
};
|
||||
}
|
||||
|
||||
function readParentSessionKeys(entry: SessionEntry | undefined): string[] {
|
||||
const keys = new Set<string>();
|
||||
for (const value of [entry?.parentSessionKey, entry?.spawnedBy]) {
|
||||
if (typeof value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (trimmed) {
|
||||
keys.add(trimmed);
|
||||
}
|
||||
}
|
||||
return [...keys];
|
||||
}
|
||||
|
||||
function collectCronGeneratedSessionKeys(
|
||||
summaries: readonly SessionEntrySummary[],
|
||||
): ReadonlySet<string> {
|
||||
// Build the cron-generated closure once so active entries and archive
|
||||
// artifacts share the same lineage classification.
|
||||
const entriesByKey = new Map(summaries.map((summary) => [summary.sessionKey, summary.entry]));
|
||||
const cronGeneratedKeys = new Set<string>();
|
||||
const cache = new Map<string, boolean>();
|
||||
const resolving = new Set<string>();
|
||||
|
||||
const isCronGenerated = (sessionKey: string, entry: SessionEntry | undefined): boolean => {
|
||||
if (isCronRunSessionKey(sessionKey)) {
|
||||
cache.set(sessionKey, true);
|
||||
cronGeneratedKeys.add(sessionKey);
|
||||
return true;
|
||||
}
|
||||
const cached = cache.get(sessionKey);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
if (resolving.has(sessionKey)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
resolving.add(sessionKey);
|
||||
const generated = readParentSessionKeys(entry).some(
|
||||
(parentKey) =>
|
||||
// Parent rows can be pruned before child rows; a cron-shaped parent key
|
||||
// still carries cron lineage without requiring a store entry.
|
||||
isCronRunSessionKey(parentKey) || isCronGenerated(parentKey, entriesByKey.get(parentKey)),
|
||||
);
|
||||
resolving.delete(sessionKey);
|
||||
cache.set(sessionKey, generated);
|
||||
if (generated) {
|
||||
cronGeneratedKeys.add(sessionKey);
|
||||
}
|
||||
return generated;
|
||||
};
|
||||
|
||||
for (const summary of summaries) {
|
||||
isCronGenerated(summary.sessionKey, summary.entry);
|
||||
}
|
||||
return cronGeneratedKeys;
|
||||
}
|
||||
|
||||
function isRegularSessionTranscriptFile(absPath: string): boolean {
|
||||
try {
|
||||
return fsSync.lstatSync(absPath).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionStoreCorpusEntry(
|
||||
agentId: string,
|
||||
sessionsDir: string,
|
||||
summary: SessionEntrySummary,
|
||||
cronGeneratedSessionKeys: ReadonlySet<string>,
|
||||
): SessionTranscriptCorpusEntry | null {
|
||||
const sessionFile = resolveSessionStoreTranscriptCorpusPath(agentId, sessionsDir, summary.entry);
|
||||
if (!sessionFile || !isUsageCountedSessionTranscriptFileName(path.basename(sessionFile))) {
|
||||
return null;
|
||||
}
|
||||
const sessionId =
|
||||
typeof summary.entry.sessionId === "string" && summary.entry.sessionId.trim()
|
||||
? summary.entry.sessionId.trim()
|
||||
: parseUsageCountedSessionIdFromFileName(path.basename(sessionFile));
|
||||
if (!sessionId) {
|
||||
return null;
|
||||
}
|
||||
const sessionKey = summary.sessionKey.trim();
|
||||
const classification = classifySessionEntry(
|
||||
summary.sessionKey,
|
||||
summary.entry,
|
||||
cronGeneratedSessionKeys,
|
||||
);
|
||||
return {
|
||||
agentId,
|
||||
artifactKind: "active-session",
|
||||
sessionFile,
|
||||
sessionId,
|
||||
...(sessionKey ? { sessionKey } : {}),
|
||||
...(classification.generatedByDreamingNarrative ? { generatedByDreamingNarrative: true } : {}),
|
||||
...(classification.generatedByCronRun ? { generatedByCronRun: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function listSessionTranscriptArtifactFiles(sessionsDir: string): string[] {
|
||||
try {
|
||||
return fsSync
|
||||
.readdirSync(sessionsDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => entry.name)
|
||||
.filter((name) => isUsageCountedSessionTranscriptFileName(name))
|
||||
.map((name) => path.join(sessionsDir, name));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function classifyTranscriptArtifact(
|
||||
artifactPath: string,
|
||||
activeEntriesByPath: ReadonlyMap<string, SessionTranscriptCorpusEntry>,
|
||||
): {
|
||||
generatedByDreamingNarrative: boolean;
|
||||
generatedByCronRun: boolean;
|
||||
} {
|
||||
const directEntry = activeEntriesByPath.get(normalizeRealComparablePath(artifactPath));
|
||||
if (directEntry) {
|
||||
return {
|
||||
generatedByDreamingNarrative: directEntry.generatedByDreamingNarrative === true,
|
||||
generatedByCronRun: directEntry.generatedByCronRun === true,
|
||||
};
|
||||
}
|
||||
const sessionsDir = path.dirname(artifactPath);
|
||||
const primarySessionId = parseUsageCountedSessionIdFromFileName(path.basename(artifactPath));
|
||||
const primaryEntry =
|
||||
primarySessionId && isSessionArchiveArtifactName(path.basename(artifactPath))
|
||||
? activeEntriesByPath.get(
|
||||
normalizeRealComparablePath(path.join(sessionsDir, `${primarySessionId}.jsonl`)),
|
||||
)
|
||||
: undefined;
|
||||
return {
|
||||
generatedByDreamingNarrative: primaryEntry?.generatedByDreamingNarrative === true,
|
||||
generatedByCronRun: primaryEntry?.generatedByCronRun === true,
|
||||
};
|
||||
}
|
||||
|
||||
function toArtifactCorpusEntry(
|
||||
agentId: string,
|
||||
artifactPath: string,
|
||||
activeEntriesByPath: ReadonlyMap<string, SessionTranscriptCorpusEntry>,
|
||||
): SessionTranscriptCorpusEntry | null {
|
||||
const sessionId = parseUsageCountedSessionIdFromFileName(path.basename(artifactPath));
|
||||
if (!sessionId) {
|
||||
return null;
|
||||
}
|
||||
const artifactKind = isSessionArchiveArtifactName(path.basename(artifactPath))
|
||||
? "archive-artifact"
|
||||
: "orphan-file-artifact";
|
||||
const classification = classifyTranscriptArtifact(artifactPath, activeEntriesByPath);
|
||||
return {
|
||||
agentId,
|
||||
artifactKind,
|
||||
sessionFile: artifactPath,
|
||||
sessionId,
|
||||
...(classification.generatedByDreamingNarrative ? { generatedByDreamingNarrative: true } : {}),
|
||||
...(classification.generatedByCronRun ? { generatedByCronRun: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function listSessionTranscriptCorpusEntriesForAgentSync(
|
||||
agentId: string,
|
||||
): SessionTranscriptCorpusEntry[] {
|
||||
const normalizedAgentId = normalizeAgentId(agentId);
|
||||
const cfg = getRuntimeConfig();
|
||||
const configuredStore = cfg.session?.store;
|
||||
const storePath = resolveStorePath(configuredStore, {
|
||||
agentId: normalizedAgentId,
|
||||
});
|
||||
const sessionsDir = path.dirname(storePath);
|
||||
const fixedStoreOwnerAgentId = extractAgentIdFromSessionsDir(sessionsDir);
|
||||
const isAgentOwnedFixedStore =
|
||||
fixedStoreOwnerAgentId !== null &&
|
||||
normalizeAgentId(fixedStoreOwnerAgentId) === normalizedAgentId;
|
||||
const isSharedFixedStore =
|
||||
typeof configuredStore === "string" &&
|
||||
configuredStore.trim().length > 0 &&
|
||||
!configuredStore.includes("{agentId}") &&
|
||||
!isAgentOwnedFixedStore;
|
||||
const activeEntriesByPath = new Map<string, SessionTranscriptCorpusEntry>();
|
||||
const activeEntryOwnersByPath = new Map<string, string>();
|
||||
const artifactDirsByPath = new Map<string, string>();
|
||||
rememberArtifactDir(artifactDirsByPath, sessionsDir);
|
||||
const sessionEntries = listSessionEntries({
|
||||
agentId: normalizedAgentId,
|
||||
hydrateSkillPromptRefs: false,
|
||||
storePath,
|
||||
});
|
||||
const cronGeneratedSessionKeys = collectCronGeneratedSessionKeys(sessionEntries);
|
||||
for (const summary of sessionEntries) {
|
||||
const sessionKey = isSharedFixedStore
|
||||
? summary.sessionKey
|
||||
: canonicalizeMainSessionAlias({
|
||||
cfg,
|
||||
agentId: normalizedAgentId,
|
||||
sessionKey: summary.sessionKey,
|
||||
});
|
||||
const ownerAgentId = resolveSessionAgentId({
|
||||
config: cfg,
|
||||
sessionKey,
|
||||
...(isSharedFixedStore ? {} : { fallbackAgentId: normalizedAgentId }),
|
||||
});
|
||||
const entry = toSessionStoreCorpusEntry(
|
||||
ownerAgentId,
|
||||
sessionsDir,
|
||||
summary,
|
||||
cronGeneratedSessionKeys,
|
||||
);
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
const normalizedEntryPath = normalizeRealComparablePath(entry.sessionFile);
|
||||
activeEntryOwnersByPath.set(normalizedEntryPath, ownerAgentId);
|
||||
rememberArtifactDir(artifactDirsByPath, path.dirname(entry.sessionFile));
|
||||
if (ownerAgentId === normalizedAgentId) {
|
||||
activeEntriesByPath.set(normalizedEntryPath, entry);
|
||||
}
|
||||
}
|
||||
const includeUnownedArtifacts = !isSharedFixedStore;
|
||||
const corpusEntries = [...activeEntriesByPath.values()].filter((entry) =>
|
||||
isRegularSessionTranscriptFile(entry.sessionFile),
|
||||
);
|
||||
const scannedArtifactPaths = new Set<string>();
|
||||
for (const artifactDir of artifactDirsByPath.values()) {
|
||||
for (const artifactPath of listSessionTranscriptArtifactFiles(artifactDir)) {
|
||||
const normalizedArtifactPath = normalizeRealComparablePath(artifactPath);
|
||||
if (scannedArtifactPaths.has(normalizedArtifactPath)) {
|
||||
continue;
|
||||
}
|
||||
scannedArtifactPaths.add(normalizedArtifactPath);
|
||||
if (activeEntriesByPath.has(normalizedArtifactPath)) {
|
||||
continue;
|
||||
}
|
||||
const artifactOwner = activeEntryOwnersByPath.get(normalizedArtifactPath);
|
||||
if (artifactOwner) {
|
||||
continue;
|
||||
}
|
||||
const primarySessionId = parseUsageCountedSessionIdFromFileName(path.basename(artifactPath));
|
||||
const primaryOwner =
|
||||
primarySessionId && isSessionArchiveArtifactName(path.basename(artifactPath))
|
||||
? activeEntryOwnersByPath.get(
|
||||
normalizeRealComparablePath(
|
||||
path.join(path.dirname(artifactPath), `${primarySessionId}.jsonl`),
|
||||
),
|
||||
)
|
||||
: undefined;
|
||||
if (primaryOwner && primaryOwner !== normalizedAgentId) {
|
||||
continue;
|
||||
}
|
||||
if (!primaryOwner && !includeUnownedArtifacts) {
|
||||
continue;
|
||||
}
|
||||
const entry = toArtifactCorpusEntry(normalizedAgentId, artifactPath, activeEntriesByPath);
|
||||
if (entry) {
|
||||
corpusEntries.push(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
return corpusEntries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists transcript corpus entries for QMD/memory indexing.
|
||||
*
|
||||
* Active sessions come from the session accessor seam; retained reset/delete
|
||||
* transcript artifacts remain explicit file artifacts until core owns archive
|
||||
* artifact enumeration.
|
||||
*/
|
||||
export async function listSessionTranscriptCorpusEntriesForAgent(
|
||||
agentId: string,
|
||||
): Promise<SessionTranscriptCorpusEntry[]> {
|
||||
return listSessionTranscriptCorpusEntriesForAgentSync(agentId);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Memory Host SDK module implements sqlite vec platform variant behavior.
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
// Resolves optional sqlite-vec native extension packages for the current platform.
|
||||
|
||||
/** Package/file pair for one sqlite-vec native platform build. */
|
||||
type PlatformVariant = { readonly pkg: string; readonly file: string };
|
||||
|
||||
const PLATFORM_VARIANTS: Readonly<Record<string, PlatformVariant | undefined>> = {
|
||||
"linux-x64": { pkg: "sqlite-vec-linux-x64", file: "vec0.so" },
|
||||
"linux-arm64": { pkg: "sqlite-vec-linux-arm64", file: "vec0.so" },
|
||||
"darwin-x64": { pkg: "sqlite-vec-darwin-x64", file: "vec0.dylib" },
|
||||
"darwin-arm64": { pkg: "sqlite-vec-darwin-arm64", file: "vec0.dylib" },
|
||||
"win32-x64": { pkg: "sqlite-vec-windows-x64", file: "vec0.dll" },
|
||||
};
|
||||
|
||||
/** Resolve the installed sqlite-vec native extension for the current platform if present. */
|
||||
export function resolveSqliteVecPlatformVariant():
|
||||
| { pkg: string; extensionPath: string }
|
||||
| undefined {
|
||||
const entry = PLATFORM_VARIANTS[`${process.platform}-${process.arch}`];
|
||||
if (!entry) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const requireForResolve = createRequire(import.meta.url);
|
||||
const extensionPath = requireForResolve.resolve(`${entry.pkg}/${entry.file}`);
|
||||
return { pkg: entry.pkg, extensionPath };
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
258
packages/memory-host-sdk/src/host/sqlite-vec.test.ts
Normal file
258
packages/memory-host-sdk/src/host/sqlite-vec.test.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
// Memory Host SDK tests cover sqlite vec behavior.
|
||||
import { existsSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
function mockMissingSqliteVecPackage(): void {
|
||||
vi.doMock("sqlite-vec", () => {
|
||||
const err = new Error("Cannot find package 'sqlite-vec' imported from sqlite-vec.test.ts");
|
||||
Object.assign(err, { code: "ERR_MODULE_NOT_FOUND" });
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
|
||||
function mockFailingSqliteVecPackage(): void {
|
||||
vi.doMock("sqlite-vec", () => ({
|
||||
getLoadablePath: () => "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
load: () => {
|
||||
throw new Error("bundled sqlite-vec load failed");
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
function mockPlatformVariantResolver(
|
||||
value: { pkg: string; extensionPath: string } | undefined,
|
||||
): void {
|
||||
vi.doMock("./sqlite-vec-platform-variant.js", () => ({
|
||||
resolveSqliteVecPlatformVariant: () => value,
|
||||
}));
|
||||
}
|
||||
|
||||
async function importLoader() {
|
||||
return import("./sqlite-vec.js");
|
||||
}
|
||||
|
||||
function createDbMock(params?: { readonly healthError?: Error }) {
|
||||
const get = vi.fn(() => ({ version: "v0.1.9" }));
|
||||
const prepare = vi.fn(() => {
|
||||
if (params?.healthError) {
|
||||
throw params.healthError;
|
||||
}
|
||||
return { get };
|
||||
});
|
||||
return {
|
||||
db: {
|
||||
enableLoadExtension: vi.fn(),
|
||||
loadExtension: vi.fn(),
|
||||
prepare,
|
||||
},
|
||||
get,
|
||||
prepare,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.doUnmock("sqlite-vec");
|
||||
vi.doUnmock("./sqlite-vec-platform-variant.js");
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
const CURRENT_PLATFORM_VARIANTS: Readonly<
|
||||
Record<string, { readonly pkg: string; readonly file: string } | undefined>
|
||||
> = {
|
||||
"linux-x64": { pkg: "sqlite-vec-linux-x64", file: "vec0.so" },
|
||||
"linux-arm64": { pkg: "sqlite-vec-linux-arm64", file: "vec0.so" },
|
||||
"darwin-x64": { pkg: "sqlite-vec-darwin-x64", file: "vec0.dylib" },
|
||||
"darwin-arm64": { pkg: "sqlite-vec-darwin-arm64", file: "vec0.dylib" },
|
||||
"win32-x64": { pkg: "sqlite-vec-windows-x64", file: "vec0.dll" },
|
||||
};
|
||||
|
||||
function isMissingModuleError(err: unknown): boolean {
|
||||
const code =
|
||||
err && typeof err === "object" && "code" in err ? (err as { code?: unknown }).code : undefined;
|
||||
return code === "MODULE_NOT_FOUND" || code === "ERR_MODULE_NOT_FOUND";
|
||||
}
|
||||
|
||||
describe("loadSqliteVecExtension", () => {
|
||||
it("loads explicit extensionPath without importing bundled sqlite-vec", async () => {
|
||||
mockMissingSqliteVecPackage();
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db, prepare } = createDbMock();
|
||||
|
||||
await expect(
|
||||
loadSqliteVecExtension({
|
||||
db: db as never,
|
||||
extensionPath: "/opt/openclaw/sqlite-vec.so",
|
||||
}),
|
||||
).resolves.toEqual({ ok: true, extensionPath: "/opt/openclaw/sqlite-vec.so" });
|
||||
expect(db.enableLoadExtension).toHaveBeenCalledWith(true);
|
||||
expect(db.loadExtension).toHaveBeenCalledWith("/opt/openclaw/sqlite-vec.so");
|
||||
expect(prepare).toHaveBeenCalledWith("SELECT vec_version() AS version");
|
||||
});
|
||||
|
||||
it("rejects a loaded extension when sqlite-vec functions are unavailable", async () => {
|
||||
mockMissingSqliteVecPackage();
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db } = createDbMock({ healthError: new Error("no such function: vec_version") });
|
||||
|
||||
const result = await loadSqliteVecExtension({
|
||||
db: db as never,
|
||||
extensionPath: "/opt/openclaw/sqlite-vec.so",
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
error:
|
||||
"sqlite-vec health check failed after loading /opt/openclaw/sqlite-vec.so | no such function: vec_version",
|
||||
});
|
||||
expect(db.loadExtension).toHaveBeenCalledWith("/opt/openclaw/sqlite-vec.so");
|
||||
});
|
||||
|
||||
it("returns a valid memorySearch extensionPath hint when sqlite-vec is absent", async () => {
|
||||
mockMissingSqliteVecPackage();
|
||||
mockPlatformVariantResolver(undefined);
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db } = createDbMock();
|
||||
|
||||
const result = await loadSqliteVecExtension({ db: db as never });
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
error: expect.stringMatching(
|
||||
/^sqlite-vec package is not installed\. Set agents\.defaults\.memorySearch\.store\.vector\.extensionPath, or an agent-specific memorySearch\.store\.vector\.extensionPath, to a sqlite-vec loadable extension path\. Original error: (?:\[vitest\] There was an error when mocking a module\. If you are using "vi\.mock" factory, make sure there are no top level variables inside, since this call is hoisted to top of the file\. Read more: https:\/\/vitest\.dev\/api\/vi\.html#vi-mock \| )?Cannot find package 'sqlite-vec' imported from sqlite-vec\.test\.ts$/u,
|
||||
),
|
||||
});
|
||||
expect(result.error).not.toContain("memory.store.vector.extensionPath");
|
||||
expect(db.enableLoadExtension).toHaveBeenCalledWith(true);
|
||||
expect(db.loadExtension).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls back to the platform-specific sqlite-vec variant when only that package is installed", async () => {
|
||||
mockMissingSqliteVecPackage();
|
||||
mockPlatformVariantResolver({
|
||||
pkg: "sqlite-vec-linux-x64",
|
||||
extensionPath: "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
});
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db, prepare } = createDbMock();
|
||||
|
||||
const result = await loadSqliteVecExtension({ db: db as never });
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: true,
|
||||
extensionPath: "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
});
|
||||
expect(db.enableLoadExtension).toHaveBeenCalledWith(true);
|
||||
expect(db.loadExtension).toHaveBeenCalledWith(
|
||||
"/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
);
|
||||
expect(prepare).toHaveBeenCalledWith("SELECT vec_version() AS version");
|
||||
});
|
||||
|
||||
it("falls back to the platform variant when bundled sqlite-vec load fails", async () => {
|
||||
mockFailingSqliteVecPackage();
|
||||
mockPlatformVariantResolver({
|
||||
pkg: "sqlite-vec-linux-x64",
|
||||
extensionPath: "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
});
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db, prepare } = createDbMock();
|
||||
|
||||
const result = await loadSqliteVecExtension({ db: db as never });
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: true,
|
||||
extensionPath: "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
});
|
||||
expect(db.loadExtension).toHaveBeenCalledWith(
|
||||
"/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
);
|
||||
expect(prepare).toHaveBeenCalledWith("SELECT vec_version() AS version");
|
||||
});
|
||||
|
||||
it("resolves the installed platform variant through its exported vec0 subpath", async () => {
|
||||
const entry = CURRENT_PLATFORM_VARIANTS[`${process.platform}-${process.arch}`];
|
||||
if (!entry) {
|
||||
return;
|
||||
}
|
||||
|
||||
const requireForResolve = createRequire(import.meta.url);
|
||||
let expectedPath: string;
|
||||
try {
|
||||
expectedPath = requireForResolve.resolve(`${entry.pkg}/${entry.file}`);
|
||||
} catch (err) {
|
||||
if (isMissingModuleError(err)) {
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const { resolveSqliteVecPlatformVariant } = await import("./sqlite-vec-platform-variant.js");
|
||||
|
||||
expect(resolveSqliteVecPlatformVariant()).toEqual({
|
||||
pkg: entry.pkg,
|
||||
extensionPath: expectedPath,
|
||||
});
|
||||
expect(existsSync(expectedPath)).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves the extensionPath config hint when the platform variant loadExtension call throws", async () => {
|
||||
mockMissingSqliteVecPackage();
|
||||
mockPlatformVariantResolver({
|
||||
pkg: "sqlite-vec-linux-x64",
|
||||
extensionPath: "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
});
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db } = createDbMock();
|
||||
db.loadExtension.mockImplementation(() => {
|
||||
throw new Error("dlopen failed: file not found");
|
||||
});
|
||||
|
||||
const result = await loadSqliteVecExtension({ db: db as never });
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
error:
|
||||
"sqlite-vec platform variant sqlite-vec-linux-x64 failed to load from /install/node_modules/sqlite-vec-linux-x64/vec0.so. Set agents.defaults.memorySearch.store.vector.extensionPath, or an agent-specific memorySearch.store.vector.extensionPath, to a sqlite-vec loadable extension path. Original error: dlopen failed: file not found",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a platform variant when sqlite-vec functions are unavailable", async () => {
|
||||
mockMissingSqliteVecPackage();
|
||||
mockPlatformVariantResolver({
|
||||
pkg: "sqlite-vec-linux-x64",
|
||||
extensionPath: "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
});
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db } = createDbMock({ healthError: new Error("no such function: vec_version") });
|
||||
|
||||
const result = await loadSqliteVecExtension({ db: db as never });
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
error:
|
||||
"sqlite-vec platform variant sqlite-vec-linux-x64 failed to load from /install/node_modules/sqlite-vec-linux-x64/vec0.so. Set agents.defaults.memorySearch.store.vector.extensionPath, or an agent-specific memorySearch.store.vector.extensionPath, to a sqlite-vec loadable extension path. Original error: sqlite-vec health check failed after loading /install/node_modules/sqlite-vec-linux-x64/vec0.so | no such function: vec_version",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves bundled sqlite-vec and platform variant errors when both fail", async () => {
|
||||
mockFailingSqliteVecPackage();
|
||||
mockPlatformVariantResolver({
|
||||
pkg: "sqlite-vec-linux-x64",
|
||||
extensionPath: "/install/node_modules/sqlite-vec-linux-x64/vec0.so",
|
||||
});
|
||||
const { loadSqliteVecExtension } = await importLoader();
|
||||
const { db } = createDbMock();
|
||||
db.loadExtension.mockImplementation(() => {
|
||||
throw new Error("platform variant failed");
|
||||
});
|
||||
|
||||
const result = await loadSqliteVecExtension({ db: db as never });
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
error:
|
||||
"sqlite-vec package failed to load, and platform variant sqlite-vec-linux-x64 failed to load from /install/node_modules/sqlite-vec-linux-x64/vec0.so. Set agents.defaults.memorySearch.store.vector.extensionPath, or an agent-specific memorySearch.store.vector.extensionPath, to a sqlite-vec loadable extension path. Package error: bundled sqlite-vec load failed. Variant error: platform variant failed",
|
||||
});
|
||||
});
|
||||
});
|
||||
105
packages/memory-host-sdk/src/host/sqlite-vec.ts
Normal file
105
packages/memory-host-sdk/src/host/sqlite-vec.ts
Normal file
@@ -0,0 +1,105 @@
|
||||
// Memory Host SDK module implements sqlite vec behavior.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { formatErrorMessage } from "./error-utils.js";
|
||||
import { resolveSqliteVecPlatformVariant } from "./sqlite-vec-platform-variant.js";
|
||||
import { normalizeOptionalString } from "./string-utils.js";
|
||||
|
||||
type SqliteVecModule = {
|
||||
getLoadablePath: () => string;
|
||||
load: (db: DatabaseSync) => void;
|
||||
};
|
||||
|
||||
const SQLITE_VEC_MODULE_ID = "sqlite-vec";
|
||||
const SQLITE_VEC_CONFIG_HINT =
|
||||
"Set agents.defaults.memorySearch.store.vector.extensionPath, or an agent-specific memorySearch.store.vector.extensionPath, to a sqlite-vec loadable extension path.";
|
||||
|
||||
async function loadSqliteVecModule(): Promise<SqliteVecModule> {
|
||||
return import(SQLITE_VEC_MODULE_ID) as Promise<SqliteVecModule>;
|
||||
}
|
||||
|
||||
function isMissingSqliteVecPackageError(err: unknown): boolean {
|
||||
const message = formatErrorMessage(err);
|
||||
const code =
|
||||
err && typeof err === "object" && "code" in err ? (err as { code?: unknown }).code : undefined;
|
||||
const missingSqliteVec = /Cannot find (?:package|module) ['"]sqlite-vec['"]/u.test(message);
|
||||
return (
|
||||
missingSqliteVec &&
|
||||
(code === undefined || code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND")
|
||||
);
|
||||
}
|
||||
|
||||
function assertSqliteVecAvailable(db: DatabaseSync, source: string): void {
|
||||
try {
|
||||
const row = db.prepare("SELECT vec_version() AS version").get() as
|
||||
| { version?: unknown }
|
||||
| undefined;
|
||||
if (typeof row?.version !== "string" || row.version.trim().length === 0) {
|
||||
throw new Error("vec_version() did not return a version");
|
||||
}
|
||||
} catch (err) {
|
||||
throw new Error(`sqlite-vec health check failed after loading ${source}`, { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
function loadExtensionAndVerify(db: DatabaseSync, extensionPath: string): void {
|
||||
db.loadExtension(extensionPath);
|
||||
assertSqliteVecAvailable(db, extensionPath);
|
||||
}
|
||||
|
||||
export async function loadSqliteVecExtension(params: {
|
||||
db: DatabaseSync;
|
||||
extensionPath?: string;
|
||||
}): Promise<{ ok: boolean; extensionPath?: string; error?: string }> {
|
||||
try {
|
||||
const resolvedPath = normalizeOptionalString(params.extensionPath);
|
||||
params.db.enableLoadExtension(true);
|
||||
if (resolvedPath) {
|
||||
loadExtensionAndVerify(params.db, resolvedPath);
|
||||
return { ok: true, extensionPath: resolvedPath };
|
||||
}
|
||||
|
||||
try {
|
||||
const sqliteVec = await loadSqliteVecModule();
|
||||
const extensionPath = sqliteVec.getLoadablePath();
|
||||
sqliteVec.load(params.db);
|
||||
assertSqliteVecAvailable(params.db, extensionPath);
|
||||
return { ok: true, extensionPath };
|
||||
} catch (err) {
|
||||
// Optional-dep installs sometimes land only the platform-specific variant
|
||||
// (e.g. sqlite-vec-linux-x64) without the meta sqlite-vec package. Load
|
||||
// the loadable extension straight from the variant when we can find it.
|
||||
// Bundled runtimes can also fail the meta-package import while the native
|
||||
// variant is still present, so try the concrete extension before failing.
|
||||
const variant = resolveSqliteVecPlatformVariant();
|
||||
if (!variant) {
|
||||
if (!isMissingSqliteVecPackageError(err)) {
|
||||
throw err;
|
||||
}
|
||||
const message = formatErrorMessage(err);
|
||||
return {
|
||||
ok: false,
|
||||
error: `sqlite-vec package is not installed. ${SQLITE_VEC_CONFIG_HINT} Original error: ${message}`,
|
||||
};
|
||||
}
|
||||
try {
|
||||
loadExtensionAndVerify(params.db, variant.extensionPath);
|
||||
return { ok: true, extensionPath: variant.extensionPath };
|
||||
} catch (variantErr) {
|
||||
const message = formatErrorMessage(variantErr);
|
||||
if (!isMissingSqliteVecPackageError(err)) {
|
||||
const packageMessage = formatErrorMessage(err);
|
||||
return {
|
||||
ok: false,
|
||||
error: `sqlite-vec package failed to load, and platform variant ${variant.pkg} failed to load from ${variant.extensionPath}. ${SQLITE_VEC_CONFIG_HINT} Package error: ${packageMessage}. Variant error: ${message}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
error: `sqlite-vec platform variant ${variant.pkg} failed to load from ${variant.extensionPath}. ${SQLITE_VEC_CONFIG_HINT} Original error: ${message}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return { ok: false, error: formatErrorMessage(err) };
|
||||
}
|
||||
}
|
||||
14
packages/memory-host-sdk/src/host/sqlite-wal.ts
Normal file
14
packages/memory-host-sdk/src/host/sqlite-wal.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
// Public SQLite WAL maintenance facade for memory database callers.
|
||||
|
||||
export {
|
||||
DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES,
|
||||
DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS,
|
||||
DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS,
|
||||
configureSqliteConnectionPragmas,
|
||||
configureSqliteWalMaintenance,
|
||||
} from "./openclaw-runtime-io.js";
|
||||
export type {
|
||||
SqliteConnectionPragmaOptions,
|
||||
SqliteWalMaintenance,
|
||||
SqliteWalMaintenanceOptions,
|
||||
} from "./openclaw-runtime-io.js";
|
||||
55
packages/memory-host-sdk/src/host/sqlite.ts
Normal file
55
packages/memory-host-sdk/src/host/sqlite.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
// Memory Host SDK module implements sqlite behavior.
|
||||
import { createRequire } from "node:module";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { formatErrorMessage } from "./error-utils.js";
|
||||
import {
|
||||
configureSqliteConnectionPragmas,
|
||||
configureSqliteWalMaintenance,
|
||||
type SqliteConnectionPragmaOptions,
|
||||
type SqliteWalMaintenance,
|
||||
type SqliteWalMaintenanceOptions,
|
||||
} from "./sqlite-wal.js";
|
||||
import { installProcessWarningFilter } from "./warning-filter.js";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const sqliteWalMaintenanceByDb = new WeakMap<DatabaseSync, SqliteWalMaintenance>();
|
||||
|
||||
export function requireNodeSqlite(): typeof import("node:sqlite") {
|
||||
installProcessWarningFilter();
|
||||
try {
|
||||
return require("node:sqlite") as typeof import("node:sqlite");
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
// Node distributions can ship without the experimental builtin SQLite module.
|
||||
// Surface an actionable error instead of the generic "unknown builtin module".
|
||||
throw new Error(
|
||||
`SQLite support is unavailable in this Node runtime (missing node:sqlite). ${message}`,
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function configureMemorySqliteWalMaintenance(
|
||||
db: DatabaseSync,
|
||||
options?: SqliteWalMaintenanceOptions & Pick<SqliteConnectionPragmaOptions, "busyTimeoutMs">,
|
||||
): SqliteWalMaintenance {
|
||||
const existing = sqliteWalMaintenanceByDb.get(db);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const maintenance =
|
||||
options?.busyTimeoutMs === undefined
|
||||
? configureSqliteWalMaintenance(db, options)
|
||||
: configureSqliteConnectionPragmas(db, options);
|
||||
sqliteWalMaintenanceByDb.set(db, maintenance);
|
||||
return maintenance;
|
||||
}
|
||||
|
||||
export function closeMemorySqliteWalMaintenance(db: DatabaseSync): boolean {
|
||||
const maintenance = sqliteWalMaintenanceByDb.get(db);
|
||||
if (!maintenance) {
|
||||
return true;
|
||||
}
|
||||
sqliteWalMaintenanceByDb.delete(db);
|
||||
return maintenance.close();
|
||||
}
|
||||
11
packages/memory-host-sdk/src/host/ssrf-policy.ts
Normal file
11
packages/memory-host-sdk/src/host/ssrf-policy.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
// Public SSRF policy shape accepted by memory host remote HTTP helpers.
|
||||
|
||||
/** Host/network allowlist policy forwarded to the runtime SSRF guard. */
|
||||
export type SsrFPolicy = {
|
||||
allowPrivateNetwork?: boolean;
|
||||
dangerouslyAllowPrivateNetwork?: boolean;
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
allowIpv6UniqueLocalRange?: boolean;
|
||||
allowedHostnames?: string[];
|
||||
hostnameAllowlist?: string[];
|
||||
};
|
||||
44
packages/memory-host-sdk/src/host/status-format.ts
Normal file
44
packages/memory-host-sdk/src/host/status-format.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
// Shared status text/tone formatter for memory health summaries.
|
||||
|
||||
/** Display tone used by memory status renderers. */
|
||||
export type Tone = "ok" | "warn" | "muted";
|
||||
|
||||
/** Resolve vector indexing state from enabled and availability flags. */
|
||||
export function resolveMemoryVectorState(vector: { enabled: boolean; available?: boolean }): {
|
||||
tone: Tone;
|
||||
state: "ready" | "unavailable" | "disabled" | "unknown";
|
||||
} {
|
||||
if (!vector.enabled) {
|
||||
return { tone: "muted", state: "disabled" };
|
||||
}
|
||||
if (vector.available === true) {
|
||||
return { tone: "ok", state: "ready" };
|
||||
}
|
||||
if (vector.available === false) {
|
||||
return { tone: "warn", state: "unavailable" };
|
||||
}
|
||||
return { tone: "muted", state: "unknown" };
|
||||
}
|
||||
|
||||
/** Resolve full-text search state from enabled and availability flags. */
|
||||
export function resolveMemoryFtsState(fts: { enabled: boolean; available: boolean }): {
|
||||
tone: Tone;
|
||||
state: "ready" | "unavailable" | "disabled";
|
||||
} {
|
||||
if (!fts.enabled) {
|
||||
return { tone: "muted", state: "disabled" };
|
||||
}
|
||||
return fts.available ? { tone: "ok", state: "ready" } : { tone: "warn", state: "unavailable" };
|
||||
}
|
||||
|
||||
/** Format cache state as concise status text with optional entry count. */
|
||||
export function resolveMemoryCacheSummary(cache: { enabled: boolean; entries?: number }): {
|
||||
tone: Tone;
|
||||
text: string;
|
||||
} {
|
||||
if (!cache.enabled) {
|
||||
return { tone: "muted", text: "cache off" };
|
||||
}
|
||||
const suffix = typeof cache.entries === "number" ? ` (${cache.entries})` : "";
|
||||
return { tone: "ok", text: `cache on${suffix}` };
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user