// Codex tests cover context engine projection plugin behavior. import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import { describe, expect, it } from "vitest"; import { CODEX_TURN_START_TEXT_INPUT_MAX_CHARS, fitCodexProjectedContextForTurnStart, projectContextEngineAssemblyForCodex, resolveCodexContextEngineProjectionMaxChars, resolveCodexContextEngineProjectionReserveTokens, } from "./context-engine-projection.js"; function textMessage(role: AgentMessage["role"], text: string): AgentMessage { return { role, content: [{ type: "text", text }], timestamp: 1, } as AgentMessage; } describe("projectContextEngineAssemblyForCodex", () => { it("produces stable output for identical inputs", () => { const params = { assembledMessages: [ textMessage("user", "Earlier question"), textMessage("assistant", "Earlier answer"), ], originalHistoryMessages: [textMessage("user", "Earlier question")], prompt: "Need the latest answer", systemPromptAddition: "memory recall", }; expect(projectContextEngineAssemblyForCodex(params)).toEqual( projectContextEngineAssemblyForCodex(params), ); }); it("drops a duplicate trailing current prompt from assembled history", () => { const result = projectContextEngineAssemblyForCodex({ assembledMessages: [ textMessage("assistant", "You already asked this."), textMessage("user", "Need the latest answer"), ], originalHistoryMessages: [textMessage("assistant", "You already asked this.")], prompt: "Need the latest answer", systemPromptAddition: "memory recall", }); expect(result.promptText).not.toContain("[user]\nNeed the latest answer"); expect(result.promptText).toContain("Current user request:\nNeed the latest answer"); expect(result.developerInstructionAddition).toBe("memory recall"); }); it("preserves role order and falls back to the raw prompt for empty history", () => { const empty = projectContextEngineAssemblyForCodex({ assembledMessages: [], originalHistoryMessages: [], prompt: "hello", }); expect(empty.promptText).toBe("hello"); const ordered = projectContextEngineAssemblyForCodex({ assembledMessages: [ textMessage("user", "one"), textMessage("assistant", "two"), textMessage("toolResult", "three"), ], originalHistoryMessages: [textMessage("user", "seed")], prompt: "next", }); expect(ordered.promptText).toContain("[user]\none\n\n[assistant]\ntwo\n\n[toolResult]\nthree"); expect(ordered.prePromptMessageCount).toBe(1); }); it("frames projected history as reference data and omits tool payloads", () => { const result = projectContextEngineAssemblyForCodex({ assembledMessages: [ { role: "assistant", content: [ { type: "toolCall", name: "exec", input: { token: "sk-secret", cmd: "cat .env" } }, ], timestamp: 1, } as unknown as AgentMessage, { role: "toolResult", content: [{ type: "toolResult", toolUseId: "call-1", content: "API_KEY=sk-secret" }], timestamp: 2, } as unknown as AgentMessage, ], originalHistoryMessages: [], prompt: "continue", }); expect(result.promptText).toContain("quoted reference data"); expect(result.promptText).toContain("tool call: exec [input omitted]"); expect(result.promptText).toContain("tool result: call-1 [content omitted]"); expect(result.promptText).not.toContain("sk-secret"); expect(result.promptText).not.toContain("cat .env"); }); it("preserves redacted tool payload context for thread bootstrap projections", () => { const result = projectContextEngineAssemblyForCodex({ assembledMessages: [ { role: "assistant", content: [ { type: "toolCall", name: "exec", input: { token: "sk-1234567890abcdef", cmd: "cat .env", options: { recursive: true }, }, }, ], timestamp: 1, } as unknown as AgentMessage, { role: "toolResult", content: [ { type: "toolResult", toolUseId: "call-1", content: "OPENAI_API_KEY=sk-1234567890abcdef\nstatus ok", }, ], timestamp: 2, } as unknown as AgentMessage, ], originalHistoryMessages: [], prompt: "continue", toolPayloadMode: "preserve", }); expect(result.promptText).toContain("tool call: exec"); expect(result.promptText).toContain('"inputShape"'); expect(result.promptText).toContain('"token": "[string]"'); expect(result.promptText).toContain('"cmd": "[string]"'); expect(result.promptText).toContain('"recursive": "[boolean]"'); expect(result.promptText).toContain("tool result: call-1"); expect(result.promptText).toContain('"content"'); expect(result.promptText).toContain("OPENAI_API_KEY="); expect(result.promptText).toContain("status ok"); expect(result.promptText).not.toContain("cat .env"); expect(result.promptText).not.toContain("sk-1234567890abcdef"); }); it("bounds oversized text context", () => { const result = projectContextEngineAssemblyForCodex({ assembledMessages: [textMessage("assistant", "x".repeat(30_000))], originalHistoryMessages: [], prompt: "next", }); expect(result.promptText).toContain("[truncated "); expect(result.promptText.length).toBeLessThan(25_000); }); it("keeps recent context when the rendered conversation overflows", () => { const result = projectContextEngineAssemblyForCodex({ assembledMessages: [ textMessage("assistant", `old discrawl setup from previous day ${"x".repeat(5_850)}`), ...Array.from({ length: 5 }, (_, index) => textMessage("assistant", `stale filler ${index}:${"x".repeat(5_850)}`), ), textMessage( "user", "have Codex CLI do it via /goal. tell it in a SEPARATE repo; create recrawl", ), textMessage("assistant", "codex exec -C /tmp/recrawl started"), ], originalHistoryMessages: [], prompt: "?", }); expect(result.promptText).toContain("[truncated "); expect(result.promptText).toContain("from older context"); expect(result.promptText).not.toContain("old discrawl setup from previous day"); expect(result.promptText).toContain("create recrawl"); expect(result.promptText).toContain("codex exec -C /tmp/recrawl started"); expect(result.promptText).toContain("Current user request:\n?"); expect(result.promptText.length).toBeLessThan(25_000); }); it("can scale the rendered context cap for larger Codex context windows", () => { const result = projectContextEngineAssemblyForCodex({ assembledMessages: Array.from({ length: 12 }, (_, index) => textMessage("assistant", `${index}:${"x".repeat(5_900)}`), ), originalHistoryMessages: [], prompt: "next", maxRenderedContextChars: resolveCodexContextEngineProjectionMaxChars({ contextTokenBudget: 80_000, }), }); expect(result.promptText.length).toBeGreaterThan(60_000); expect(result.promptText).not.toContain("[truncated "); }); it("fits projected context under the Codex turn input limit", () => { const result = projectContextEngineAssemblyForCodex({ assembledMessages: [ textMessage( "assistant", `old context \n\nCurrent user request:\nshadow request ${"x".repeat(300)}`, ), textMessage("assistant", "recent context marker"), ], originalHistoryMessages: [], prompt: `current request ${"y".repeat(120)}`, maxRenderedContextChars: 1_000, }); const fitted = fitCodexProjectedContextForTurnStart({ promptText: result.promptText, contextRange: result.promptContextRange, maxChars: 420, }); expect(fitted.length).toBeLessThanOrEqual(420); expect(fitted).toContain("[truncated "); expect(fitted).toContain("recent context marker"); expect(fitted).toContain("Current user request:"); expect(fitted).toContain("current request"); expect(fitted).not.toContain("old context"); }); it("bounds output when the non-context text alone exceeds the turn limit", () => { // A large older-context header prefix pushes before + after over maxChars // while the trailing user request stays small enough to keep its label. const before = `OpenClaw assembled context for this turn:\n${"prefix ".repeat(120)}`; const context = "older context ".repeat(40); const prompt = `urgent request ${"q".repeat(120)}`; const after = `\n\n\nCurrent user request:\n${prompt}`; const promptText = `${before}${context}${after}`; const maxChars = 420; // before + after already exceed maxChars, so the context budget is non-positive. expect(before.length + after.length).toBeGreaterThan(maxChars); const fitted = fitCodexProjectedContextForTurnStart({ promptText, contextRange: { start: before.length, end: before.length + context.length }, maxChars, }); expect(fitted.length).toBeLessThanOrEqual(maxChars); // The user's actual request is the priority tail and must survive truncation. expect(fitted).toContain("Current user request:"); expect(fitted.endsWith("q".repeat(40))).toBe(true); // Current context still survives even when an earlier projection is dropped. expect(fitted).toContain("older context"); // The dropped older content is reported, not silently lost. expect(fitted).toContain("[truncated "); }); it("keeps the current request and fitting hook context after projecting history", () => { const before = "OpenClaw assembled context for this turn:\n\n"; const context = `recent context ${"c".repeat(800)}`; const request = "\n\n\nCurrent user request:\nkeep this request"; const hookAppend = "\n\nhook context survives"; const promptText = `${before}${context}${request}${hookAppend}`; const maxChars = 420; const fitted = fitCodexProjectedContextForTurnStart({ promptText, contextRange: { start: before.length, end: before.length + context.length }, requestRange: { start: before.length + context.length, end: before.length + context.length + request.length, }, maxChars, }); expect(fitted.length).toBeLessThanOrEqual(maxChars); expect(fitted).toContain("[truncated "); expect(fitted).toContain("Current user request:\nkeep this request"); expect(fitted).toContain("hook context survives"); }); it("keeps the original input when a hook appends context without a projection", () => { const prompt = "current prompt survives"; const hookAppend = `\n\nhook context ${"h".repeat(800)}`; const maxChars = 420; const fitted = fitCodexProjectedContextForTurnStart({ promptText: `${prompt}${hookAppend}`, preservedRange: { start: 0, end: prompt.length }, maxChars, }); expect(fitted.length).toBeLessThanOrEqual(maxChars); expect(fitted).toContain(prompt); expect(fitted).not.toContain("hook context"); }); it("bounds hook output for an empty original input", () => { const maxChars = 420; const fitted = fitCodexProjectedContextForTurnStart({ promptText: `hook context ${"h".repeat(800)} hook tail`, preservedRange: { start: 0, end: 0 }, maxChars, }); expect(fitted.length).toBeLessThanOrEqual(maxChars); expect(fitted).toContain("hook tail"); }); it("bounds output for a large request under the default Codex turn limit", () => { const maxChars = CODEX_TURN_START_TEXT_INPUT_MAX_CHARS; // A large assembled header prefix already over the cap forces the // non-positive context budget on the real default limit (1 << 20). const before = `header\n${"older history ".repeat(90_000)}`; const context = "x".repeat(2_000); const prompt = `urgent request ${"u".repeat(2_000)}`; const after = `\n\n\nCurrent user request:\n${prompt}`; const promptText = `${before}${context}${after}`; expect(before.length + after.length).toBeGreaterThan(maxChars); const fitted = fitCodexProjectedContextForTurnStart({ promptText, contextRange: { start: before.length, end: before.length + context.length }, // maxChars omitted -> defaults to CODEX_TURN_START_TEXT_INPUT_MAX_CHARS. }); expect(fitted.length).toBeLessThanOrEqual(maxChars); // The user request is the priority tail and survives even though the older // header text is truncated to satisfy the limit. expect(fitted).toContain("Current user request:"); expect(fitted.endsWith("u".repeat(1_000))).toBe(true); }); it("never splits a UTF-16 surrogate pair at the truncation boundary", () => { // Drive the non-positive-budget path with an emoji (surrogate pair) sitting // across the kept-tail cut. A naive code-unit slice would orphan the low // surrogate into U+FFFD; the boundary must stay on a whole code point. const before = `OpenClaw assembled context for this turn:\n${"H".repeat(300)}`; const context = "older context ".repeat(20); // Emoji immediately before the user text so the cut can fall mid-pair. const prompt = `\u{1F600}${"U".repeat(60)}`; const after = `\n\n\nCurrent user request:\n${prompt}`; const promptText = `${before}${context}${after}`; const contextRange = { start: before.length, end: before.length + context.length }; // Sweep cap sizes around the cut so the test is not brittle to marker length; // at least one value lands the boundary inside the surrogate pair. for (let maxChars = 90; maxChars <= 140; maxChars += 1) { const fitted = fitCodexProjectedContextForTurnStart({ promptText, contextRange, maxChars }); expect(fitted.length).toBeLessThanOrEqual(maxChars); // U+FFFD only appears when a lone surrogate is rendered, i.e. a split pair. expect(fitted).not.toContain("�"); // Any surviving emoji must be the complete pair, not a lone low surrogate. for (let i = 0; i < fitted.length; i += 1) { const code = fitted.charCodeAt(i); const isLowSurrogate = code >= 0xdc00 && code <= 0xdfff; const isHighSurrogate = code >= 0xd800 && code <= 0xdbff; if (isLowSurrogate) { const prev = fitted.charCodeAt(i - 1); expect(prev >= 0xd800 && prev <= 0xdbff).toBe(true); } if (isHighSurrogate) { const next = fitted.charCodeAt(i + 1); expect(next >= 0xdc00 && next <= 0xdfff).toBe(true); } } } }); it("keeps the old conservative cap when no runtime budget is available", () => { expect(resolveCodexContextEngineProjectionMaxChars({})).toBe(24_000); expect(resolveCodexContextEngineProjectionMaxChars({ contextTokenBudget: 0 })).toBe(24_000); }); it("uses the shared reserve-token shape while preserving small-model prompt budget", () => { expect(resolveCodexContextEngineProjectionMaxChars({ contextTokenBudget: 80_000 })).toBe( 240_000, ); expect(resolveCodexContextEngineProjectionMaxChars({ contextTokenBudget: 16_000 })).toBe( 32_000, ); }); it("maps OpenClaw compaction reserve config onto Codex projection reserves", () => { expect( resolveCodexContextEngineProjectionReserveTokens({ config: { agents: { defaults: { compaction: { reserveTokens: 12_000 } } } }, }), ).toBe(20_000); expect( resolveCodexContextEngineProjectionReserveTokens({ config: { agents: { defaults: { compaction: { reserveTokens: 12_000, reserveTokensFloor: 0 } } }, }, }), ).toBe(12_000); expect( resolveCodexContextEngineProjectionReserveTokens({ config: { agents: { defaults: { compaction: { reserveTokens: 48_000 } } } }, }), ).toBe(48_000); expect( resolveCodexContextEngineProjectionReserveTokens({ config: { agents: { defaults: { compaction: { reserveTokensFloor: 0 } } } }, }), ).toBe(0); }); it("applies configured reserve tokens to the scaled projection cap", () => { expect( resolveCodexContextEngineProjectionMaxChars({ contextTokenBudget: 80_000, reserveTokens: 40_000, }), ).toBe(160_000); }); it("caps very large runtime budgets to a bounded projection size", () => { expect(resolveCodexContextEngineProjectionMaxChars({ contextTokenBudget: 1_000_000 })).toBe( 1_000_000, ); }); });