Vendor OpenClaw source as Adolf fork baseline
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Some checks failed
ClawSweeper Dispatch / dispatch (push) Has been cancelled
CodeQL / Security High (actions) (push) Has been cancelled
CodeQL / Security High (channel-runtime-boundary) (push) Has been cancelled
CodeQL / Security High (core-auth-secrets) (push) Has been cancelled
CodeQL / Security High (mcp-process-tool-boundary) (push) Has been cancelled
CodeQL / Security High (network-ssrf-boundary) (push) Has been cancelled
CodeQL / Security High (plugin-trust-boundary) (push) Has been cancelled
CodeQL / Security High (process-exec-boundary) (push) Has been cancelled
Docs Sync Publish Repo / sync-publish-repo (push) Has been cancelled
Docs / docs (push) Has been cancelled
OpenClaw Stable Main Closeout / Resolve stable release closeout inputs (push) Has been cancelled
OpenClaw Stable Main Closeout / Verify stable main closeout (push) Has been cancelled
Workflow Sanity / no-tabs (push) Has been cancelled
Workflow Sanity / actionlint (push) Has been cancelled
Workflow Sanity / generated-doc-baselines (push) Has been cancelled
CI / runner-admission (push) Has been cancelled
CI / preflight (push) Has been cancelled
CI / security-fast (push) Has been cancelled
CI / pnpm-store-warmup (push) Has been cancelled
CI / build-artifacts (push) Has been cancelled
CI / native-i18n (push) Has been cancelled
CI / ${{ matrix.check_name }} (push) Has been cancelled
CI / ${{ matrix.checkName }} (push) Has been cancelled
CI / checks-node-compat-node22 (push) Has been cancelled
CI / check-bundled-channel-config-metadata (push) Has been cancelled
CI / check-dependencies (push) Has been cancelled
CI / check-guards (push) Has been cancelled
CI / check-lint (push) Has been cancelled
CI / check-prod-types (push) Has been cancelled
CI / check-shrinkwrap (push) Has been cancelled
CI / check-test-types (push) Has been cancelled
CI / check-additional-boundaries-a (push) Has been cancelled
CI / check-additional-boundaries-bcd (push) Has been cancelled
CI / check-additional-extension-bundled (push) Has been cancelled
CI / check-additional-extension-channels (push) Has been cancelled
CI / check-additional-extension-package-boundary (push) Has been cancelled
CI / check-additional-runtime-topology-architecture (push) Has been cancelled
CI / check-session-accessor-boundary (push) Has been cancelled
CI / check-session-transcript-reader-boundary (push) Has been cancelled
CI / check-docs (push) Has been cancelled
CI / skills-python (push) Has been cancelled
CI / macos-swift (push) Has been cancelled
CI / ios-build (push) Has been cancelled
CI / ci-timings-summary (push) Has been cancelled
Native App Locale Refresh / Refresh native fa (push) Has been cancelled
Native App Locale Refresh / Refresh native fr (push) Has been cancelled
Native App Locale Refresh / Refresh native hi (push) Has been cancelled
Native App Locale Refresh / Refresh native id (push) Has been cancelled
Native App Locale Refresh / Refresh native it (push) Has been cancelled
Native App Locale Refresh / Refresh native ja-JP (push) Has been cancelled
Control UI Locale Refresh / plan (push) Has been cancelled
Control UI Locale Refresh / Refresh ${{ matrix.locale }} (push) Has been cancelled
Control UI Locale Refresh / Commit control UI locale refresh (push) Has been cancelled
Live Media Runner Image / Build live media runner image (push) Has been cancelled
Native App Locale Refresh / Refresh native ar (push) Has been cancelled
Native App Locale Refresh / Refresh native de (push) Has been cancelled
Native App Locale Refresh / Refresh native es (push) Has been cancelled
Native App Locale Refresh / Refresh native ko (push) Has been cancelled
Native App Locale Refresh / Refresh native nl (push) Has been cancelled
Native App Locale Refresh / Refresh native pl (push) Has been cancelled
Native App Locale Refresh / Refresh native pt-BR (push) Has been cancelled
Native App Locale Refresh / Refresh native ru (push) Has been cancelled
Native App Locale Refresh / Refresh native sv (push) Has been cancelled
Native App Locale Refresh / Refresh native th (push) Has been cancelled
Native App Locale Refresh / Refresh native tr (push) Has been cancelled
Native App Locale Refresh / Refresh native uk (push) Has been cancelled
Native App Locale Refresh / Refresh native vi (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-CN (push) Has been cancelled
Native App Locale Refresh / Refresh native zh-TW (push) Has been cancelled
Native App Locale Refresh / Commit native locale refresh (push) Has been cancelled
Plugin Init Scaffold Validation / Validate provider scaffold (push) Has been cancelled
Plugin NPM Release / preview_plugins_npm (push) Has been cancelled
Plugin NPM Release / Validate release publish approval (push) Has been cancelled
Plugin NPM Release / preview_plugin_pack (push) Has been cancelled
Plugin NPM Release / publish_plugins_npm (push) Has been cancelled
Sandbox Common Smoke / sandbox-common-smoke (push) Has been cancelled
Website Installer Sync / static (push) Has been cancelled
Website Installer Sync / linux-docker (push) Has been cancelled
Website Installer Sync / macos-installer (push) Has been cancelled
Website Installer Sync / windows-installer (push) Has been cancelled
Website Installer Sync / sync-website (push) Has been cancelled
Adolf is a fork/vendored clone of github.com/openclaw/openclaw (v2026.6.11), free to diverge. Tree copied sans upstream .git; upstream remote added for future syncs. Node pinned to 24 (.nvmrc); engines already require >=22.19. Preserves docs/ARCHITECTURE.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LeqyaxJF2nbRXJtae2kNB2
This commit is contained in:
278
scripts/docs-i18n/behavior_baseline_test.go
Normal file
278
scripts/docs-i18n/behavior_baseline_test.go
Normal file
@@ -0,0 +1,278 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type behaviorReplacePair struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
}
|
||||
|
||||
type behaviorRule struct {
|
||||
Method string `json:"method"`
|
||||
MatchAll []string `json:"match_all"`
|
||||
ResponseFile string `json:"response_file,omitempty"`
|
||||
ReplacePairs []behaviorReplacePair `json:"replace_pairs,omitempty"`
|
||||
}
|
||||
|
||||
type behaviorFixture struct {
|
||||
Name string `json:"name"`
|
||||
Mode string `json:"mode"`
|
||||
RelPath string `json:"rel_path"`
|
||||
SourceFile string `json:"source_file"`
|
||||
ExpectedFile string `json:"expected_file,omitempty"`
|
||||
ExpectedErrorContains string `json:"expected_error_contains,omitempty"`
|
||||
ExpectedOutputContains []string `json:"expected_output_contains,omitempty"`
|
||||
ExpectedOutputNotContains []string `json:"expected_output_not_contains,omitempty"`
|
||||
Rules []behaviorRule `json:"rules"`
|
||||
}
|
||||
|
||||
type behaviorFixtureTranslator struct {
|
||||
t *testing.T
|
||||
dir string
|
||||
rules []behaviorRule
|
||||
}
|
||||
|
||||
func (tr *behaviorFixtureTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return tr.run("masked", text), nil
|
||||
}
|
||||
|
||||
func (tr *behaviorFixtureTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
return tr.run("raw", text), nil
|
||||
}
|
||||
|
||||
func (tr *behaviorFixtureTranslator) Close() {}
|
||||
|
||||
func (tr *behaviorFixtureTranslator) run(method, text string) string {
|
||||
tr.t.Helper()
|
||||
for _, rule := range tr.rules {
|
||||
if rule.Method != method {
|
||||
continue
|
||||
}
|
||||
if !matchesAll(text, rule.MatchAll) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case rule.ResponseFile != "":
|
||||
return readFixtureTextInDir(tr.t, tr.dir, rule.ResponseFile)
|
||||
case len(rule.ReplacePairs) > 0:
|
||||
out := text
|
||||
for _, pair := range rule.ReplacePairs {
|
||||
out = strings.ReplaceAll(out, pair.From, pair.To)
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return text
|
||||
}
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func matchesAll(text string, fragments []string) bool {
|
||||
for _, fragment := range fragments {
|
||||
if !strings.Contains(text, fragment) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestDocsI18nBehaviorBaselines(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := filepath.Join("testdata", "behavior")
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir(%q): %v", root, err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
dir := filepath.Join(root, entry.Name())
|
||||
fixture := loadBehaviorFixture(t, dir)
|
||||
name := fixture.Name
|
||||
if name == "" {
|
||||
name = entry.Name()
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
runBehaviorFixture(t, dir, fixture)
|
||||
})
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Fatalf("no behavior fixtures found under %s", root)
|
||||
}
|
||||
}
|
||||
|
||||
func loadBehaviorFixture(t *testing.T, dir string) behaviorFixture {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(dir, "case.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(case.json): %v", err)
|
||||
}
|
||||
var fixture behaviorFixture
|
||||
if err := json.Unmarshal(data, &fixture); err != nil {
|
||||
t.Fatalf("Unmarshal(case.json): %v", err)
|
||||
}
|
||||
return fixture
|
||||
}
|
||||
|
||||
func runBehaviorFixture(t *testing.T, dir string, fixture behaviorFixture) {
|
||||
t.Helper()
|
||||
|
||||
source := readFixtureTextInDir(t, dir, fixture.SourceFile)
|
||||
translator := &behaviorFixtureTranslator{
|
||||
t: t,
|
||||
dir: dir,
|
||||
rules: fixture.Rules,
|
||||
}
|
||||
|
||||
var (
|
||||
got string
|
||||
err error
|
||||
)
|
||||
|
||||
switch fixture.Mode {
|
||||
case "doc_body_chunked":
|
||||
got, err = translateDocBodyChunked(context.Background(), translator, fixture.RelPath, source, "en", "zh-CN")
|
||||
case "frontmatter_scalar":
|
||||
got, err = translateSnippet(
|
||||
context.Background(),
|
||||
translator,
|
||||
&TranslationMemory{entries: map[string]TMEntry{}},
|
||||
fixture.RelPath+":frontmatter:title",
|
||||
source,
|
||||
"en",
|
||||
"zh-CN",
|
||||
)
|
||||
default:
|
||||
t.Fatalf("unsupported fixture mode %q", fixture.Mode)
|
||||
}
|
||||
|
||||
if fixture.ExpectedErrorContains != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q, got nil", fixture.ExpectedErrorContains)
|
||||
}
|
||||
if !strings.Contains(err.Error(), fixture.ExpectedErrorContains) {
|
||||
t.Fatalf("expected error containing %q, got %v", fixture.ExpectedErrorContains, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if fixture.ExpectedFile != "" {
|
||||
want := readFixtureTextInDir(t, dir, fixture.ExpectedFile)
|
||||
if normalizeBehaviorText(got) != normalizeBehaviorText(want) {
|
||||
t.Fatalf("unexpected output\nwant:\n%s\n\ngot:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
for _, fragment := range fixture.ExpectedOutputContains {
|
||||
if !strings.Contains(got, fragment) {
|
||||
t.Fatalf("expected output to contain %q\noutput:\n%s", fragment, got)
|
||||
}
|
||||
}
|
||||
for _, fragment := range fixture.ExpectedOutputNotContains {
|
||||
if strings.Contains(got, fragment) {
|
||||
t.Fatalf("expected output to exclude %q\noutput:\n%s", fragment, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readFixtureText(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q): %v", path, err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func readFixtureTextInDir(t *testing.T, dir, name string) string {
|
||||
t.Helper()
|
||||
resolvedPath, err := resolveFixturePathInDir(dir, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return readFixtureText(t, resolvedPath)
|
||||
}
|
||||
|
||||
func resolveFixturePathInDir(dir, name string) (string, error) {
|
||||
if filepath.IsAbs(name) {
|
||||
return "", fmt.Errorf("absolute fixture paths are not allowed: %q", name)
|
||||
}
|
||||
clean := filepath.Clean(name)
|
||||
if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("fixture path escapes dir: %q", name)
|
||||
}
|
||||
|
||||
joined := filepath.Join(dir, clean)
|
||||
resolvedDir, err := filepath.EvalSymlinks(dir)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("EvalSymlinks(%q): %w", dir, err)
|
||||
}
|
||||
resolvedPath, err := filepath.EvalSymlinks(joined)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("EvalSymlinks(%q): %w", joined, err)
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(resolvedDir, resolvedPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Rel(%q, %q): %w", resolvedDir, resolvedPath, err)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("fixture path resolves outside dir: %q", name)
|
||||
}
|
||||
|
||||
return resolvedPath, nil
|
||||
}
|
||||
|
||||
func normalizeBehaviorText(value string) string {
|
||||
return strings.TrimSpace(strings.ReplaceAll(value, "\r\n", "\n"))
|
||||
}
|
||||
|
||||
func TestResolveFixturePathInDirRejectsSymlinkEscape(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
fixtureDir := filepath.Join(root, "fixture")
|
||||
if err := os.MkdirAll(fixtureDir, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", fixtureDir, err)
|
||||
}
|
||||
|
||||
outsidePath := filepath.Join(root, "outside.txt")
|
||||
if err := os.WriteFile(outsidePath, []byte("outside\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q): %v", outsidePath, err)
|
||||
}
|
||||
|
||||
linkPath := filepath.Join(fixtureDir, "outside-link.txt")
|
||||
if err := os.Symlink(outsidePath, linkPath); err != nil {
|
||||
if os.IsPermission(err) || runtime.GOOS == "windows" {
|
||||
t.Skipf("symlink creation unavailable in this test environment: %v", err)
|
||||
}
|
||||
t.Fatalf("Symlink(%q, %q): %v", outsidePath, linkPath, err)
|
||||
}
|
||||
|
||||
_, err := resolveFixturePathInDir(fixtureDir, "outside-link.txt")
|
||||
if err == nil {
|
||||
t.Fatal("expected symlink escape to fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "resolves outside dir") {
|
||||
t.Fatalf("expected outside-dir error, got %v", err)
|
||||
}
|
||||
}
|
||||
25
scripts/docs-i18n/codex_command_unix.go
Normal file
25
scripts/docs-i18n/codex_command_unix.go
Normal file
@@ -0,0 +1,25 @@
|
||||
//go:build !windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func configureCodexPromptCommand(command *exec.Cmd) {
|
||||
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
command.Cancel = func() error {
|
||||
if command.Process == nil {
|
||||
return os.ErrProcessDone
|
||||
}
|
||||
err := syscall.Kill(-command.Process.Pid, syscall.SIGKILL)
|
||||
if errors.Is(err, syscall.ESRCH) {
|
||||
return os.ErrProcessDone
|
||||
}
|
||||
return err
|
||||
}
|
||||
command.WaitDelay = docsI18nCommandWaitDelay()
|
||||
}
|
||||
67
scripts/docs-i18n/codex_command_windows.go
Normal file
67
scripts/docs-i18n/codex_command_windows.go
Normal file
@@ -0,0 +1,67 @@
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultWindowsSystemRoot = `C:\Windows`
|
||||
|
||||
func resolveWindowsTaskkillPath() string {
|
||||
systemRoot := normalizeWindowsSystemRoot(os.Getenv("SystemRoot"))
|
||||
if systemRoot == "" {
|
||||
systemRoot = normalizeWindowsSystemRoot(os.Getenv("WINDIR"))
|
||||
}
|
||||
if systemRoot == "" {
|
||||
systemRoot = defaultWindowsSystemRoot
|
||||
}
|
||||
return filepath.Join(systemRoot, "System32", "taskkill.exe")
|
||||
}
|
||||
|
||||
func normalizeWindowsSystemRoot(raw string) string {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
if trimmed == "" ||
|
||||
strings.ContainsAny(trimmed, "\x00\r\n;") ||
|
||||
strings.HasPrefix(trimmed, `\\`) ||
|
||||
!filepath.IsAbs(trimmed) {
|
||||
return ""
|
||||
}
|
||||
cleaned := filepath.Clean(trimmed)
|
||||
volume := filepath.VolumeName(cleaned)
|
||||
if volume == "" || len(cleaned) <= len(volume)+1 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimRight(cleaned, `\/`)
|
||||
}
|
||||
|
||||
var runWindowsTaskkill = func(pid int) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), docsI18nCommandWaitDelay())
|
||||
defer cancel()
|
||||
return exec.CommandContext(ctx, resolveWindowsTaskkillPath(), "/T", "/F", "/PID", strconv.Itoa(pid)).Run()
|
||||
}
|
||||
|
||||
func configureCodexPromptCommand(command *exec.Cmd) {
|
||||
command.Cancel = func() error {
|
||||
if command.Process == nil {
|
||||
return os.ErrProcessDone
|
||||
}
|
||||
if err := runWindowsTaskkill(command.Process.Pid); err != nil {
|
||||
killErr := command.Process.Kill()
|
||||
if errors.Is(killErr, os.ErrProcessDone) {
|
||||
return os.ErrProcessDone
|
||||
}
|
||||
if killErr != nil {
|
||||
return errors.Join(err, killErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
command.WaitDelay = docsI18nCommandWaitDelay()
|
||||
}
|
||||
83
scripts/docs-i18n/codex_command_windows_test.go
Normal file
83
scripts/docs-i18n/codex_command_windows_test.go
Normal file
@@ -0,0 +1,83 @@
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestResolveWindowsTaskkillPath(t *testing.T) {
|
||||
t.Setenv("SystemRoot", `C:\Windows`)
|
||||
t.Setenv("WINDIR", `D:\Ignored`)
|
||||
|
||||
got := resolveWindowsTaskkillPath()
|
||||
want := filepath.Join(`C:\Windows`, "System32", "taskkill.exe")
|
||||
if got != want {
|
||||
t.Fatalf("expected %q, got %q", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWindowsTaskkillPathFallsBackToWindir(t *testing.T) {
|
||||
t.Setenv("SystemRoot", `relative\windows`)
|
||||
t.Setenv("WINDIR", `D:\Windows`)
|
||||
|
||||
got := resolveWindowsTaskkillPath()
|
||||
want := filepath.Join(`D:\Windows`, "System32", "taskkill.exe")
|
||||
if got != want {
|
||||
t.Fatalf("expected %q, got %q", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWindowsTaskkillPathRejectsUnsafeRoots(t *testing.T) {
|
||||
t.Setenv("SystemRoot", `\\server\share`)
|
||||
t.Setenv("WINDIR", `C:\Windows;taskkill.exe`)
|
||||
|
||||
got := resolveWindowsTaskkillPath()
|
||||
want := filepath.Join(defaultWindowsSystemRoot, "System32", "taskkill.exe")
|
||||
if got != want {
|
||||
t.Fatalf("expected %q, got %q", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureCodexPromptCommandWindowsCancelsProcessTree(t *testing.T) {
|
||||
t.Setenv(envDocsI18nCommandWaitDelay, "25ms")
|
||||
previousRunTaskkill := runWindowsTaskkill
|
||||
defer func() { runWindowsTaskkill = previousRunTaskkill }()
|
||||
|
||||
var gotPID int
|
||||
runWindowsTaskkill = func(pid int) error {
|
||||
gotPID = pid
|
||||
return nil
|
||||
}
|
||||
|
||||
command := exec.Command("codex")
|
||||
configureCodexPromptCommand(command)
|
||||
command.Process = &os.Process{Pid: 1234}
|
||||
|
||||
if command.WaitDelay != 25*time.Millisecond {
|
||||
t.Fatalf("expected WaitDelay override, got %s", command.WaitDelay)
|
||||
}
|
||||
if command.Cancel == nil {
|
||||
t.Fatal("expected Cancel to be configured")
|
||||
}
|
||||
if err := command.Cancel(); err != nil {
|
||||
t.Fatalf("Cancel returned error: %v", err)
|
||||
}
|
||||
if gotPID != 1234 {
|
||||
t.Fatalf("expected taskkill for pid 1234, got %d", gotPID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureCodexPromptCommandWindowsCancelBeforeStart(t *testing.T) {
|
||||
command := exec.Command("codex")
|
||||
configureCodexPromptCommand(command)
|
||||
|
||||
if err := command.Cancel(); !errors.Is(err, os.ErrProcessDone) {
|
||||
t.Fatalf("expected os.ErrProcessDone, got %v", err)
|
||||
}
|
||||
}
|
||||
825
scripts/docs-i18n/doc_chunked_raw.go
Normal file
825
scripts/docs-i18n/doc_chunked_raw.go
Normal file
@@ -0,0 +1,825 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultDocChunkMaxBytes = 12000
|
||||
const defaultDocChunkPromptBudget = 15000
|
||||
|
||||
var (
|
||||
docsFenceRE = regexp.MustCompile(`^\s*(` + "```" + `|~~~)`)
|
||||
docsComponentTagRE = regexp.MustCompile(`<(/?)([A-Z][A-Za-z0-9]*)\b[^>]*?/?>`)
|
||||
)
|
||||
|
||||
var docsProtocolTokens = []string{
|
||||
frontmatterTagStart,
|
||||
frontmatterTagEnd,
|
||||
bodyTagStart,
|
||||
bodyTagEnd,
|
||||
"[[[FM_",
|
||||
}
|
||||
|
||||
type docChunkStructure struct {
|
||||
fenceCount int
|
||||
tagCounts map[string]int
|
||||
}
|
||||
|
||||
type docChunkSplitPlan struct {
|
||||
groups [][]string
|
||||
reason string
|
||||
}
|
||||
|
||||
func translateDocBodyChunked(ctx context.Context, translator docsTranslator, relPath, body, srcLang, tgtLang string) (string, error) {
|
||||
if strings.TrimSpace(body) == "" {
|
||||
return body, nil
|
||||
}
|
||||
blocks := splitDocBodyIntoBlocks(body)
|
||||
groups := groupDocBlocks(blocks, docsI18nDocChunkMaxBytes())
|
||||
logDocChunkPlan(relPath, blocks, groups)
|
||||
out := strings.Builder{}
|
||||
for index, group := range groups {
|
||||
chunkID := fmt.Sprintf("%s.chunk-%03d", relPath, index+1)
|
||||
translated, err := translateDocBlockGroup(ctx, translator, chunkID, group, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out.WriteString(translated)
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, srcLang, tgtLang string) (string, error) {
|
||||
source := strings.Join(blocks, "")
|
||||
if strings.TrimSpace(source) == "" {
|
||||
return source, nil
|
||||
}
|
||||
if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
|
||||
logDocChunkPlanSplit(chunkID, plan, source)
|
||||
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
|
||||
}
|
||||
normalizedSource, commonIndent := stripCommonIndent(source)
|
||||
log.Printf("docs-i18n: chunk start %s blocks=%d bytes=%d", chunkID, len(blocks), len(source))
|
||||
translated, err := translator.TranslateRaw(ctx, normalizedSource, srcLang, tgtLang)
|
||||
if err == nil {
|
||||
translated = sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
translated = reapplyCommonIndent(translated, commonIndent)
|
||||
if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil {
|
||||
log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated))
|
||||
return translated, nil
|
||||
} else {
|
||||
err = validationErr
|
||||
}
|
||||
}
|
||||
if len(blocks) <= 1 {
|
||||
if fallback, fallbackErr := translateDocLeafBlock(ctx, translator, chunkID, source, srcLang, tgtLang); fallbackErr == nil {
|
||||
return fallback, nil
|
||||
}
|
||||
if plan, ok := planSingletonDocChunkRetry(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
|
||||
logDocChunkPlanSplit(chunkID, plan, source)
|
||||
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
|
||||
}
|
||||
return "", fmt.Errorf("%s: %w", chunkID, err)
|
||||
}
|
||||
if plan, ok := planDocChunkSplit(blocks, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
|
||||
logDocChunkSplit(chunkID, len(blocks), err)
|
||||
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
|
||||
}
|
||||
if plan, ok := splitDocChunkBlocksMidpointSimple(blocks); ok {
|
||||
logDocChunkSplit(chunkID, len(blocks), err)
|
||||
return translatePlannedDocChunkGroups(ctx, translator, chunkID, plan.groups, srcLang, tgtLang)
|
||||
}
|
||||
return "", fmt.Errorf("%s: %w", chunkID, err)
|
||||
}
|
||||
|
||||
func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source, srcLang, tgtLang string) (string, error) {
|
||||
sourceStructure := summarizeDocChunkStructure(source)
|
||||
if sourceStructure.fenceCount != 0 {
|
||||
return "", fmt.Errorf("%s: raw leaf fallback not applicable", chunkID)
|
||||
}
|
||||
normalizedSource, commonIndent := stripCommonIndent(source)
|
||||
maskedSource, placeholders := maskDocComponentTags(normalizedSource)
|
||||
translated, err := translator.Translate(ctx, maskedSource, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
translated, err = restoreDocComponentTags(translated, placeholders)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
translated = sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
translated = reapplyCommonIndent(translated, commonIndent)
|
||||
if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil {
|
||||
return "", validationErr
|
||||
}
|
||||
log.Printf("docs-i18n: chunk leaf-fallback done %s out_bytes=%d", chunkID, len(translated))
|
||||
return translated, nil
|
||||
}
|
||||
|
||||
func splitDocBodyIntoBlocks(body string) []string {
|
||||
if body == "" {
|
||||
return nil
|
||||
}
|
||||
lines := strings.SplitAfter(body, "\n")
|
||||
blocks := make([]string, 0, len(lines))
|
||||
var current strings.Builder
|
||||
fenceDelimiter := ""
|
||||
for _, line := range lines {
|
||||
current.WriteString(line)
|
||||
fenceDelimiter, _ = updateFenceDelimiter(fenceDelimiter, line)
|
||||
inFence := fenceDelimiter != ""
|
||||
if !inFence && strings.TrimSpace(line) == "" {
|
||||
blocks = append(blocks, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
}
|
||||
if current.Len() > 0 {
|
||||
blocks = append(blocks, current.String())
|
||||
}
|
||||
if len(blocks) == 0 {
|
||||
return []string{body}
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
func groupDocBlocks(blocks []string, maxBytes int) [][]string {
|
||||
if len(blocks) == 0 {
|
||||
return nil
|
||||
}
|
||||
if maxBytes <= 0 {
|
||||
maxBytes = defaultDocChunkMaxBytes
|
||||
}
|
||||
groups := make([][]string, 0, len(blocks))
|
||||
current := make([]string, 0, 8)
|
||||
currentBytes := 0
|
||||
flush := func() {
|
||||
if len(current) == 0 {
|
||||
return
|
||||
}
|
||||
groups = append(groups, current)
|
||||
current = make([]string, 0, 8)
|
||||
currentBytes = 0
|
||||
}
|
||||
for _, block := range blocks {
|
||||
blockBytes := len(block)
|
||||
if len(current) > 0 && currentBytes+blockBytes > maxBytes {
|
||||
flush()
|
||||
}
|
||||
if blockBytes > maxBytes {
|
||||
groups = append(groups, []string{block})
|
||||
continue
|
||||
}
|
||||
current = append(current, block)
|
||||
currentBytes += blockBytes
|
||||
}
|
||||
flush()
|
||||
return groups
|
||||
}
|
||||
|
||||
func validateDocChunkTranslation(source, translated string) error {
|
||||
if hasUnexpectedTopLevelProtocolWrapper(source, translated) {
|
||||
return fmt.Errorf("protocol token leaked: top-level wrapper")
|
||||
}
|
||||
if err := validateNoTranslationTranscriptArtifacts(source, translated); err != nil {
|
||||
return err
|
||||
}
|
||||
sourceLower := strings.ToLower(source)
|
||||
translatedLower := strings.ToLower(translated)
|
||||
for _, token := range docsProtocolTokens {
|
||||
tokenLower := strings.ToLower(token)
|
||||
if strings.Contains(sourceLower, tokenLower) {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(translatedLower, tokenLower) {
|
||||
return fmt.Errorf("protocol token leaked: %s", token)
|
||||
}
|
||||
}
|
||||
sourceStructure := summarizeDocChunkStructure(source)
|
||||
translatedStructure := summarizeDocChunkStructure(translated)
|
||||
if sourceStructure.fenceCount != translatedStructure.fenceCount {
|
||||
return fmt.Errorf("code fence mismatch: source=%d translated=%d", sourceStructure.fenceCount, translatedStructure.fenceCount)
|
||||
}
|
||||
if !slices.Equal(sortedKeys(sourceStructure.tagCounts), sortedKeys(translatedStructure.tagCounts)) {
|
||||
return fmt.Errorf("component tag set mismatch")
|
||||
}
|
||||
for _, key := range sortedKeys(sourceStructure.tagCounts) {
|
||||
if sourceStructure.tagCounts[key] != translatedStructure.tagCounts[key] {
|
||||
return fmt.Errorf("component tag mismatch for %s: source=%d translated=%d", key, sourceStructure.tagCounts[key], translatedStructure.tagCounts[key])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeDocChunkProtocolWrappers(source, translated string) string {
|
||||
if !containsProtocolWrapperToken(translated) {
|
||||
return translated
|
||||
}
|
||||
trimmedTranslated := strings.TrimSpace(translated)
|
||||
if !hasUnexpectedTopLevelProtocolWrapper(source, trimmedTranslated) {
|
||||
return translated
|
||||
}
|
||||
if !hasAmbiguousTaggedBodyClose(source, trimmedTranslated) {
|
||||
_, body, err := parseTaggedDocument(trimmedTranslated)
|
||||
if err == nil {
|
||||
if strings.TrimSpace(body) == "" {
|
||||
return translated
|
||||
}
|
||||
return body
|
||||
}
|
||||
}
|
||||
body, ok := stripBodyOnlyWrapper(source, trimmedTranslated)
|
||||
if !ok || strings.TrimSpace(body) == "" {
|
||||
return translated
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func stripBodyOnlyWrapper(source, text string) (string, bool) {
|
||||
sourceLower := strings.ToLower(source)
|
||||
// When the source itself documents <body> tokens, a bare body-only payload is
|
||||
// ambiguous: the trailing </body> can be literal translated content instead of
|
||||
// a real wrapper close. Keep it for validation/retry instead of truncating.
|
||||
if strings.Contains(sourceLower, strings.ToLower(bodyTagStart)) || strings.Contains(sourceLower, strings.ToLower(bodyTagEnd)) {
|
||||
return "", false
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
bodyStartLower := strings.ToLower(bodyTagStart)
|
||||
bodyEndLower := strings.ToLower(bodyTagEnd)
|
||||
if !strings.HasPrefix(lower, bodyStartLower) || !strings.HasSuffix(lower, bodyEndLower) {
|
||||
return "", false
|
||||
}
|
||||
body := text[len(bodyTagStart) : len(text)-len(bodyTagEnd)]
|
||||
bodyLower := lower[len(bodyTagStart) : len(lower)-len(bodyTagEnd)]
|
||||
if strings.Contains(bodyLower, bodyStartLower) || strings.Contains(bodyLower, bodyEndLower) {
|
||||
return "", false
|
||||
}
|
||||
return trimTagNewlines(body), true
|
||||
}
|
||||
|
||||
func hasAmbiguousTaggedBodyClose(source, translated string) bool {
|
||||
sourceLower := strings.ToLower(source)
|
||||
if !strings.Contains(sourceLower, strings.ToLower(bodyTagStart)) && !strings.Contains(sourceLower, strings.ToLower(bodyTagEnd)) {
|
||||
return false
|
||||
}
|
||||
translatedLower := strings.ToLower(translated)
|
||||
if !strings.Contains(translatedLower, strings.ToLower(frontmatterTagStart)) {
|
||||
return false
|
||||
}
|
||||
return strings.Count(translatedLower, strings.ToLower(bodyTagEnd)) == 1
|
||||
}
|
||||
|
||||
func maskDocComponentTags(text string) (string, []string) {
|
||||
placeholders := make([]string, 0, 4)
|
||||
masked := docsComponentTagRE.ReplaceAllStringFunc(text, func(match string) string {
|
||||
placeholder := fmt.Sprintf("__OC_DOC_TAG_%03d__", len(placeholders))
|
||||
placeholders = append(placeholders, match)
|
||||
return placeholder
|
||||
})
|
||||
return masked, placeholders
|
||||
}
|
||||
|
||||
func restoreDocComponentTags(text string, placeholders []string) (string, error) {
|
||||
restored := text
|
||||
for index, original := range placeholders {
|
||||
placeholder := fmt.Sprintf("__OC_DOC_TAG_%03d__", index)
|
||||
if !strings.Contains(restored, placeholder) {
|
||||
return "", fmt.Errorf("component tag placeholder missing: %s", placeholder)
|
||||
}
|
||||
restored = strings.ReplaceAll(restored, placeholder, original)
|
||||
}
|
||||
return restored, nil
|
||||
}
|
||||
|
||||
func logDocChunkSplit(chunkID string, blockCount int, err error) {
|
||||
if docsI18nVerboseLogs() || blockCount >= 16 {
|
||||
log.Printf("docs-i18n: chunk split %s blocks=%d err=%v", chunkID, blockCount, err)
|
||||
}
|
||||
}
|
||||
|
||||
func logDocChunkPlanSplit(chunkID string, plan docChunkSplitPlan, source string) {
|
||||
if plan.reason == "" {
|
||||
plan.reason = "unknown"
|
||||
}
|
||||
log.Printf("docs-i18n: chunk pre-split %s reason=%s groups=%d bytes=%d", chunkID, plan.reason, len(plan.groups), len(source))
|
||||
}
|
||||
|
||||
func summarizeDocChunkStructure(text string) docChunkStructure {
|
||||
counts := map[string]int{}
|
||||
lines := strings.Split(text, "\n")
|
||||
fenceDelimiter := ""
|
||||
for _, line := range lines {
|
||||
var toggled bool
|
||||
fenceDelimiter, toggled = updateFenceDelimiter(fenceDelimiter, line)
|
||||
if toggled {
|
||||
counts["__fence_toggle__"]++
|
||||
}
|
||||
for _, match := range docsComponentTagRE.FindAllStringSubmatch(line, -1) {
|
||||
if len(match) < 3 {
|
||||
continue
|
||||
}
|
||||
fullToken := match[0]
|
||||
tagName := match[2]
|
||||
direction := "open"
|
||||
if match[1] == "/" {
|
||||
direction = "close"
|
||||
}
|
||||
if strings.HasSuffix(fullToken, "/>") {
|
||||
direction = "self"
|
||||
}
|
||||
counts[tagName+":"+direction]++
|
||||
}
|
||||
}
|
||||
return docChunkStructure{
|
||||
fenceCount: counts["__fence_toggle__"],
|
||||
tagCounts: countsWithoutFence(counts),
|
||||
}
|
||||
}
|
||||
|
||||
func countsWithoutFence(counts map[string]int) map[string]int {
|
||||
filtered := map[string]int{}
|
||||
for key, value := range counts {
|
||||
if key == "__fence_toggle__" {
|
||||
continue
|
||||
}
|
||||
filtered[key] = value
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func sortedKeys(counts map[string]int) []string {
|
||||
keys := make([]string, 0, len(counts))
|
||||
for key := range counts {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func updateFenceDelimiter(current, line string) (string, bool) {
|
||||
delimiter := leadingFenceDelimiter(line)
|
||||
if delimiter == "" {
|
||||
return current, false
|
||||
}
|
||||
if current == "" {
|
||||
return delimiter, true
|
||||
}
|
||||
if delimiter[0] == current[0] && len(delimiter) >= len(current) && isClosingFenceLine(line, delimiter) {
|
||||
return "", true
|
||||
}
|
||||
return current, false
|
||||
}
|
||||
|
||||
func leadingFenceDelimiter(line string) string {
|
||||
trimmed := strings.TrimLeft(line, " \t")
|
||||
if len(trimmed) < 3 {
|
||||
return ""
|
||||
}
|
||||
switch trimmed[0] {
|
||||
case '`', '~':
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
marker := trimmed[0]
|
||||
index := 0
|
||||
for index < len(trimmed) && trimmed[index] == marker {
|
||||
index++
|
||||
}
|
||||
if index < 3 {
|
||||
return ""
|
||||
}
|
||||
return trimmed[:index]
|
||||
}
|
||||
|
||||
func isClosingFenceLine(line, delimiter string) bool {
|
||||
trimmed := strings.TrimLeft(line, " \t")
|
||||
if !strings.HasPrefix(trimmed, delimiter) {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(trimmed[len(delimiter):]) == ""
|
||||
}
|
||||
|
||||
func hasUnexpectedTopLevelProtocolWrapper(source, translated string) bool {
|
||||
sourceTrimmed := strings.ToLower(strings.TrimSpace(source))
|
||||
translatedTrimmed := strings.ToLower(strings.TrimSpace(translated))
|
||||
checks := []struct {
|
||||
token string
|
||||
match func(string) bool
|
||||
}{
|
||||
{token: frontmatterTagStart, match: func(text string) bool { return strings.HasPrefix(text, strings.ToLower(frontmatterTagStart)) }},
|
||||
{token: bodyTagStart, match: func(text string) bool { return strings.HasPrefix(text, strings.ToLower(bodyTagStart)) }},
|
||||
{token: frontmatterTagEnd, match: func(text string) bool { return strings.HasSuffix(text, strings.ToLower(frontmatterTagEnd)) }},
|
||||
{token: bodyTagEnd, match: func(text string) bool { return strings.HasSuffix(text, strings.ToLower(bodyTagEnd)) }},
|
||||
}
|
||||
for _, check := range checks {
|
||||
if check.match(translatedTrimmed) && !check.match(sourceTrimmed) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsProtocolWrapperToken(text string) bool {
|
||||
lower := strings.ToLower(text)
|
||||
return strings.Contains(lower, strings.ToLower(bodyTagStart)) || strings.Contains(lower, strings.ToLower(frontmatterTagStart))
|
||||
}
|
||||
|
||||
func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID string, groups [][]string, srcLang, tgtLang string) (string, error) {
|
||||
var out strings.Builder
|
||||
for index, group := range groups {
|
||||
translated, err := translateDocBlockGroup(ctx, translator, fmt.Sprintf("%s.%02d", chunkID, index+1), group, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out.WriteString(translated)
|
||||
}
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
func planDocChunkSplit(blocks []string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
|
||||
if len(blocks) == 0 {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
source := strings.Join(blocks, "")
|
||||
if strings.TrimSpace(source) == "" {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
normalizedSource, _ := stripCommonIndent(source)
|
||||
estimatedPromptCost := estimateDocPromptCost(normalizedSource)
|
||||
if len(blocks) > 1 && promptBudget > 0 && estimatedPromptCost > promptBudget {
|
||||
return splitDocChunkBlocksMidpoint(blocks, estimatedPromptCost, promptBudget)
|
||||
}
|
||||
if len(blocks) == 1 {
|
||||
return planSingletonDocChunk(blocks[0], maxBytes, promptBudget)
|
||||
}
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
|
||||
func splitDocChunkBlocksMidpoint(blocks []string, estimatedPromptCost, promptBudget int) (docChunkSplitPlan, bool) {
|
||||
if len(blocks) <= 1 {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
mid := len(blocks) / 2
|
||||
if mid <= 0 || mid >= len(blocks) {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: [][]string{blocks[:mid], blocks[mid:]},
|
||||
reason: fmt.Sprintf("prompt-budget:%d>%d", estimatedPromptCost, promptBudget),
|
||||
}, true
|
||||
}
|
||||
|
||||
func splitDocChunkBlocksMidpointSimple(blocks []string) (docChunkSplitPlan, bool) {
|
||||
if len(blocks) <= 1 {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
mid := len(blocks) / 2
|
||||
if mid <= 0 || mid >= len(blocks) {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: [][]string{blocks[:mid], blocks[mid:]},
|
||||
reason: "retry-midpoint",
|
||||
}, true
|
||||
}
|
||||
|
||||
func planSingletonDocChunk(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
|
||||
normalizedBlock, _ := stripCommonIndent(block)
|
||||
estimatedPromptCost := estimateDocPromptCost(normalizedBlock)
|
||||
overBytes := maxBytes > 0 && len(block) > maxBytes
|
||||
overPrompt := promptBudget > 0 && estimatedPromptCost > promptBudget
|
||||
if !overBytes && !overPrompt {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
|
||||
return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, false)
|
||||
}
|
||||
|
||||
func planSingletonDocChunkRetry(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
|
||||
return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, true)
|
||||
}
|
||||
|
||||
func planSingletonDocChunkWithMode(block string, maxBytes, promptBudget int, force bool) (docChunkSplitPlan, bool) {
|
||||
if sections := splitDocBlockSections(block); len(sections) > 1 {
|
||||
if groups := wrapDocChunkSections(sections); len(groups) > 1 {
|
||||
reason := "singleton-structural"
|
||||
if force {
|
||||
reason = "singleton-retry-structural"
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: groups,
|
||||
reason: reason,
|
||||
}, true
|
||||
}
|
||||
}
|
||||
|
||||
if groups, ok := splitPureFencedDocSectionWithMode(block, maxBytes, promptBudget, force); ok {
|
||||
reason := "singleton-fence"
|
||||
if force {
|
||||
reason = "singleton-retry-fence"
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: groups,
|
||||
reason: reason,
|
||||
}, true
|
||||
}
|
||||
|
||||
if groups, ok := splitPlainDocSectionWithMode(block, maxBytes, promptBudget, force); ok {
|
||||
reason := "singleton-lines"
|
||||
if force {
|
||||
reason = "singleton-retry-lines"
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: groups,
|
||||
reason: reason,
|
||||
}, true
|
||||
}
|
||||
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
|
||||
func wrapDocChunkSections(sections []string) [][]string {
|
||||
groups := make([][]string, 0, len(sections))
|
||||
for _, section := range sections {
|
||||
if strings.TrimSpace(section) == "" {
|
||||
continue
|
||||
}
|
||||
groups = append(groups, []string{section})
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func splitDocBlockSections(block string) []string {
|
||||
lines := strings.SplitAfter(block, "\n")
|
||||
if len(lines) == 0 {
|
||||
return nil
|
||||
}
|
||||
sections := make([]string, 0, len(lines))
|
||||
var current strings.Builder
|
||||
fenceDelimiter := ""
|
||||
for _, line := range lines {
|
||||
lineDelimiter := leadingFenceDelimiter(line)
|
||||
if fenceDelimiter == "" && lineDelimiter != "" {
|
||||
if current.Len() > 0 {
|
||||
sections = append(sections, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
current.WriteString(line)
|
||||
fenceDelimiter = lineDelimiter
|
||||
continue
|
||||
}
|
||||
|
||||
current.WriteString(line)
|
||||
if fenceDelimiter != "" {
|
||||
if lineDelimiter != "" && lineDelimiter[0] == fenceDelimiter[0] && len(lineDelimiter) >= len(fenceDelimiter) && isClosingFenceLine(line, fenceDelimiter) {
|
||||
sections = append(sections, current.String())
|
||||
current.Reset()
|
||||
fenceDelimiter = ""
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.TrimSpace(line) == "" {
|
||||
sections = append(sections, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
}
|
||||
if current.Len() > 0 {
|
||||
sections = append(sections, current.String())
|
||||
}
|
||||
if len(sections) <= 1 {
|
||||
return nil
|
||||
}
|
||||
return sections
|
||||
}
|
||||
|
||||
func splitPureFencedDocSectionWithMode(block string, maxBytes, promptBudget int, force bool) ([][]string, bool) {
|
||||
lines := strings.SplitAfter(block, "\n")
|
||||
if len(lines) < 2 {
|
||||
return nil, false
|
||||
}
|
||||
openingIndex := firstNonEmptyLineIndex(lines)
|
||||
closingIndex := lastNonEmptyLineIndex(lines)
|
||||
if openingIndex == -1 || closingIndex <= openingIndex {
|
||||
return nil, false
|
||||
}
|
||||
opening := lines[openingIndex]
|
||||
delimiter := leadingFenceDelimiter(opening)
|
||||
if delimiter == "" || !isClosingFenceLine(lines[closingIndex], delimiter) {
|
||||
return nil, false
|
||||
}
|
||||
prefix := strings.Join(lines[:openingIndex], "")
|
||||
suffix := strings.Join(lines[closingIndex+1:], "")
|
||||
if strings.TrimSpace(prefix) != "" || strings.TrimSpace(suffix) != "" {
|
||||
return nil, false
|
||||
}
|
||||
closing := lines[closingIndex]
|
||||
inner := strings.Join(lines[openingIndex+1:closingIndex], "")
|
||||
groups, ok := splitPlainDocSectionWithMode(inner, maxBytes-len(opening)-len(closing), promptBudget, force)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
for index, group := range groups {
|
||||
joined := strings.Join(group, "")
|
||||
groups[index] = []string{opening + joined + closing}
|
||||
}
|
||||
return groups, true
|
||||
}
|
||||
|
||||
func splitPlainDocSectionWithMode(text string, maxBytes, promptBudget int, force bool) ([][]string, bool) {
|
||||
if maxBytes <= 0 {
|
||||
maxBytes = len(text)
|
||||
}
|
||||
if promptBudget <= 0 {
|
||||
promptBudget = defaultDocChunkPromptBudget
|
||||
}
|
||||
lines := strings.SplitAfter(text, "\n")
|
||||
if len(lines) <= 1 {
|
||||
return nil, false
|
||||
}
|
||||
groups := make([][]string, 0, len(lines))
|
||||
var current strings.Builder
|
||||
currentBytes := 0
|
||||
currentPrompt := 0
|
||||
for _, line := range lines {
|
||||
linePrompt := estimateDocPromptCost(line)
|
||||
if len(line) > maxBytes || linePrompt > promptBudget {
|
||||
return nil, false
|
||||
}
|
||||
if currentBytes > 0 && (currentBytes+len(line) > maxBytes || currentPrompt+linePrompt > promptBudget) {
|
||||
groups = append(groups, []string{current.String()})
|
||||
current.Reset()
|
||||
currentBytes = 0
|
||||
currentPrompt = 0
|
||||
}
|
||||
current.WriteString(line)
|
||||
currentBytes += len(line)
|
||||
currentPrompt += linePrompt
|
||||
}
|
||||
if current.Len() > 0 {
|
||||
groups = append(groups, []string{current.String()})
|
||||
}
|
||||
if len(groups) <= 1 {
|
||||
if !force {
|
||||
return nil, false
|
||||
}
|
||||
return splitPlainDocSectionMidpoint(lines)
|
||||
}
|
||||
return groups, true
|
||||
}
|
||||
|
||||
func splitPlainDocSectionMidpoint(lines []string) ([][]string, bool) {
|
||||
if len(lines) <= 1 {
|
||||
return nil, false
|
||||
}
|
||||
mid := len(lines) / 2
|
||||
if mid <= 0 || mid >= len(lines) {
|
||||
return nil, false
|
||||
}
|
||||
left := strings.Join(lines[:mid], "")
|
||||
right := strings.Join(lines[mid:], "")
|
||||
if strings.TrimSpace(left) == "" || strings.TrimSpace(right) == "" {
|
||||
return nil, false
|
||||
}
|
||||
return [][]string{{left}, {right}}, true
|
||||
}
|
||||
|
||||
func firstNonEmptyLineIndex(lines []string) int {
|
||||
for index, line := range lines {
|
||||
if strings.TrimSpace(line) != "" {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func lastNonEmptyLineIndex(lines []string) int {
|
||||
for index := len(lines) - 1; index >= 0; index-- {
|
||||
if strings.TrimSpace(lines[index]) != "" {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func docsI18nDocChunkMaxBytes() int {
|
||||
value := strings.TrimSpace(os.Getenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES"))
|
||||
if value == "" {
|
||||
return defaultDocChunkMaxBytes
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return defaultDocChunkMaxBytes
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func docsI18nDocChunkPromptBudget() int {
|
||||
value := strings.TrimSpace(os.Getenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET"))
|
||||
if value == "" {
|
||||
return defaultDocChunkPromptBudget
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return defaultDocChunkPromptBudget
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func estimateDocPromptCost(text string) int {
|
||||
cost := len(text)
|
||||
cost += strings.Count(text, "`") * 6
|
||||
cost += strings.Count(text, "|") * 4
|
||||
cost += strings.Count(text, "{") * 4
|
||||
cost += strings.Count(text, "}") * 4
|
||||
cost += strings.Count(text, "[") * 4
|
||||
cost += strings.Count(text, "]") * 4
|
||||
cost += strings.Count(text, ":") * 2
|
||||
cost += strings.Count(text, "<") * 4
|
||||
cost += strings.Count(text, ">") * 4
|
||||
return cost
|
||||
}
|
||||
|
||||
func stripCommonIndent(text string) (string, string) {
|
||||
lines := strings.SplitAfter(text, "\n")
|
||||
common := ""
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimRight(line, "\r\n")
|
||||
if strings.TrimSpace(trimmed) == "" {
|
||||
continue
|
||||
}
|
||||
indent := leadingIndent(trimmed)
|
||||
if common == "" {
|
||||
common = indent
|
||||
continue
|
||||
}
|
||||
common = commonIndentPrefix(common, indent)
|
||||
if common == "" {
|
||||
return text, ""
|
||||
}
|
||||
}
|
||||
if common == "" {
|
||||
return text, ""
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimRight(line, "\r\n")
|
||||
if strings.TrimSpace(trimmed) == "" {
|
||||
out.WriteString(line)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, common) {
|
||||
out.WriteString(strings.TrimPrefix(line, common))
|
||||
continue
|
||||
}
|
||||
out.WriteString(line)
|
||||
}
|
||||
return out.String(), common
|
||||
}
|
||||
|
||||
func reapplyCommonIndent(text, indent string) string {
|
||||
if indent == "" || text == "" {
|
||||
return text
|
||||
}
|
||||
lines := strings.SplitAfter(text, "\n")
|
||||
var out strings.Builder
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimRight(line, "\r\n")
|
||||
if strings.TrimSpace(trimmed) == "" {
|
||||
out.WriteString(line)
|
||||
continue
|
||||
}
|
||||
out.WriteString(indent)
|
||||
out.WriteString(line)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func leadingIndent(line string) string {
|
||||
index := 0
|
||||
for index < len(line) {
|
||||
if line[index] != ' ' && line[index] != '\t' {
|
||||
break
|
||||
}
|
||||
index++
|
||||
}
|
||||
return line[:index]
|
||||
}
|
||||
|
||||
func commonIndentPrefix(a, b string) string {
|
||||
limit := len(a)
|
||||
if len(b) < limit {
|
||||
limit = len(b)
|
||||
}
|
||||
index := 0
|
||||
for index < limit && a[index] == b[index] {
|
||||
index++
|
||||
}
|
||||
return a[:index]
|
||||
}
|
||||
238
scripts/docs-i18n/doc_mode.go
Normal file
238
scripts/docs-i18n/doc_mode.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
frontmatterTagStart = "<frontmatter>"
|
||||
frontmatterTagEnd = "</frontmatter>"
|
||||
bodyTagStart = "<body>"
|
||||
bodyTagEnd = "</body>"
|
||||
)
|
||||
|
||||
type docOutputStatus int
|
||||
|
||||
const (
|
||||
docOutputNeedsTranslation docOutputStatus = iota
|
||||
docOutputReady
|
||||
docOutputNeedsPostprocess
|
||||
)
|
||||
|
||||
func processFileDoc(ctx context.Context, translator docsTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool) (bool, string, error) {
|
||||
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(absPath)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
currentHash := hashBytes(content)
|
||||
|
||||
outputPath := filepath.Join(docsRoot, tgtLang, relPath)
|
||||
if !overwrite {
|
||||
status, err := classifyDocOutput(outputPath, currentHash, tgtLang)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
switch status {
|
||||
case docOutputReady:
|
||||
return true, "", nil
|
||||
case docOutputNeedsPostprocess:
|
||||
return true, outputPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
sourceFront, sourceBody := splitFrontMatter(string(content))
|
||||
frontData := map[string]any{}
|
||||
if strings.TrimSpace(sourceFront) != "" {
|
||||
if err := yaml.Unmarshal([]byte(sourceFront), &frontData); err != nil {
|
||||
return false, "", fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
|
||||
}
|
||||
}
|
||||
docTM := &TranslationMemory{entries: map[string]TMEntry{}}
|
||||
if err := translateFrontMatter(ctx, translator, docTM, frontData, relPath, srcLang, tgtLang); err != nil {
|
||||
return false, "", fmt.Errorf("frontmatter translation failed for %s: %w", relPath, err)
|
||||
}
|
||||
updatedFront, err := encodeFrontMatter(frontData, relPath, content)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
translatedBody, err := translateDocBodyChunked(ctx, translator, relPath, sourceBody, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("body translate failed for %s: %w", relPath, err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
output := updatedFront + translatedBody
|
||||
return false, outputPath, os.WriteFile(outputPath, []byte(output), 0o644)
|
||||
}
|
||||
|
||||
func parseTaggedDocument(text string) (string, string, error) {
|
||||
frontStart := strings.Index(text, frontmatterTagStart)
|
||||
if frontStart == -1 {
|
||||
return "", "", fmt.Errorf("missing %s", frontmatterTagStart)
|
||||
}
|
||||
frontStart += len(frontmatterTagStart)
|
||||
frontEnd := strings.Index(text[frontStart:], frontmatterTagEnd)
|
||||
if frontEnd == -1 {
|
||||
return "", "", fmt.Errorf("missing %s", frontmatterTagEnd)
|
||||
}
|
||||
frontEnd += frontStart
|
||||
|
||||
bodyStart := strings.Index(text[frontEnd:], bodyTagStart)
|
||||
if bodyStart == -1 {
|
||||
return "", "", fmt.Errorf("missing %s", bodyTagStart)
|
||||
}
|
||||
bodyStart += frontEnd + len(bodyTagStart)
|
||||
|
||||
bodyEnd := findTaggedBodyEnd(text, bodyStart)
|
||||
if bodyEnd == -1 {
|
||||
return "", "", fmt.Errorf("missing %s", bodyTagEnd)
|
||||
}
|
||||
body := trimTagNewlines(text[bodyStart:bodyEnd])
|
||||
suffix := strings.TrimSpace(text[bodyEnd+len(bodyTagEnd):])
|
||||
|
||||
prefix := strings.TrimSpace(text[:frontStart-len(frontmatterTagStart)])
|
||||
if prefix != "" || suffix != "" {
|
||||
return "", "", fmt.Errorf("unexpected text outside tagged sections")
|
||||
}
|
||||
|
||||
frontMatter := trimTagNewlines(text[frontStart:frontEnd])
|
||||
return frontMatter, body, nil
|
||||
}
|
||||
|
||||
func findTaggedBodyEnd(text string, bodyStart int) int {
|
||||
if bodyStart < 0 || bodyStart > len(text) {
|
||||
return -1
|
||||
}
|
||||
search := text[bodyStart:]
|
||||
candidate := -1
|
||||
offset := 0
|
||||
for {
|
||||
index := strings.Index(search[offset:], bodyTagEnd)
|
||||
if index == -1 {
|
||||
return candidate
|
||||
}
|
||||
index += offset
|
||||
absolute := bodyStart + index
|
||||
suffix := strings.TrimSpace(text[absolute+len(bodyTagEnd):])
|
||||
if suffix == "" {
|
||||
candidate = absolute
|
||||
}
|
||||
offset = index + len(bodyTagEnd)
|
||||
if offset >= len(search) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func trimTagNewlines(value string) string {
|
||||
value = strings.TrimPrefix(value, "\n")
|
||||
value = strings.TrimSuffix(value, "\n")
|
||||
return value
|
||||
}
|
||||
|
||||
func classifyDocOutput(outputPath string, sourceHash string, targetLang string) (docOutputStatus, error) {
|
||||
data, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
return docOutputNeedsTranslation, err
|
||||
}
|
||||
frontMatter, _ := splitFrontMatter(string(data))
|
||||
if frontMatter == "" {
|
||||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
frontData := map[string]any{}
|
||||
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
|
||||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
storedHash := extractSourceHash(frontData)
|
||||
if storedHash == "" {
|
||||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
if !strings.EqualFold(storedHash, sourceHash) {
|
||||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(targetLang), "en") {
|
||||
return docOutputReady, nil
|
||||
}
|
||||
|
||||
postprocessVersion := extractPostprocessVersion(frontData)
|
||||
if strings.EqualFold(postprocessVersion, localizedLinkPostprocessVersion) {
|
||||
return docOutputReady, nil
|
||||
}
|
||||
return docOutputNeedsPostprocess, nil
|
||||
}
|
||||
|
||||
func extractSourceHash(frontData map[string]any) string {
|
||||
xi, ok := extractXI18N(frontData)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
value, ok := xi["source_hash"].(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func extractPostprocessVersion(frontData map[string]any) string {
|
||||
xi, ok := extractXI18N(frontData)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
value, ok := xi["postprocess_version"].(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func extractXI18N(frontData map[string]any) (map[string]any, bool) {
|
||||
xi, ok := frontData["x-i18n"].(map[string]any)
|
||||
if ok {
|
||||
return xi, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func logDocChunkPlan(relPath string, blocks []string, groups [][]string) {
|
||||
totalBytes := 0
|
||||
for _, block := range blocks {
|
||||
totalBytes += len(block)
|
||||
}
|
||||
log.Printf("docs-i18n: body-chunks %s blocks=%d groups=%d bytes=%d", relPath, len(blocks), len(groups), totalBytes)
|
||||
}
|
||||
|
||||
func resolveDocsPath(docsRoot, filePath string) (string, string, error) {
|
||||
absPath, err := filepath.Abs(filePath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
relPath, err := filepath.Rel(docsRoot, absPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if relPath == "." || relPath == "" {
|
||||
return "", "", fmt.Errorf("file %s resolves to docs root %s", absPath, docsRoot)
|
||||
}
|
||||
if filepath.IsAbs(relPath) || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) {
|
||||
return "", "", fmt.Errorf("file %s not under docs root %s", absPath, docsRoot)
|
||||
}
|
||||
return absPath, relPath, nil
|
||||
}
|
||||
919
scripts/docs-i18n/doc_mode_test.go
Normal file
919
scripts/docs-i18n/doc_mode_test.go
Normal file
@@ -0,0 +1,919 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type docChunkTranslator struct{}
|
||||
|
||||
func (docChunkTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (docChunkTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
switch {
|
||||
case strings.Contains(text, "Alpha block") && strings.Contains(text, "Beta block"):
|
||||
return strings.ReplaceAll(text, "</Accordion>", ""), nil
|
||||
default:
|
||||
replacer := strings.NewReplacer(
|
||||
"Alpha block", "阿尔法段",
|
||||
"Beta block", "贝塔段",
|
||||
"Code sample", "代码示例",
|
||||
)
|
||||
return replacer.Replace(text), nil
|
||||
}
|
||||
}
|
||||
|
||||
func (docChunkTranslator) Close() {}
|
||||
|
||||
type docLeafFallbackTranslator struct{}
|
||||
|
||||
func (docLeafFallbackTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
replacer := strings.NewReplacer(
|
||||
"Gateway refuses to start unless `local`.", "Gateway 只有在 `local` 时才会启动。",
|
||||
"`gateway.auth.mode: \"trusted-proxy\"`", "`gateway.auth.mode: \"trusted-proxy\"`",
|
||||
)
|
||||
return replacer.Replace(text), nil
|
||||
}
|
||||
|
||||
func (docLeafFallbackTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
if strings.Contains(text, "Gateway refuses to start unless `local`.") {
|
||||
return strings.Replace(text, "Gateway refuses to start unless `local`.", "<Tip>Gateway only starts in local mode.</Tip>", 1), nil
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (docLeafFallbackTranslator) Close() {}
|
||||
|
||||
type docFrontmatterTranslator struct{}
|
||||
|
||||
func (docFrontmatterTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
replacer := strings.NewReplacer(
|
||||
"Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS", "在 Fly.io 上逐步部署 OpenClaw,包含持久化存储和 HTTPS",
|
||||
"Deploying OpenClaw on Fly.io", "在 Fly.io 上部署 OpenClaw",
|
||||
"Setting up Fly volumes, secrets, and first-run config", "设置 Fly volume、密钥和首次运行配置",
|
||||
)
|
||||
return replacer.Replace(text), nil
|
||||
}
|
||||
|
||||
func (docFrontmatterTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
return "extra text outside tagged sections", nil
|
||||
}
|
||||
|
||||
func (docFrontmatterTranslator) Close() {}
|
||||
|
||||
type docFrontmatterFallbackTranslator struct{}
|
||||
|
||||
func (docFrontmatterFallbackTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
switch text {
|
||||
case "Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS":
|
||||
return strings.Join([]string{
|
||||
"<frontmatter>",
|
||||
"title: Fly.io",
|
||||
"summary: \"在 Fly.io 上部署 OpenClaw 的逐步指南,包含持久化存储和 HTTPS 设置\"",
|
||||
"read_when:",
|
||||
" - 在 Fly.io 上部署 OpenClaw",
|
||||
" - 设置 Fly 卷、机密和初始运行配置",
|
||||
"</frontmatter>",
|
||||
"",
|
||||
"<body>",
|
||||
"# Fly.io 部署",
|
||||
"</body>",
|
||||
}, "\n"), nil
|
||||
case "Deploying OpenClaw on Fly.io":
|
||||
return "在 Fly.io 上部署 OpenClaw", nil
|
||||
case "Setting up Fly volumes, secrets, and first-run config":
|
||||
return "设置 Fly 卷、机密和初始运行配置", nil
|
||||
default:
|
||||
return text, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (docFrontmatterFallbackTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (docFrontmatterFallbackTranslator) Close() {}
|
||||
|
||||
type docProtocolLeakTranslator struct{}
|
||||
|
||||
func (docProtocolLeakTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (docProtocolLeakTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
switch {
|
||||
case strings.Contains(text, "First chunk") && strings.Contains(text, "Second chunk"):
|
||||
return strings.Join([]string{
|
||||
"<frontmatter>",
|
||||
"title: leaked",
|
||||
"</frontmatter>",
|
||||
"",
|
||||
"<body>",
|
||||
"First translated",
|
||||
"",
|
||||
"Second translated",
|
||||
"</body>",
|
||||
}, "\n"), nil
|
||||
default:
|
||||
replacer := strings.NewReplacer(
|
||||
"First chunk", "First translated",
|
||||
"Second chunk", "Second translated",
|
||||
)
|
||||
return replacer.Replace(text), nil
|
||||
}
|
||||
}
|
||||
|
||||
func (docProtocolLeakTranslator) Close() {}
|
||||
|
||||
type docWrappedLeafTranslator struct{}
|
||||
|
||||
func (docWrappedLeafTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (docWrappedLeafTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
return strings.Join([]string{
|
||||
"<frontmatter>",
|
||||
"title: leaked",
|
||||
"</frontmatter>",
|
||||
"",
|
||||
"<body>",
|
||||
"# Fly.io 部署",
|
||||
"</body>",
|
||||
}, "\n"), nil
|
||||
}
|
||||
|
||||
func (docWrappedLeafTranslator) Close() {}
|
||||
|
||||
type docComponentLeafFallbackTranslator struct{}
|
||||
|
||||
func (docComponentLeafFallbackTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return strings.ReplaceAll(text, "Yes.", "是的。"), nil
|
||||
}
|
||||
|
||||
func (docComponentLeafFallbackTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
if strings.Contains(text, "Can I use Claude Max subscription without an API key?") {
|
||||
return strings.ReplaceAll(text, "Yes.\n", "Yes.\n</Accordion>\n"), nil
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (docComponentLeafFallbackTranslator) Close() {}
|
||||
|
||||
type docPromptBudgetTranslator struct {
|
||||
rawInputs []string
|
||||
}
|
||||
|
||||
func (t *docPromptBudgetTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (t *docPromptBudgetTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
t.rawInputs = append(t.rawInputs, text)
|
||||
replacer := strings.NewReplacer(
|
||||
"First chunk with `json5` and { braces }", "第一块,含 `json5` 和 { braces }",
|
||||
"Second chunk with | table | pipes |", "第二块,含 | table | pipes |",
|
||||
)
|
||||
return replacer.Replace(text), nil
|
||||
}
|
||||
|
||||
func (t *docPromptBudgetTranslator) Close() {}
|
||||
|
||||
type uppercaseWrapperTranslator struct{}
|
||||
|
||||
func (uppercaseWrapperTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (uppercaseWrapperTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
return "<BODY>\n" + strings.ReplaceAll(text, "Regular paragraph.", "Translated paragraph.") + "\n</BODY>\n", nil
|
||||
}
|
||||
|
||||
func (uppercaseWrapperTranslator) Close() {}
|
||||
|
||||
type oversizedBlockTranslator struct {
|
||||
rawInputs []string
|
||||
}
|
||||
|
||||
func (t *oversizedBlockTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (t *oversizedBlockTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
t.rawInputs = append(t.rawInputs, text)
|
||||
return strings.ReplaceAll(text, "Line ", "Translated line "), nil
|
||||
}
|
||||
|
||||
func (t *oversizedBlockTranslator) Close() {}
|
||||
|
||||
type singletonFenceRetryTranslator struct {
|
||||
rawInputs []string
|
||||
}
|
||||
|
||||
func (t *singletonFenceRetryTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (t *singletonFenceRetryTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
t.rawInputs = append(t.rawInputs, text)
|
||||
if strings.Contains(text, "Line 01") && strings.Contains(text, "Line 04") {
|
||||
return strings.Replace(text, "\n```\n", "\n", 1), nil
|
||||
}
|
||||
return strings.ReplaceAll(text, "Line ", "Translated line "), nil
|
||||
}
|
||||
|
||||
func (t *singletonFenceRetryTranslator) Close() {}
|
||||
|
||||
func TestParseTaggedDocumentRejectsMissingBodyCloseAtEOF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := "<frontmatter>\ntitle: Test\n</frontmatter>\n<body>\nTranslated body\n"
|
||||
|
||||
_, _, err := parseTaggedDocument(input)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing </body>")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTaggedDocumentRejectsTrailingTextOutsideTags(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := "<frontmatter>\ntitle: Test\n</frontmatter>\n<body>\nTranslated body\n</body>\nextra"
|
||||
|
||||
_, _, err := parseTaggedDocument(input)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for trailing text")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindTaggedBodyEndSearchesFromBodyStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
text := strings.Join([]string{
|
||||
"<frontmatter>",
|
||||
"summary: literal </body> token in frontmatter",
|
||||
"</frontmatter>",
|
||||
"<body>",
|
||||
"Translated body",
|
||||
"</body>",
|
||||
}, "\n")
|
||||
bodyStart := strings.Index(text, bodyTagStart)
|
||||
if bodyStart == -1 {
|
||||
t.Fatal("expected body tag in test input")
|
||||
}
|
||||
bodyStart += len(bodyTagStart)
|
||||
|
||||
bodyEnd := findTaggedBodyEnd(text, bodyStart)
|
||||
if bodyEnd == -1 {
|
||||
t.Fatal("expected closing body tag to be found")
|
||||
}
|
||||
body := trimTagNewlines(text[bodyStart:bodyEnd])
|
||||
if body != "Translated body" {
|
||||
t.Fatalf("expected body slice to ignore pre-body literal token, got %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitDocBodyIntoBlocksKeepsFenceTogether(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := strings.Join([]string{
|
||||
"<Accordion title=\"Alpha block\">",
|
||||
"",
|
||||
"Code sample:",
|
||||
"```ts",
|
||||
"console.log('hello')",
|
||||
"```",
|
||||
"",
|
||||
"Beta block",
|
||||
"",
|
||||
"</Accordion>",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
blocks := splitDocBodyIntoBlocks(body)
|
||||
if len(blocks) != 4 {
|
||||
t.Fatalf("expected 4 blocks, got %d", len(blocks))
|
||||
}
|
||||
if !strings.Contains(blocks[1], "```ts") || !strings.Contains(blocks[1], "```") {
|
||||
t.Fatalf("expected code fence to stay in a single block:\n%s", blocks[1])
|
||||
}
|
||||
if !strings.Contains(blocks[2], "Beta block") {
|
||||
t.Fatalf("expected Beta paragraph in its own block:\n%s", blocks[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitDocBodyIntoBlocksKeepsNestedTripleBackticksInsideFourBacktickFence(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := strings.Join([]string{
|
||||
"````md",
|
||||
"```ts",
|
||||
"console.log('nested example')",
|
||||
"```",
|
||||
"````",
|
||||
"",
|
||||
"Outside paragraph",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
blocks := splitDocBodyIntoBlocks(body)
|
||||
if len(blocks) != 2 {
|
||||
t.Fatalf("expected 2 blocks, got %d", len(blocks))
|
||||
}
|
||||
if !strings.Contains(blocks[0], "console.log('nested example')") || !strings.Contains(blocks[0], "````") {
|
||||
t.Fatalf("expected the full fenced example to stay in one block:\n%s", blocks[0])
|
||||
}
|
||||
if !strings.Contains(blocks[1], "Outside paragraph") {
|
||||
t.Fatalf("expected trailing paragraph in second block:\n%s", blocks[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeDocChunkProtocolWrappersStripsOuterWrapperAroundBodyExamples(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := strings.Join([]string{
|
||||
"Paragraph mentioning literal tokens `<body>` and `</body>`.",
|
||||
"",
|
||||
"<html>",
|
||||
" <body>",
|
||||
" literal example",
|
||||
" </body>",
|
||||
"</html>",
|
||||
}, "\n")
|
||||
translated := strings.Join([]string{
|
||||
"<frontmatter>",
|
||||
"title: leaked",
|
||||
"</frontmatter>",
|
||||
"",
|
||||
"<body>",
|
||||
"提到字面量 `<body>` 和 `</body>` 的段落。",
|
||||
"",
|
||||
"<html>",
|
||||
" <body>",
|
||||
" literal example",
|
||||
" </body>",
|
||||
"</html>",
|
||||
"</body>",
|
||||
}, "\n")
|
||||
|
||||
sanitized := sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
if strings.Contains(sanitized, frontmatterTagStart) || strings.HasPrefix(strings.TrimSpace(sanitized), bodyTagStart) {
|
||||
t.Fatalf("expected outer wrapper stripped, got:\n%s", sanitized)
|
||||
}
|
||||
if !strings.Contains(sanitized, "<html>") || !strings.Contains(sanitized, "<body>") || !strings.Contains(sanitized, "</body>") {
|
||||
t.Fatalf("expected inner HTML example preserved, got:\n%s", sanitized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedFallsBackToSmallerChunks(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
"<Accordion title=\"Alpha block\">",
|
||||
"Alpha block",
|
||||
"</Accordion>",
|
||||
"",
|
||||
"Beta block",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
translated, err := translateDocBodyChunked(context.Background(), docChunkTranslator{}, "help/faq.md", body, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if !strings.Contains(translated, "阿尔法段") || !strings.Contains(translated, "贝塔段") {
|
||||
t.Fatalf("expected translated text after chunk split, got:\n%s", translated)
|
||||
}
|
||||
if strings.Count(translated, "</Accordion>") != 1 {
|
||||
t.Fatalf("expected closing Accordion tag to be preserved after fallback split:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripAndReapplyCommonIndent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := strings.Join([]string{
|
||||
" <Step title=\"Example\">",
|
||||
" - item one",
|
||||
" - item two",
|
||||
" </Step>",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
normalized, indent := stripCommonIndent(source)
|
||||
if indent != " " {
|
||||
t.Fatalf("expected common indent of four spaces, got %q", indent)
|
||||
}
|
||||
if strings.HasPrefix(normalized, " ") {
|
||||
t.Fatalf("expected normalized text without common indent:\n%s", normalized)
|
||||
}
|
||||
roundTrip := reapplyCommonIndent(normalized, indent)
|
||||
if roundTrip != source {
|
||||
t.Fatalf("expected indent round-trip to preserve source\nwant:\n%s\ngot:\n%s", source, roundTrip)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedFallsBackToMaskedTranslateForLeafValidationFailure(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
"- `mode`: `local` or `remote`. Gateway refuses to start unless `local`.",
|
||||
"- `gateway.auth.mode: \"trusted-proxy\"`: delegate auth to a reverse proxy.",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
translated, err := translateDocBodyChunked(
|
||||
context.Background(),
|
||||
docLeafFallbackTranslator{},
|
||||
"gateway/configuration-reference.md",
|
||||
body,
|
||||
"en",
|
||||
"zh-CN",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if strings.Contains(translated, "<Tip>") {
|
||||
t.Fatalf("expected masked fallback to remove hallucinated component tags:\n%s", translated)
|
||||
}
|
||||
if !strings.Contains(translated, "Gateway 只有在 `local` 时才会启动。") {
|
||||
t.Fatalf("expected fallback translation to be applied:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocChunkTranslationRejectsProtocolTokenLeakage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Regular paragraph.\n\n"
|
||||
translated := "<frontmatter>\ntitle: leaked\n</frontmatter>\n<body>\nRegular paragraph.\n</body>\n"
|
||||
|
||||
err := validateDocChunkTranslation(source, translated)
|
||||
if err == nil {
|
||||
t.Fatal("expected protocol token leakage to be rejected")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "protocol token leaked") {
|
||||
t.Fatalf("expected protocol token leakage error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocChunkTranslationRejectsTranscriptArtifact(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Regular paragraph.\n\n"
|
||||
translated := `Regular paragraph. assistant to=functions.read commentary {"path":"/home/runner/work/docs/docs/source/AGENTS.md"} code`
|
||||
|
||||
err := validateDocChunkTranslation(source, translated)
|
||||
if err == nil {
|
||||
t.Fatal("expected transcript artifact to be rejected")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "agent transcript artifact") {
|
||||
t.Fatalf("expected transcript artifact error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocChunkTranslationRejectsTopLevelBodyWrapperLeakEvenWhenSourceMentionsBodyTag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Use `<body>` in examples, but keep prose outside wrappers.\n"
|
||||
translated := "<body>\nTranslated paragraph.\n"
|
||||
|
||||
err := validateDocChunkTranslation(source, translated)
|
||||
if err == nil {
|
||||
t.Fatal("expected top-level wrapper leakage to be rejected")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "protocol token leaked") {
|
||||
t.Fatalf("expected protocol token leakage error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedSplitsOnProtocolTokenLeakage(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
"First chunk",
|
||||
"",
|
||||
"Second chunk",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
translated, err := translateDocBodyChunked(context.Background(), docProtocolLeakTranslator{}, "gateway/configuration-reference.md", body, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if strings.Contains(translated, "<frontmatter>") || strings.Contains(translated, "<body>") || strings.Contains(translated, "[[[FM_") {
|
||||
t.Fatalf("expected protocol wrapper leakage to be removed after split:\n%s", translated)
|
||||
}
|
||||
if !strings.Contains(translated, "First translated") || !strings.Contains(translated, "Second translated") {
|
||||
t.Fatalf("expected split chunks to translate successfully:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedStripsUppercaseBodyWrapper(t *testing.T) {
|
||||
body := "Regular paragraph.\n"
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
translated, err := translateDocBodyChunked(context.Background(), uppercaseWrapperTranslator{}, "gateway/configuration-reference.md", body, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(translated), "<body>") {
|
||||
t.Fatalf("expected uppercase wrapper to be stripped:\n%s", translated)
|
||||
}
|
||||
if !strings.Contains(translated, "Translated paragraph.") {
|
||||
t.Fatalf("expected translated body content to survive unwrap:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeDocChunkProtocolWrappersKeepsBodyOnlyWrapperWhenSourceMentionsBodyTag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Use `<body>` and `</body>` in examples, but keep the paragraph text plain.\n"
|
||||
translated := "<body>\nTranslated paragraph.\n</body>\n"
|
||||
|
||||
got := sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
if got != translated {
|
||||
t.Fatalf("expected ambiguous body-only wrapper to remain unchanged for retry\nwant:\n%s\ngot:\n%s", translated, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeDocChunkProtocolWrappersKeepsLegitimateTopLevelBodyBlock(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "<body>\nLiteral HTML block.\n</body>\n"
|
||||
translated := "<body>\nLiteral HTML block.\n</body>\n"
|
||||
|
||||
got := sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
if got != translated {
|
||||
t.Fatalf("expected legitimate top-level body block to remain unchanged\nwant:\n%s\ngot:\n%s", translated, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeDocChunkProtocolWrappersStripsBodyOnlyWrapperWhenSourceHasNoBodyTokens(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Regular paragraph.\n"
|
||||
translated := "<body>\nTranslated paragraph.\n</body>\n"
|
||||
|
||||
got := sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
if strings.Contains(got, "<body>") || strings.Contains(got, "</body>") {
|
||||
t.Fatalf("expected body-only wrapper to be stripped, got %q", got)
|
||||
}
|
||||
if strings.TrimSpace(got) != "Translated paragraph." {
|
||||
t.Fatalf("unexpected sanitized body %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeDocChunkProtocolWrappersKeepsAmbiguousTaggedWrapperForRetry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := strings.Join([]string{
|
||||
"Paragraph mentioning literal tokens `<body>` and `</body>`.",
|
||||
"",
|
||||
"Closing example:",
|
||||
"</body>",
|
||||
}, "\n")
|
||||
translated := strings.Join([]string{
|
||||
"<frontmatter>",
|
||||
"title: leaked",
|
||||
"</frontmatter>",
|
||||
"",
|
||||
"<body>",
|
||||
"提到字面量 `<body>` 和 `</body>` 的段落。",
|
||||
}, "\n")
|
||||
|
||||
got := sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
if got != translated {
|
||||
t.Fatalf("expected ambiguous tagged wrapper to remain unchanged for retry\nwant:\n%s\ngot:\n%s", translated, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitDocBodyIntoBlocksKeepsInfoStringExampleInsideFence(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := strings.Join([]string{
|
||||
"```md",
|
||||
"```ts",
|
||||
"console.log('inside example')",
|
||||
"```",
|
||||
"",
|
||||
"Outside paragraph",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
blocks := splitDocBodyIntoBlocks(body)
|
||||
if len(blocks) != 2 {
|
||||
t.Fatalf("expected 2 blocks, got %d", len(blocks))
|
||||
}
|
||||
if !strings.Contains(blocks[0], "console.log('inside example')") || !strings.Contains(blocks[0], "```ts") {
|
||||
t.Fatalf("expected fenced example to stay together:\n%s", blocks[0])
|
||||
}
|
||||
if !strings.Contains(blocks[1], "Outside paragraph") {
|
||||
t.Fatalf("expected trailing paragraph in second block:\n%s", blocks[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedPreSplitsOversizedPromptBudget(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
"First chunk with `json5` and { braces }",
|
||||
"",
|
||||
"Second chunk with | table | pipes |",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET", "60")
|
||||
|
||||
translator := &docPromptBudgetTranslator{}
|
||||
translated, err := translateDocBodyChunked(
|
||||
context.Background(),
|
||||
translator,
|
||||
"gateway/configuration-reference.md",
|
||||
body,
|
||||
"en",
|
||||
"zh-CN",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
for _, input := range translator.rawInputs {
|
||||
if strings.Contains(input, "First chunk with `json5` and { braces }") && strings.Contains(input, "Second chunk with | table | pipes |") {
|
||||
t.Fatalf("expected prompt budget guard to split before raw translation, saw combined input:\n%s", input)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(translated, "第一块") || !strings.Contains(translated, "第二块") {
|
||||
t.Fatalf("expected split chunks to translate successfully:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedSplitsOversizedSingletonBlock(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
"Line 01",
|
||||
"Line 02",
|
||||
"Line 03",
|
||||
"Line 04",
|
||||
"Line 05",
|
||||
"Line 06",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "24")
|
||||
translator := &oversizedBlockTranslator{}
|
||||
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if len(translator.rawInputs) < 2 {
|
||||
t.Fatalf("expected oversized singleton block to be split before translation, saw %d input(s)", len(translator.rawInputs))
|
||||
}
|
||||
for _, input := range translator.rawInputs {
|
||||
if len(input) > 24 {
|
||||
t.Fatalf("expected split chunk under byte budget, got %d bytes:\n%s", len(input), input)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(translated, "Translated line 01") || !strings.Contains(translated, "Translated line 06") {
|
||||
t.Fatalf("expected translated singleton parts to be reassembled:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedSplitsSingletonBlockWhenPromptBudgetExceeded(t *testing.T) {
|
||||
lineA := "Alpha chunk with { braces }\n"
|
||||
lineB := "Beta chunk with | pipes |\n"
|
||||
body := lineA + lineB + "\n"
|
||||
budget := max(estimateDocPromptCost(lineA), estimateDocPromptCost(lineB)) + 1
|
||||
if estimateDocPromptCost(body) <= budget {
|
||||
t.Fatalf("test setup expected combined singleton prompt cost to exceed budget; cost=%d budget=%d", estimateDocPromptCost(body), budget)
|
||||
}
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET", strconv.Itoa(budget))
|
||||
translator := &oversizedBlockTranslator{}
|
||||
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if len(translator.rawInputs) < 2 {
|
||||
t.Fatalf("expected prompt-budget singleton split before translation, saw %d input(s)", len(translator.rawInputs))
|
||||
}
|
||||
for _, input := range translator.rawInputs {
|
||||
if estimateDocPromptCost(input) > budget {
|
||||
t.Fatalf("expected split chunk under prompt budget, got cost=%d budget=%d:\n%s", estimateDocPromptCost(input), budget, input)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(translated, "Alpha chunk") || !strings.Contains(translated, "Beta chunk") {
|
||||
t.Fatalf("expected translated singleton parts to be reassembled:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedSplitsOversizedFenceBeforeTrailingProse(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
"```md",
|
||||
"Line 01",
|
||||
"Line 02",
|
||||
"Line 03",
|
||||
"Line 04",
|
||||
"```",
|
||||
"Trailing paragraph after the fence.",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "24")
|
||||
translator := &oversizedBlockTranslator{}
|
||||
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if len(translator.rawInputs) < 3 {
|
||||
t.Fatalf("expected oversized fenced block with trailing prose to split, saw %d input(s)", len(translator.rawInputs))
|
||||
}
|
||||
for _, input := range translator.rawInputs {
|
||||
if strings.Contains(input, "Line 01") || strings.Contains(input, "Line 02") || strings.Contains(input, "Line 03") || strings.Contains(input, "Line 04") {
|
||||
if !strings.Contains(input, "```md") || !strings.Contains(input, "```") {
|
||||
t.Fatalf("expected fenced split input to keep matched fence wrappers:\n%s", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(translated, "Translated line 01") || !strings.Contains(translated, "Trailing paragraph after the fence.") {
|
||||
t.Fatalf("expected fence content and trailing prose to survive split:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedRetriesSingletonFenceAfterValidationFailure(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
"```md",
|
||||
"Line 01",
|
||||
"Line 02",
|
||||
"Line 03",
|
||||
"Line 04",
|
||||
"```",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_PROMPT_BUDGET", "4096")
|
||||
|
||||
translator := &singletonFenceRetryTranslator{}
|
||||
translated, err := translateDocBodyChunked(context.Background(), translator, "gateway/configuration-reference.md", body, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if len(translator.rawInputs) < 3 {
|
||||
t.Fatalf("expected singleton fence retry to split after validation failure, saw %d input(s)", len(translator.rawInputs))
|
||||
}
|
||||
if !strings.Contains(translator.rawInputs[0], "Line 01") || !strings.Contains(translator.rawInputs[0], "Line 04") {
|
||||
t.Fatalf("expected first raw attempt to include the original fenced block:\n%s", translator.rawInputs[0])
|
||||
}
|
||||
for _, input := range translator.rawInputs[1:] {
|
||||
if strings.Contains(input, "Line 01") || strings.Contains(input, "Line 02") || strings.Contains(input, "Line 03") || strings.Contains(input, "Line 04") {
|
||||
if !strings.Contains(input, "```md") || !strings.Contains(input, "```") {
|
||||
t.Fatalf("expected split retry inputs to preserve fence wrappers:\n%s", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(translated, "Translated line 01") || !strings.Contains(translated, "Translated line 04") {
|
||||
t.Fatalf("expected singleton fence retry to reassemble translated output:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedUnwrapsTaggedLeafProtocolLeakage(t *testing.T) {
|
||||
body := "# Fly.io Deployment\n\n"
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
translated, err := translateDocBodyChunked(
|
||||
context.Background(),
|
||||
docWrappedLeafTranslator{},
|
||||
"install/fly.md",
|
||||
body,
|
||||
"en",
|
||||
"zh-CN",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if strings.Contains(translated, "<frontmatter>") || strings.Contains(translated, "<body>") {
|
||||
t.Fatalf("expected wrapped leaf translation to unwrap protocol tags:\n%s", translated)
|
||||
}
|
||||
if !strings.Contains(translated, "# Fly.io 部署") {
|
||||
t.Fatalf("expected unwrapped body translation:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateDocBodyChunkedFallsBackForComponentLeafValidationFailure(t *testing.T) {
|
||||
body := " <Accordion title=\"Can I use Claude Max subscription without an API key?\">\n Yes.\n\n"
|
||||
|
||||
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "4096")
|
||||
translated, err := translateDocBodyChunked(
|
||||
context.Background(),
|
||||
docComponentLeafFallbackTranslator{},
|
||||
"help/faq.md",
|
||||
body,
|
||||
"en",
|
||||
"zh-CN",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("translateDocBodyChunked returned error: %v", err)
|
||||
}
|
||||
if strings.Contains(translated, "</Accordion>") {
|
||||
t.Fatalf("expected component leaf fallback to avoid hallucinated closing tag:\n%s", translated)
|
||||
}
|
||||
if !strings.Contains(translated, "是的。") {
|
||||
t.Fatalf("expected body text to be translated after component leaf fallback:\n%s", translated)
|
||||
}
|
||||
if !strings.Contains(translated, "<Accordion title=\"Can I use Claude Max subscription without an API key?\">") {
|
||||
t.Fatalf("expected Accordion opening tag to be preserved:\n%s", translated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(docsRoot, "install")
|
||||
if err := os.MkdirAll(sourcePath, 0o755); err != nil {
|
||||
t.Fatalf("mkdir failed: %v", err)
|
||||
}
|
||||
sourceFile := filepath.Join(sourcePath, "fly.md")
|
||||
source := strings.Join([]string{
|
||||
"---",
|
||||
"title: Fly.io",
|
||||
"summary: \"Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS\"",
|
||||
"read_when:",
|
||||
" - Deploying OpenClaw on Fly.io",
|
||||
" - Setting up Fly volumes, secrets, and first-run config",
|
||||
"---",
|
||||
"",
|
||||
}, "\n")
|
||||
if err := os.WriteFile(sourceFile, []byte(source), 0o644); err != nil {
|
||||
t.Fatalf("write failed: %v", err)
|
||||
}
|
||||
|
||||
skipped, outputPath, err := processFileDoc(context.Background(), docFrontmatterTranslator{}, docsRoot, sourceFile, "en", "zh-CN", true)
|
||||
if err != nil {
|
||||
t.Fatalf("processFileDoc returned error: %v", err)
|
||||
}
|
||||
if skipped {
|
||||
t.Fatal("expected file to be processed")
|
||||
}
|
||||
if outputPath == "" {
|
||||
t.Fatal("expected output path")
|
||||
}
|
||||
output, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output failed: %v", err)
|
||||
}
|
||||
text := string(output)
|
||||
if !strings.Contains(text, "在 Fly.io 上逐步部署 OpenClaw,包含持久化存储和 HTTPS") {
|
||||
t.Fatalf("expected translated summary in output:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
|
||||
t.Fatalf("expected translated read_when entry in output:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFileDocRejectsSuspiciousFrontmatterScalarExpansion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(docsRoot, "install")
|
||||
if err := os.MkdirAll(sourcePath, 0o755); err != nil {
|
||||
t.Fatalf("mkdir failed: %v", err)
|
||||
}
|
||||
sourceFile := filepath.Join(sourcePath, "fly.md")
|
||||
source := strings.Join([]string{
|
||||
"---",
|
||||
"title: Fly.io",
|
||||
"summary: \"Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS\"",
|
||||
"read_when:",
|
||||
" - Deploying OpenClaw on Fly.io",
|
||||
" - Setting up Fly volumes, secrets, and first-run config",
|
||||
"---",
|
||||
"",
|
||||
}, "\n")
|
||||
if err := os.WriteFile(sourceFile, []byte(source), 0o644); err != nil {
|
||||
t.Fatalf("write failed: %v", err)
|
||||
}
|
||||
|
||||
skipped, outputPath, err := processFileDoc(context.Background(), docFrontmatterFallbackTranslator{}, docsRoot, sourceFile, "en", "zh-CN", true)
|
||||
if err != nil {
|
||||
t.Fatalf("processFileDoc returned error: %v", err)
|
||||
}
|
||||
if skipped {
|
||||
t.Fatal("expected file to be processed")
|
||||
}
|
||||
output, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output failed: %v", err)
|
||||
}
|
||||
text := string(output)
|
||||
if strings.Contains(text, "<frontmatter>") || strings.Contains(text, "<body>") {
|
||||
t.Fatalf("expected suspicious frontmatter expansion to be rejected:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, "summary: Step-by-step Fly.io deployment for OpenClaw with persistent storage and HTTPS") {
|
||||
t.Fatalf("expected original summary to be preserved after fallback:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
|
||||
t.Fatalf("expected read_when translation to survive fallback:\n%s", text)
|
||||
}
|
||||
}
|
||||
29
scripts/docs-i18n/glossary.go
Normal file
29
scripts/docs-i18n/glossary.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
type GlossaryEntry struct {
|
||||
Source string `json:"source"`
|
||||
Target string `json:"target"`
|
||||
}
|
||||
|
||||
func LoadGlossary(path string) ([]GlossaryEntry, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var entries []GlossaryEntry
|
||||
if err := json.Unmarshal(data, &entries); err != nil {
|
||||
return nil, fmt.Errorf("glossary parse failed: %w", err)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
9
scripts/docs-i18n/go.mod
Normal file
9
scripts/docs-i18n/go.mod
Normal file
@@ -0,0 +1,9 @@
|
||||
module github.com/openclaw/openclaw/scripts/docs-i18n
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/yuin/goldmark v1.8.2
|
||||
golang.org/x/net v0.53.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
8
scripts/docs-i18n/go.sum
Normal file
8
scripts/docs-i18n/go.sum
Normal file
@@ -0,0 +1,8 @@
|
||||
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
160
scripts/docs-i18n/html_translate.go
Normal file
160
scripts/docs-i18n/html_translate.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/ast"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
"github.com/yuin/goldmark/text"
|
||||
"golang.org/x/net/html"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type htmlReplacement struct {
|
||||
Start int
|
||||
Stop int
|
||||
Value string
|
||||
}
|
||||
|
||||
func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, srcLang, tgtLang string) (string, error) {
|
||||
source := []byte(body)
|
||||
r := text.NewReader(source)
|
||||
md := goldmark.New(
|
||||
goldmark.WithExtensions(extension.GFM),
|
||||
)
|
||||
doc := md.Parser().Parse(r)
|
||||
|
||||
replacements := make([]htmlReplacement, 0, 8)
|
||||
|
||||
_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
block, ok := n.(*ast.HTMLBlock)
|
||||
if !ok {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
start, stop, ok := htmlBlockSpan(block, source)
|
||||
if !ok {
|
||||
return ast.WalkSkipChildren, nil
|
||||
}
|
||||
htmlText := string(source[start:stop])
|
||||
translated, err := translateHTMLBlock(ctx, translator, htmlText, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return ast.WalkStop, err
|
||||
}
|
||||
replacements = append(replacements, htmlReplacement{Start: start, Stop: stop, Value: translated})
|
||||
return ast.WalkSkipChildren, nil
|
||||
})
|
||||
|
||||
if len(replacements) == 0 {
|
||||
return body, nil
|
||||
}
|
||||
|
||||
return applyHTMLReplacements(body, replacements), nil
|
||||
}
|
||||
|
||||
func htmlBlockSpan(block *ast.HTMLBlock, source []byte) (int, int, bool) {
|
||||
lines := block.Lines()
|
||||
if lines.Len() == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
start := lines.At(0).Start
|
||||
stop := lines.At(lines.Len() - 1).Stop
|
||||
if start >= stop {
|
||||
return 0, 0, false
|
||||
}
|
||||
return start, stop, true
|
||||
}
|
||||
|
||||
func applyHTMLReplacements(body string, replacements []htmlReplacement) string {
|
||||
if len(replacements) == 0 {
|
||||
return body
|
||||
}
|
||||
sortHTMLReplacements(replacements)
|
||||
var out strings.Builder
|
||||
last := 0
|
||||
for _, rep := range replacements {
|
||||
if rep.Start < last {
|
||||
continue
|
||||
}
|
||||
out.WriteString(body[last:rep.Start])
|
||||
out.WriteString(rep.Value)
|
||||
last = rep.Stop
|
||||
}
|
||||
out.WriteString(body[last:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func sortHTMLReplacements(replacements []htmlReplacement) {
|
||||
sort.Slice(replacements, func(i, j int) bool {
|
||||
return replacements[i].Start < replacements[j].Start
|
||||
})
|
||||
}
|
||||
|
||||
func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText, srcLang, tgtLang string) (string, error) {
|
||||
tokenizer := html.NewTokenizer(strings.NewReader(htmlText))
|
||||
var out strings.Builder
|
||||
skipDepth := 0
|
||||
|
||||
for {
|
||||
tt := tokenizer.Next()
|
||||
if tt == html.ErrorToken {
|
||||
if err := tokenizer.Err(); err != nil && err != io.EOF {
|
||||
return "", err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
raw := string(tokenizer.Raw())
|
||||
tok := tokenizer.Token()
|
||||
|
||||
switch tt {
|
||||
case html.StartTagToken:
|
||||
out.WriteString(raw)
|
||||
if isSkipTag(strings.ToLower(tok.Data)) {
|
||||
skipDepth++
|
||||
}
|
||||
case html.EndTagToken:
|
||||
out.WriteString(raw)
|
||||
if isSkipTag(strings.ToLower(tok.Data)) && skipDepth > 0 {
|
||||
skipDepth--
|
||||
}
|
||||
case html.SelfClosingTagToken:
|
||||
out.WriteString(raw)
|
||||
case html.TextToken:
|
||||
if shouldTranslateHTMLText(skipDepth, raw) {
|
||||
translated, err := translator.Translate(ctx, raw, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out.WriteString(translated)
|
||||
} else {
|
||||
out.WriteString(raw)
|
||||
}
|
||||
default:
|
||||
out.WriteString(raw)
|
||||
}
|
||||
}
|
||||
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
func shouldTranslateHTMLText(skipDepth int, text string) bool {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return false
|
||||
}
|
||||
return skipDepth == 0
|
||||
}
|
||||
|
||||
func isSkipTag(tag string) bool {
|
||||
switch tag {
|
||||
case "code", "pre", "script", "style":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
408
scripts/docs-i18n/localized_links.go
Normal file
408
scripts/docs-i18n/localized_links.go
Normal file
@@ -0,0 +1,408 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type routeIndex struct {
|
||||
targetLang string
|
||||
redirects map[string]string
|
||||
sourceRoutes map[string]struct{}
|
||||
localizedRoutes map[string]struct{}
|
||||
localePrefixes map[string]struct{}
|
||||
}
|
||||
|
||||
type docsConfig struct {
|
||||
Redirects []docsRedirect `json:"redirects"`
|
||||
}
|
||||
|
||||
type docsRedirect struct {
|
||||
Source string `json:"source"`
|
||||
Destination string `json:"destination"`
|
||||
}
|
||||
|
||||
var (
|
||||
localeDirRe = regexp.MustCompile(`^[a-z]{2,3}(?:-[A-Za-z0-9]{2,8})?$`)
|
||||
fencedBacktickCodeBlock = regexp.MustCompile("(?ms)(^|\\n)[ \\t]*```[^\\n]*\\n.*?\\n[ \\t]*```[ \\t]*(?:\\n|$)")
|
||||
fencedTildeCodeBlock = regexp.MustCompile("(?ms)(^|\\n)[ \\t]*~~~[^\\n]*\\n.*?\\n[ \\t]*~~~[ \\t]*(?:\\n|$)")
|
||||
markdownLinkTargetRe = regexp.MustCompile(`!?\[[^\]]*\]\(([^)]+)\)`)
|
||||
hrefDoubleQuotedValueRe = regexp.MustCompile(`\bhref\s*=\s*"([^"]*)"`)
|
||||
hrefSingleQuotedValueRe = regexp.MustCompile(`\bhref\s*=\s*'([^']*)'`)
|
||||
)
|
||||
|
||||
func loadRouteIndex(docsRoot, targetLang string) (*routeIndex, error) {
|
||||
index := &routeIndex{
|
||||
targetLang: strings.TrimSpace(targetLang),
|
||||
redirects: map[string]string{},
|
||||
sourceRoutes: map[string]struct{}{},
|
||||
localizedRoutes: map[string]struct{}{},
|
||||
localePrefixes: map[string]struct{}{},
|
||||
}
|
||||
|
||||
if err := index.loadRedirects(filepath.Join(docsRoot, "docs.json")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := index.loadRoutes(docsRoot); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func (ri *routeIndex) loadRedirects(configPath string) error {
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var config docsConfig
|
||||
if err := json.Unmarshal(data, &config); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range config.Redirects {
|
||||
source := normalizeRoute(item.Source)
|
||||
destination := normalizeRoute(item.Destination)
|
||||
if source == "" || destination == "" {
|
||||
continue
|
||||
}
|
||||
ri.redirects[source] = destination
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ri *routeIndex) loadRoutes(docsRoot string) error {
|
||||
localePrefixes, err := discoverLocalePrefixes(docsRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ri.targetLang != "" {
|
||||
localePrefixes[ri.targetLang] = struct{}{}
|
||||
}
|
||||
ri.localePrefixes = localePrefixes
|
||||
|
||||
return filepath.WalkDir(docsRoot, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !isMarkdownFile(path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(docsRoot, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relPath = normalizeSlashes(relPath)
|
||||
firstSegment := firstPathSegment(relPath)
|
||||
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
permalinks := extractPermalinks(content)
|
||||
|
||||
switch {
|
||||
case firstSegment == ri.targetLang:
|
||||
trimmedRel := strings.TrimPrefix(relPath, firstSegment+"/")
|
||||
addRouteCandidates(ri.localizedRoutes, trimmedRel, permalinks)
|
||||
case ri.isLocalePrefix(firstSegment):
|
||||
return nil
|
||||
default:
|
||||
addRouteCandidates(ri.sourceRoutes, relPath, permalinks)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func discoverLocalePrefixes(docsRoot string) (map[string]struct{}, error) {
|
||||
entries, err := os.ReadDir(docsRoot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
locales := map[string]struct{}{}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if !localeDirRe.MatchString(name) {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, name, ".i18n", "README.md")); err != nil {
|
||||
continue
|
||||
}
|
||||
locales[name] = struct{}{}
|
||||
}
|
||||
return locales, nil
|
||||
}
|
||||
|
||||
func isMarkdownFile(path string) bool {
|
||||
return strings.HasSuffix(path, ".md") || strings.HasSuffix(path, ".mdx")
|
||||
}
|
||||
|
||||
func normalizeSlashes(path string) string {
|
||||
return strings.ReplaceAll(path, "\\", "/")
|
||||
}
|
||||
|
||||
func firstPathSegment(relPath string) string {
|
||||
if relPath == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.SplitN(relPath, "/", 2)
|
||||
return parts[0]
|
||||
}
|
||||
|
||||
func addRouteCandidates(routes map[string]struct{}, relPath string, permalinks []string) {
|
||||
base := strings.TrimSuffix(strings.TrimSuffix(relPath, ".md"), ".mdx")
|
||||
if base != relPath {
|
||||
addRoute(routes, normalizeRoute(base))
|
||||
switch {
|
||||
case base == "index":
|
||||
addRoute(routes, "/")
|
||||
case strings.HasSuffix(base, "/index"):
|
||||
addRoute(routes, normalizeRoute(strings.TrimSuffix(base, "/index")))
|
||||
}
|
||||
}
|
||||
|
||||
for _, permalink := range permalinks {
|
||||
addRoute(routes, normalizeRoute(permalink))
|
||||
}
|
||||
}
|
||||
|
||||
func addRoute(routes map[string]struct{}, route string) {
|
||||
if route == "" {
|
||||
return
|
||||
}
|
||||
routes[route] = struct{}{}
|
||||
}
|
||||
|
||||
func extractPermalinks(content []byte) []string {
|
||||
frontMatter, _ := splitFrontMatter(string(content))
|
||||
if strings.TrimSpace(frontMatter) == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
data := map[string]any{}
|
||||
if err := yaml.Unmarshal([]byte(frontMatter), &data); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
raw, ok := data["permalink"].(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
permalink := strings.TrimSpace(raw)
|
||||
if permalink == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{permalink}
|
||||
}
|
||||
|
||||
func normalizeRoute(path string) string {
|
||||
trimmed := strings.TrimSpace(path)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
stripped := strings.Trim(trimmed, "/")
|
||||
if stripped == "" {
|
||||
return "/"
|
||||
}
|
||||
return "/" + stripped
|
||||
}
|
||||
|
||||
func (ri *routeIndex) localizeBodyLinks(body string) string {
|
||||
if ri == nil || ri.targetLang == "" || strings.EqualFold(ri.targetLang, "en") {
|
||||
return body
|
||||
}
|
||||
|
||||
state := NewPlaceholderState(body)
|
||||
placeholders := make([]string, 0, 8)
|
||||
mapping := map[string]string{}
|
||||
masked := maskMatches(body, fencedBacktickCodeBlock, state.Next, &placeholders, mapping)
|
||||
masked = maskMatches(masked, fencedTildeCodeBlock, state.Next, &placeholders, mapping)
|
||||
masked = maskMatches(masked, inlineCodeRe, state.Next, &placeholders, mapping)
|
||||
|
||||
masked = rewriteMarkdownLinkTargets(masked, ri)
|
||||
masked = rewriteHrefTargets(masked, ri)
|
||||
|
||||
return unmaskMarkdown(masked, placeholders, mapping)
|
||||
}
|
||||
|
||||
func rewriteMarkdownLinkTargets(text string, ri *routeIndex) string {
|
||||
matches := markdownLinkTargetRe.FindAllStringSubmatchIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
fullStart, targetStart, targetEnd := span[0], span[2], span[3]
|
||||
if fullStart < pos {
|
||||
continue
|
||||
}
|
||||
|
||||
out.WriteString(text[pos:targetStart])
|
||||
target := text[targetStart:targetEnd]
|
||||
if text[fullStart] == '!' {
|
||||
out.WriteString(target)
|
||||
} else {
|
||||
out.WriteString(ri.localizeURL(target))
|
||||
}
|
||||
pos = targetEnd
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func rewriteHrefTargets(text string, ri *routeIndex) string {
|
||||
text = rewriteCapturedTargets(text, hrefDoubleQuotedValueRe, 2, ri)
|
||||
text = rewriteCapturedTargets(text, hrefSingleQuotedValueRe, 2, ri)
|
||||
return text
|
||||
}
|
||||
|
||||
func rewriteCapturedTargets(text string, re *regexp.Regexp, groupIndex int, ri *routeIndex) string {
|
||||
matches := re.FindAllStringSubmatchIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
start, end := span[groupIndex], span[groupIndex+1]
|
||||
if start < pos || start < 0 || end < 0 {
|
||||
continue
|
||||
}
|
||||
out.WriteString(text[pos:start])
|
||||
out.WriteString(ri.localizeURL(text[start:end]))
|
||||
pos = end
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func (ri *routeIndex) localizeURL(raw string) string {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
if trimmed == "" {
|
||||
return raw
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "//") {
|
||||
return raw
|
||||
}
|
||||
if hasURLScheme(trimmed) {
|
||||
return raw
|
||||
}
|
||||
|
||||
pathPart, suffix := splitURLSuffix(trimmed)
|
||||
if !strings.HasPrefix(pathPart, "/") {
|
||||
return raw
|
||||
}
|
||||
|
||||
normalized := normalizeRoute(pathPart)
|
||||
if ri.routeHasLocalePrefix(normalized) {
|
||||
return raw
|
||||
}
|
||||
|
||||
canonical, ok := ri.resolveRoute(normalized)
|
||||
if !ok {
|
||||
return raw
|
||||
}
|
||||
if _, ok := ri.localizedRoutes[canonical]; !ok {
|
||||
return raw
|
||||
}
|
||||
|
||||
return prefixLocaleRoute(ri.targetLang, canonical) + suffix
|
||||
}
|
||||
|
||||
func hasURLScheme(raw string) bool {
|
||||
switch {
|
||||
case hasSchemePrefix(raw, "http://"), hasSchemePrefix(raw, "https://"):
|
||||
return true
|
||||
case hasSchemePrefix(raw, "mailto:"), hasSchemePrefix(raw, "tel:"):
|
||||
return true
|
||||
case hasSchemePrefix(raw, "data:"), hasSchemePrefix(raw, "javascript:"), hasSchemePrefix(raw, "vbscript:"):
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func hasSchemePrefix(raw, prefix string) bool {
|
||||
if len(raw) < len(prefix) {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(raw[:len(prefix)], prefix)
|
||||
}
|
||||
|
||||
func splitURLSuffix(raw string) (string, string) {
|
||||
index := strings.IndexAny(raw, "?#")
|
||||
if index == -1 {
|
||||
return raw, ""
|
||||
}
|
||||
return raw[:index], raw[index:]
|
||||
}
|
||||
|
||||
func prefixLocaleRoute(lang, route string) string {
|
||||
if route == "/" {
|
||||
return "/" + lang
|
||||
}
|
||||
return "/" + lang + route
|
||||
}
|
||||
|
||||
func (ri *routeIndex) routeHasLocalePrefix(route string) bool {
|
||||
if route == "/" {
|
||||
return false
|
||||
}
|
||||
firstSegment := strings.TrimPrefix(route, "/")
|
||||
firstSegment = strings.SplitN(firstSegment, "/", 2)[0]
|
||||
return ri.isLocalePrefix(firstSegment)
|
||||
}
|
||||
|
||||
func (ri *routeIndex) isLocalePrefix(segment string) bool {
|
||||
if segment == "" {
|
||||
return false
|
||||
}
|
||||
_, ok := ri.localePrefixes[segment]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (ri *routeIndex) resolveRoute(route string) (string, bool) {
|
||||
current := normalizeRoute(route)
|
||||
if current == "" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{current: {}}
|
||||
for {
|
||||
next, ok := ri.redirects[current]
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
current = next
|
||||
if _, ok := seen[current]; ok {
|
||||
return "", false
|
||||
}
|
||||
seen[current] = struct{}{}
|
||||
}
|
||||
|
||||
if current == "/" {
|
||||
_, ok := ri.localizedRoutes[current]
|
||||
return current, ok
|
||||
}
|
||||
if _, ok := ri.sourceRoutes[current]; ok {
|
||||
return current, true
|
||||
}
|
||||
if _, ok := ri.localizedRoutes[current]; ok {
|
||||
return current, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
182
scripts/docs-i18n/localized_links_test.go
Normal file
182
scripts/docs-i18n/localized_links_test.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocalizeBodyLinks(t *testing.T) {
|
||||
docsRoot := setupDocsTree(t)
|
||||
routes, err := loadRouteIndex(docsRoot, "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("loadRouteIndex failed: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "markdown link",
|
||||
input: `See [Config](/gateway/configuration).`,
|
||||
want: `See [Config](/zh-CN/gateway/configuration).`,
|
||||
},
|
||||
{
|
||||
name: "href attribute",
|
||||
input: `<Card href="/gateway/configuration" title="Config" />`,
|
||||
want: `<Card href="/zh-CN/gateway/configuration" title="Config" />`,
|
||||
},
|
||||
{
|
||||
name: "redirect source resolves to canonical localized page",
|
||||
input: `See [Sandbox](/sandboxing).`,
|
||||
want: `See [Sandbox](/zh-CN/gateway/sandboxing).`,
|
||||
},
|
||||
{
|
||||
name: "fragment is preserved",
|
||||
input: `See [Hooks](/gateway/configuration#hooks).`,
|
||||
want: `See [Hooks](/zh-CN/gateway/configuration#hooks).`,
|
||||
},
|
||||
{
|
||||
name: "images stay unchanged",
|
||||
input: ``,
|
||||
want: ``,
|
||||
},
|
||||
{
|
||||
name: "already localized stays unchanged",
|
||||
input: `See [Config](/zh-CN/gateway/configuration).`,
|
||||
want: `See [Config](/zh-CN/gateway/configuration).`,
|
||||
},
|
||||
{
|
||||
name: "vbscript scheme stays unchanged",
|
||||
input: `<a href="vbscript:msgbox(1)">bad</a>`,
|
||||
want: `<a href="vbscript:msgbox(1)">bad</a>`,
|
||||
},
|
||||
{
|
||||
name: "mixed-case javascript scheme stays unchanged",
|
||||
input: `<a href="Javascript:alert(1)">bad</a>`,
|
||||
want: `<a href="Javascript:alert(1)">bad</a>`,
|
||||
},
|
||||
{
|
||||
name: "missing localized page stays unchanged",
|
||||
input: `See [FAQ](/help/faq).`,
|
||||
want: `See [FAQ](/help/faq).`,
|
||||
},
|
||||
{
|
||||
name: "permalink route localizes",
|
||||
input: `See [Formal verification](/security/formal-verification).`,
|
||||
want: `See [Formal verification](/zh-CN/security/formal-verification).`,
|
||||
},
|
||||
{
|
||||
name: "inline code stays unchanged",
|
||||
input: "Use `[Config](/gateway/configuration)` in examples.\n\n" +
|
||||
"See [Config](/gateway/configuration).",
|
||||
want: "Use `[Config](/gateway/configuration)` in examples.\n\n" +
|
||||
"See [Config](/zh-CN/gateway/configuration).",
|
||||
},
|
||||
{
|
||||
name: "fenced code block stays unchanged",
|
||||
input: "```md\n[Config](/gateway/configuration)\n```\n\n" +
|
||||
"See [Config](/gateway/configuration).",
|
||||
want: "```md\n[Config](/gateway/configuration)\n```\n\n" +
|
||||
"See [Config](/zh-CN/gateway/configuration).",
|
||||
},
|
||||
{
|
||||
name: "inline code does not swallow later paragraphs",
|
||||
input: strings.Join([]string{
|
||||
"Use `channels.matrix.accounts` and `name`.",
|
||||
"",
|
||||
"See [Config](/gateway/configuration).",
|
||||
"",
|
||||
"Then review [Troubleshooting](/channels/troubleshooting).",
|
||||
}, "\n"),
|
||||
want: strings.Join([]string{
|
||||
"Use `channels.matrix.accounts` and `name`.",
|
||||
"",
|
||||
"See [Config](/zh-CN/gateway/configuration).",
|
||||
"",
|
||||
"Then review [Troubleshooting](/zh-CN/channels/troubleshooting).",
|
||||
}, "\n"),
|
||||
},
|
||||
{
|
||||
name: "indented fenced code block does not swallow later paragraphs",
|
||||
input: strings.Join([]string{
|
||||
"1. Setup:",
|
||||
"",
|
||||
" ```bash",
|
||||
" echo hi",
|
||||
" ```",
|
||||
"",
|
||||
"Use `channels.matrix.accounts` and `name`.",
|
||||
"",
|
||||
"For triage: [/channels/troubleshooting](/channels/troubleshooting).",
|
||||
"See [Config](/gateway/configuration).",
|
||||
}, "\n"),
|
||||
want: strings.Join([]string{
|
||||
"1. Setup:",
|
||||
"",
|
||||
" ```bash",
|
||||
" echo hi",
|
||||
" ```",
|
||||
"",
|
||||
"Use `channels.matrix.accounts` and `name`.",
|
||||
"",
|
||||
"For triage: [/channels/troubleshooting](/zh-CN/channels/troubleshooting).",
|
||||
"See [Config](/zh-CN/gateway/configuration).",
|
||||
}, "\n"),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := routes.localizeBodyLinks(tt.input)
|
||||
if got != tt.want {
|
||||
t.Fatalf("unexpected rewrite\nwant: %q\ngot: %q", tt.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func setupDocsTree(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
root := t.TempDir()
|
||||
writeFile(t, filepath.Join(root, "docs.json"), `{
|
||||
"redirects": [
|
||||
{ "source": "/sandboxing", "destination": "/gateway/sandboxing" }
|
||||
]
|
||||
}`)
|
||||
|
||||
files := map[string]string{
|
||||
"index.md": "# Home\n",
|
||||
"channels/troubleshooting.md": "# Troubleshooting\n",
|
||||
"gateway/configuration.md": "# Config\n",
|
||||
"gateway/sandboxing.md": "# Sandboxing\n",
|
||||
"security/formal-verification.md": "---\npermalink: /security/formal-verification/\n---\n\n# Formal verification\n",
|
||||
"help/faq.md": "# FAQ\n",
|
||||
"zh-CN/index.md": "# Home\n",
|
||||
"zh-CN/channels/troubleshooting.md": "# Troubleshooting\n",
|
||||
"zh-CN/gateway/configuration.md": "# Config\n",
|
||||
"zh-CN/gateway/sandboxing.md": "# Sandboxing\n",
|
||||
"zh-CN/security/formal-verification.md": "---\npermalink: /security/formal-verification/\n---\n\n# Formal verification\n",
|
||||
"ja-JP/index.md": "# Home\n",
|
||||
}
|
||||
|
||||
for relPath, content := range files {
|
||||
writeFile(t, filepath.Join(root, relPath), content)
|
||||
}
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path string, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir failed for %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write failed for %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
386
scripts/docs-i18n/main.go
Normal file
386
scripts/docs-i18n/main.go
Normal file
@@ -0,0 +1,386 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type docJob struct {
|
||||
index int
|
||||
path string
|
||||
rel string
|
||||
}
|
||||
|
||||
type docResult struct {
|
||||
index int
|
||||
rel string
|
||||
output string
|
||||
duration time.Duration
|
||||
skipped bool
|
||||
err error
|
||||
}
|
||||
|
||||
type runConfig struct {
|
||||
targetLang string
|
||||
sourceLang string
|
||||
docsRoot string
|
||||
tmPath string
|
||||
mode string
|
||||
thinking string
|
||||
overwrite bool
|
||||
allowPartial bool
|
||||
maxFiles int
|
||||
parallel int
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
|
||||
sourceLang = flag.String("src", "en", "source language")
|
||||
docsRoot = flag.String("docs", "docs", "docs root")
|
||||
tmPath = flag.String("tm", "", "translation memory path")
|
||||
mode = flag.String("mode", "segment", "translation mode (segment|doc)")
|
||||
thinking = flag.String("thinking", "high", "thinking level (low|medium|high|xhigh)")
|
||||
overwrite = flag.Bool("overwrite", false, "overwrite existing translations")
|
||||
allowPartial = flag.Bool("allow-partial", false, "write successful doc-mode outputs even when another file fails")
|
||||
maxFiles = flag.Int("max", 0, "max files to process (0 = all)")
|
||||
parallel = flag.Int("parallel", 1, "parallel workers for doc mode")
|
||||
)
|
||||
flag.Parse()
|
||||
files := flag.Args()
|
||||
if len(files) == 0 {
|
||||
fatal(fmt.Errorf("no doc files provided"))
|
||||
}
|
||||
|
||||
if err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: *targetLang,
|
||||
sourceLang: *sourceLang,
|
||||
docsRoot: *docsRoot,
|
||||
tmPath: *tmPath,
|
||||
mode: *mode,
|
||||
thinking: *thinking,
|
||||
overwrite: *overwrite,
|
||||
allowPartial: *allowPartial,
|
||||
maxFiles: *maxFiles,
|
||||
parallel: *parallel,
|
||||
}, files, func(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (docsTranslator, error) {
|
||||
return NewCodexTranslator(srcLang, tgtLang, glossary, thinking)
|
||||
}); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslator docsTranslatorFactory) error {
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("no doc files provided")
|
||||
}
|
||||
|
||||
resolvedDocsRoot, err := filepath.Abs(cfg.docsRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmPath := cfg.tmPath
|
||||
if tmPath == "" {
|
||||
tmPath = filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("%s.tm.jsonl", cfg.targetLang))
|
||||
}
|
||||
|
||||
glossaryPath := filepath.Join(resolvedDocsRoot, ".i18n", fmt.Sprintf("glossary.%s.json", cfg.targetLang))
|
||||
glossary, err := LoadGlossary(glossaryPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tm, err := LoadTranslationMemory(tmPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ordered, err := orderFiles(resolvedDocsRoot, files)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
totalFiles := len(ordered)
|
||||
preSkipped := 0
|
||||
prePostprocessFiles := []string{}
|
||||
if cfg.mode == "doc" && !cfg.overwrite {
|
||||
filtered, skipped, existingOutputs, err := filterDocQueue(resolvedDocsRoot, cfg.targetLang, ordered, cfg.maxFiles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ordered = filtered
|
||||
preSkipped = skipped
|
||||
prePostprocessFiles = append(prePostprocessFiles, existingOutputs...)
|
||||
}
|
||||
if (cfg.mode != "doc" || cfg.overwrite) && cfg.maxFiles > 0 && cfg.maxFiles < len(ordered) {
|
||||
ordered = ordered[:cfg.maxFiles]
|
||||
}
|
||||
|
||||
parallel := cfg.parallel
|
||||
if parallel < 1 {
|
||||
parallel = 1
|
||||
}
|
||||
|
||||
log.SetFlags(log.LstdFlags)
|
||||
start := time.Now()
|
||||
processed := 0
|
||||
skipped := 0
|
||||
localizedFiles := append([]string{}, prePostprocessFiles...)
|
||||
var translationErr error
|
||||
|
||||
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel)
|
||||
switch cfg.mode {
|
||||
case "doc":
|
||||
if parallel > 1 {
|
||||
proc, skip, outputs, err := runDocParallel(ctx, ordered, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial, parallel, glossary, cfg.thinking, newTranslator)
|
||||
processed += proc
|
||||
skipped += skip
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
translationErr = err
|
||||
}
|
||||
} else {
|
||||
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer translator.Close()
|
||||
proc, skip, outputs, err := runDocSequential(ctx, ordered, translator, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial)
|
||||
processed += proc
|
||||
skipped += skip
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
translationErr = err
|
||||
}
|
||||
}
|
||||
case "segment":
|
||||
if parallel > 1 {
|
||||
return fmt.Errorf("parallel processing is only supported in doc mode")
|
||||
}
|
||||
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer translator.Close()
|
||||
proc, outputs, err := runSegmentSequential(ctx, ordered, translator, tm, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang)
|
||||
processed += proc
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
translationErr = err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown mode: %s", cfg.mode)
|
||||
}
|
||||
|
||||
if err := tm.Save(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil {
|
||||
return err
|
||||
}
|
||||
elapsed := time.Since(start).Round(time.Millisecond)
|
||||
log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed)
|
||||
if translationErr != nil && cfg.allowPartial && cfg.mode == "doc" && processed > 0 {
|
||||
if ctx.Err() != nil || errors.Is(translationErr, context.Canceled) || errors.Is(translationErr, context.DeadlineExceeded) {
|
||||
return translationErr
|
||||
}
|
||||
log.Printf("docs-i18n: allowing partial doc output after translation error: %v", translationErr)
|
||||
return nil
|
||||
}
|
||||
return translationErr
|
||||
}
|
||||
|
||||
func runDocSequential(ctx context.Context, ordered []string, translator docsTranslator, docsRoot, srcLang, tgtLang string, overwrite, allowPartial bool) (int, int, []string, error) {
|
||||
processed := 0
|
||||
skipped := 0
|
||||
outputs := []string{}
|
||||
var firstErr error
|
||||
for index, file := range ordered {
|
||||
relPath := resolveRelPath(docsRoot, file)
|
||||
log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
|
||||
start := time.Now()
|
||||
skip, outputPath, err := processFileDoc(ctx, translator, docsRoot, file, srcLang, tgtLang, overwrite)
|
||||
if err != nil {
|
||||
if shouldStopDocRun(ctx, err, allowPartial) {
|
||||
return processed, skipped, outputs, err
|
||||
}
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
log.Printf("docs-i18n: [%d/%d] failed %s (%s): %v", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond), err)
|
||||
continue
|
||||
}
|
||||
if skip {
|
||||
skipped++
|
||||
if outputPath != "" {
|
||||
outputs = append(outputs, outputPath)
|
||||
}
|
||||
log.Printf("docs-i18n: [%d/%d] skipped %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
|
||||
} else {
|
||||
processed++
|
||||
outputs = append(outputs, outputPath)
|
||||
log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
}
|
||||
return processed, skipped, outputs, firstErr
|
||||
}
|
||||
|
||||
func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tgtLang string, overwrite, allowPartial bool, parallel int, glossary []GlossaryEntry, thinking string, newTranslator docsTranslatorFactory) (int, int, []string, error) {
|
||||
jobs := make(chan docJob)
|
||||
results := make(chan docResult, len(ordered))
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for worker := 0; worker < parallel; worker++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
translator, err := newTranslator(srcLang, tgtLang, glossary, thinking)
|
||||
if err != nil {
|
||||
results <- docResult{err: err}
|
||||
return
|
||||
}
|
||||
defer translator.Close()
|
||||
for job := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
log.Printf("docs-i18n: [w%d %d/%d] start %s", workerID, job.index, len(ordered), job.rel)
|
||||
start := time.Now()
|
||||
skip, outputPath, err := processFileDoc(ctx, translator, docsRoot, job.path, srcLang, tgtLang, overwrite)
|
||||
results <- docResult{
|
||||
index: job.index,
|
||||
rel: job.rel,
|
||||
output: outputPath,
|
||||
duration: time.Since(start),
|
||||
skipped: skip,
|
||||
err: err,
|
||||
}
|
||||
if err != nil && shouldStopDocRun(ctx, err, allowPartial) {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}(worker + 1)
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(jobs)
|
||||
for index, file := range ordered {
|
||||
job := docJob{index: index + 1, path: file, rel: resolveRelPath(docsRoot, file)}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case jobs <- job:
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
processed := 0
|
||||
skipped := 0
|
||||
outputs := []string{}
|
||||
var firstErr error
|
||||
for result := range results {
|
||||
if result.err != nil && firstErr == nil {
|
||||
firstErr = result.err
|
||||
}
|
||||
if result.skipped {
|
||||
skipped++
|
||||
if result.output != "" {
|
||||
outputs = append(outputs, result.output)
|
||||
}
|
||||
log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
|
||||
} else if result.err != nil {
|
||||
log.Printf("docs-i18n: [w* %d/%d] failed %s (%s): %v", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond), result.err)
|
||||
} else if result.err == nil {
|
||||
processed++
|
||||
outputs = append(outputs, result.output)
|
||||
log.Printf("docs-i18n: [w* %d/%d] done %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
|
||||
}
|
||||
}
|
||||
return processed, skipped, outputs, firstErr
|
||||
}
|
||||
|
||||
func shouldStopDocRun(ctx context.Context, err error, allowPartial bool) bool {
|
||||
if !allowPartial {
|
||||
return true
|
||||
}
|
||||
return ctx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
func runSegmentSequential(ctx context.Context, ordered []string, translator docsTranslator, tm *TranslationMemory, docsRoot, srcLang, tgtLang string) (int, []string, error) {
|
||||
processed := 0
|
||||
outputs := []string{}
|
||||
for index, file := range ordered {
|
||||
relPath := resolveRelPath(docsRoot, file)
|
||||
log.Printf("docs-i18n: [%d/%d] start %s", index+1, len(ordered), relPath)
|
||||
start := time.Now()
|
||||
_, outputPath, err := processFile(ctx, translator, tm, docsRoot, file, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return processed, outputs, err
|
||||
}
|
||||
processed++
|
||||
outputs = append(outputs, outputPath)
|
||||
log.Printf("docs-i18n: [%d/%d] done %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
return processed, outputs, nil
|
||||
}
|
||||
|
||||
func resolveRelPath(docsRoot, file string) string {
|
||||
relPath := file
|
||||
if _, rel, err := resolveDocsPath(docsRoot, file); err == nil {
|
||||
relPath = rel
|
||||
}
|
||||
return relPath
|
||||
}
|
||||
|
||||
func filterDocQueue(docsRoot, targetLang string, ordered []string, maxFiles int) ([]string, int, []string, error) {
|
||||
pending := make([]string, 0, len(ordered))
|
||||
existingOutputs := []string{}
|
||||
skipped := 0
|
||||
for _, file := range ordered {
|
||||
absPath, relPath, err := resolveDocsPath(docsRoot, file)
|
||||
if err != nil {
|
||||
return nil, skipped, nil, err
|
||||
}
|
||||
content, err := os.ReadFile(absPath)
|
||||
if err != nil {
|
||||
return nil, skipped, nil, err
|
||||
}
|
||||
sourceHash := hashBytes(content)
|
||||
outputPath := filepath.Join(docsRoot, targetLang, relPath)
|
||||
status, err := classifyDocOutput(outputPath, sourceHash, targetLang)
|
||||
if err != nil {
|
||||
return nil, skipped, nil, err
|
||||
}
|
||||
switch status {
|
||||
case docOutputReady:
|
||||
skipped++
|
||||
case docOutputNeedsPostprocess:
|
||||
if maxFiles > 0 && len(pending)+len(existingOutputs) >= maxFiles {
|
||||
continue
|
||||
}
|
||||
skipped++
|
||||
existingOutputs = append(existingOutputs, outputPath)
|
||||
case docOutputNeedsTranslation:
|
||||
if maxFiles > 0 && len(pending)+len(existingOutputs) >= maxFiles {
|
||||
continue
|
||||
}
|
||||
pending = append(pending, file)
|
||||
}
|
||||
}
|
||||
return pending, skipped, existingOutputs, nil
|
||||
}
|
||||
787
scripts/docs-i18n/main_test.go
Normal file
787
scripts/docs-i18n/main_test.go
Normal file
@@ -0,0 +1,787 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeDocsTranslator struct{}
|
||||
|
||||
func (fakeDocsTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (fakeDocsTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
// Keep the fake translator deterministic so this test exercises the
|
||||
// docs-i18n pipeline wiring and final link relocalization, not model output.
|
||||
replaced := strings.NewReplacer(
|
||||
"Gateway", "网关",
|
||||
"See ", "参见 ",
|
||||
).Replace(text)
|
||||
return replaced, nil
|
||||
}
|
||||
|
||||
func (fakeDocsTranslator) Close() {}
|
||||
|
||||
type invalidFrontmatterTranslator struct{}
|
||||
|
||||
func (invalidFrontmatterTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return "<body>\n" + text + "\n</body>\n", nil
|
||||
}
|
||||
|
||||
func (invalidFrontmatterTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (invalidFrontmatterTranslator) Close() {}
|
||||
|
||||
type transcriptFrontmatterTranslator struct{}
|
||||
|
||||
func (transcriptFrontmatterTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text + ` analysis to=functions.read {"path":"/home/runner/work/docs/docs/source/.agents/skills/openclaw-pr-maintainer/SKILL.md"} code`, nil
|
||||
}
|
||||
|
||||
func (transcriptFrontmatterTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (transcriptFrontmatterTranslator) Close() {}
|
||||
|
||||
type errorTranslator struct{}
|
||||
|
||||
func (errorTranslator) Translate(context.Context, string, string, string) (string, error) {
|
||||
return "", errors.New("codex exec failed: exit status 1")
|
||||
}
|
||||
|
||||
func (errorTranslator) TranslateRaw(context.Context, string, string, string) (string, error) {
|
||||
return "", errors.New("codex exec failed: exit status 1")
|
||||
}
|
||||
|
||||
func (errorTranslator) Close() {}
|
||||
|
||||
type partialFailTranslator struct{}
|
||||
|
||||
func (partialFailTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
if strings.Contains(text, "FAIL") {
|
||||
return "", errors.New("translation failed")
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (partialFailTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
if strings.Contains(text, "FAIL") {
|
||||
return "", errors.New("translation failed")
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (partialFailTranslator) Close() {}
|
||||
|
||||
type partialFailSlowTranslator struct{}
|
||||
|
||||
func (partialFailSlowTranslator) Translate(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
|
||||
if strings.Contains(text, "SLOW") {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return partialFailTranslator{}.Translate(ctx, text, srcLang, tgtLang)
|
||||
}
|
||||
|
||||
func (partialFailSlowTranslator) TranslateRaw(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
|
||||
if strings.Contains(text, "SLOW") {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return partialFailTranslator{}.TranslateRaw(ctx, text, srcLang, tgtLang)
|
||||
}
|
||||
|
||||
func (partialFailSlowTranslator) Close() {}
|
||||
|
||||
type cancelAwareTranslator struct{}
|
||||
|
||||
func (cancelAwareTranslator) Translate(ctx context.Context, text, _, _ string) (string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (cancelAwareTranslator) TranslateRaw(ctx context.Context, text, _, _ string) (string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (cancelAwareTranslator) Close() {}
|
||||
|
||||
type contextErrorTranslator struct{}
|
||||
|
||||
func (contextErrorTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
if strings.Contains(text, "CANCEL") {
|
||||
return "", context.Canceled
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (contextErrorTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
|
||||
if strings.Contains(text, "CANCEL") {
|
||||
return "", context.Canceled
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (contextErrorTranslator) Close() {}
|
||||
|
||||
type cancelAfterFirstDocTranslator struct {
|
||||
cancel context.CancelFunc
|
||||
calls int
|
||||
}
|
||||
|
||||
func (t *cancelAfterFirstDocTranslator) Translate(ctx context.Context, text, _, _ string) (string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (t *cancelAfterFirstDocTranslator) TranslateRaw(ctx context.Context, text, _, _ string) (string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
t.calls++
|
||||
if t.calls == 1 {
|
||||
t.cancel()
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (t *cancelAfterFirstDocTranslator) Close() {}
|
||||
|
||||
func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), stringsJoin(
|
||||
"---",
|
||||
"title: Gateway",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
"",
|
||||
"See [Example provider](/providers/example-provider).",
|
||||
))
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "providers", "example-provider.md"), "# Example provider\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "providers", "example-provider.md"), "# 示例 provider\n")
|
||||
|
||||
// This is the higher-level regression for the bug fixed in this PR:
|
||||
// if the pipeline stops wiring postprocess through the main flow, the final
|
||||
// localized output page will keep stale English-root links and this test fails.
|
||||
err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
parallel: 1,
|
||||
}, []string{filepath.Join(docsRoot, "gateway", "index.md")}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return fakeDocsTranslator{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runDocsI18N failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"))
|
||||
expected := []string{
|
||||
"参见 [Troubleshooting](/zh-CN/gateway/troubleshooting).",
|
||||
"参见 [Example provider](/zh-CN/providers/example-provider).",
|
||||
}
|
||||
for _, want := range expected {
|
||||
if !containsLine(got, want) {
|
||||
t.Fatalf("expected final localized page link %q in output:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NDoesNotSkipOutputAfterPostprocessFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(docsRoot, "gateway", "index.md")
|
||||
writeFile(t, sourcePath, stringsJoin(
|
||||
"---",
|
||||
"title: Gateway",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
skip, outputPath, err := processFileDoc(context.Background(), fakeDocsTranslator{}, docsRoot, sourcePath, "en", "zh-CN", true)
|
||||
if err != nil {
|
||||
t.Fatalf("processFileDoc failed: %v", err)
|
||||
}
|
||||
if skip {
|
||||
t.Fatal("processFileDoc unexpectedly skipped translation")
|
||||
}
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{outputPath}); err == nil {
|
||||
t.Fatal("expected missing docs.json to fail postprocess")
|
||||
}
|
||||
|
||||
sourceBytes, err := os.ReadFile(sourcePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read source failed: %v", err)
|
||||
}
|
||||
status, err := classifyDocOutput(outputPath, hashBytes(sourceBytes), "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("classifyDocOutput failed: %v", err)
|
||||
}
|
||||
if status != docOutputNeedsPostprocess {
|
||||
t.Fatalf("expected failed-postprocess output to need postprocess, got %v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NOnlyBecomesSkippableAfterPostprocessSucceeds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
sourcePath := filepath.Join(docsRoot, "gateway", "index.md")
|
||||
writeFile(t, sourcePath, stringsJoin(
|
||||
"---",
|
||||
"title: Gateway",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
|
||||
skip, outputPath, err := processFileDoc(context.Background(), fakeDocsTranslator{}, docsRoot, sourcePath, "en", "zh-CN", true)
|
||||
if err != nil {
|
||||
t.Fatalf("processFileDoc failed: %v", err)
|
||||
}
|
||||
if skip {
|
||||
t.Fatal("processFileDoc unexpectedly skipped translation")
|
||||
}
|
||||
|
||||
sourceBytes, err := os.ReadFile(sourcePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read source failed: %v", err)
|
||||
}
|
||||
status, err := classifyDocOutput(outputPath, hashBytes(sourceBytes), "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("classifyDocOutput before postprocess failed: %v", err)
|
||||
}
|
||||
if status != docOutputNeedsPostprocess {
|
||||
t.Fatalf("expected pending postprocess output to need postprocess, got %v", status)
|
||||
}
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{outputPath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
status, err = classifyDocOutput(outputPath, hashBytes(sourceBytes), "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("classifyDocOutput after postprocess failed: %v", err)
|
||||
}
|
||||
if status != docOutputReady {
|
||||
t.Fatalf("expected postprocessed output to be ready, got %v:\n%s", status, mustReadFile(t, outputPath))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyDocOutputKeepsEnglishTargetsHashOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(docsRoot, "gateway", "index.md")
|
||||
writeFile(t, sourcePath, stringsJoin(
|
||||
"---",
|
||||
"title: Gateway",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
outputPath := filepath.Join(docsRoot, "en", "gateway", "index.md")
|
||||
writeFile(t, outputPath, stringsJoin(
|
||||
"---",
|
||||
"title: Gateway",
|
||||
"x-i18n:",
|
||||
" source_hash: "+hashBytes([]byte(mustReadFile(t, sourcePath))),
|
||||
" postprocess_version: "+localizedLinkPostprocessPending,
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
status, err := classifyDocOutput(outputPath, hashBytes([]byte(mustReadFile(t, sourcePath))), "en")
|
||||
if err != nil {
|
||||
t.Fatalf("classifyDocOutput for English target failed: %v", err)
|
||||
}
|
||||
if status != docOutputReady {
|
||||
t.Fatalf("expected English target to remain ready with matching source hash, got %v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterDocQueueSchedulesLegacyOutputsForPostprocessOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(docsRoot, "gateway", "index.md")
|
||||
writeFile(t, sourcePath, "# Gateway\n")
|
||||
outputPath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
writeFile(t, outputPath, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"x-i18n:",
|
||||
" source_hash: "+hashBytes([]byte(mustReadFile(t, sourcePath))),
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
pending, skipped, existingOutputs, err := filterDocQueue(docsRoot, "zh-CN", []string{sourcePath}, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("filterDocQueue failed: %v", err)
|
||||
}
|
||||
if len(pending) != 0 {
|
||||
t.Fatalf("expected legacy matching output to skip translation, got pending=%v", pending)
|
||||
}
|
||||
if skipped != 1 {
|
||||
t.Fatalf("expected one skipped translation, got %d", skipped)
|
||||
}
|
||||
if len(existingOutputs) != 1 || existingOutputs[0] != outputPath {
|
||||
t.Fatalf("expected existing output to be queued for postprocess, got %v", existingOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterDocQueueHonorsMaxAcrossPostprocessOutputs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
firstSource := filepath.Join(docsRoot, "gateway", "index.md")
|
||||
secondSource := filepath.Join(docsRoot, "providers", "example-provider.md")
|
||||
writeFile(t, firstSource, "# Gateway\n")
|
||||
writeFile(t, secondSource, "# Example provider\n")
|
||||
firstOutput := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
secondOutput := filepath.Join(docsRoot, "zh-CN", "providers", "example-provider.md")
|
||||
writeFile(t, firstOutput, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"x-i18n:",
|
||||
" source_hash: "+hashBytes([]byte(mustReadFile(t, firstSource))),
|
||||
"---",
|
||||
"",
|
||||
"# 网关",
|
||||
))
|
||||
writeFile(t, secondOutput, stringsJoin(
|
||||
"---",
|
||||
"title: 示例 provider",
|
||||
"x-i18n:",
|
||||
" source_hash: "+hashBytes([]byte(mustReadFile(t, secondSource))),
|
||||
"---",
|
||||
"",
|
||||
"# 示例 provider",
|
||||
))
|
||||
|
||||
pending, skipped, existingOutputs, err := filterDocQueue(docsRoot, "zh-CN", []string{firstSource, secondSource}, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("filterDocQueue failed: %v", err)
|
||||
}
|
||||
if len(pending) != 0 {
|
||||
t.Fatalf("expected no translations to be queued, got %v", pending)
|
||||
}
|
||||
if skipped != 1 {
|
||||
t.Fatalf("expected one bounded postprocess-only skip, got %d", skipped)
|
||||
}
|
||||
if len(existingOutputs) != 1 || existingOutputs[0] != firstOutput {
|
||||
t.Fatalf("expected only first output to be queued for postprocess, got %v", existingOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NAllowPartialKeepsEarlierSuccessfulDocOutputs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
okPath := filepath.Join(docsRoot, "aaa-ok.md")
|
||||
failPath := filepath.Join(docsRoot, "zzz-fail.md")
|
||||
writeFile(t, okPath, "# Gateway\n")
|
||||
writeFile(t, failPath, "# FAIL\n")
|
||||
|
||||
err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
allowPartial: true,
|
||||
parallel: 1,
|
||||
}, []string{okPath, failPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return partialFailTranslator{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runDocsI18N failed despite partial output: %v", err)
|
||||
}
|
||||
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "aaa-ok.md")); !strings.Contains(got, "# Gateway") {
|
||||
t.Fatalf("expected successful output to be written, got:\n%s", got)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "zzz-fail.md")); err == nil {
|
||||
t.Fatal("did not expect failed output to be written")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NAllowPartialContinuesAfterFailedDoc(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
failPath := filepath.Join(docsRoot, "aaa-fail.md")
|
||||
okPath := filepath.Join(docsRoot, "zzz-ok.md")
|
||||
writeFile(t, failPath, "# FAIL\n")
|
||||
writeFile(t, okPath, "# Gateway\n")
|
||||
|
||||
err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
allowPartial: true,
|
||||
parallel: 1,
|
||||
}, []string{failPath, okPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return partialFailTranslator{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runDocsI18N failed despite later partial output: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "aaa-fail.md")); err == nil {
|
||||
t.Fatal("did not expect failed output to be written")
|
||||
}
|
||||
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "zzz-ok.md")); !strings.Contains(got, "# Gateway") {
|
||||
t.Fatalf("expected later successful output to be written, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NAllowPartialParallelKeepsQueuedDocsAfterFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
failPath := filepath.Join(docsRoot, "aaa-fail.md")
|
||||
slowPath := filepath.Join(docsRoot, "bbb-slow.md")
|
||||
okPath := filepath.Join(docsRoot, "zzz-ok.md")
|
||||
writeFile(t, failPath, "# FAIL\n")
|
||||
writeFile(t, slowPath, "# SLOW\n")
|
||||
writeFile(t, okPath, "# Gateway\n")
|
||||
|
||||
err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
allowPartial: true,
|
||||
parallel: 2,
|
||||
}, []string{failPath, slowPath, okPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return partialFailSlowTranslator{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runDocsI18N failed despite later parallel output: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "aaa-fail.md")); err == nil {
|
||||
t.Fatal("did not expect failed output to be written")
|
||||
}
|
||||
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "bbb-slow.md")); !strings.Contains(got, "# SLOW") {
|
||||
t.Fatalf("expected in-flight output to be written after a failed doc, got:\n%s", got)
|
||||
}
|
||||
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "zzz-ok.md")); !strings.Contains(got, "# Gateway") {
|
||||
t.Fatalf("expected queued output to be written after a failed doc, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NAllowPartialStopsAfterRunCancellation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
firstPath := filepath.Join(docsRoot, "aaa-first.md")
|
||||
secondPath := filepath.Join(docsRoot, "zzz-second.md")
|
||||
writeFile(t, firstPath, "# Gateway\n")
|
||||
writeFile(t, secondPath, "# Gateway\n")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := runDocsI18N(ctx, runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
allowPartial: true,
|
||||
parallel: 1,
|
||||
}, []string{firstPath, secondPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return cancelAwareTranslator{}, nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected canceled run to fail even with allowPartial=true")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "zzz-second.md")); err == nil {
|
||||
t.Fatal("did not expect later output to be written after run cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NAllowPartialReturnsCancellationAfterPartialSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
firstPath := filepath.Join(docsRoot, "aaa-first.md")
|
||||
secondPath := filepath.Join(docsRoot, "zzz-second.md")
|
||||
writeFile(t, firstPath, "# Gateway\n")
|
||||
writeFile(t, secondPath, "# Gateway\n")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
err := runDocsI18N(ctx, runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
allowPartial: true,
|
||||
parallel: 1,
|
||||
}, []string{firstPath, secondPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return &cancelAfterFirstDocTranslator{cancel: cancel}, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected canceled run after partial success, got %v", err)
|
||||
}
|
||||
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "aaa-first.md")); !strings.Contains(got, "# Gateway") {
|
||||
t.Fatalf("expected first output to be written before cancellation, got:\n%s", got)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "zzz-second.md")); err == nil {
|
||||
t.Fatal("did not expect later output to be written after run cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NAllowPartialStopsAfterContextError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
firstPath := filepath.Join(docsRoot, "aaa-first.md")
|
||||
cancelPath := filepath.Join(docsRoot, "bbb-cancel.md")
|
||||
laterPath := filepath.Join(docsRoot, "zzz-later.md")
|
||||
writeFile(t, firstPath, "# Gateway\n")
|
||||
writeFile(t, cancelPath, "# CANCEL\n")
|
||||
writeFile(t, laterPath, "# Gateway\n")
|
||||
|
||||
err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: "zh-CN",
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "high",
|
||||
overwrite: true,
|
||||
allowPartial: true,
|
||||
parallel: 1,
|
||||
}, []string{firstPath, cancelPath, laterPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
|
||||
return contextErrorTranslator{}, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected cancellation error to remain terminal, got %v", err)
|
||||
}
|
||||
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "aaa-first.md")); !strings.Contains(got, "# Gateway") {
|
||||
t.Fatalf("expected first output to be written before cancellation, got:\n%s", got)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "zzz-later.md")); err == nil {
|
||||
t.Fatal("did not expect later output to be written after context error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDocsI18NRewritesLineTitleFromExactGlossaryWithoutModel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
linePath := filepath.Join(docsRoot, "channels", "line.md")
|
||||
writeFile(t, linePath, stringsJoin(
|
||||
"---",
|
||||
"title: LINE",
|
||||
"---",
|
||||
"",
|
||||
))
|
||||
|
||||
locales := []string{"zh-CN", "zh-TW", "de", "es"}
|
||||
for _, locale := range locales {
|
||||
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary."+locale+".json"), `[{"source":"LINE","target":"LINE"}]`)
|
||||
writeFile(t, filepath.Join(docsRoot, locale, "channels", "line.md"), stringsJoin(
|
||||
"---",
|
||||
"title: 行",
|
||||
"---",
|
||||
"",
|
||||
))
|
||||
|
||||
err := runDocsI18N(context.Background(), runConfig{
|
||||
targetLang: locale,
|
||||
sourceLang: "en",
|
||||
docsRoot: docsRoot,
|
||||
mode: "doc",
|
||||
thinking: "low",
|
||||
overwrite: true,
|
||||
parallel: 1,
|
||||
}, []string{linePath}, func(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (docsTranslator, error) {
|
||||
translator, err := NewCodexTranslator(srcLang, tgtLang, glossary, thinking)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
translator.runPrompt = func(context.Context, codexPromptRequest) (string, error) {
|
||||
t.Fatalf("exact LINE title for %s should not call Codex", tgtLang)
|
||||
return "", nil
|
||||
}
|
||||
return translator, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runDocsI18N(%s) failed: %v", locale, err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, filepath.Join(docsRoot, locale, "channels", "line.md"))
|
||||
if !containsLine(got, "title: LINE") {
|
||||
t.Fatalf("expected %s title to stay LINE, got:\n%s", locale, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateSnippetDoesNotCacheFallbackToSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tm := &TranslationMemory{entries: map[string]TMEntry{}}
|
||||
source := "Gateway"
|
||||
|
||||
translated, err := translateSnippet(context.Background(), invalidFrontmatterTranslator{}, tm, "gateway/index.md:frontmatter:title", source, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateSnippet returned error: %v", err)
|
||||
}
|
||||
if translated != source {
|
||||
t.Fatalf("expected fallback to source text, got %q", translated)
|
||||
}
|
||||
|
||||
cacheKey := cacheKey(cacheNamespace(), "en", "zh-CN", "gateway/index.md:frontmatter:title", hashText(source))
|
||||
if _, ok := tm.Get(cacheKey); ok {
|
||||
t.Fatalf("expected fallback translation not to be cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateSnippetRejectsTranscriptArtifact(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tm := &TranslationMemory{entries: map[string]TMEntry{}}
|
||||
source := "Working with reactions across channels"
|
||||
|
||||
translated, err := translateSnippet(context.Background(), transcriptFrontmatterTranslator{}, tm, "tools/reactions.md:frontmatter:read_when:0", source, "en", "th")
|
||||
if err != nil {
|
||||
t.Fatalf("translateSnippet returned error: %v", err)
|
||||
}
|
||||
if translated != source {
|
||||
t.Fatalf("expected fallback to source text, got %q", translated)
|
||||
}
|
||||
|
||||
cacheKey := cacheKey(cacheNamespace(), "en", "th", "tools/reactions.md:frontmatter:read_when:0", hashText(source))
|
||||
if _, ok := tm.Get(cacheKey); ok {
|
||||
t.Fatalf("expected fallback translation not to be cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateSnippetFallsBackWhenFrontmatterTranslatorFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tm := &TranslationMemory{entries: map[string]TMEntry{}}
|
||||
source := "LINE Messaging API plugin setup, config, and usage"
|
||||
|
||||
translated, err := translateSnippet(context.Background(), errorTranslator{}, tm, "channels/line.md:frontmatter:summary", source, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateSnippet returned error: %v", err)
|
||||
}
|
||||
if translated != source {
|
||||
t.Fatalf("expected fallback to source text, got %q", translated)
|
||||
}
|
||||
|
||||
cacheKey := cacheKey(cacheNamespace(), "en", "zh-CN", "channels/line.md:frontmatter:summary", hashText(source))
|
||||
if _, ok := tm.Get(cacheKey); ok {
|
||||
t.Fatalf("expected failed frontmatter translation not to be cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateSnippetCachesDocumentSourcePath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tm := &TranslationMemory{entries: map[string]TMEntry{}}
|
||||
source := "Gateway"
|
||||
segmentID := "gateway/index.md:frontmatter:title"
|
||||
|
||||
translated, err := translateSnippet(context.Background(), fakeDocsTranslator{}, tm, segmentID, source, "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("translateSnippet returned error: %v", err)
|
||||
}
|
||||
if translated != source {
|
||||
t.Fatalf("unexpected translation %q", translated)
|
||||
}
|
||||
|
||||
cacheKey := cacheKey(cacheNamespace(), "en", "zh-CN", segmentID, hashText(source))
|
||||
entry, ok := tm.Get(cacheKey)
|
||||
if !ok {
|
||||
t.Fatal("expected successful frontmatter translation to be cached")
|
||||
}
|
||||
if entry.SourcePath != "gateway/index.md" {
|
||||
t.Fatalf("expected document source path, got %q", entry.SourcePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNoTranslationTranscriptArtifacts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []string{
|
||||
`表情回应 analysis to=functions.read {"path":"/home/runner/work/docs/docs/source/.agents/skills/openclaw-qa-testing/SKILL.md"} code`,
|
||||
"<openclaw_docs_i18n_input>\nTranslated\n</openclaw_docs_i18n_input>",
|
||||
`กำลังทำงานกับ reactions to=functions.read commentary  ̄第四色json 皇平台`,
|
||||
`คุณต้องการแผนที่เอกสาร analysis to=final code omitted`,
|
||||
`Potrzebujesz listy funkcji TUI force_parallel: false} code`,
|
||||
`กำลังตัดสินใจว่าจะกำหนดค่าผู้ให้บริการสื่อรายใด 全民彩票 casino`,
|
||||
}
|
||||
for _, translated := range tests {
|
||||
if err := validateNoTranslationTranscriptArtifacts("Working with reactions across channels", translated); err == nil {
|
||||
t.Fatalf("expected artifact to be rejected: %q", translated)
|
||||
}
|
||||
}
|
||||
|
||||
source := "Document `functions.read` examples exactly."
|
||||
if err := validateNoTranslationTranscriptArtifacts(source, "Document `functions.read` examples exactly."); err != nil {
|
||||
t.Fatalf("expected source-owned token to be allowed: %v", err)
|
||||
}
|
||||
}
|
||||
131
scripts/docs-i18n/markdown_segments.go
Normal file
131
scripts/docs-i18n/markdown_segments.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/ast"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
"github.com/yuin/goldmark/text"
|
||||
)
|
||||
|
||||
func extractSegments(body, relPath string) ([]Segment, error) {
|
||||
source := []byte(body)
|
||||
r := text.NewReader(source)
|
||||
md := goldmark.New(
|
||||
goldmark.WithExtensions(extension.GFM),
|
||||
)
|
||||
doc := md.Parser().Parse(r)
|
||||
|
||||
segments := make([]Segment, 0, 128)
|
||||
skipDepth := 0
|
||||
var lastBlock ast.Node
|
||||
|
||||
err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
switch n.(type) {
|
||||
case *ast.CodeBlock, *ast.FencedCodeBlock, *ast.CodeSpan, *ast.HTMLBlock, *ast.RawHTML:
|
||||
if entering {
|
||||
skipDepth++
|
||||
} else {
|
||||
skipDepth--
|
||||
}
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
|
||||
if !entering || skipDepth > 0 {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
|
||||
textNode, ok := n.(*ast.Text)
|
||||
if !ok {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
block := blockParent(textNode)
|
||||
if block == nil {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
textValue := string(textNode.Segment.Value(source))
|
||||
if strings.TrimSpace(textValue) == "" {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
|
||||
start := textNode.Segment.Start
|
||||
stop := textNode.Segment.Stop
|
||||
if len(segments) > 0 && lastBlock == block {
|
||||
last := &segments[len(segments)-1]
|
||||
gap := string(source[last.Stop:start])
|
||||
if strings.TrimSpace(gap) == "" {
|
||||
last.Stop = stop
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
}
|
||||
|
||||
segments = append(segments, Segment{Start: start, Stop: stop})
|
||||
lastBlock = block
|
||||
return ast.WalkContinue, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filtered := make([]Segment, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
textValue := string(source[seg.Start:seg.Stop])
|
||||
trimmed := strings.TrimSpace(textValue)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
textHash := hashText(textValue)
|
||||
segmentID := segmentID(relPath, textHash)
|
||||
filtered = append(filtered, Segment{
|
||||
Start: seg.Start,
|
||||
Stop: seg.Stop,
|
||||
Text: textValue,
|
||||
TextHash: textHash,
|
||||
SegmentID: segmentID,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(filtered, func(i, j int) bool {
|
||||
return filtered[i].Start < filtered[j].Start
|
||||
})
|
||||
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
func blockParent(n ast.Node) ast.Node {
|
||||
for node := n.Parent(); node != nil; node = node.Parent() {
|
||||
if isTranslatableBlock(node) {
|
||||
return node
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isTranslatableBlock(n ast.Node) bool {
|
||||
switch n.(type) {
|
||||
case *ast.Paragraph, *ast.Heading, *ast.ListItem:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func applyTranslations(body string, segments []Segment) string {
|
||||
if len(segments) == 0 {
|
||||
return body
|
||||
}
|
||||
var out strings.Builder
|
||||
last := 0
|
||||
for _, seg := range segments {
|
||||
if seg.Start < last {
|
||||
continue
|
||||
}
|
||||
out.WriteString(body[last:seg.Start])
|
||||
out.WriteString(seg.Translated)
|
||||
last = seg.Stop
|
||||
}
|
||||
out.WriteString(body[last:])
|
||||
return out.String()
|
||||
}
|
||||
89
scripts/docs-i18n/masking.go
Normal file
89
scripts/docs-i18n/masking.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
inlineCodeRe = regexp.MustCompile("`[^`]+`")
|
||||
angleLinkRe = regexp.MustCompile(`<https?://[^>]+>`)
|
||||
linkURLRe = regexp.MustCompile(`\[[^\]]*\]\(([^)]+)\)`)
|
||||
placeholderRe = regexp.MustCompile(`__OC_I18N_\d+__`)
|
||||
)
|
||||
|
||||
func maskMarkdown(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
|
||||
masked := maskMatches(text, inlineCodeRe, nextPlaceholder, placeholders, mapping)
|
||||
masked = maskMatches(masked, angleLinkRe, nextPlaceholder, placeholders, mapping)
|
||||
masked = maskLinkURLs(masked, nextPlaceholder, placeholders, mapping)
|
||||
return masked
|
||||
}
|
||||
|
||||
func maskMatches(text string, re *regexp.Regexp, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
|
||||
matches := re.FindAllStringIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
}
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
start, end := span[0], span[1]
|
||||
if start < pos {
|
||||
continue
|
||||
}
|
||||
out.WriteString(text[pos:start])
|
||||
placeholder := nextPlaceholder()
|
||||
mapping[placeholder] = text[start:end]
|
||||
*placeholders = append(*placeholders, placeholder)
|
||||
out.WriteString(placeholder)
|
||||
pos = end
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func maskLinkURLs(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
|
||||
matches := linkURLRe.FindAllStringSubmatchIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
}
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
fullStart := span[0]
|
||||
urlStart, urlEnd := span[2], span[3]
|
||||
if urlStart < 0 || urlEnd < 0 {
|
||||
continue
|
||||
}
|
||||
if fullStart < pos {
|
||||
continue
|
||||
}
|
||||
out.WriteString(text[pos:urlStart])
|
||||
placeholder := nextPlaceholder()
|
||||
mapping[placeholder] = text[urlStart:urlEnd]
|
||||
*placeholders = append(*placeholders, placeholder)
|
||||
out.WriteString(placeholder)
|
||||
pos = urlEnd
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func unmaskMarkdown(text string, placeholders []string, mapping map[string]string) string {
|
||||
out := text
|
||||
for _, placeholder := range placeholders {
|
||||
original := mapping[placeholder]
|
||||
out = strings.ReplaceAll(out, placeholder, original)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validatePlaceholders(text string, placeholders []string) error {
|
||||
for _, placeholder := range placeholders {
|
||||
if !strings.Contains(text, placeholder) {
|
||||
return fmt.Errorf("placeholder missing: %s", placeholder)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
37
scripts/docs-i18n/order.go
Normal file
37
scripts/docs-i18n/order.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type orderedFile struct {
|
||||
path string
|
||||
rel string
|
||||
}
|
||||
|
||||
func orderFiles(docsRoot string, files []string) ([]string, error) {
|
||||
entries := make([]orderedFile, 0, len(files))
|
||||
for _, file := range files {
|
||||
abs, err := filepath.Abs(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rel, err := filepath.Rel(docsRoot, abs)
|
||||
if err != nil {
|
||||
rel = abs
|
||||
}
|
||||
entries = append(entries, orderedFile{path: file, rel: rel})
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].rel < entries[j].rel
|
||||
})
|
||||
ordered := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
ordered = append(ordered, entry.path)
|
||||
}
|
||||
return ordered, nil
|
||||
}
|
||||
30
scripts/docs-i18n/placeholders.go
Normal file
30
scripts/docs-i18n/placeholders.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type PlaceholderState struct {
|
||||
counter int
|
||||
used map[string]struct{}
|
||||
}
|
||||
|
||||
func NewPlaceholderState(text string) *PlaceholderState {
|
||||
used := map[string]struct{}{}
|
||||
for _, hit := range placeholderRe.FindAllString(text, -1) {
|
||||
used[hit] = struct{}{}
|
||||
}
|
||||
return &PlaceholderState{counter: 900000, used: used}
|
||||
}
|
||||
|
||||
func (s *PlaceholderState) Next() string {
|
||||
for {
|
||||
candidate := fmt.Sprintf("__OC_I18N_%d__", s.counter)
|
||||
s.counter++
|
||||
if _, ok := s.used[candidate]; ok {
|
||||
continue
|
||||
}
|
||||
s.used[candidate] = struct{}{}
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
293
scripts/docs-i18n/process.go
Normal file
293
scripts/docs-i18n/process.go
Normal file
@@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
localizedLinkPostprocessPending = "pending"
|
||||
localizedLinkPostprocessVersion = "locale-links-v1"
|
||||
)
|
||||
|
||||
func processFile(ctx context.Context, translator docsTranslator, tm *TranslationMemory, docsRoot, filePath, srcLang, tgtLang string) (bool, string, error) {
|
||||
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(absPath)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
frontMatter, body := splitFrontMatter(string(content))
|
||||
frontData := map[string]any{}
|
||||
if frontMatter != "" {
|
||||
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
|
||||
return false, "", fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := translateFrontMatter(ctx, translator, tm, frontData, relPath, srcLang, tgtLang); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
body, err = translateHTMLBlocks(ctx, translator, body, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
segments, err := extractSegments(body, relPath)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
namespace := cacheNamespace()
|
||||
for i := range segments {
|
||||
seg := &segments[i]
|
||||
seg.CacheKey = cacheKey(namespace, srcLang, tgtLang, seg.SegmentID, seg.TextHash)
|
||||
if entry, ok := tm.Get(seg.CacheKey); ok {
|
||||
seg.Translated = entry.Translated
|
||||
continue
|
||||
}
|
||||
translated, err := translator.Translate(ctx, seg.Text, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("translate failed (%s): %w", relPath, err)
|
||||
}
|
||||
seg.Translated = translated
|
||||
entry := TMEntry{
|
||||
CacheKey: seg.CacheKey,
|
||||
SegmentID: seg.SegmentID,
|
||||
SourcePath: relPath,
|
||||
TextHash: seg.TextHash,
|
||||
Text: seg.Text,
|
||||
Translated: translated,
|
||||
Provider: docsI18nProvider(),
|
||||
Model: docsI18nModel(),
|
||||
SrcLang: srcLang,
|
||||
TgtLang: tgtLang,
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
tm.Put(entry)
|
||||
}
|
||||
|
||||
translatedBody := applyTranslations(body, segments)
|
||||
updatedFront, err := encodeFrontMatter(frontData, relPath, content)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(docsRoot, tgtLang, relPath)
|
||||
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
output := updatedFront + translatedBody
|
||||
return false, outputPath, os.WriteFile(outputPath, []byte(output), 0o644)
|
||||
}
|
||||
|
||||
func splitFrontMatter(content string) (string, string) {
|
||||
if !strings.HasPrefix(content, "---") {
|
||||
return "", content
|
||||
}
|
||||
lines := strings.Split(content, "\n")
|
||||
if len(lines) < 2 {
|
||||
return "", content
|
||||
}
|
||||
endIndex := -1
|
||||
for i := 1; i < len(lines); i++ {
|
||||
if strings.TrimSpace(lines[i]) == "---" {
|
||||
endIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if endIndex == -1 {
|
||||
return "", content
|
||||
}
|
||||
front := strings.Join(lines[1:endIndex], "\n")
|
||||
body := strings.Join(lines[endIndex+1:], "\n")
|
||||
if strings.HasPrefix(body, "\n") {
|
||||
body = body[1:]
|
||||
}
|
||||
return front, body
|
||||
}
|
||||
|
||||
func encodeFrontMatter(frontData map[string]any, relPath string, source []byte) (string, error) {
|
||||
if frontData == nil {
|
||||
frontData = map[string]any{}
|
||||
}
|
||||
frontData["x-i18n"] = map[string]any{
|
||||
"source_path": relPath,
|
||||
"source_hash": hashBytes(source),
|
||||
"provider": docsI18nProvider(),
|
||||
"model": docsI18nModel(),
|
||||
"workflow": workflowVersion,
|
||||
"generated_at": time.Now().UTC().Format(time.RFC3339),
|
||||
"postprocess_version": localizedLinkPostprocessPending,
|
||||
}
|
||||
encoded, err := yaml.Marshal(frontData)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("---\n%s---\n\n", string(encoded)), nil
|
||||
}
|
||||
|
||||
func translateFrontMatter(ctx context.Context, translator docsTranslator, tm *TranslationMemory, data map[string]any, relPath, srcLang, tgtLang string) error {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if summary, ok := data["summary"].(string); ok {
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter start %s field=summary bytes=%d", relPath, len(summary))
|
||||
}
|
||||
translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:summary", summary, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter done %s field=summary out_bytes=%d", relPath, len(translated))
|
||||
}
|
||||
data["summary"] = translated
|
||||
}
|
||||
if title, ok := data["title"].(string); ok {
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter start %s field=title bytes=%d", relPath, len(title))
|
||||
}
|
||||
translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:title", title, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter done %s field=title out_bytes=%d", relPath, len(translated))
|
||||
}
|
||||
data["title"] = translated
|
||||
}
|
||||
if readWhen, ok := data["read_when"].([]any); ok {
|
||||
translated := make([]any, 0, len(readWhen))
|
||||
for idx, item := range readWhen {
|
||||
textValue, ok := item.(string)
|
||||
if !ok {
|
||||
translated = append(translated, item)
|
||||
continue
|
||||
}
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter start %s field=read_when[%d] bytes=%d", relPath, idx, len(textValue))
|
||||
}
|
||||
value, err := translateSnippet(ctx, translator, tm, fmt.Sprintf("%s:frontmatter:read_when:%d", relPath, idx), textValue, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter done %s field=read_when[%d] out_bytes=%d", relPath, idx, len(value))
|
||||
}
|
||||
translated = append(translated, value)
|
||||
}
|
||||
data["read_when"] = translated
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func docsI18nVerboseLogs() bool {
|
||||
value := strings.TrimSpace(os.Getenv("OPENCLAW_DOCS_I18N_VERBOSE_LOGS"))
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(value) {
|
||||
case "1", "true", "yes", "on", "debug", "verbose":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func translateSnippet(ctx context.Context, translator docsTranslator, tm *TranslationMemory, segmentID, textValue, srcLang, tgtLang string) (string, error) {
|
||||
if strings.TrimSpace(textValue) == "" {
|
||||
return textValue, nil
|
||||
}
|
||||
namespace := cacheNamespace()
|
||||
textHash := hashText(textValue)
|
||||
ck := cacheKey(namespace, srcLang, tgtLang, segmentID, textHash)
|
||||
if entry, ok := tm.Get(ck); ok {
|
||||
return entry.Translated, nil
|
||||
}
|
||||
translated, err := translator.Translate(ctx, textValue, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
log.Printf("docs-i18n: frontmatter fallback %s reason=%v", segmentID, err)
|
||||
return textValue, nil
|
||||
}
|
||||
shouldCache := true
|
||||
if validationErr := validateFrontmatterScalarTranslation(textValue, translated); validationErr != nil {
|
||||
log.Printf("docs-i18n: frontmatter fallback %s reason=%v", segmentID, validationErr)
|
||||
translated = textValue
|
||||
shouldCache = false
|
||||
}
|
||||
sourcePath := segmentID
|
||||
if path, _, ok := strings.Cut(segmentID, ":frontmatter:"); ok {
|
||||
sourcePath = path
|
||||
}
|
||||
entry := TMEntry{
|
||||
CacheKey: ck,
|
||||
SegmentID: segmentID,
|
||||
SourcePath: sourcePath,
|
||||
TextHash: textHash,
|
||||
Text: textValue,
|
||||
Translated: translated,
|
||||
Provider: docsI18nProvider(),
|
||||
Model: docsI18nModel(),
|
||||
SrcLang: srcLang,
|
||||
TgtLang: tgtLang,
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if shouldCache {
|
||||
tm.Put(entry)
|
||||
}
|
||||
return translated, nil
|
||||
}
|
||||
|
||||
func validateFrontmatterScalarTranslation(source, translated string) error {
|
||||
trimmed := strings.TrimSpace(translated)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("empty translation")
|
||||
}
|
||||
lower := strings.ToLower(trimmed)
|
||||
if strings.Contains(lower, "<frontmatter>") || strings.Contains(lower, "</frontmatter>") || strings.Contains(lower, "<body>") || strings.Contains(lower, "</body>") {
|
||||
return fmt.Errorf("tagged document wrapper detected")
|
||||
}
|
||||
if err := validateNoTranslationTranscriptArtifacts(source, trimmed); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.Contains(trimmed, "[[[FM_") {
|
||||
return fmt.Errorf("frontmatter marker leaked into scalar translation")
|
||||
}
|
||||
if strings.Contains(trimmed, "\n---\n") || strings.HasPrefix(trimmed, "---\n") {
|
||||
return fmt.Errorf("yaml document boundary detected")
|
||||
}
|
||||
if !strings.Contains(source, "\n") && strings.Count(trimmed, "\n") >= 3 {
|
||||
return fmt.Errorf("unexpected multiline expansion")
|
||||
}
|
||||
sourceLen := len(strings.TrimSpace(source))
|
||||
translatedLen := len(trimmed)
|
||||
if sourceLen > 0 {
|
||||
limit := sourceLen*8 + 256
|
||||
if limit < 512 {
|
||||
limit = 512
|
||||
}
|
||||
if translatedLen > limit {
|
||||
return fmt.Errorf("unexpected size expansion source=%d translated=%d", sourceLen, translatedLen)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"title:", "summary:", "read_when:"} {
|
||||
if strings.Contains(lower, "\n"+key) || strings.HasPrefix(lower, key) {
|
||||
return fmt.Errorf("frontmatter key leaked into scalar translation")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
203
scripts/docs-i18n/prompt.go
Normal file
203
scripts/docs-i18n/prompt.go
Normal file
@@ -0,0 +1,203 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func prettyLanguageLabel(lang string) string {
|
||||
trimmed := strings.TrimSpace(lang)
|
||||
if trimmed == "" {
|
||||
return lang
|
||||
}
|
||||
switch {
|
||||
case strings.EqualFold(trimmed, "en"):
|
||||
return "English"
|
||||
case strings.EqualFold(trimmed, "zh-CN"):
|
||||
return "Simplified Chinese"
|
||||
case strings.EqualFold(trimmed, "zh-TW"):
|
||||
return "Traditional Chinese"
|
||||
case strings.EqualFold(trimmed, "ja-JP"):
|
||||
return "Japanese"
|
||||
case strings.EqualFold(trimmed, "es"):
|
||||
return "Spanish"
|
||||
case strings.EqualFold(trimmed, "pt-BR"):
|
||||
return "Brazilian Portuguese"
|
||||
case strings.EqualFold(trimmed, "ko"):
|
||||
return "Korean"
|
||||
case strings.EqualFold(trimmed, "fr"):
|
||||
return "French"
|
||||
case strings.EqualFold(trimmed, "ar"):
|
||||
return "Arabic"
|
||||
case strings.EqualFold(trimmed, "it"):
|
||||
return "Italian"
|
||||
case strings.EqualFold(trimmed, "vi"):
|
||||
return "Vietnamese"
|
||||
case strings.EqualFold(trimmed, "nl"):
|
||||
return "Dutch"
|
||||
case strings.EqualFold(trimmed, "fa"):
|
||||
return "Persian"
|
||||
case strings.EqualFold(trimmed, "tr"):
|
||||
return "Turkish"
|
||||
case strings.EqualFold(trimmed, "de"):
|
||||
return "German"
|
||||
case strings.EqualFold(trimmed, "th"):
|
||||
return "Thai"
|
||||
case strings.EqualFold(trimmed, "uk"):
|
||||
return "Ukrainian"
|
||||
case strings.EqualFold(trimmed, "id"):
|
||||
return "Indonesian"
|
||||
case strings.EqualFold(trimmed, "pl"):
|
||||
return "Polish"
|
||||
default:
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
|
||||
func translationPrompt(srcLang, tgtLang string, glossary []GlossaryEntry) string {
|
||||
srcLabel := prettyLanguageLabel(srcLang)
|
||||
tgtLabel := prettyLanguageLabel(tgtLang)
|
||||
glossaryBlock := buildGlossaryPrompt(glossary)
|
||||
|
||||
switch {
|
||||
case strings.EqualFold(tgtLang, "zh-CN"):
|
||||
// Keep this prompt as stable as possible; it has lots of tuning baked into the wording.
|
||||
return strings.TrimSpace(fmt.Sprintf(zhCNPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
|
||||
case strings.EqualFold(tgtLang, "ja-JP"):
|
||||
return strings.TrimSpace(fmt.Sprintf(jaJPPromptTemplate, srcLabel, tgtLabel, glossaryBlock))
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprintf(genericPromptTemplate, srcLabel, tgtLabel, localePromptRules(tgtLang), glossaryBlock))
|
||||
}
|
||||
}
|
||||
|
||||
func localePromptRules(tgtLang string) string {
|
||||
switch {
|
||||
case strings.EqualFold(tgtLang, "de"):
|
||||
return "- For German docs, use formal address consistently: “Sie/Ihr/Ihnen”. Avoid informal “du/dein/dir”.\n- Use established technical German; keep “Provider” where it is clearer than “Anbieter”, and avoid awkward mixed compounds."
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
const zhCNPromptTemplate = `You are a translation function, not a chat assistant.
|
||||
Translate from %s to %s.
|
||||
|
||||
Rules:
|
||||
- Output ONLY the translated text. No preamble, no questions, no commentary.
|
||||
- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
|
||||
- All prose must be Chinese. If any English sentence remains outside code/URLs/product names, it is wrong.
|
||||
- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
|
||||
- Translate only the contents inside those tags.
|
||||
- Preserve YAML structure inside <frontmatter>; translate only values.
|
||||
- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
|
||||
- Translate headings/labels like "Exit codes" and "Optional scripts".
|
||||
- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
|
||||
- Preserve HTML tags and attributes exactly.
|
||||
- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
|
||||
- Do not alter URLs or anchors.
|
||||
- Preserve placeholders exactly: __OC_I18N_####__.
|
||||
- Do not remove, reorder, or summarize content.
|
||||
- Use fluent, idiomatic technical Chinese; avoid slang or jokes.
|
||||
- Use neutral documentation tone; prefer “你/你的”, avoid “您/您的”.
|
||||
- Glossary terms are mandatory. When a source term matches a glossary entry, use
|
||||
the glossary target exactly, including headings, link labels, and short
|
||||
UI-style labels.
|
||||
- If a glossary target is identical to the source text, preserve that term in
|
||||
English exactly as written.
|
||||
- Insert a space between Latin characters and CJK text (W3C CLREQ), e.g., “Gateway 网关”, “Skills 配置”.
|
||||
- Use Chinese quotation marks “ and ” for Chinese prose; keep ASCII quotes inside code spans/blocks or literal CLI/keys.
|
||||
- Keep product names in English: OpenClaw, Raspberry Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
|
||||
- For the OpenClaw Gateway, use “Gateway 网关”.
|
||||
- Keep these terms in English: Skills, local loopback, Tailscale.
|
||||
- Never output an empty response; if unsure, return the source text unchanged.
|
||||
|
||||
%s
|
||||
|
||||
If the input is empty, output empty.
|
||||
If the input contains only placeholders, output it unchanged.`
|
||||
|
||||
const jaJPPromptTemplate = `You are a translation function, not a chat assistant.
|
||||
Translate from %s to %s.
|
||||
|
||||
Rules:
|
||||
- Output ONLY the translated text. No preamble, no questions, no commentary.
|
||||
- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
|
||||
- All prose must be Japanese. If any English sentence remains outside code/URLs/product names, it is wrong.
|
||||
- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
|
||||
- Translate only the contents inside those tags.
|
||||
- Preserve YAML structure inside <frontmatter>; translate only values.
|
||||
- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
|
||||
- Translate headings/labels like "Exit codes" and "Optional scripts".
|
||||
- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
|
||||
- Preserve HTML tags and attributes exactly.
|
||||
- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
|
||||
- Do not alter URLs or anchors.
|
||||
- Preserve placeholders exactly: __OC_I18N_####__.
|
||||
- Do not remove, reorder, or summarize content.
|
||||
- Use fluent, idiomatic technical Japanese; avoid slang or jokes.
|
||||
- Use neutral documentation tone; avoid overly formal honorifics (e.g., avoid “〜でございます”).
|
||||
- Glossary terms are mandatory. When a source term matches a glossary entry, use
|
||||
the glossary target exactly, including headings, link labels, and short
|
||||
UI-style labels.
|
||||
- If a glossary target is identical to the source text, preserve that term in
|
||||
English exactly as written.
|
||||
- Use Japanese quotation marks 「 and 」 for Japanese prose; keep ASCII quotes inside code spans/blocks or literal CLI/keys.
|
||||
- Do not add or remove spacing around Latin text just because it borders Japanese; keep spacing stable unless required by Japanese grammar.
|
||||
- Keep product names in English: OpenClaw, Raspberry Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
|
||||
- Keep these terms in English: Skills, local loopback, Tailscale.
|
||||
- Never output an empty response; if unsure, return the source text unchanged.
|
||||
|
||||
%s
|
||||
|
||||
If the input is empty, output empty.
|
||||
If the input contains only placeholders, output it unchanged.`
|
||||
|
||||
const genericPromptTemplate = `You are a translation function, not a chat assistant.
|
||||
Translate from %s to %s.
|
||||
|
||||
Rules:
|
||||
- Output ONLY the translated text. No preamble, no questions, no commentary.
|
||||
- Translate all English prose; do not leave English unless it is code, a URL, or a product name.
|
||||
- If any English sentence remains outside code/URLs/product names, it is likely wrong.
|
||||
- If the input contains <frontmatter> and <body> tags, keep them exactly and output exactly one of each.
|
||||
- Translate only the contents inside those tags.
|
||||
- Preserve YAML structure inside <frontmatter>; translate only values.
|
||||
- Preserve all [[[FM_*]]] markers exactly and translate only the text between each START/END pair.
|
||||
- Translate headings/labels like "Exit codes" and "Optional scripts".
|
||||
- Preserve Markdown syntax exactly (headings, lists, tables, emphasis).
|
||||
- Preserve HTML tags and attributes exactly.
|
||||
- Do not translate code spans/blocks, config keys, CLI flags, or env vars.
|
||||
- Do not alter URLs or anchors.
|
||||
- Preserve placeholders exactly: __OC_I18N_####__.
|
||||
- Do not remove, reorder, or summarize content.
|
||||
- Use fluent, idiomatic technical language in the target language; avoid slang or jokes.
|
||||
- Use neutral documentation tone.
|
||||
%s
|
||||
- Glossary terms are mandatory. When a source term matches a glossary entry, use
|
||||
the glossary target exactly, including headings, link labels, and short
|
||||
UI-style labels.
|
||||
- If a glossary target is identical to the source text, preserve that term in
|
||||
English exactly as written.
|
||||
- Keep product names in English: OpenClaw, Raspberry Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal.
|
||||
- Keep these terms in English: Skills, local loopback, Tailscale.
|
||||
- Never output an empty response; if unsure, return the source text unchanged.
|
||||
|
||||
%s
|
||||
|
||||
If the input is empty, output empty.
|
||||
If the input contains only placeholders, output it unchanged.`
|
||||
|
||||
func buildGlossaryPrompt(glossary []GlossaryEntry) string {
|
||||
if len(glossary) == 0 {
|
||||
return ""
|
||||
}
|
||||
var lines []string
|
||||
lines = append(lines, "Required terminology (use exactly when the source term matches):")
|
||||
for _, entry := range glossary {
|
||||
if entry.Source == "" || entry.Target == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("- %s -> %s", entry.Source, entry.Target))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
22
scripts/docs-i18n/prompt_test.go
Normal file
22
scripts/docs-i18n/prompt_test.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTranslationPromptAddsGermanStyleRules(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
prompt := translationPrompt("en", "de", nil)
|
||||
|
||||
for _, want := range []string{
|
||||
"Translate from English to German.",
|
||||
"Sie/Ihr/Ihnen",
|
||||
"Avoid informal “du/dein/dir”",
|
||||
} {
|
||||
if !strings.Contains(prompt, want) {
|
||||
t.Fatalf("expected %q in German prompt:\n%s", want, prompt)
|
||||
}
|
||||
}
|
||||
}
|
||||
94
scripts/docs-i18n/relocalize.go
Normal file
94
scripts/docs-i18n/relocalize.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func postprocessLocalizedDocs(docsRoot, targetLang string, localizedFiles []string) error {
|
||||
if targetLang == "" || targetLang == "en" || len(localizedFiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
routes, err := loadRouteIndex(docsRoot, targetLang)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, path := range localizedFiles {
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
frontMatter, body := splitFrontMatter(string(content))
|
||||
rewrittenBody := routes.localizeBodyLinks(body)
|
||||
updatedFrontMatter := setPostprocessVersion(frontMatter, localizedLinkPostprocessVersion)
|
||||
if rewrittenBody == body && updatedFrontMatter == frontMatter {
|
||||
continue
|
||||
}
|
||||
|
||||
output := rewrittenBody
|
||||
if updatedFrontMatter != "" {
|
||||
output = "---\n" + updatedFrontMatter + "\n---\n\n" + rewrittenBody
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, []byte(output), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func setPostprocessVersion(frontMatter, version string) string {
|
||||
if strings.TrimSpace(frontMatter) == "" {
|
||||
return frontMatter
|
||||
}
|
||||
|
||||
lines := strings.Split(frontMatter, "\n")
|
||||
inXI18N := false
|
||||
xi18nLine := -1
|
||||
insertAt := -1
|
||||
childIndent := " "
|
||||
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "x-i18n:" {
|
||||
inXI18N = true
|
||||
xi18nLine = i
|
||||
insertAt = len(lines)
|
||||
continue
|
||||
}
|
||||
if !inXI18N {
|
||||
continue
|
||||
}
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
indent := leadingWhitespace(line)
|
||||
if len(indent) <= len(leadingWhitespace(lines[xi18nLine])) {
|
||||
insertAt = i
|
||||
break
|
||||
}
|
||||
childIndent = indent
|
||||
if strings.HasPrefix(trimmed, "postprocess_version:") {
|
||||
lines[i] = indent + "postprocess_version: " + version
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
}
|
||||
|
||||
if xi18nLine == -1 {
|
||||
return frontMatter
|
||||
}
|
||||
if insertAt == -1 {
|
||||
insertAt = len(lines)
|
||||
}
|
||||
|
||||
lines = append(lines[:insertAt], append([]string{childIndent + "postprocess_version: " + version}, lines[insertAt:]...)...)
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func leadingWhitespace(text string) string {
|
||||
return text[:len(text)-len(strings.TrimLeft(text, " \t"))]
|
||||
}
|
||||
291
scripts/docs-i18n/relocalize_test.go
Normal file
291
scripts/docs-i18n/relocalize_test.go
Normal file
@@ -0,0 +1,291 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPostprocessLocalizedDocsFixesStaleLinksAfterLaterPagesExist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), "# Gateway\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"), stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), stringsJoin(
|
||||
"---",
|
||||
"title: 故障排除",
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
"---",
|
||||
"",
|
||||
"# 故障排除",
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{
|
||||
filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"),
|
||||
filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"),
|
||||
}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "index.md"))
|
||||
if !strings.Contains(got, "---\ntitle: 网关\nx-i18n:\n source_hash: test\n postprocess_version: "+localizedLinkPostprocessVersion+"\n---\n\n") {
|
||||
t.Fatalf("front matter corrupted after rewrite:\n%s", got)
|
||||
}
|
||||
want := "See [Troubleshooting](/zh-CN/gateway/troubleshooting)."
|
||||
if !containsLine(got, want) {
|
||||
t.Fatalf("expected rewritten localized link %q in output:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsRewritesPublishedPageLinksForEachLocale(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
lang string
|
||||
title string
|
||||
wantPrefix string
|
||||
}{
|
||||
{name: "zh-CN", lang: "zh-CN", title: "网关", wantPrefix: "/zh-CN"},
|
||||
{name: "ja-JP", lang: "ja-JP", title: "ゲートウェイ", wantPrefix: "/ja-JP"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "index.md"), "# Gateway\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "providers", "example-provider.md"), "# Example provider\n")
|
||||
writeFile(t, filepath.Join(docsRoot, tt.lang, "gateway", "troubleshooting.md"), "# Localized troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, tt.lang, "providers", "example-provider.md"), "# Localized example provider\n")
|
||||
|
||||
pagePath := filepath.Join(docsRoot, tt.lang, "gateway", "index.md")
|
||||
writeFile(t, pagePath, stringsJoin(
|
||||
"---",
|
||||
"title: "+tt.title,
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
"",
|
||||
"See [Example provider](/providers/example-provider).",
|
||||
"",
|
||||
`<Card href="/gateway/troubleshooting" title="Troubleshooting" />`,
|
||||
`<Card href="`+tt.wantPrefix+`/providers/example-provider" title="Example provider" />`,
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, tt.lang, []string{pagePath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, pagePath)
|
||||
expectedLinks := []string{
|
||||
"See [Troubleshooting](" + tt.wantPrefix + "/gateway/troubleshooting).",
|
||||
"See [Example provider](" + tt.wantPrefix + "/providers/example-provider).",
|
||||
`<Card href="` + tt.wantPrefix + `/gateway/troubleshooting" title="Troubleshooting" />`,
|
||||
`<Card href="` + tt.wantPrefix + `/providers/example-provider" title="Example provider" />`,
|
||||
}
|
||||
for _, want := range expectedLinks {
|
||||
if !containsLine(got, want) {
|
||||
t.Fatalf("expected rewritten link %q in output:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsDoesNotTreatThreeLetterSourceDirsAsLocales(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "cli", "index.md"), "# CLI\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "cli", "AGENTS.md"), "# CLI docs guide\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "web", "index.md"), "# Web\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "AGENTS.md"), "# zh-CN\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", ".i18n", "README.md"), "# zh-CN i18n\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "cli", "index.md"), "# CLI 本地化\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "web", "index.md"), "# Web 本地化\n")
|
||||
|
||||
pagePath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
writeFile(t, pagePath, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
"---",
|
||||
"",
|
||||
"See [CLI](/cli).",
|
||||
"",
|
||||
"See [Web](/web).",
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{pagePath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, pagePath)
|
||||
for _, want := range []string{
|
||||
"See [CLI](/zh-CN/cli).",
|
||||
"See [Web](/zh-CN/web).",
|
||||
} {
|
||||
if !containsLine(got, want) {
|
||||
t.Fatalf("expected rewritten link %q in output:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsOnlyTouchesScopedFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
|
||||
|
||||
scopedPath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
unscopedPath := filepath.Join(docsRoot, "zh-CN", "help", "index.md")
|
||||
|
||||
writeFile(t, scopedPath, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
writeFile(t, unscopedPath, stringsJoin(
|
||||
"---",
|
||||
"title: 帮助",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
beforeUnscoped := mustReadFile(t, unscopedPath)
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{scopedPath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
gotScoped := mustReadFile(t, scopedPath)
|
||||
if !containsLine(gotScoped, "See [Troubleshooting](/zh-CN/gateway/troubleshooting).") {
|
||||
t.Fatalf("expected scoped file rewrite, got:\n%s", gotScoped)
|
||||
}
|
||||
|
||||
afterUnscoped := mustReadFile(t, unscopedPath)
|
||||
if afterUnscoped != beforeUnscoped {
|
||||
t.Fatalf("expected unscoped file to remain unchanged\nbefore:\n%s\nafter:\n%s", beforeUnscoped, afterUnscoped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsContinuesAfterUnchangedFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, filepath.Join(docsRoot, "gateway", "troubleshooting.md"), "# Troubleshooting\n")
|
||||
writeFile(t, filepath.Join(docsRoot, "zh-CN", "gateway", "troubleshooting.md"), "# 故障排除\n")
|
||||
|
||||
unchangedPath := filepath.Join(docsRoot, "zh-CN", "gateway", "already-localized.md")
|
||||
needsRewritePath := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
|
||||
writeFile(t, unchangedPath, stringsJoin(
|
||||
"---",
|
||||
"title: 已本地化",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/zh-CN/gateway/troubleshooting).",
|
||||
))
|
||||
writeFile(t, needsRewritePath, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{unchangedPath, needsRewritePath}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, needsRewritePath)
|
||||
if !containsLine(got, "See [Troubleshooting](/zh-CN/gateway/troubleshooting).") {
|
||||
t.Fatalf("expected later file rewrite after unchanged file, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostprocessLocalizedDocsFinalizesPostprocessVersionWithoutBodyRewrite(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
docsRoot := t.TempDir()
|
||||
path := filepath.Join(docsRoot, "zh-CN", "gateway", "index.md")
|
||||
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
|
||||
writeFile(t, path, stringsJoin(
|
||||
"---",
|
||||
"title: 网关",
|
||||
"x-i18n:",
|
||||
" source_hash: test",
|
||||
" postprocess_version: "+localizedLinkPostprocessPending,
|
||||
"---",
|
||||
"",
|
||||
"See [Troubleshooting](/zh-CN/gateway/troubleshooting).",
|
||||
))
|
||||
|
||||
if err := postprocessLocalizedDocs(docsRoot, "zh-CN", []string{path}); err != nil {
|
||||
t.Fatalf("postprocessLocalizedDocs failed: %v", err)
|
||||
}
|
||||
|
||||
got := mustReadFile(t, path)
|
||||
if !strings.Contains(got, " postprocess_version: "+localizedLinkPostprocessVersion) {
|
||||
t.Fatalf("expected postprocess version marker to be finalized:\n%s", got)
|
||||
}
|
||||
if !containsLine(got, "See [Troubleshooting](/zh-CN/gateway/troubleshooting).") {
|
||||
t.Fatalf("expected localized link to remain unchanged, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func stringsJoin(lines ...string) string {
|
||||
result := ""
|
||||
for i, line := range lines {
|
||||
if i > 0 {
|
||||
result += "\n"
|
||||
}
|
||||
result += line
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func mustReadFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read failed for %s: %v", path, err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func containsLine(text, want string) bool {
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if line == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
11
scripts/docs-i18n/segment.go
Normal file
11
scripts/docs-i18n/segment.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package main
|
||||
|
||||
type Segment struct {
|
||||
Start int
|
||||
Stop int
|
||||
Text string
|
||||
TextHash string
|
||||
SegmentID string
|
||||
Translated string
|
||||
CacheKey string
|
||||
}
|
||||
24
scripts/docs-i18n/testdata/behavior/fenced-singleton-retry/case.json
vendored
Normal file
24
scripts/docs-i18n/testdata/behavior/fenced-singleton-retry/case.json
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "fenced singleton retries after malformed raw output",
|
||||
"mode": "doc_body_chunked",
|
||||
"rel_path": "gateway/configuration-reference.md",
|
||||
"source_file": "source.txt",
|
||||
"expected_output_contains": [
|
||||
"Translated line 01",
|
||||
"Translated line 02",
|
||||
"Translated line 03",
|
||||
"Translated line 04"
|
||||
],
|
||||
"expected_output_not_contains": ["Line 01", "Line 02", "Line 03", "Line 04"],
|
||||
"rules": [
|
||||
{
|
||||
"method": "raw",
|
||||
"match_all": ["Line 01", "Line 04"],
|
||||
"response_file": "raw-malformed.txt"
|
||||
},
|
||||
{
|
||||
"method": "raw",
|
||||
"replace_pairs": [{ "from": "Line ", "to": "Translated line " }]
|
||||
}
|
||||
]
|
||||
}
|
||||
6
scripts/docs-i18n/testdata/behavior/fenced-singleton-retry/raw-malformed.txt
vendored
Normal file
6
scripts/docs-i18n/testdata/behavior/fenced-singleton-retry/raw-malformed.txt
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
```md
|
||||
Line 01
|
||||
Line 02
|
||||
Line 03
|
||||
Line 04
|
||||
|
||||
7
scripts/docs-i18n/testdata/behavior/fenced-singleton-retry/source.txt
vendored
Normal file
7
scripts/docs-i18n/testdata/behavior/fenced-singleton-retry/source.txt
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
```md
|
||||
Line 01
|
||||
Line 02
|
||||
Line 03
|
||||
Line 04
|
||||
```
|
||||
|
||||
14
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/case.json
vendored
Normal file
14
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/case.json
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "frontmatter scalar falls back instead of keeping tagged wrapper",
|
||||
"mode": "frontmatter_scalar",
|
||||
"rel_path": "install/fly.md",
|
||||
"source_file": "source.txt",
|
||||
"expected_file": "expected.txt",
|
||||
"rules": [
|
||||
{
|
||||
"method": "masked",
|
||||
"match_all": ["Deploying OpenClaw on Fly.io"],
|
||||
"response_file": "masked-tagged-wrapper.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
1
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/expected.txt
vendored
Normal file
1
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/expected.txt
vendored
Normal file
@@ -0,0 +1 @@
|
||||
Deploying OpenClaw on Fly.io
|
||||
7
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/masked-tagged-wrapper.txt
vendored
Normal file
7
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/masked-tagged-wrapper.txt
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
<frontmatter>
|
||||
title: Fly.io
|
||||
</frontmatter>
|
||||
|
||||
<body>
|
||||
# Fly.io 部署
|
||||
</body>
|
||||
1
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/source.txt
vendored
Normal file
1
scripts/docs-i18n/testdata/behavior/frontmatter-fallback/source.txt
vendored
Normal file
@@ -0,0 +1 @@
|
||||
Deploying OpenClaw on Fly.io
|
||||
25
scripts/docs-i18n/testdata/behavior/protocol-leak-split/case.json
vendored
Normal file
25
scripts/docs-i18n/testdata/behavior/protocol-leak-split/case.json
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "protocol leak retries on smaller chunks",
|
||||
"mode": "doc_body_chunked",
|
||||
"rel_path": "gateway/configuration-reference.md",
|
||||
"source_file": "source.txt",
|
||||
"expected_file": "expected.txt",
|
||||
"expected_output_not_contains": ["<frontmatter>", "<body>", "[[[FM_"],
|
||||
"rules": [
|
||||
{
|
||||
"method": "raw",
|
||||
"match_all": ["First chunk", "Second chunk"],
|
||||
"response_file": "raw-leaked.txt"
|
||||
},
|
||||
{
|
||||
"method": "raw",
|
||||
"match_all": ["First chunk"],
|
||||
"response_file": "raw-first.txt"
|
||||
},
|
||||
{
|
||||
"method": "raw",
|
||||
"match_all": ["Second chunk"],
|
||||
"response_file": "raw-second.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
4
scripts/docs-i18n/testdata/behavior/protocol-leak-split/expected.txt
vendored
Normal file
4
scripts/docs-i18n/testdata/behavior/protocol-leak-split/expected.txt
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
First translated
|
||||
|
||||
Second translated
|
||||
|
||||
2
scripts/docs-i18n/testdata/behavior/protocol-leak-split/raw-first.txt
vendored
Normal file
2
scripts/docs-i18n/testdata/behavior/protocol-leak-split/raw-first.txt
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
First translated
|
||||
|
||||
9
scripts/docs-i18n/testdata/behavior/protocol-leak-split/raw-leaked.txt
vendored
Normal file
9
scripts/docs-i18n/testdata/behavior/protocol-leak-split/raw-leaked.txt
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
<frontmatter>
|
||||
title: leaked
|
||||
</frontmatter>
|
||||
|
||||
<body>
|
||||
First translated
|
||||
|
||||
Second translated
|
||||
</body>
|
||||
2
scripts/docs-i18n/testdata/behavior/protocol-leak-split/raw-second.txt
vendored
Normal file
2
scripts/docs-i18n/testdata/behavior/protocol-leak-split/raw-second.txt
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
Second translated
|
||||
|
||||
4
scripts/docs-i18n/testdata/behavior/protocol-leak-split/source.txt
vendored
Normal file
4
scripts/docs-i18n/testdata/behavior/protocol-leak-split/source.txt
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
First chunk
|
||||
|
||||
Second chunk
|
||||
|
||||
14
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/case.json
vendored
Normal file
14
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/case.json
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "uppercase body wrapper is stripped",
|
||||
"mode": "doc_body_chunked",
|
||||
"rel_path": "help/testing.md",
|
||||
"source_file": "source.txt",
|
||||
"expected_file": "expected.txt",
|
||||
"rules": [
|
||||
{
|
||||
"method": "raw",
|
||||
"match_all": ["Regular paragraph."],
|
||||
"response_file": "raw-uppercase-wrapper.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
1
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/expected.txt
vendored
Normal file
1
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/expected.txt
vendored
Normal file
@@ -0,0 +1 @@
|
||||
Translated paragraph.
|
||||
3
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/raw-uppercase-wrapper.txt
vendored
Normal file
3
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/raw-uppercase-wrapper.txt
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
<BODY>
|
||||
Translated paragraph.
|
||||
</BODY>
|
||||
1
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/source.txt
vendored
Normal file
1
scripts/docs-i18n/testdata/behavior/uppercase-wrapper-strip/source.txt
vendored
Normal file
@@ -0,0 +1 @@
|
||||
Regular paragraph.
|
||||
132
scripts/docs-i18n/tm.go
Normal file
132
scripts/docs-i18n/tm.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type TMEntry struct {
|
||||
CacheKey string `json:"cache_key"`
|
||||
SegmentID string `json:"segment_id"`
|
||||
SourcePath string `json:"source_path"`
|
||||
TextHash string `json:"text_hash"`
|
||||
Text string `json:"text"`
|
||||
Translated string `json:"translated"`
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
SrcLang string `json:"src_lang"`
|
||||
TgtLang string `json:"tgt_lang"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type TranslationMemory struct {
|
||||
path string
|
||||
entries map[string]TMEntry
|
||||
}
|
||||
|
||||
func LoadTranslationMemory(path string) (*TranslationMemory, error) {
|
||||
tm := &TranslationMemory{path: path, entries: map[string]TMEntry{}}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return tm, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
reader := bufio.NewReader(file)
|
||||
for {
|
||||
line, err := reader.ReadBytes('\n')
|
||||
if len(line) > 0 {
|
||||
trimmed := strings.TrimSpace(string(line))
|
||||
if trimmed != "" {
|
||||
var entry TMEntry
|
||||
if err := json.Unmarshal([]byte(trimmed), &entry); err != nil {
|
||||
return nil, fmt.Errorf("translation memory decode failed: %w", err)
|
||||
}
|
||||
if entry.CacheKey != "" && strings.TrimSpace(entry.Translated) != "" {
|
||||
tm.entries[entry.CacheKey] = entry
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return tm, nil
|
||||
}
|
||||
|
||||
func (tm *TranslationMemory) Get(cacheKey string) (TMEntry, bool) {
|
||||
entry, ok := tm.entries[cacheKey]
|
||||
if !ok {
|
||||
return TMEntry{}, false
|
||||
}
|
||||
if strings.TrimSpace(entry.Translated) == "" {
|
||||
return TMEntry{}, false
|
||||
}
|
||||
return entry, true
|
||||
}
|
||||
|
||||
func (tm *TranslationMemory) Put(entry TMEntry) {
|
||||
if entry.CacheKey == "" {
|
||||
return
|
||||
}
|
||||
tm.entries[entry.CacheKey] = entry
|
||||
}
|
||||
|
||||
func (tm *TranslationMemory) Save() error {
|
||||
if tm.path == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(tm.path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmpPath := tm.path + ".tmp"
|
||||
file, err := os.Create(tmpPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(tm.entries))
|
||||
for key := range tm.entries {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
writer := bufio.NewWriter(file)
|
||||
for _, key := range keys {
|
||||
entry := tm.entries[key]
|
||||
payload, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := writer.Write(payload); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := writer.WriteString("\n"); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmpPath, tm.path)
|
||||
}
|
||||
374
scripts/docs-i18n/translator.go
Normal file
374
scripts/docs-i18n/translator.go
Normal file
@@ -0,0 +1,374 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
translateMaxAttempts = 3
|
||||
translateBaseDelay = 15 * time.Second
|
||||
defaultPromptTimeout = 2 * time.Minute
|
||||
defaultCommandWaitDelay = 15 * time.Second
|
||||
envDocsI18nPromptTimeout = "OPENCLAW_DOCS_I18N_PROMPT_TIMEOUT"
|
||||
envDocsI18nCommandWaitDelay = "OPENCLAW_DOCS_I18N_COMMAND_WAIT_DELAY"
|
||||
envDocsI18nCodexExecutable = "OPENCLAW_DOCS_I18N_CODEX_EXECUTABLE"
|
||||
)
|
||||
|
||||
var errEmptyTranslation = errors.New("empty translation")
|
||||
|
||||
var translateRetryDelay = func(attempt int) time.Duration {
|
||||
return translateBaseDelay * time.Duration(attempt)
|
||||
}
|
||||
|
||||
type CodexTranslator struct {
|
||||
systemPrompt string
|
||||
exactGlossaryMappings map[string]string
|
||||
thinking string
|
||||
runPrompt codexPromptRunner
|
||||
}
|
||||
|
||||
type docsTranslator interface {
|
||||
Translate(context.Context, string, string, string) (string, error)
|
||||
TranslateRaw(context.Context, string, string, string) (string, error)
|
||||
Close()
|
||||
}
|
||||
|
||||
type docsTranslatorFactory func(string, string, []GlossaryEntry, string) (docsTranslator, error)
|
||||
|
||||
type codexPromptRunner func(context.Context, codexPromptRequest) (string, error)
|
||||
|
||||
type codexPromptRequest struct {
|
||||
SystemPrompt string
|
||||
Message string
|
||||
Model string
|
||||
Thinking string
|
||||
}
|
||||
|
||||
func NewCodexTranslator(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (*CodexTranslator, error) {
|
||||
return &CodexTranslator{
|
||||
systemPrompt: translationPrompt(srcLang, tgtLang, glossary),
|
||||
exactGlossaryMappings: exactGlossaryMappings(glossary),
|
||||
thinking: normalizeThinking(thinking),
|
||||
runPrompt: runCodexExecPrompt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) Translate(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
|
||||
return t.translate(ctx, text, t.translateMasked)
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) TranslateRaw(ctx context.Context, text, srcLang, tgtLang string) (string, error) {
|
||||
return t.translate(ctx, text, t.translateRaw)
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) translate(ctx context.Context, text string, run func(context.Context, string) (string, error)) (string, error) {
|
||||
prefix, core, suffix := splitWhitespace(text)
|
||||
if core == "" {
|
||||
return text, nil
|
||||
}
|
||||
if translated, ok := t.exactGlossaryMappings[core]; ok {
|
||||
return prefix + translated + suffix, nil
|
||||
}
|
||||
translated, err := t.translateWithRetry(ctx, func(ctx context.Context) (string, error) {
|
||||
return run(ctx, core)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return prefix + translated + suffix, nil
|
||||
}
|
||||
|
||||
func exactGlossaryMappings(glossary []GlossaryEntry) map[string]string {
|
||||
mappings := map[string]string{}
|
||||
for _, entry := range glossary {
|
||||
source := strings.TrimSpace(entry.Source)
|
||||
target := strings.TrimSpace(entry.Target)
|
||||
if source == "" || target == "" {
|
||||
continue
|
||||
}
|
||||
mappings[source] = target
|
||||
}
|
||||
return mappings
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) translateWithRetry(ctx context.Context, run func(context.Context) (string, error)) (string, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < translateMaxAttempts; attempt++ {
|
||||
translated, err := run(ctx)
|
||||
if err == nil {
|
||||
return translated, nil
|
||||
}
|
||||
if !isRetryableTranslateError(err) {
|
||||
return "", err
|
||||
}
|
||||
lastErr = err
|
||||
if attempt+1 < translateMaxAttempts {
|
||||
delay := translateRetryDelay(attempt + 1)
|
||||
if err := sleepWithContext(ctx, delay); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", lastErr
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) translateMasked(ctx context.Context, core string) (string, error) {
|
||||
state := NewPlaceholderState(core)
|
||||
placeholders := make([]string, 0, 8)
|
||||
mapping := map[string]string{}
|
||||
masked := maskMarkdown(core, state.Next, &placeholders, mapping)
|
||||
resText, err := t.prompt(ctx, masked)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
translated := stripCodexI18nInputWrappers(strings.TrimSpace(resText))
|
||||
if translated == "" {
|
||||
return "", errEmptyTranslation
|
||||
}
|
||||
if err := validatePlaceholders(translated, placeholders); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return unmaskMarkdown(translated, placeholders, mapping), nil
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) translateRaw(ctx context.Context, core string) (string, error) {
|
||||
resText, err := t.prompt(ctx, core)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
translated := stripCodexI18nInputWrappers(strings.TrimSpace(resText))
|
||||
if translated == "" {
|
||||
return "", errEmptyTranslation
|
||||
}
|
||||
return translated, nil
|
||||
}
|
||||
|
||||
func stripCodexI18nInputWrappers(text string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"<openclaw_docs_i18n_input>", "",
|
||||
"</openclaw_docs_i18n_input>", "",
|
||||
)
|
||||
return strings.TrimSpace(replacer.Replace(text))
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) prompt(ctx context.Context, message string) (string, error) {
|
||||
if t.runPrompt == nil {
|
||||
return "", errors.New("codex prompt runner unavailable")
|
||||
}
|
||||
promptCtx, cancel := context.WithTimeout(ctx, docsI18nPromptTimeout())
|
||||
defer cancel()
|
||||
return t.runPrompt(promptCtx, codexPromptRequest{
|
||||
SystemPrompt: t.systemPrompt,
|
||||
Message: message,
|
||||
Model: docsI18nModel(),
|
||||
Thinking: t.thinking,
|
||||
})
|
||||
}
|
||||
|
||||
func isRetryableTranslateError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, errEmptyTranslation) {
|
||||
return true
|
||||
}
|
||||
message := strings.ToLower(err.Error())
|
||||
if strings.Contains(message, "authentication failed") || strings.Contains(message, "invalid_api_key") || strings.Contains(message, "api key") {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(message, "placeholder missing") ||
|
||||
strings.Contains(message, "rate limit") ||
|
||||
strings.Contains(message, "429") ||
|
||||
strings.Contains(message, "500") ||
|
||||
strings.Contains(message, "502") ||
|
||||
strings.Contains(message, "503") ||
|
||||
strings.Contains(message, "504") ||
|
||||
strings.Contains(message, "temporarily unavailable") ||
|
||||
strings.Contains(message, "connection reset") ||
|
||||
strings.Contains(message, "stream")
|
||||
}
|
||||
|
||||
func runCodexExecPrompt(ctx context.Context, req codexPromptRequest) (string, error) {
|
||||
outputFile, err := os.CreateTemp("", "openclaw-docs-i18n-codex-*.txt")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
outputPath := outputFile.Name()
|
||||
_ = outputFile.Close()
|
||||
defer os.Remove(outputPath)
|
||||
|
||||
codexHomeBase, err := isolatedCodexHomeBase()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
codexHome, err := os.MkdirTemp(codexHomeBase, "codex-home-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(codexHome)
|
||||
if err := writeCodexAuthFile(codexHome); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"exec",
|
||||
"--model", req.Model,
|
||||
"-c", fmt.Sprintf("model_reasoning_effort=%q", normalizeThinking(req.Thinking)),
|
||||
"-c", `service_tier="fast"`,
|
||||
"--sandbox", "read-only",
|
||||
"--ignore-rules",
|
||||
"--skip-git-repo-check",
|
||||
"--output-last-message", outputPath,
|
||||
"-",
|
||||
}
|
||||
command := exec.CommandContext(ctx, docsCodexExecutable(), args...)
|
||||
configureCodexPromptCommand(command)
|
||||
command.Stdin = strings.NewReader(buildCodexTranslationPrompt(req.SystemPrompt, req.Message))
|
||||
command.Env = append(os.Environ(), "CODEX_HOME="+codexHome)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
command.Stdout = &stdout
|
||||
command.Stderr = &stderr
|
||||
if err := command.Run(); err != nil {
|
||||
if translated, readErr := readCodexOutputLastMessage(outputPath); readErr == nil {
|
||||
return translated, nil
|
||||
}
|
||||
return "", fmt.Errorf("codex exec failed: %w (%s)", err, previewCommandOutput(stdout.String(), stderr.String()))
|
||||
}
|
||||
|
||||
return readCodexOutputLastMessage(outputPath)
|
||||
}
|
||||
|
||||
func readCodexOutputLastMessage(outputPath string) (string, error) {
|
||||
data, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
translated := strings.TrimSpace(string(data))
|
||||
if translated == "" {
|
||||
return "", errEmptyTranslation
|
||||
}
|
||||
return translated, nil
|
||||
}
|
||||
|
||||
func writeCodexAuthFile(codexHome string) error {
|
||||
apiKey := strings.TrimSpace(os.Getenv("OPENAI_API_KEY"))
|
||||
if apiKey == "" {
|
||||
return nil
|
||||
}
|
||||
data, err := json.Marshal(map[string]string{
|
||||
"auth_mode": "apikey",
|
||||
"OPENAI_API_KEY": apiKey,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(codexHome, "auth.json"), append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func isolatedCodexHomeBase() (string, error) {
|
||||
cacheDir, err := os.UserCacheDir()
|
||||
if err != nil || strings.TrimSpace(cacheDir) == "" {
|
||||
homeDir, homeErr := os.UserHomeDir()
|
||||
if homeErr != nil {
|
||||
return "", err
|
||||
}
|
||||
cacheDir = filepath.Join(homeDir, ".cache")
|
||||
}
|
||||
base := filepath.Join(cacheDir, "openclaw-docs-i18n")
|
||||
if err := os.MkdirAll(base, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
func docsCodexExecutable() string {
|
||||
if executable := strings.TrimSpace(os.Getenv(envDocsI18nCodexExecutable)); executable != "" {
|
||||
return executable
|
||||
}
|
||||
return "codex"
|
||||
}
|
||||
|
||||
func buildCodexTranslationPrompt(systemPrompt, message string) string {
|
||||
return strings.TrimSpace(systemPrompt) + "\n\n" +
|
||||
"Translate the exact input below. Return only the translated text, with no code fences, no tool calls, no reasoning, and no commentary.\n\n" +
|
||||
"<openclaw_docs_i18n_input>\n" +
|
||||
message +
|
||||
"\n</openclaw_docs_i18n_input>\n"
|
||||
}
|
||||
|
||||
func previewCommandOutput(stdout, stderr string) string {
|
||||
combined := strings.TrimSpace(strings.Join([]string{stdout, stderr}, "\n"))
|
||||
if combined == "" {
|
||||
return "no output"
|
||||
}
|
||||
combined = strings.Join(strings.Fields(combined), " ")
|
||||
const (
|
||||
limit = 1200
|
||||
headLength = 300
|
||||
tailLength = 800
|
||||
)
|
||||
if len(combined) <= limit {
|
||||
return combined
|
||||
}
|
||||
// Codex prints API failures after its header and prompt, so the tail carries the actionable error.
|
||||
return combined[:headLength] + " ... [truncated] ... " + combined[len(combined)-tailLength:]
|
||||
}
|
||||
|
||||
func sleepWithContext(ctx context.Context, delay time.Duration) error {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (t *CodexTranslator) Close() {}
|
||||
|
||||
func normalizeThinking(value string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "low", "medium", "high", "xhigh":
|
||||
return strings.ToLower(strings.TrimSpace(value))
|
||||
default:
|
||||
return "high"
|
||||
}
|
||||
}
|
||||
|
||||
func docsI18nPromptTimeout() time.Duration {
|
||||
value := strings.TrimSpace(os.Getenv(envDocsI18nPromptTimeout))
|
||||
if value == "" {
|
||||
return defaultPromptTimeout
|
||||
}
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return defaultPromptTimeout
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func docsI18nCommandWaitDelay() time.Duration {
|
||||
value := strings.TrimSpace(os.Getenv(envDocsI18nCommandWaitDelay))
|
||||
if value == "" {
|
||||
return defaultCommandWaitDelay
|
||||
}
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return defaultCommandWaitDelay
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
365
scripts/docs-i18n/translator_test.go
Normal file
365
scripts/docs-i18n/translator_test.go
Normal file
@@ -0,0 +1,365 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCodexTranslatorAddsTimeout(t *testing.T) {
|
||||
var deadline time.Time
|
||||
translator := &CodexTranslator{
|
||||
systemPrompt: "Translate from English to Chinese.",
|
||||
thinking: "high",
|
||||
runPrompt: func(ctx context.Context, req codexPromptRequest) (string, error) {
|
||||
var ok bool
|
||||
deadline, ok = ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("expected prompt deadline")
|
||||
}
|
||||
if req.Message != "Translate me" {
|
||||
t.Fatalf("unexpected message %q", req.Message)
|
||||
}
|
||||
if req.Model != defaultOpenAIModel {
|
||||
t.Fatalf("unexpected model %q", req.Model)
|
||||
}
|
||||
if req.Thinking != "high" {
|
||||
t.Fatalf("unexpected thinking %q", req.Thinking)
|
||||
}
|
||||
return "translated", nil
|
||||
},
|
||||
}
|
||||
|
||||
got, err := translator.TranslateRaw(context.Background(), "Translate me", "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("TranslateRaw returned error: %v", err)
|
||||
}
|
||||
if got != "translated" {
|
||||
t.Fatalf("unexpected translation %q", got)
|
||||
}
|
||||
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= time.Minute || remaining > docsI18nPromptTimeout() {
|
||||
t.Fatalf("unexpected timeout window %s", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocsI18nPromptTimeoutUsesEnvOverride(t *testing.T) {
|
||||
t.Setenv(envDocsI18nPromptTimeout, "5m")
|
||||
|
||||
if got := docsI18nPromptTimeout(); got != 5*time.Minute {
|
||||
t.Fatalf("expected 5m timeout, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocsI18nCommandWaitDelayUsesEnvOverride(t *testing.T) {
|
||||
t.Setenv(envDocsI18nCommandWaitDelay, "50ms")
|
||||
|
||||
if got := docsI18nCommandWaitDelay(); got != 50*time.Millisecond {
|
||||
t.Fatalf("expected 50ms wait delay, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableTranslateErrorRejectsDeadlineExceeded(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if isRetryableTranslateError(context.DeadlineExceeded) {
|
||||
t.Fatal("deadline exceeded should not retry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableTranslateErrorRejectsAuthenticationFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if isRetryableTranslateError(errors.New(`Authentication failed for "openai"`)) {
|
||||
t.Fatal("auth failures should not retry")
|
||||
}
|
||||
if isRetryableTranslateError(errors.New("invalid_api_key")) {
|
||||
t.Fatal("API key failures should not retry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableTranslateErrorRetriesTransientCodexFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, message := range []string{
|
||||
"codex exec failed: rate limit 429",
|
||||
"codex exec failed: stream disconnected",
|
||||
"codex exec failed: 503 temporarily unavailable",
|
||||
} {
|
||||
if !isRetryableTranslateError(errors.New(message)) {
|
||||
t.Fatalf("expected retryable error for %q", message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodexTranslatorRetriesTransientFailure(t *testing.T) {
|
||||
previousDelay := translateRetryDelay
|
||||
translateRetryDelay = func(int) time.Duration { return 0 }
|
||||
defer func() { translateRetryDelay = previousDelay }()
|
||||
|
||||
attempts := 0
|
||||
translator := &CodexTranslator{
|
||||
systemPrompt: "Translate from English to Chinese.",
|
||||
thinking: "high",
|
||||
runPrompt: func(context.Context, codexPromptRequest) (string, error) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
return "", errors.New("codex exec failed: stream disconnected")
|
||||
}
|
||||
return "translated", nil
|
||||
},
|
||||
}
|
||||
|
||||
got, err := translator.TranslateRaw(context.Background(), "Translate me", "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("TranslateRaw returned error: %v", err)
|
||||
}
|
||||
if got != "translated" {
|
||||
t.Fatalf("unexpected translation %q", got)
|
||||
}
|
||||
if attempts != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodexTranslatorStripsInputWrapperEcho(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
translator := &CodexTranslator{
|
||||
systemPrompt: "Translate from English to German.",
|
||||
thinking: "high",
|
||||
runPrompt: func(context.Context, codexPromptRequest) (string, error) {
|
||||
return "<openclaw_docs_i18n_input>\nÜbersetzt\n</openclaw_docs_i18n_input>", nil
|
||||
},
|
||||
}
|
||||
|
||||
got, err := translator.TranslateRaw(context.Background(), "Translate me", "en", "de")
|
||||
if err != nil {
|
||||
t.Fatalf("TranslateRaw returned error: %v", err)
|
||||
}
|
||||
if got != "Übersetzt" {
|
||||
t.Fatalf("unexpected translation %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodexTranslatorUsesExactGlossaryMatchWithoutPrompt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
translator, err := NewCodexTranslator("en", "zh-CN", []GlossaryEntry{
|
||||
{Source: "LINE", Target: "LINE"},
|
||||
}, "low")
|
||||
if err != nil {
|
||||
t.Fatalf("NewCodexTranslator returned error: %v", err)
|
||||
}
|
||||
translator.runPrompt = func(context.Context, codexPromptRequest) (string, error) {
|
||||
t.Fatal("exact glossary matches should not call Codex")
|
||||
return "", nil
|
||||
}
|
||||
|
||||
got, err := translator.TranslateRaw(context.Background(), " LINE ", "en", "zh-CN")
|
||||
if err != nil {
|
||||
t.Fatalf("TranslateRaw returned error: %v", err)
|
||||
}
|
||||
if got != " LINE " {
|
||||
t.Fatalf("unexpected translation %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCodexTranslationPromptIncludesGuardrailsAndInput(t *testing.T) {
|
||||
prompt := buildCodexTranslationPrompt("System prompt.", "Hello\nworld")
|
||||
|
||||
for _, want := range []string{
|
||||
"System prompt.",
|
||||
"Return only the translated text",
|
||||
"<openclaw_docs_i18n_input>",
|
||||
"Hello\nworld",
|
||||
"</openclaw_docs_i18n_input>",
|
||||
} {
|
||||
if !strings.Contains(prompt, want) {
|
||||
t.Fatalf("expected %q in prompt:\n%s", want, prompt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCodexExecPromptUsesOutputLastMessage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fakeCodex := filepath.Join(dir, "codex")
|
||||
if err := os.WriteFile(fakeCodex, []byte(`#!/bin/sh
|
||||
set -eu
|
||||
out=""
|
||||
saw_effort=0
|
||||
saw_service=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--output-last-message)
|
||||
shift
|
||||
out="$1"
|
||||
;;
|
||||
-c|--config)
|
||||
shift
|
||||
case "$1" in
|
||||
model_reasoning_effort=\"high\")
|
||||
saw_effort=1
|
||||
;;
|
||||
service_tier=\"fast\")
|
||||
saw_service=1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
shift || true
|
||||
done
|
||||
cat >/dev/null
|
||||
if [ "$saw_effort" != "1" ]; then
|
||||
echo "missing high reasoning effort config" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$saw_service" != "1" ]; then
|
||||
echo "missing fast service tier config" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${CODEX_HOME:-}" ]; then
|
||||
echo "missing CODEX_HOME" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$CODEX_HOME/auth.json" ]; then
|
||||
echo "missing auth.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q '"auth_mode":"apikey"' "$CODEX_HOME/auth.json"; then
|
||||
echo "auth.json missing apikey mode" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q '"OPENAI_API_KEY":"test-openai-key"' "$CODEX_HOME/auth.json"; then
|
||||
echo "auth.json missing API key" >&2
|
||||
exit 1
|
||||
fi
|
||||
case "$CODEX_HOME" in
|
||||
/tmp/*)
|
||||
echo "CODEX_HOME must not be under /tmp" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
printf 'translated from codex\n' > "$out"
|
||||
`), 0o755); err != nil {
|
||||
t.Fatalf("write fake codex: %v", err)
|
||||
}
|
||||
t.Setenv(envDocsI18nCodexExecutable, fakeCodex)
|
||||
t.Setenv("OPENAI_API_KEY", "test-openai-key")
|
||||
|
||||
got, err := runCodexExecPrompt(context.Background(), codexPromptRequest{
|
||||
SystemPrompt: "Translate.",
|
||||
Message: "Hello",
|
||||
Model: "gpt-5.5",
|
||||
Thinking: "high",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runCodexExecPrompt returned error: %v", err)
|
||||
}
|
||||
if got != "translated from codex" {
|
||||
t.Fatalf("unexpected output %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCodexExecPromptUsesOutputLastMessageAfterNonZeroExit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fakeCodex := filepath.Join(dir, "codex")
|
||||
if err := os.WriteFile(fakeCodex, []byte(`#!/bin/sh
|
||||
set -eu
|
||||
out=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--output-last-message)
|
||||
shift
|
||||
out="$1"
|
||||
;;
|
||||
esac
|
||||
shift || true
|
||||
done
|
||||
cat >/dev/null
|
||||
printf 'translated despite nonzero\n' > "$out"
|
||||
echo "transient Codex shutdown failure" >&2
|
||||
exit 1
|
||||
`), 0o755); err != nil {
|
||||
t.Fatalf("write fake codex: %v", err)
|
||||
}
|
||||
t.Setenv(envDocsI18nCodexExecutable, fakeCodex)
|
||||
|
||||
got, err := runCodexExecPrompt(context.Background(), codexPromptRequest{
|
||||
SystemPrompt: "Translate.",
|
||||
Message: "Hello",
|
||||
Model: "gpt-5.5",
|
||||
Thinking: "high",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runCodexExecPrompt returned error: %v", err)
|
||||
}
|
||||
if got != "translated despite nonzero" {
|
||||
t.Fatalf("unexpected output %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCodexExecPromptDoesNotHangOnInheritedPipesAfterTimeout(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fakeCodex := filepath.Join(dir, "codex")
|
||||
if err := os.WriteFile(fakeCodex, []byte(`#!/bin/sh
|
||||
set -eu
|
||||
(sleep 10) &
|
||||
sleep 10
|
||||
`), 0o755); err != nil {
|
||||
t.Fatalf("write fake codex: %v", err)
|
||||
}
|
||||
t.Setenv(envDocsI18nCodexExecutable, fakeCodex)
|
||||
t.Setenv(envDocsI18nCommandWaitDelay, "20ms")
|
||||
t.Setenv("OPENAI_API_KEY", "test-openai-key")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
_, err := runCodexExecPrompt(ctx, codexPromptRequest{
|
||||
SystemPrompt: "Translate.",
|
||||
Message: "Hello",
|
||||
Model: "gpt-5.5",
|
||||
Thinking: "high",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error")
|
||||
}
|
||||
if elapsed := time.Since(started); elapsed > 2*time.Second {
|
||||
t.Fatalf("expected bounded timeout, took %s", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewCommandOutputFlattensAndTruncates(t *testing.T) {
|
||||
input := "line one\n\nline two\tline three " + strings.Repeat("x", 1200) + " final api error 429"
|
||||
preview := previewCommandOutput(input, "")
|
||||
if strings.Contains(preview, "\n") {
|
||||
t.Fatalf("expected flattened whitespace, got %q", preview)
|
||||
}
|
||||
if !strings.HasPrefix(preview, "line one line two line three ") {
|
||||
t.Fatalf("unexpected preview prefix: %q", preview)
|
||||
}
|
||||
if !strings.Contains(preview, "... [truncated] ...") {
|
||||
t.Fatalf("expected truncation marker, got %q", preview)
|
||||
}
|
||||
if !strings.HasSuffix(preview, "final api error 429") {
|
||||
t.Fatalf("expected retained error tail, got %q", preview)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewCommandOutputRetainsStderrTail(t *testing.T) {
|
||||
stdout := "startup banner " + strings.Repeat("x", 1200)
|
||||
stderr := "provider api error 429"
|
||||
preview := previewCommandOutput(stdout, stderr)
|
||||
if !strings.HasPrefix(preview, "startup banner ") {
|
||||
t.Fatalf("unexpected preview prefix: %q", preview)
|
||||
}
|
||||
if !strings.HasSuffix(preview, stderr) {
|
||||
t.Fatalf("expected retained stderr tail, got %q", preview)
|
||||
}
|
||||
}
|
||||
128
scripts/docs-i18n/util.go
Normal file
128
scripts/docs-i18n/util.go
Normal file
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
workflowVersion = 16
|
||||
docsI18nEngineName = "codex"
|
||||
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
|
||||
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"
|
||||
defaultOpenAIModel = "gpt-5.5"
|
||||
defaultFallbackProvider = "openai"
|
||||
defaultFallbackModelName = defaultOpenAIModel
|
||||
)
|
||||
|
||||
var translationTranscriptArtifactRE = regexp.MustCompile(`(?i)(?:\b(?:analysis|commentary|final|assistant|user)\s+to\s*=\s*(?:functions\.[a-z0-9_-]+|[a-z_]+)|\bto\s*=\s*(?:functions\.[a-z0-9_-]+|analysis|commentary|final)\b|\bfunctions\.[a-z0-9_-]+\b|/home/runner/work/|\.agents/skills/|\bforce_parallel\s*:|\bcode\s+omitted\b|\bomitted\s+reasoning\b|全民彩票|娱乐平台开户|娱乐平台|皇平台|彩票平台|一本道|毛片|高清视频免费|不卡免费播放)`)
|
||||
|
||||
func cacheNamespace() string {
|
||||
return fmt.Sprintf(
|
||||
"wf=%d|engine=%s|provider=%s|model=%s",
|
||||
workflowVersion,
|
||||
docsI18nEngineName,
|
||||
docsI18nProvider(),
|
||||
docsI18nModel(),
|
||||
)
|
||||
}
|
||||
|
||||
func cacheKey(namespace, srcLang, tgtLang, segmentID, textHash string) string {
|
||||
raw := fmt.Sprintf("%s|%s|%s|%s|%s", namespace, srcLang, tgtLang, segmentID, textHash)
|
||||
hash := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func hashText(text string) string {
|
||||
normalized := normalizeText(text)
|
||||
hash := sha256.Sum256([]byte(normalized))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func hashBytes(data []byte) string {
|
||||
hash := sha256.Sum256(data)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func normalizeText(text string) string {
|
||||
return strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
|
||||
}
|
||||
|
||||
func docsI18nProvider() string {
|
||||
if value := strings.TrimSpace(os.Getenv(envDocsI18nProvider)); strings.EqualFold(value, "openai") {
|
||||
return value
|
||||
}
|
||||
return defaultFallbackProvider
|
||||
}
|
||||
|
||||
func docsI18nModel() string {
|
||||
if value := strings.TrimSpace(os.Getenv(envDocsI18nModel)); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultFallbackModelName
|
||||
}
|
||||
|
||||
func segmentID(relPath, textHash string) string {
|
||||
shortHash := textHash
|
||||
if len(shortHash) > 16 {
|
||||
shortHash = shortHash[:16]
|
||||
}
|
||||
return fmt.Sprintf("%s:%s", relPath, shortHash)
|
||||
}
|
||||
|
||||
func splitWhitespace(text string) (string, string, string) {
|
||||
if text == "" {
|
||||
return "", "", ""
|
||||
}
|
||||
start := 0
|
||||
for start < len(text) && isWhitespace(text[start]) {
|
||||
start++
|
||||
}
|
||||
end := len(text)
|
||||
for end > start && isWhitespace(text[end-1]) {
|
||||
end--
|
||||
}
|
||||
return text[:start], text[start:end], text[end:]
|
||||
}
|
||||
|
||||
func isWhitespace(b byte) bool {
|
||||
switch b {
|
||||
case ' ', '\t', '\n', '\r':
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validateNoTranslationTranscriptArtifacts(source, translated string) error {
|
||||
sourceLower := strings.ToLower(source)
|
||||
for _, token := range []string{"<openclaw_docs_i18n_input>", "</openclaw_docs_i18n_input>"} {
|
||||
if strings.Contains(strings.ToLower(translated), token) && !strings.Contains(sourceLower, token) {
|
||||
return fmt.Errorf("agent transcript artifact leaked into translation: %q", token)
|
||||
}
|
||||
}
|
||||
for _, match := range translationTranscriptArtifactRE.FindAllString(translated, -1) {
|
||||
match = strings.TrimSpace(match)
|
||||
if match == "" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(sourceLower, strings.ToLower(match)) {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("agent transcript artifact leaked into translation: %q", match)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(os.Stderr, err.Error()+"\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
28
scripts/docs-i18n/util_test.go
Normal file
28
scripts/docs-i18n/util_test.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDocsI18nProviderUsesOpenAI(t *testing.T) {
|
||||
t.Setenv(envDocsI18nProvider, "anthropic")
|
||||
t.Setenv("ANTHROPIC_API_KEY", "anthropic-key")
|
||||
|
||||
if got := docsI18nProvider(); got != "openai" {
|
||||
t.Fatalf("expected OpenAI provider, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocsI18nModelKeepsOpenAIDefaultAtGPT55(t *testing.T) {
|
||||
t.Setenv(envDocsI18nModel, "")
|
||||
|
||||
if got := docsI18nModel(); got != defaultOpenAIModel {
|
||||
t.Fatalf("expected OpenAI default model %q, got %q", defaultOpenAIModel, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocsI18nModelPrefersExplicitOverride(t *testing.T) {
|
||||
t.Setenv(envDocsI18nModel, "__test_model_override__")
|
||||
|
||||
if got := docsI18nModel(); got != "__test_model_override__" {
|
||||
t.Fatalf("expected explicit model override, got %q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user