const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const matter = require('gray-matter'); const https = require('https'); // Configuration const DEEPL_API_BASE_URL_FREE = 'https://api-free.deepl.com'; const DEEPL_API_BASE_URL_PRO = 'https://api.deepl.com'; const DEEPL_TRANSLATE_PATH = '/v2/translate'; const DEEPL_GLOSSARIES_PATH = '/v3/glossaries'; const DEEPL_BATCH_LIMIT = 50; const DEEPL_CONTEXT_LIMIT = 8000; const DEFAULT_GLOSSARY_NAME = 'CBDocs Lock Terms'; const DEFAULT_GLOSSARY_SYNC_LOCALES = ['fr', 'es', 'no', 'it']; const DEFAULT_CUSTOM_INSTRUCTIONS = [ 'Use professional language for financial technical documentation.', 'Prefer natural target-language terminology over literal English calques.', 'Use terminology appropriate for institutional credit-risk readers.', 'Preserve product names such as Credit Benchmark and Consensus Credit Rating.', 'Treat Consensus PD and CCR as Credit Benchmark product/domain terms, not generic phrases.', 'Translate headings and list labels naturally for documentation navigation.', ]; const UI_SKIP_MESSAGES = new Set(); const UI_SKIP_KEYS = new Set(); const NAV_LOCALIZABLE_KEYS = new Set([ 'tab', 'group', 'dropdown', 'product', 'item', 'anchor', 'name', 'description', ]); const TRANSLATABLE_JSX_ATTRS = new Set(['title']); // Locale mapping (DeepL format) const LOCALE_MAP = { fr: 'FR', es: 'ES', de: 'DE', no: 'NB', nb: 'NB', it: 'IT', pt: 'PT', pl: 'PL', ru: 'RU', ja: 'JA', zh: 'ZH', }; // Directories to translate (Mintlify content tree) const TRANSLATE_DIRS = ['']; // Directories to exclude const EXCLUDE_DIRS = [ ...Object.keys(LOCALE_MAP), 'images', 'fonts', 'openapi', 'interactive-api', 'scripts', 'node_modules', ]; // File extensions to process const TRANSLATABLE_EXTENSIONS = ['.md', '.mdx']; const FRONTMATTER_TRANSLATABLE_KEYS = new Set([ 'title', 'sidebar_label', 'description', 'summary', ]); const PLACEHOLDER_PATTERN = /__(?:CODE_BLOCK|INLINE_CODE|MATH_BLOCK|MATH_INLINE|ADMONITION|TRUNCATE|MDX_IMPORT|MDX_TAG|MD_TABLE_PIPE|MD_HEADING|DEEPL_IMAGE)_\d+__/g; const CACHE_VERSION = 2; const CONTENT_TRANSLATION_VERSION = 5; const CACHE_FILE = path.join(__dirname, '..', '.deepl-cache.json'); let translationCache = null; let cacheReadEnabled = true; let cacheStats = null; let lockTerms = null; let deeplConfig = null; let warnedAboutFormatting = false; const MAX_RETRIES = 5; const BASE_RETRY_DELAY_MS = 1000; /** * Parse command line arguments */ function parseArgs() { const args = process.argv.slice(2); let locale = null; let testOnly = false; let syncGlossaries = false; let noCache = false; let clearCache = false; let filePath = null; let overwrite = false; for (let i = 0; i < args.length; i++) { if ((args[i] === '--locale' || args[i] === '-l') && i + 1 < args.length) { locale = args[i + 1]; i++; } else if (args[i].startsWith('--locale=')) { locale = args[i].split('=')[1]; } else if (args[i].startsWith('-l')) { locale = args[i].substring(2); } else if (args[i] === '--test' || args[i] === '--test-only' || args[i] === '-t') { testOnly = true; } else if (args[i] === '--sync-glossaries') { syncGlossaries = true; } else if (args[i] === '--no-cache') { noCache = true; } else if (args[i] === '--clear-cache') { clearCache = true; } else if (args[i] === '--overwrite') { overwrite = true; } else if ((args[i] === '--file' || args[i] === '-f') && i + 1 < args.length) { filePath = args[i + 1]; i++; } else if (args[i].startsWith('--file=')) { filePath = args[i].split('=')[1]; } else if (args[i].startsWith('--test=')) { const value = args[i].split('=')[1]; testOnly = value !== 'false' && value !== '0'; } else if (args[i].startsWith('-t') && args[i].length > 2) { const value = args[i].substring(2); testOnly = value !== 'false' && value !== '0'; } } if (!testOnly && !syncGlossaries && !locale) { console.error('Error: --locale or -l flag is required'); console.error('Usage: npm run translate:fr'); console.error(' npm run translate:es'); console.error(' npm run translate:no'); console.error(' npm run translate:it'); console.error(' npm run sync:glossaries'); console.error(' npm run sync:glossaries -- --locale fr'); console.error(' node scripts/translate-docs.js --locale fr --file methodology/intro.mdx'); console.error(' node scripts/translate-docs.js --locale fr --overwrite'); console.error(' node scripts/translate-docs.js --test'); console.error(' node scripts/translate-docs.js --sync-glossaries'); console.error(' node scripts/translate-docs.js --locale fr --no-cache'); console.error(' node scripts/translate-docs.js --locale fr --clear-cache'); process.exit(1); } return { locale, testOnly, syncGlossaries, noCache, clearCache, filePath, overwrite }; } /** * Load DeepL API key from config file */ function loadConfig() { const configPaths = [ path.join(__dirname, '..', '.deeplrc.json'), path.join(__dirname, '..', '..', 'website', '.deeplrc.json'), ]; const configPath = configPaths.find((candidate) => fs.existsSync(candidate)); if (!configPath) { console.error(`Error: Config file not found at ${configPaths[0]}`); console.error(`Checked fallback path: ${configPaths[1]}`); console.error('Please create .deeplrc.json with your API key. See .deeplrc.json.example for template.'); process.exit(1); } try { const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); if (!config.apiKey) { console.error('Error: apiKey not found in .deeplrc.json'); process.exit(1); } return config; } catch (error) { console.error(`Error reading config file: ${error.message}`); process.exit(1); } } /** * Check if file should be excluded */ function shouldExclude(filePath) { return EXCLUDE_DIRS.some(excludeDir => { const fullExcludePath = path.join(__dirname, '..', excludeDir); return filePath === fullExcludePath || filePath.startsWith(`${fullExcludePath}${path.sep}`); }); } function getContentTarget(filePath, locale, baseDir) { const normalized = filePath.replace(/\//g, path.sep).replace(new RegExp(`^\\.${escapeRegExp(path.sep)}`), ''); const targetBase = path.join(baseDir, locale.toLowerCase()); const targetDir = path.join(targetBase, path.dirname(normalized)); const targetPath = path.join(targetDir, path.basename(normalized)); return { type: 'content', relativePath: normalized, targetDir, targetPath, displayPath: path.join(locale.toLowerCase(), normalized), }; } function getDeepLApiBaseUrl(apiKey) { return apiKey.endsWith(':fx') ? DEEPL_API_BASE_URL_FREE : DEEPL_API_BASE_URL_PRO; } function getTargetLanguageCode(locale) { const localeKey = locale ? locale.toLowerCase() : ''; return LOCALE_MAP[localeKey] || localeKey.toUpperCase(); } function getGlossaryName(config) { if (config && typeof config.glossaryName === 'string' && config.glossaryName.trim() !== '') { return config.glossaryName.trim(); } return DEFAULT_GLOSSARY_NAME; } function getGlossaryNameForLocale(config, locale) { return `${getGlossaryName(config)} (${getTargetLanguageCode(locale)})`; } function getStyleId(config) { if (config && typeof config.styleId === 'string' && config.styleId.trim() !== '') { return config.styleId.trim(); } if (config && typeof config.style_id === 'string' && config.style_id.trim() !== '') { return config.style_id.trim(); } return null; } function getCustomInstructions(config, locale) { const localeKey = locale ? locale.toLowerCase() : ''; const configured = config && config.customInstructions; if (Array.isArray(configured)) { return configured.filter((item) => typeof item === 'string' && item.trim() !== '').map((item) => item.trim()); } if (configured && typeof configured === 'object') { const localeInstructions = configured[localeKey] || configured.default; if (Array.isArray(localeInstructions)) { return localeInstructions.filter((item) => typeof item === 'string' && item.trim() !== '').map((item) => item.trim()); } } return DEFAULT_CUSTOM_INSTRUCTIONS; } function getGlossarySyncLocales(locale) { if (locale) { return [locale.toLowerCase()]; } return DEFAULT_GLOSSARY_SYNC_LOCALES; } function getGlossaryDirCandidates() { return [ path.join(__dirname, '..', 'glossary'), path.join(__dirname, '..', '..', 'website', 'glossary'), ]; } function findLockTermCsvPath() { const glossaryPaths = [ path.join(__dirname, '..', 'glossary', 'lockterm.csv'), path.join(__dirname, '..', 'glossary', 'lock-terms.csv'), path.join(__dirname, '..', '..', 'website', 'glossary', 'lockterm.csv'), path.join(__dirname, '..', '..', 'website', 'glossary', 'lock-terms.csv'), ]; return glossaryPaths.find((candidate) => fs.existsSync(candidate)) || null; } function findLocaleTermCsvPath(locale) { const localeKey = locale ? locale.toLowerCase() : ''; if (!localeKey) { return null; } const glossaryPaths = getGlossaryDirCandidates().flatMap((dir) => [ path.join(dir, `translation-terms.${localeKey}.csv`), path.join(dir, `lock-terms.${localeKey}.csv`), ]); return glossaryPaths.find((candidate) => fs.existsSync(candidate)) || null; } function parseCsvRows(raw) { const rows = []; let row = []; let field = ''; let inQuotes = false; for (let i = 0; i < raw.length; i++) { const char = raw[i]; const next = raw[i + 1]; if (inQuotes) { if (char === '"' && next === '"') { field += '"'; i++; } else if (char === '"') { inQuotes = false; } else { field += char; } continue; } if (char === '"') { inQuotes = true; continue; } if (char === ',') { row.push(field); field = ''; continue; } if (char === '\n') { row.push(field); rows.push(row); row = []; field = ''; continue; } if (char === '\r') { continue; } field += char; } if (field !== '' || row.length > 0) { row.push(field); rows.push(row); } return rows.filter((currentRow) => currentRow.some((value) => value.trim() !== '')); } function normalizeLockTerms(terms) { const deduped = new Map(); let duplicateCount = 0; terms.forEach((term) => { if (deduped.has(term.source)) { duplicateCount += 1; } deduped.set(term.source, term); }); if (duplicateCount > 0) { console.warn(`Warning: Found ${duplicateCount} duplicate lock term row(s). Keeping the last value for each source term.`); } return Array.from(deduped.values()).sort((a, b) => b.source.length - a.source.length); } function loadTermsFromCsv(glossaryPath) { if (!glossaryPath) { return []; } const raw = fs.readFileSync(glossaryPath, 'utf8').replace(/^\uFEFF/, ''); const rows = parseCsvRows(raw); const terms = []; rows.forEach((parts, index) => { if (parts.length < 2) { return; } const source = parts[0].trim(); const target = parts[1].trim(); const isHeaderRow = index === 0 && source.toLowerCase() === 'source' && target.toLowerCase() === 'target'; if (isHeaderRow) { return; } if (!source || !target) { return; } terms.push({ source, target }); }); return terms; } function loadLockTerms(locale = null) { const terms = [ ...loadTermsFromCsv(findLockTermCsvPath()), ...loadTermsFromCsv(findLocaleTermCsvPath(locale)), ]; return normalizeLockTerms(terms); } function buildGlossaryEntries(terms) { return terms.map(({ source, target }) => `${source}\t${target}`).join('\n'); } function selectGlossary(glossaries, glossaryName) { const matching = glossaries .filter((glossary) => glossary.name === glossaryName) .sort((a, b) => new Date(b.creation_time || 0) - new Date(a.creation_time || 0)); if (matching.length > 1) { console.warn(`Warning: Found ${matching.length} DeepL glossaries named "${glossaryName}". Using the most recent one.`); } return matching[0] || null; } async function sendDeepLJsonRequest(method, requestPath, apiKey, payload = null, expectedStatusCodes = [200]) { return new Promise((resolve, reject) => { const body = payload === null ? null : JSON.stringify(payload); const url = new URL(requestPath, getDeepLApiBaseUrl(apiKey)); const headers = { 'Authorization': `DeepL-Auth-Key ${apiKey}`, }; if (body !== null) { headers['Content-Type'] = 'application/json'; headers['Content-Length'] = Buffer.byteLength(body); } const options = { hostname: url.hostname, port: 443, path: url.pathname + url.search, method, headers, }; const req = https.request(options, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { if (expectedStatusCodes.includes(res.statusCode)) { if (data === '') { resolve(null); return; } try { resolve(JSON.parse(data)); } catch (error) { reject(new Error(`Failed to parse DeepL API response: ${error.message}`)); } return; } const error = new Error(`DeepL API error (${res.statusCode}): ${data}`); error.statusCode = res.statusCode; error.data = data; reject(error); }); }); req.on('error', (error) => { reject(new Error(`Request failed: ${error.message}`)); }); if (body !== null) { req.write(body); } req.end(); }); } async function listDeepLGlossaries(apiKey) { const response = await sendDeepLJsonRequest('GET', DEEPL_GLOSSARIES_PATH, apiKey); return Array.isArray(response && response.glossaries) ? response.glossaries : []; } async function listDeepLGlossariesV2(apiKey) { const response = await sendDeepLJsonRequest('GET', '/v2/glossaries', apiKey); return Array.isArray(response && response.glossaries) ? response.glossaries : []; } async function createDeepLGlossary(apiKey, glossaryName, sourceLang, targetLang, entries) { return sendDeepLJsonRequest('POST', DEEPL_GLOSSARIES_PATH, apiKey, { name: glossaryName, dictionaries: [ { source_lang: sourceLang.toLowerCase(), target_lang: targetLang.toLowerCase(), entries, entries_format: 'tsv', }, ], }, [201]); } async function upsertDeepLGlossaryDictionary(apiKey, glossaryId, sourceLang, targetLang, entries) { return sendDeepLJsonRequest('PUT', `${DEEPL_GLOSSARIES_PATH}/${glossaryId}/dictionaries`, apiKey, { source_lang: sourceLang.toLowerCase(), target_lang: targetLang.toLowerCase(), entries, entries_format: 'tsv', }); } async function createDeepLGlossaryV2(apiKey, glossaryName, sourceLang, targetLang, entries) { return sendDeepLJsonRequest('POST', '/v2/glossaries', apiKey, { name: glossaryName, source_lang: sourceLang.toUpperCase(), target_lang: targetLang.toUpperCase(), entries, entries_format: 'tsv', }, [201]); } async function deleteDeepLGlossaryV2(apiKey, glossaryId) { return sendDeepLJsonRequest('DELETE', `/v2/glossaries/${glossaryId}`, apiKey, null, [204]); } function selectGlossariesV2(glossaries, glossaryName, sourceLang, targetLang) { return glossaries .filter((glossary) => glossary.name === glossaryName && String(glossary.source_lang || '').toUpperCase() === sourceLang.toUpperCase() && String(glossary.target_lang || '').toUpperCase() === targetLang.toUpperCase()) .sort((a, b) => new Date(b.creation_time || 0) - new Date(a.creation_time || 0)); } async function ensureDeepLGlossaryForLocalesV2(locales, apiKey, config, terms) { const normalizedLocales = Array.from(new Set(locales.map((locale) => locale.toLowerCase()))); if (normalizedLocales.length === 0 || !terms || terms.length === 0) { return null; } const sourceLang = 'EN'; const entries = buildGlossaryEntries(terms); const existingGlossaries = await listDeepLGlossariesV2(apiKey); const glossaryIdsByLocale = new Map(); for (const locale of normalizedLocales) { const targetLang = getTargetLanguageCode(locale); const glossaryName = getGlossaryNameForLocale(config, locale); const matches = selectGlossariesV2(existingGlossaries, glossaryName, sourceLang, targetLang); for (const match of matches) { await deleteDeepLGlossaryV2(apiKey, match.glossary_id); } const created = await createDeepLGlossaryV2(apiKey, glossaryName, sourceLang, targetLang, entries); console.log(` [ok] Synced DeepL v2 glossary "${glossaryName}" (${created.glossary_id}) for ${locale.toUpperCase()}`); glossaryIdsByLocale.set(locale, created.glossary_id); } if (normalizedLocales.length === 1) { return glossaryIdsByLocale.get(normalizedLocales[0]) || null; } return glossaryIdsByLocale; } async function ensureDeepLGlossaryForLocales(locales, apiKey, config, terms) { const normalizedLocales = Array.from(new Set(locales.map((locale) => locale.toLowerCase()))); if (normalizedLocales.length === 0 || !terms || terms.length === 0) { return null; } try { const glossaryName = getGlossaryName(config); const entries = buildGlossaryEntries(terms); const sourceLang = 'EN'; const glossaries = await listDeepLGlossaries(apiKey); let glossary = selectGlossary(glossaries, glossaryName); if (!glossary) { const firstLocale = normalizedLocales[0]; const firstTargetLang = getTargetLanguageCode(firstLocale); glossary = await createDeepLGlossary(apiKey, glossaryName, sourceLang, firstTargetLang, entries); console.log(` [ok] Created DeepL glossary "${glossaryName}" (${glossary.glossary_id}) for ${firstLocale.toUpperCase()}`); normalizedLocales.shift(); } for (const locale of normalizedLocales) { const targetLang = getTargetLanguageCode(locale); await upsertDeepLGlossaryDictionary(apiKey, glossary.glossary_id, sourceLang, targetLang, entries); console.log(` [ok] Synced DeepL glossary "${glossaryName}" (${glossary.glossary_id}) for ${locale.toUpperCase()}`); } return glossary.glossary_id; } catch (error) { if (error && error.statusCode === 403) { console.warn('Warning: DeepL v3 glossary endpoints are not available for this account. Falling back to v2 glossaries.'); return ensureDeepLGlossaryForLocalesV2(normalizedLocales, apiKey, config, terms); } throw error; } } async function resolveGlossaryId(locale, apiKey, config, terms) { if (!terms || terms.length === 0) { return null; } return ensureDeepLGlossaryForLocales([locale], apiKey, config, terms); } async function syncGlossariesForConfiguredLocales(locale, apiKey, config) { const locales = getGlossarySyncLocales(locale); for (const glossaryLocale of locales) { const terms = config.lockTerms === false ? [] : loadLockTerms(glossaryLocale); if (!terms || terms.length === 0) { console.log(`No lock terms found for ${glossaryLocale.toUpperCase()}. Skipping.`); continue; } await ensureDeepLGlossaryForLocales([glossaryLocale], apiKey, config, terms); } console.log(`Synced DeepL glossary for ${locales.length} locale(s).`); } function warnGlossarySyncFailure(locale, error) { console.warn(`Warning: Failed to sync DeepL glossary for locale "${locale}". Continuing without glossary.`); console.warn(` ${error.message}`); } function applyTermLocks(text, terms) { if (!terms || terms.length === 0) { return { text, map: new Map() }; } let updated = text; const map = new Map(); terms.filter(shouldUsePlaceholderTermLock).forEach((term, index) => { // Put the HTML-escaped term inside the ignored tag. This gives DeepL semantic // context (it sees e.g. "S&P" as a proper noun and integrates grammar correctly), // while ignoreTags prevents the content from being translated. Opaque placeholders // like empty tags or "__TERM_N__" caused DeepL to add stray articles/suffixes // (e.g. "d'S&P s et de Fitch") because it had no hint about the term's grammar. const placeholder = `${escapeHtml(term.target)}`; map.set(placeholder, term.target); const escaped = escapeRegExp(term.source); const useWordBoundary = /^[A-Za-z0-9_]+$/.test(term.source); const regex = useWordBoundary ? new RegExp(`\\b${escaped}\\b`, 'g') : new RegExp(escaped, 'g'); updated = updated.replace(regex, placeholder); }); return { text: updated, map }; } function shouldUsePlaceholderTermLock(term) { if (!term || term.source !== term.target) { return false; } const source = term.source; if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(source) && !/&/.test(source)) { return false; } const hasIdentifierMarker = /[_0-9]/.test(source) || /[a-z][A-Z]/.test(source) || /^[A-Z]{2,}[A-Z][a-z]/.test(source) || /^[A-Za-z]+&[A-Za-z]+$/.test(source); const isShortAcronym = /^[A-Z]{2,5}s?$/.test(source); return hasIdentifierMarker && !isShortAcronym; } function restoreTermLocks(text, map) { if (!map || map.size === 0) { return text; } const wordChar = /[\p{L}\p{N}]/u; const spaced = (target, match, offset, str) => { const before = offset > 0 ? str[offset - 1] : ''; const after = offset + match.length < str.length ? str[offset + match.length] : ''; return (before && wordChar.test(before) ? ' ' : '') + target + (after && wordChar.test(after) ? ' ' : ''); }; let restored = text; map.forEach((target, placeholder) => { const escapedPlaceholder = new RegExp(escapeRegExp(placeholder), 'g'); restored = restored.replace(escapedPlaceholder, (match, offset, str) => spaced(target, match, offset, str)); const indexMatch = placeholder.match(/data-i="(\d+)"/); if (indexMatch) { const tagRegex = new RegExp(`]*data-i=(?:"${indexMatch[1]}"|'${indexMatch[1]}'|${indexMatch[1]})[^>]*>[\\s\\S]*?<\\/cb-lock>`, 'g'); restored = restored.replace(tagRegex, (match, offset, str) => spaced(target, match, offset, str)); } }); return restored; } function normalizeFileArg(fileArg, baseDir) { let normalized = fileArg.trim(); if (path.isAbsolute(normalized)) { normalized = path.relative(baseDir, normalized); } normalized = normalized.replace(/\//g, path.sep); if (normalized.startsWith(`.${path.sep}`)) { normalized = normalized.slice(2); } return normalized; } function deepClone(value) { return JSON.parse(JSON.stringify(value)); } function prefixLocalePages(value, locale) { if (Array.isArray(value)) { return value.map((item) => prefixLocalePages(item, locale)); } if (!value || typeof value !== 'object') { return value; } const out = {}; Object.entries(value).forEach(([key, inner]) => { if (key === 'pages' && Array.isArray(inner)) { out[key] = inner.map((page) => { if (typeof page !== 'string') { return prefixLocalePages(page, locale); } if (/^(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) /.test(page)) { return page; } return page.startsWith(`${locale}/`) ? page : `${locale}/${page}`; }); return; } out[key] = prefixLocalePages(inner, locale); }); return out; } function collectLocalizableFields(node, fields = []) { if (Array.isArray(node)) { node.forEach((item) => collectLocalizableFields(item, fields)); return fields; } if (!node || typeof node !== 'object') { return fields; } Object.entries(node).forEach(([key, value]) => { if (NAV_LOCALIZABLE_KEYS.has(key) && typeof value === 'string' && value.trim() !== '') { fields.push({ node, key, value }); } collectLocalizableFields(value, fields); }); return fields; } function loadTranslationCache() { if (!fs.existsSync(CACHE_FILE)) { return { version: CACHE_VERSION, entries: {}, uiFiles: {}, contentFiles: {} }; } try { const raw = fs.readFileSync(CACHE_FILE, 'utf8'); const parsed = JSON.parse(raw); if (!parsed || typeof parsed !== 'object') { return { version: CACHE_VERSION, entries: {}, uiFiles: {}, contentFiles: {} }; } if (!parsed.entries || typeof parsed.entries !== 'object') { parsed.entries = {}; } if (!parsed.uiFiles || typeof parsed.uiFiles !== 'object') { parsed.uiFiles = {}; } if (!parsed.contentFiles || typeof parsed.contentFiles !== 'object') { parsed.contentFiles = {}; } if (parsed.version !== CACHE_VERSION) { return { version: CACHE_VERSION, entries: parsed.entries, uiFiles: parsed.uiFiles, contentFiles: parsed.contentFiles, }; } return parsed; } catch (error) { return { version: CACHE_VERSION, entries: {}, uiFiles: {}, contentFiles: {} }; } } function saveTranslationCache(cache) { fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2), 'utf8'); } function makeCacheKey(text, sourceLang, targetLang, options = {}) { const hash = crypto.createHash('sha256') .update(sourceLang) .update('\n') .update(targetLang) .update('\n') .update(JSON.stringify({ context: options.context || '', modelType: options.modelType || '', formality: options.formality || '', customInstructions: options.customInstructions || [], styleId: options.styleId || '', tagHandling: options.tagHandling || '', tagHandlingVersion: options.tagHandlingVersion || '', ignoreTags: options.ignoreTags || [], translationVersion: CONTENT_TRANSLATION_VERSION, })) .update('\n') .update(text) .digest('hex'); return `${sourceLang}:${targetLang}:${hash}`; } function makeUiHash(entries) { const hash = crypto.createHash('sha256'); entries.forEach(({ key, message }) => { hash.update(key); hash.update('\n'); hash.update(message || ''); hash.update('\n'); }); return hash.digest('hex'); } function makeContentHash(content) { return crypto.createHash('sha256').update(content).digest('hex'); } function getUiMeta() { if (!cacheReadEnabled || !translationCache) { return null; } if (!translationCache.uiFiles || typeof translationCache.uiFiles !== 'object') { translationCache.uiFiles = {}; } return translationCache.uiFiles; } function getContentMeta() { if (!cacheReadEnabled || !translationCache) { return null; } if (!translationCache.contentFiles || typeof translationCache.contentFiles !== 'object') { translationCache.contentFiles = {}; } return translationCache.contentFiles; } function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } function escapeHtml(value) { return String(value) .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"'); } function decodeHtmlEntities(value) { return String(value) .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16))) .replace(/&#(\d+);/g, (_, decimal) => String.fromCodePoint(Number.parseInt(decimal, 10))) .replace(/"/g, '"') .replace(/'/g, "'") .replace(/'/g, "'") .replace(/</g, '<') .replace(/>/g, '>') .replace(/&/g, '&'); } function makePlaceholderRegex(placeholder) { return new RegExp(`${escapeRegExp(placeholder)}(?:_(?!_))?`, 'g'); } /** * Check if file is translatable (has correct extension) */ function isTranslatable(filePath) { const ext = path.extname(filePath); return TRANSLATABLE_EXTENSIONS.includes(ext); } /** * Recursively find all translatable files in a directory */ function findTranslatableFiles(dir, baseDir) { const files = []; const fullPath = path.join(baseDir, dir); if (!fs.existsSync(fullPath)) { return files; } const entries = fs.readdirSync(fullPath, { withFileTypes: true }); for (const entry of entries) { const entryPath = path.join(fullPath, entry.name); const relativePath = path.join(dir, entry.name); if (shouldExclude(entryPath)) { continue; } if (entry.isDirectory()) { files.push(...findTranslatableFiles(relativePath, baseDir)); } else if (entry.isFile() && isTranslatable(entryPath)) { files.push(relativePath); } } return files; } function findLocaleTranslatedFiles(locale, baseDir, subDir = '') { const files = []; const localeKey = locale.toLowerCase(); const localeRoot = path.join(baseDir, localeKey); const currentDir = path.join(localeRoot, subDir); if (!fs.existsSync(currentDir)) { return files; } const entries = fs.readdirSync(currentDir, { withFileTypes: true }); for (const entry of entries) { if (entry.name === '_ui') { continue; } const entryPath = path.join(currentDir, entry.name); const entryRelativePath = subDir ? path.join(subDir, entry.name) : entry.name; if (entry.isDirectory()) { files.push(...findLocaleTranslatedFiles(localeKey, baseDir, entryRelativePath)); } else if (entry.isFile() && isTranslatable(entryPath)) { files.push(path.join(localeKey, entryRelativePath)); } } return files; } function removeEmptyDirectoriesUpward(startDir, stopDir) { const stopPath = path.resolve(stopDir); let currentDir = path.resolve(startDir); while (currentDir.startsWith(stopPath) && currentDir !== stopPath) { if (!fs.existsSync(currentDir)) { currentDir = path.dirname(currentDir); continue; } const entries = fs.readdirSync(currentDir); if (entries.length > 0) { break; } fs.rmdirSync(currentDir); currentDir = path.dirname(currentDir); } } function pruneStaleTranslatedFiles(locale, baseDir, sourceFiles) { const localeKey = locale.toLowerCase(); const localeRoot = path.join(baseDir, localeKey); const existingFiles = findLocaleTranslatedFiles(localeKey, baseDir); const expectedSourceFiles = new Set(sourceFiles); const expectedTargetFiles = new Set( sourceFiles .map((filePath) => getContentTarget(filePath, localeKey, baseDir)) .filter(Boolean) .map((targetInfo) => targetInfo.displayPath) ); const contentMeta = getContentMeta(); let deletedFiles = 0; existingFiles.forEach((displayPath) => { if (expectedTargetFiles.has(displayPath)) { return; } const targetPath = path.join(baseDir, displayPath); fs.unlinkSync(targetPath); deletedFiles += 1; console.log(`[delete] Deleted stale translated file ${displayPath}`); removeEmptyDirectoriesUpward(path.dirname(targetPath), localeRoot); }); if (contentMeta) { Object.keys(contentMeta).forEach((metaKey) => { if (!metaKey.startsWith(`${localeKey}:`)) { return; } const sourcePath = metaKey.slice(localeKey.length + 1); if (expectedSourceFiles.has(sourcePath)) { return; } delete contentMeta[metaKey]; }); } return { deletedFiles }; } /** * Extract and replace code blocks with placeholders */ function preserveCodeBlocks(content) { const codeBlocks = []; const codeBlockRegex = /```[\s\S]*?```/g; const inlineCodeRegex = /`[^`\n]+`/g; // Replace code blocks let processed = content.replace(codeBlockRegex, (match) => { const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`; codeBlocks.push(match); return placeholder; }); // Replace inline code processed = processed.replace(inlineCodeRegex, (match) => { const index = codeBlocks.length; const inner = match.slice(1, -1); codeBlocks.push(match); return `${escapeHtml(inner)}`; }); return { processed, codeBlocks }; } /** * Restore code blocks from placeholders */ function restoreCodeBlocks(content, codeBlocks) { let restored = content; codeBlocks.forEach((codeBlock, index) => { const codeBlockPlaceholder = makePlaceholderRegex(`__CODE_BLOCK_${index}__`); const inlineCodePlaceholder = makePlaceholderRegex(`__INLINE_CODE_${index}__`); const inlineCodeTag = new RegExp(`]*data-cb-code=(?:"${index}"|'${index}'|${index})[^>]*>[\\s\\S]*?<\\/code>`, 'g'); restored = restored.replace(codeBlockPlaceholder, () => codeBlock); restored = restored.replace(inlineCodePlaceholder, () => codeBlock); restored = restored.replace(inlineCodeTag, () => codeBlock); }); return restored; } /** * Preserve KaTeX math equations (inline $...$ and block $$...$$) */ function preserveMath(content) { const mathElements = []; // Match block math equations $$...$$ const blockMathRegex = /\$\$[\s\S]*?\$\$/g; // Match inline math equations $...$ (but not $$) const inlineMathRegex = /(? { const placeholder = `__MATH_BLOCK_${mathElements.length}__`; mathElements.push({ type: 'block', content: match }); return placeholder; }); // Then replace inline math processed = processed.replace(inlineMathRegex, (match) => { const placeholder = `__MATH_INLINE_${mathElements.length}__`; mathElements.push({ type: 'inline', content: match }); return placeholder; }); return { processed, mathElements }; } /** * Restore KaTeX math equations */ function restoreMath(content, mathElements) { let restored = content; mathElements.forEach((element, index) => { const blockPlaceholder = makePlaceholderRegex(`__MATH_BLOCK_${index}__`); const inlinePlaceholder = makePlaceholderRegex(`__MATH_INLINE_${index}__`); restored = restored.replace(blockPlaceholder, () => element.content); restored = restored.replace(inlinePlaceholder, () => element.content); }); return restored; } /** * Preserve MDX admonitions (:::tip, :::warning, etc.) */ function preserveAdmonitions(content) { const admonitions = []; // Match Docusaurus/MDX admonitions like :::tip\ncontent\n::: const admonitionRegex = /:::(tip|warning|danger|info|note|caution)[\s\S]*?:::/g; let processed = content; processed = processed.replace(admonitionRegex, (match) => { const placeholder = `__ADMONITION_${admonitions.length}__`; admonitions.push(match); return placeholder; }); return { processed, admonitions }; } /** * Preserve Docusaurus blog truncation markers */ function preserveTruncateMarkers(content) { const markers = []; const truncateRegex = //gi; const processed = content.replace(truncateRegex, (match) => { const placeholder = `__TRUNCATE_${markers.length}__`; markers.push(match); return placeholder; }); return { processed, markers }; } /** * Restore Docusaurus blog truncation markers */ function restoreTruncateMarkers(content, markers) { let restored = content; markers.forEach((marker, index) => { const placeholder = makePlaceholderRegex(`__TRUNCATE_${index}__`); restored = restored.replace(placeholder, () => marker); }); return restored; } /** * Restore MDX admonitions */ function restoreAdmonitions(content, admonitions) { let restored = content; admonitions.forEach((admonition, index) => { const placeholder = makePlaceholderRegex(`__ADMONITION_${index}__`); restored = restored.replace(placeholder, () => admonition); }); return restored; } /** * Preserve MDX imports and JSX/React tags (not their contents) */ function preserveMDX(content) { const importRegex = /^import\s+[\s\S]*?from\s+['"][^'"]+['"];?$/gm; const jsxTagRegex = /<\/?(?:[A-Z][\w.]*|[\w]+:[A-Z][\w.]*|[a-z][\w-]*)(?:\s+[^>]*?)?>/g; const attrRegex = /\b([A-Za-z_][\w-]*)\s*=\s*(["'])(.*?)\2/g; const imports = []; const jsxTags = []; let processed = content; // Replace imports first processed = processed.replace(importRegex, (match) => { const placeholder = `__MDX_IMPORT_${imports.length}__`; imports.push({ type: 'import', content: match, placeholder }); return placeholder; }); // Replace JSX tags (leave inner text to translate) processed = processed.replace(jsxTagRegex, (match) => { const attributes = []; const updatedTag = match.replace(attrRegex, (attrMatch, attrName, quote, attrValue) => { if (!TRANSLATABLE_JSX_ATTRS.has(attrName)) { return attrMatch; } if (!attrValue || attrValue.trim() === '') { return attrMatch; } const placeholder = `__MDX_ATTR_${attributes.length}__`; attributes.push({ placeholder, value: attrValue, quote }); return `${attrName}=${quote}${placeholder}${quote}`; }); const placeholder = `__MDX_TAG_${jsxTags.length}__`; jsxTags.push({ type: 'jsxTag', content: updatedTag, placeholder, attributes }); return placeholder; }); return { processed, mdxElements: [...imports, ...jsxTags] }; } /** * Restore MDX imports and JSX components */ function restoreMDX(content, mdxElements) { let restored = content; mdxElements.forEach((element) => { if (!element.placeholder) { return; } let elementContent = element.content; if (element.attributes && element.attributes.length > 0) { element.attributes.forEach((attr) => { const placeholder = makePlaceholderRegex(attr.placeholder); const value = attr.translated || attr.value; elementContent = elementContent.replace(placeholder, () => value); }); } const placeholder = makePlaceholderRegex(element.placeholder); restored = restored.replace(placeholder, () => elementContent); }); return restored; } function preserveMarkdownEmphasis(content) { const elements = []; const strongRegex = /\*\*([^*\r\n]+?)\*\*/g; const processed = content.replace(strongRegex, (match, text) => { const index = elements.length; elements.push({ index }); return `${text}`; }); return { processed, elements }; } function restoreMarkdownEmphasis(content, normalizeSpacing = true) { const restored = content.replace(/]*data-cb-strong=(?:"\d+"|'\d+'|\d+)[^>]*>([\s\S]*?)<\/strong>/gi, (_, text) => { return `**${decodeHtmlEntities(text).trim()}**`; }); const merged = mergeAdjacentStrongMarkdown(restored); return normalizeSpacing ? normalizeMarkdownEmphasisSpacing(merged) : merged; } function mergeAdjacentStrongMarkdown(content) { let previous; let merged = content; const adjacentStrongRegex = /\*\*([^*\n]+)\*\*([\s\u00a0]+)\*\*([^*\n]+)\*\*/g; do { previous = merged; merged = merged.replace(adjacentStrongRegex, '**$1$2$3**'); } while (merged !== previous); return merged; } function normalizeMarkdownEmphasisSpacing(content) { return content .replace(/\*\*([^*\n]*?)\s+\*\*/g, '**$1**') .replace(/([,;!?—–\-])\*\*([\p{L}\p{N}])/gu, '$1 **$2') .replace(/([\p{L}\p{N}])\*\*([\p{L}\p{N}])/gu, '$1 **$2') // Closing guillemet mis-nested inside bold: **text »** → **text** » .replace(/\*\*([\p{L}\p{N}][^*\n]*?)\s*(»|›)\s*\*\*/gu, '**$1** $2') // Opening guillemet mis-nested inside bold: **«text**» → «**text**» // Exclude content that already contains a closing guillemet to avoid touching // the valid nesting **«text»**. DeepL occasionally splits the guillemet pair // across a boundary when adding target-language quotation marks. .replace(/\*\*\s*([«‹])([^*»›\n]+?)\*\*/gu, '$1**$2**') // Closing guillemet mis-nested inside bold (mirror): «**text»** → «**text**» .replace(/([«‹])\*\*([^«‹\n]+?)([»›])\s*\*\*/gu, '$1**$2**$3'); } function preserveMarkdownHeadings(content) { const headings = []; const headingRegex = /^(#{1,6})([ \t]+)(.+?)([ \t]+#+[ \t]*)?$/gm; const processed = content.replace(headingRegex, (match, marker, spacing, text, closing = '') => { const index = headings.length; const placeholder = `__MD_HEADING_${index}__`; headings.push({ marker, spacing, text: text.trim(), closing, placeholder, }); return placeholder; }); return { processed, headings }; } function restoreMarkdownHeadings(content, headings) { let restored = content; headings.forEach((heading) => { const placeholder = makePlaceholderRegex(heading.placeholder); const text = heading.translated || heading.text || ''; const closing = heading.closing || ''; restored = restored.replace(placeholder, () => `${heading.marker}${heading.spacing}${text}${closing}`); }); return restored; } /** * Preserve links and images as Markdown structure. * Images → placeholder tokens (alt text translated separately). * Links → __DEEPL_LINK_N__ so that: * 1. The placeholder is protected inside an ignored tag — DeepL cannot split or * expand it, but may reposition the whole element within the sentence. * 2. Because DEEPL_LINK is absent from PLACEHOLDER_PATTERN, splitContentForTranslation * no longer splits at link boundaries, so surrounding text stays in one sentence * context (fixing the previous post-link capitalisation bug). * Link text is translated separately in translateFile with full term-lock support. */ function preserveLinks(content) { const links = []; const imageRegex = /!\[([^\]]*)\]\(([^)]*)\)/g; const linkRegex = /(? { const placeholder = `__DEEPL_IMAGE_${links.length}__`; links.push({ type: 'image', url, text: alt, placeholder }); return placeholder; }); // Replace links with wrappers containing the HTML-escaped link text. // The visible text (English at this stage) gives DeepL grammatical context so it can // integrate the link position into the target sentence correctly — without this // context DeepL adds stray grammatical fragments around the opaque tag (e.g. // "L'[PD consensuelle](…) e est…"). ignoreTags prevents the content from being // translated, and restoration replaces the whole tag with the separately-translated // link text from link.translated (set later in translateFile). processed = processed.replace(linkRegex, (match, text, url) => { const index = links.length; const placeholder = `__DEEPL_LINK_${index}__`; links.push({ type: 'link', url, text, placeholder, index }); return `${escapeHtml(text)}`; }); return { processed, links }; } function preserveMarkdownTables(content) { const parts = content.split(/(\r?\n)/); let pipeIndex = 0; const hasPipe = (line) => line.includes('|'); const isSeparatorLine = (line) => /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/.test(line); // Wrap pipes as placeholder tokens that cause splitContentForTranslation to split // each row into per-cell segments. DeepL translates each cell independently — // this is more reliable than the row-level approach because // DeepL can't drop a placeholder token the way it sometimes drops an ignored tag // (v10 regressed the fr "Multiple Rating Systems" table — DeepL dropped internal // wrappers, merging two cells into one). Whitespace artifacts // within a cell translation are stripped in translateContentInChunks. const replacePipes = (line) => line.replace(/\|/g, () => `__MD_TABLE_PIPE_${pipeIndex++}__`); let i = 0; while (i < parts.length) { const line = parts[i]; const nextLine = parts[i + 2]; if (hasPipe(line) && nextLine !== undefined && isSeparatorLine(nextLine)) { parts[i] = replacePipes(line); parts[i + 2] = replacePipes(nextLine); let j = i + 4; while (j < parts.length) { const rowLine = parts[j]; if (rowLine !== undefined && hasPipe(rowLine)) { parts[j] = replacePipes(rowLine); j += 2; continue; } break; } i = j; continue; } i += 2; } return { processed: parts.join('') }; } function restoreMarkdownTables(content) { return content // Primary: placeholder tokens restored to pipes .replace(/__MD_TABLE_PIPE_\d+__/g, () => '|') // Legacy fallbacks (in case any cached translation still contains older forms): .replace(/]*\bdata-pipe\b[^>]*>[\s\S]*?<\/cb-lock>/g, () => '|') .replace(/\bTBLPIPE\d+\b/g, () => '|') .replace(/\{copia_\d+\}/g, () => '|'); } function preserveDataLiterals(content) { const literals = []; const protect = (match) => { const index = literals.length; literals.push(match); return `${match}`; }; const dataLiteralRegex = /\b\d+\s+bps\s*=\s*\d+\.\d+%|(? { const tagRegex = new RegExp(`]*data-cb-data=(?:"${index}"|'${index}'|${index})[^>]*>[\\s\\S]*?<\\/cb-lock>`, 'g'); restored = restored.replace(tagRegex, () => literal); }); return restored; } /** * Restore links and images after translation. * Images: plain placeholder tokens → ![alt](url) * Links: __DEEPL_LINK_N__ → [translated text](url) * Falls back to bare __DEEPL_LINK_N__ token if the cb-link wrapper was stripped. */ function restoreLinks(content, links) { let restored = content; // Restore image placeholder tokens links.forEach((link) => { if (link.type !== 'image') return; const placeholder = makePlaceholderRegex(link.placeholder); const text = link.translated || link.text || ''; restored = restored.replace(placeholder, () => `![${text}](${link.url})`); }); // Restore … → [translated text](url) // Primary: match by data-i index. Fallback: bare __DEEPL_LINK_N__ token. links.forEach((link) => { if (link.type !== 'link') return; const text = link.translated || link.text || ''; const cbLinkRegex = new RegExp( `]*\\bdata-i=(?:"${link.index}"|'${link.index}'|${link.index})[^>]*>[\\s\\S]*?<\\/cb-link>`, 'g' ); restored = restored.replace(cbLinkRegex, () => `[${text}](${link.url})`); // Fallback: bare placeholder token (if DeepL stripped the cb-link wrapper) const tokenRegex = makePlaceholderRegex(link.placeholder); restored = restored.replace(tokenRegex, () => `[${text}](${link.url})`); }); // Fix missing space around restored markdown links // (DeepL occasionally removes spaces adjacent to inline elements) restored = restored.replace(/([\p{L}\p{N}])\[/gu, '$1 ['); restored = restored.replace(/(\]\([^)]*\))([\p{L}\p{N}])/gu, '$1 $2'); // Navigation safety net: DeepL sometimes drops an entire element when // multiple ignored-tag types (cb-link + cb-lock) appear in the same sentence — // it confuses them and emits only one. The typical failure mode is that DeepL // substitutes the dropped link with a duplicated term wrapped in the // target language's quotation marks (e.g. "consulte «S&P»"). // If we detect exactly one such pattern we replace it inline; otherwise we append // the link at the end as a last-resort fallback so navigation isn't lost. links.forEach((link) => { if (link.type !== 'link') return; if (restored.includes(link.url)) return; const text = link.translated || link.text || ''; console.warn(`[link-recover] Link to ${link.url} was dropped by DeepL`); // Heuristic: a cb-lock term wrapped in target-language quotation marks AND // immediately followed by sentence-ending punctuation is almost certainly a // DeepL substitution for the dropped link (e.g. "consulte «S&P»."), // not a legitimate mid-sentence quoted term. The trailing-punctuation guard // keeps us from clobbering valid quotes like "«S&P» (por ejemplo, …)". // // CRITICAL: the middle span uses [^<]*? not [\s\S]*? — otherwise the non-greedy // engine backtracks and spans ACROSS multiple elements when the first // candidate's '»' isn't followed by '.!?', eating every line between two quoted // S&P terms. v11 destroyed 24 lines of es/rating-scales.mdx this way. const droppedPattern = /([«"„])\s*]*>[^<]*?<\/cb-lock>\s*([»"‟])(?=\s*[.!?])/g; const candidates = restored.match(droppedPattern); if (candidates && candidates.length === 1) { restored = restored.replace(droppedPattern, `[${text}](${link.url})`); return; } // Heuristic 2: try to find the translated anchor text literally in the // content and wrap it inline. DeepL sometimes drops the cb-link tag but // translates the anchor text into the sentence using identical wording. const textCandidates = [text]; if (link.text && link.text !== text) textCandidates.push(link.text); let reinserted = false; for (const candidate of textCandidates) { if (!candidate || candidate.length < 3) continue; const escaped = candidate.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const re = new RegExp(`(? wrapper in all-dashes // separator rows, producing e.g. "|-------||----------------|". .replace(/(?:\|[ \t]*){2,}(?=[ \t]*[-:])/g, '|') // DeepL-level artifacts around ignored-tag terms with '&' (e.g. S&P): // S&S&P → S&P (duplicated ampersand cluster) // S&P; → S&P (stray entity-completion semicolon) .replace(/S&S&P/g, 'S&P') .replace(/S&P;(?!\w)/g, 'S&P') // Duplicate short function words on a single line, e.g. "e e", "et et", "de de", // "la la" — DeepL occasionally duplicates a connector word when repositioning an // ignored-tag element. Use Unicode-aware whitespace lookaround (NOT \b) because // JavaScript's \b is ASCII-only: with \b, an accented letter like 'é' inside // "marché" looks like a word boundary and the regex collapses "marché é…" into // "marché…", destroying a space (produced the v8 "marchévoluent" typo). .replace(/(? { const placeholder = `__VAR_${index}__`; index += 1; map.set(placeholder, match); return placeholder; }); return { text: updated, map }; } function restoreVariablePlaceholders(text, map) { if (!map || map.size === 0) { return text; } let restored = text; map.forEach((value, placeholder) => { const placeholderRegex = new RegExp(escapeRegExp(placeholder), 'g'); restored = restored.replace(placeholderRegex, () => value); }); return restored; } /** * Translate text using DeepL API */ function translateText(text, sourceLang, targetLang, apiKey, glossaryId = null, options = {}) { return translateTextBatch([text], sourceLang, targetLang, apiKey, glossaryId, options).then((translations) => { return translations[0]; }); } function stripTranslationContextMarkdown(content) { return content .replace(/```[\s\S]*?```/g, ' ') .replace(/\$\$[\s\S]*?\$\$/g, ' ') .replace(/`[^`\n]+`/g, ' ') .replace(/!\[([^\]]*)\]\([^\)]*\)/g, '$1') .replace(/\[([^\]]*)\]\([^\)]*\)/g, '$1') .replace(/<\/?[^>]+>/g, ' ') .replace(/[#*_~>|-]+/g, ' ') .replace(/\s+/g, ' ') .trim(); } function buildDocumentTranslationContext(frontmatter, markdownContent) { const parts = []; ['title', 'description', 'summary', 'sidebar_label'].forEach((key) => { if (typeof frontmatter[key] === 'string' && frontmatter[key].trim() !== '') { parts.push(frontmatter[key].trim()); } }); const bodyContext = stripTranslationContextMarkdown(markdownContent); if (bodyContext) { parts.push(bodyContext); } const domainHint = 'Credit Benchmark documentation about credit risk, bank probability of default estimates, Consensus Credit Rating, rating scales, credit analytics, and financial data products.'; const context = `${domainHint} ${parts.join(' ')}`.trim(); return context.length > DEEPL_CONTEXT_LIMIT ? context.slice(0, DEEPL_CONTEXT_LIMIT) : context; } function sendDeepLRequest(payload, apiKey, expectedCount) { return new Promise((resolve, reject) => { const postData = JSON.stringify(payload); const url = new URL(DEEPL_TRANSLATE_PATH, getDeepLApiBaseUrl(apiKey)); const options = { hostname: url.hostname, port: 443, path: url.pathname, method: 'POST', headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(postData), 'Authorization': `DeepL-Auth-Key ${apiKey}`, }, }; const req = https.request(options, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { if (res.statusCode === 200) { try { const response = JSON.parse(data); if (response.translations && response.translations.length === expectedCount) { resolve(response.translations.map((item) => item.text)); } else { reject(new Error('Invalid response from DeepL API')); } } catch (error) { reject(new Error(`Failed to parse API response: ${error.message}`)); } } else { const error = new Error(`DeepL API error (${res.statusCode}): ${data}`); error.statusCode = res.statusCode; error.data = data; error.retryAfter = res.headers ? res.headers['retry-after'] : null; reject(error); } }); }); req.on('error', (error) => { reject(new Error(`Request failed: ${error.message}`)); }); req.write(postData); req.end(); }); } function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } function buildDeepLOptions(options = {}) { return { context: options.context || null, modelType: options.modelType || 'quality_optimized', formality: options.formality || 'prefer_more', customInstructions: options.customInstructions || [], styleId: options.styleId || null, // HTML tag handling with ignoreTags for our preservation wrappers. XML mode was // tried but rejected by our DeepL account (400 on every request), so HTML is the // effective mode. tagHandling: options.tagHandling || 'html', tagHandlingVersion: options.tagHandlingVersion || 'v2', ignoreTags: options.ignoreTags || ['cb-lock', 'cb-link', 'code'], }; } async function requestDeepL(texts, sourceLang, targetLang, apiKey, glossaryId = null, options = {}) { const deepLOptions = buildDeepLOptions(options); const payload = { text: texts, source_lang: sourceLang, target_lang: targetLang, preserve_formatting: true, split_sentences: 'nonewlines', model_type: deepLOptions.modelType, formality: deepLOptions.formality, tag_handling: deepLOptions.tagHandling, tag_handling_version: deepLOptions.tagHandlingVersion, ignore_tags: deepLOptions.ignoreTags, }; if (deepLOptions.customInstructions.length > 0) { payload.custom_instructions = deepLOptions.customInstructions; } if (deepLOptions.styleId) { payload.style_id = deepLOptions.styleId; } if (deepLOptions.context) { payload.context = deepLOptions.context; } if (glossaryId) { payload.glossary_id = glossaryId; } const expectedCount = texts.length; let attempt = 0; let currentPayload = payload; while (true) { try { return await sendDeepLRequest(currentPayload, apiKey, expectedCount); } catch (error) { const errorText = `${error.message || ''} ${error.data || ''}`; if (error.statusCode === 404 && currentPayload.glossary_id && /glossary/i.test(errorText)) { const fallbackPayload = { ...currentPayload }; delete fallbackPayload.glossary_id; console.warn('Warning: DeepL glossary was not found. Retrying without glossary_id.'); currentPayload = fallbackPayload; continue; } if (error.statusCode === 400 && errorText.includes('preserve_formatting')) { const fallbackPayload = { ...currentPayload }; delete fallbackPayload.preserve_formatting; delete fallbackPayload.split_sentences; if (!warnedAboutFormatting) { console.warn('Warning: DeepL does not support preserve_formatting. Retrying without it.'); warnedAboutFormatting = true; } currentPayload = fallbackPayload; continue; } if (error.statusCode === 400 && /model_type|quality_optimized/i.test(errorText) && currentPayload.model_type) { const fallbackPayload = { ...currentPayload }; delete fallbackPayload.model_type; console.warn('Warning: DeepL did not accept model_type. Retrying without quality_optimized.'); currentPayload = fallbackPayload; continue; } if (error.statusCode === 400 && /custom_instructions/i.test(errorText) && currentPayload.custom_instructions) { const fallbackPayload = { ...currentPayload }; delete fallbackPayload.custom_instructions; console.warn('Warning: DeepL did not accept custom_instructions. Retrying without them.'); currentPayload = fallbackPayload; continue; } if (error.statusCode === 400 && /style_id|style/i.test(errorText) && currentPayload.style_id) { const fallbackPayload = { ...currentPayload }; delete fallbackPayload.style_id; console.warn('Warning: DeepL did not accept style_id. Retrying without it.'); currentPayload = fallbackPayload; continue; } if (error.statusCode === 400 && /formality/i.test(errorText) && currentPayload.formality) { const fallbackPayload = { ...currentPayload }; delete fallbackPayload.formality; console.warn('Warning: DeepL did not accept formality. Retrying without it.'); currentPayload = fallbackPayload; continue; } if (error.statusCode === 400 && /tag_handling|tag_handling_version|ignore_tags/i.test(errorText) && currentPayload.tag_handling) { const fallbackPayload = { ...currentPayload }; delete fallbackPayload.tag_handling; delete fallbackPayload.tag_handling_version; delete fallbackPayload.ignore_tags; console.warn('Warning: DeepL did not accept tag handling. Retrying without tag handling.'); currentPayload = fallbackPayload; continue; } if (error.statusCode === 429 && attempt < MAX_RETRIES) { const retryAfterSeconds = Number.parseInt(error.retryAfter, 10); const baseDelay = Number.isFinite(retryAfterSeconds) ? retryAfterSeconds * 1000 : BASE_RETRY_DELAY_MS * Math.pow(2, attempt); const jitter = Math.floor(Math.random() * 250); const delayMs = Math.min(60000, baseDelay + jitter); console.warn(`Warning: DeepL rate limited (429). Retrying in ${Math.ceil(delayMs / 1000)}s...`); await delay(delayMs); attempt += 1; continue; } throw error; } } } async function translateTextBatch(texts, sourceLang, targetLang, apiKey, glossaryId = null, options = {}) { const results = new Array(texts.length); const uncachedTexts = []; const uncachedMeta = []; const useCache = cacheReadEnabled && translationCache; const canWriteCache = Boolean(translationCache); texts.forEach((text, index) => { if (text === '') { results[index] = text; return; } const cacheKey = canWriteCache ? makeCacheKey(text, sourceLang, targetLang, buildDeepLOptions(options)) : null; if (useCache) { const cached = translationCache.entries[cacheKey]; if (cached) { results[index] = cached; if (cacheStats) { cacheStats.hits += 1; } return; } if (cacheStats) { cacheStats.misses += 1; } uncachedTexts.push(text); uncachedMeta.push({ index, cacheKey }); return; } uncachedTexts.push(text); uncachedMeta.push({ index, cacheKey }); }); if (uncachedTexts.length === 0) { return results; } for (let start = 0; start < uncachedTexts.length; start += DEEPL_BATCH_LIMIT) { const batchTexts = uncachedTexts.slice(start, start + DEEPL_BATCH_LIMIT); const batchMeta = uncachedMeta.slice(start, start + DEEPL_BATCH_LIMIT); const translations = await requestDeepL(batchTexts, sourceLang, targetLang, apiKey, glossaryId, options); if (translations.length !== batchTexts.length) { throw new Error('DeepL API returned unexpected translation count'); } translations.forEach((translation, idx) => { const meta = batchMeta[idx]; const decodedTranslation = decodeHtmlEntities(translation); results[meta.index] = decodedTranslation; if (translationCache && meta.cacheKey) { translationCache.entries[meta.cacheKey] = decodedTranslation; } }); } return results; } function splitTextByParagraphs(text) { const segments = []; const separatorRegex = /(\r?\n)/g; let lastIndex = 0; let match; while ((match = separatorRegex.exec(text)) !== null) { if (match.index > lastIndex) { segments.push({ type: 'text', value: text.slice(lastIndex, match.index) }); } segments.push({ type: 'separator', value: match[0] }); lastIndex = match.index + match[0].length; } if (lastIndex < text.length) { segments.push({ type: 'text', value: text.slice(lastIndex) }); } return segments; } function normalizeWhitespaceAroundPlaceholders(segments) { const adjusted = []; const leadingWhitespaceRegex = /^[ \t]+/; const trailingWhitespaceRegex = /[ \t]+$/; for (let i = 0; i < segments.length; i++) { const segment = segments[i]; if (segment.type === 'placeholder') { const prev = adjusted[adjusted.length - 1]; if (prev && prev.type === 'text') { const trailing = prev.value.match(trailingWhitespaceRegex); if (trailing) { prev.value = prev.value.slice(0, -trailing[0].length); if (trailing[0]) { adjusted.push({ type: 'separator', value: trailing[0] }); } } } adjusted.push(segment); const next = segments[i + 1]; if (next && next.type === 'text') { const leading = next.value.match(leadingWhitespaceRegex); if (leading) { if (leading[0]) { adjusted.push({ type: 'separator', value: leading[0] }); } next.value = next.value.slice(leading[0].length); } } continue; } adjusted.push(segment); } return adjusted.filter((segment) => segment.value !== ''); } async function translateUiJson(filePath, locale, apiKey, glossaryId, uiMeta) { if (!fs.existsSync(filePath)) { console.warn(`Warning: UI file not found at ${filePath}`); return { skipped: true, reason: 'missing' }; } let parsed; try { parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')); } catch (error) { console.warn(`Warning: Failed to parse UI file ${filePath}: ${error.message}`); return { skipped: true, reason: 'parse-error' }; } const entries = []; const entryMap = new Map(); Object.entries(parsed).forEach(([key, value]) => { if (!value || typeof value !== 'object') { return; } const message = value.message; if (typeof message !== 'string' || message.trim() === '') { return; } entries.push({ key, message }); entryMap.set(key, value); }); if (entries.length === 0) { return { skipped: true, reason: 'empty' }; } const relativePath = path.relative(path.join(__dirname, '..'), filePath); const currentHash = makeUiHash(entries); const metaEntry = uiMeta ? uiMeta[relativePath] : null; if (metaEntry && metaEntry.translatedHash === currentHash) { return { skipped: true, reason: 'unchanged' }; } const targetLang = LOCALE_MAP[locale.toLowerCase()] || locale.toUpperCase(); const translatedByKey = new Map(); const requestTexts = []; const requestMeta = []; let skippedCount = 0; entries.forEach(({ key, message }) => { if (UI_SKIP_KEYS.has(key) || UI_SKIP_MESSAGES.has(message)) { translatedByKey.set(key, message); skippedCount += 1; return; } const parts = message.split('|'); parts.forEach((part, partIndex) => { const locked = applyTermLocks(part, lockTerms); const placeholders = applyVariablePlaceholders(locked.text); requestTexts.push(placeholders.text); requestMeta.push({ key, partIndex, totalParts: parts.length, lockMap: locked.map, placeholderMap: placeholders.map, }); }); }); if (requestTexts.length > 0) { const translated = await translateTextBatch(requestTexts, 'EN', targetLang, apiKey, glossaryId); requestMeta.forEach((meta, index) => { let restored = restoreVariablePlaceholders(translated[index], meta.placeholderMap); restored = restoreTermLocks(restored, meta.lockMap); const existing = translatedByKey.get(meta.key) || new Array(meta.totalParts).fill(''); existing[meta.partIndex] = restored; translatedByKey.set(meta.key, existing); }); } translatedByKey.forEach((value, key) => { const entry = entryMap.get(key); if (!entry) { return; } entry.message = Array.isArray(value) ? value.join('|') : value; }); fs.writeFileSync(filePath, JSON.stringify(parsed, null, 2), 'utf8'); const updatedEntries = Object.entries(parsed).map(([key, value]) => ({ key, message: value && typeof value === 'object' ? value.message || '' : '', })); if (uiMeta) { uiMeta[relativePath] = { translatedHash: makeUiHash(updatedEntries), }; } return { skipped: false, translatedCount: requestTexts.length, skippedCount }; } async function translateDocusaurusConfig(locale, apiKey, glossaryId, uiMeta) { const configPath = path.join(__dirname, '..', 'docusaurus.config.ts'); if (!fs.existsSync(configPath)) { console.warn('Warning: docusaurus.config.ts not found.'); return { skipped: true, reason: 'missing' }; } const rawConfig = fs.readFileSync(configPath, 'utf8'); const titleMatch = rawConfig.match(/title:\s*['"]([^'"]+)['"]/); const taglineMatch = rawConfig.match(/tagline:\s*['"]([^'"]+)['"]/); if (!titleMatch && !taglineMatch) { return { skipped: true, reason: 'empty' }; } const sourceEntries = []; if (titleMatch) { sourceEntries.push({ key: 'title', message: titleMatch[1] }); } if (taglineMatch) { sourceEntries.push({ key: 'tagline', message: taglineMatch[1] }); } const targetLang = LOCALE_MAP[locale.toLowerCase()] || locale.toUpperCase(); const requestTexts = []; const requestMeta = []; sourceEntries.forEach((entry) => { const locked = applyTermLocks(entry.message, lockTerms); const placeholders = applyVariablePlaceholders(locked.text); requestTexts.push(placeholders.text); requestMeta.push({ key: entry.key, lockMap: locked.map, placeholderMap: placeholders.map }); }); let translatedTexts = []; if (requestTexts.length > 0) { translatedTexts = await translateTextBatch(requestTexts, 'EN', targetLang, apiKey, glossaryId); } const translatedConfig = {}; requestMeta.forEach((meta, index) => { let restored = restoreVariablePlaceholders(translatedTexts[index], meta.placeholderMap); restored = restoreTermLocks(restored, meta.lockMap); translatedConfig[meta.key] = restored; }); const targetPath = path.join(__dirname, '..', 'i18n', locale, 'docusaurus.config.json'); const relativePath = path.relative(path.join(__dirname, '..'), targetPath); const targetEntries = Object.entries(translatedConfig).map(([key, message]) => ({ key, message })); const currentHash = makeUiHash(targetEntries); const metaEntry = uiMeta ? uiMeta[relativePath] : null; if (metaEntry && metaEntry.translatedHash === currentHash) { return { skipped: true, reason: 'unchanged' }; } if (!fs.existsSync(path.dirname(targetPath))) { fs.mkdirSync(path.dirname(targetPath), { recursive: true }); } fs.writeFileSync(targetPath, JSON.stringify(translatedConfig, null, 2), 'utf8'); if (uiMeta) { uiMeta[relativePath] = { translatedHash: currentHash, }; } return { skipped: false, translatedCount: requestTexts.length }; } function splitContentForTranslation(content) { const segments = []; const placeholderRegex = new RegExp(PLACEHOLDER_PATTERN.source, 'g'); let lastIndex = 0; let match; while ((match = placeholderRegex.exec(content)) !== null) { if (match.index > lastIndex) { const before = content.slice(lastIndex, match.index); segments.push(...splitTextByParagraphs(before)); } segments.push({ type: 'placeholder', value: match[0] }); lastIndex = match.index + match[0].length; } if (lastIndex < content.length) { const tail = content.slice(lastIndex); segments.push(...splitTextByParagraphs(tail)); } return normalizeWhitespaceAroundPlaceholders(segments); } async function translateContentInChunks(content, sourceLang, targetLang, apiKey, glossaryId = null, options = {}) { const segments = splitContentForTranslation(content); const textSegments = []; const textIndexes = []; const textAffixes = []; segments.forEach((segment, index) => { if (segment.type === 'text' && segment.value.trim() !== '') { const leading = segment.value.match(/^\s*/)[0]; const trailing = segment.value.match(/\s*$/)[0]; const trimmed = segment.value.slice(leading.length, segment.value.length - trailing.length); textIndexes.push(index); textSegments.push(trimmed); textAffixes.push({ leading, trailing }); } }); if (textSegments.length === 0) { return content; } const translatedSegments = await translateTextBatch(textSegments, sourceLang, targetLang, apiKey, glossaryId, options); textIndexes.forEach((segmentIndex, idx) => { const affix = textAffixes[idx]; // Source segments are single-line by construction (splitTextByParagraphs splits at // newlines). Any \r / \n / \t in a translation is a DeepL artifact that would // break block structure (notably table rows) — normalise to single spaces. const translated = translatedSegments[idx].replace(/[\r\n\t\f\v]+/g, ' '); segments[segmentIndex].value = `${affix.leading}${translated}${affix.trailing}`; }); return segments.map((segment) => segment.value).join(''); } async function translateFrontmatter(frontmatter, sourceLang, targetLang, apiKey, glossaryId = null, terms = [], options = {}) { const translated = { ...frontmatter }; const tasks = []; for (const [key, value] of Object.entries(frontmatter)) { if (!FRONTMATTER_TRANSLATABLE_KEYS.has(key)) { continue; } if (typeof value === 'string' && value.trim() !== '') { const locked = applyTermLocks(value, terms); tasks.push( translateText(locked.text, sourceLang, targetLang, apiKey, glossaryId, options).then((translatedValue) => { translated[key] = restoreTermLocks(translatedValue, locked.map); }) ); } else if (Array.isArray(value)) { const stringIndexes = []; const strings = []; value.forEach((item, index) => { if (typeof item === 'string' && item.trim() !== '') { stringIndexes.push(index); strings.push(item); } }); if (strings.length > 0) { const locked = strings.map((item) => applyTermLocks(item, terms)); const lockedStrings = locked.map((item) => item.text); tasks.push( translateTextBatch(lockedStrings, sourceLang, targetLang, apiKey, glossaryId, options).then((translatedStrings) => { const updated = value.slice(); translatedStrings.forEach((translatedText, idx) => { updated[stringIndexes[idx]] = restoreTermLocks(translatedText, locked[idx].map); }); translated[key] = updated; }) ); } } } if (tasks.length === 0) { return translated; } await Promise.all(tasks); return translated; } async function translateUiResources(locale, apiKey, glossaryId, baseDir) { const uiMeta = getUiMeta(); const uiFiles = [ path.join(baseDir, 'i18n', locale, 'docusaurus-theme-classic', 'navbar.json'), path.join(baseDir, 'i18n', locale, 'docusaurus-theme-classic', 'footer.json'), path.join(baseDir, 'i18n', locale, 'docusaurus-plugin-content-docs', 'current.json'), path.join(baseDir, 'i18n', locale, 'docusaurus-plugin-content-blog', 'options.json'), path.join(baseDir, 'i18n', locale, 'code.json'), ]; console.log('\nUI translation:'); for (const filePath of uiFiles) { const result = await translateUiJson(filePath, locale, apiKey, glossaryId, uiMeta); const relative = path.relative(baseDir, filePath); if (result.skipped) { console.log(` [skip] UI ${relative}${result.reason ? ` (${result.reason})` : ''}`); } else { console.log(` [ok] Translated UI ${relative} (${result.translatedCount || 0} items)`); } } const configResult = await translateDocusaurusConfig(locale, apiKey, glossaryId, uiMeta); const configRelative = path.relative(baseDir, path.join(baseDir, 'i18n', locale, 'docusaurus.config.json')); if (configResult.skipped) { console.log(` [skip] UI ${configRelative}${configResult.reason ? ` (${configResult.reason})` : ''}`); } else { console.log(` [ok] Translated UI ${configRelative} (${configResult.translatedCount || 0} items)`); } } async function translateMintlifyNavigation(locale, apiKey, glossaryId, baseDir) { const docsPath = path.join(baseDir, 'docs.json'); if (!fs.existsSync(docsPath)) { return { skipped: true, reason: 'docs.json missing' }; } let docs; try { docs = JSON.parse(fs.readFileSync(docsPath, 'utf8')); } catch (error) { return { skipped: true, reason: `docs.json parse error: ${error.message}` }; } if (!docs.navigation || !Array.isArray(docs.navigation.languages)) { return { skipped: true, reason: 'navigation.languages missing' }; } const localeKey = locale.toLowerCase(); const english = docs.navigation.languages.find((language) => language.language === 'en'); if (!english || !Array.isArray(english.tabs)) { return { skipped: true, reason: 'english tabs missing' }; } let localized = docs.navigation.languages.find((language) => language.language === localeKey); if (!localized) { localized = { language: localeKey }; docs.navigation.languages.push(localized); } const localizedTabs = prefixLocalePages(deepClone(english.tabs), localeKey); const localizableFields = collectLocalizableFields(localizedTabs); const targetLang = LOCALE_MAP[localeKey] || localeKey.toUpperCase(); const requestTexts = []; const requestMeta = []; localizableFields.forEach((field) => { if (UI_SKIP_KEYS.has(field.key) || UI_SKIP_MESSAGES.has(field.value)) { return; } const locked = applyTermLocks(field.value, lockTerms); const placeholders = applyVariablePlaceholders(locked.text); requestTexts.push(placeholders.text); requestMeta.push({ field, lockMap: locked.map, placeholderMap: placeholders.map, }); }); if (requestTexts.length > 0) { const translated = await translateTextBatch(requestTexts, 'EN', targetLang, apiKey, glossaryId); requestMeta.forEach((meta, index) => { let restored = restoreVariablePlaceholders(translated[index], meta.placeholderMap); restored = restoreTermLocks(restored, meta.lockMap); meta.field.node[meta.field.key] = restored; }); } localized.tabs = localizedTabs; if (Object.prototype.hasOwnProperty.call(localized, 'default')) { delete localized.default; } fs.writeFileSync(docsPath, `${JSON.stringify(docs, null, 2)}\n`, 'utf8'); return { skipped: false, translatedCount: requestTexts.length }; } async function translateMintlifyLocaleUiAssets(locale, apiKey, glossaryId, baseDir) { const docsPath = path.join(baseDir, 'docs.json'); if (!fs.existsSync(docsPath)) { return { skipped: true, reason: 'docs.json missing' }; } let docs; try { docs = JSON.parse(fs.readFileSync(docsPath, 'utf8')); } catch (error) { return { skipped: true, reason: `docs.json parse error: ${error.message}` }; } const localeKey = locale.toLowerCase(); const targetLang = LOCALE_MAP[localeKey] || localeKey.toUpperCase(); const uiAsset = { locale: localeKey, navbar: { primary: docs.navbar && docs.navbar.primary ? { type: docs.navbar.primary.type, label: typeof docs.navbar.primary.label === 'string' ? docs.navbar.primary.label : null, href: docs.navbar.primary.href || null, } : null, links: Array.isArray(docs.navbar && docs.navbar.links) ? docs.navbar.links.map((link) => ({ type: link.type || null, href: link.href || null, label: typeof link.label === 'string' ? link.label : null, })) : [], }, footer: { links: Array.isArray(docs.footer && docs.footer.links) ? docs.footer.links.map((column) => ({ header: typeof column.header === 'string' ? column.header : null, items: Array.isArray(column.items) ? column.items.map((item) => ({ label: typeof item.label === 'string' ? item.label : null, href: item.href || null, })) : [], })) : [], }, }; const localizableFields = []; if (uiAsset.navbar && uiAsset.navbar.primary && typeof uiAsset.navbar.primary.label === 'string') { localizableFields.push({ node: uiAsset.navbar.primary, key: 'label', value: uiAsset.navbar.primary.label }); } uiAsset.navbar.links.forEach((link) => { if (typeof link.label === 'string' && link.label.trim() !== '') { localizableFields.push({ node: link, key: 'label', value: link.label }); } }); uiAsset.footer.links.forEach((column) => { if (typeof column.header === 'string' && column.header.trim() !== '') { localizableFields.push({ node: column, key: 'header', value: column.header }); } column.items.forEach((item) => { if (typeof item.label === 'string' && item.label.trim() !== '') { localizableFields.push({ node: item, key: 'label', value: item.label }); } }); }); const requestTexts = []; const requestMeta = []; localizableFields.forEach((field) => { if (UI_SKIP_KEYS.has(field.key) || UI_SKIP_MESSAGES.has(field.value)) { return; } const locked = applyTermLocks(field.value, lockTerms); const placeholders = applyVariablePlaceholders(locked.text); requestTexts.push(placeholders.text); requestMeta.push({ field, lockMap: locked.map, placeholderMap: placeholders.map, }); }); if (requestTexts.length > 0) { const translated = await translateTextBatch(requestTexts, 'EN', targetLang, apiKey, glossaryId); requestMeta.forEach((meta, index) => { let restored = restoreVariablePlaceholders(translated[index], meta.placeholderMap); restored = restoreTermLocks(restored, meta.lockMap); meta.field.node[meta.field.key] = restored; }); } const targetDir = path.join(baseDir, localeKey, '_ui'); const targetPath = path.join(targetDir, 'labels.json'); if (!fs.existsSync(targetDir)) { fs.mkdirSync(targetDir, { recursive: true }); } fs.writeFileSync(targetPath, `${JSON.stringify(uiAsset, null, 2)}\n`, 'utf8'); return { skipped: false, translatedCount: requestTexts.length, path: path.relative(baseDir, targetPath), }; } /** * Translate a markdown file */ async function translateFile(filePath, locale, apiKey, baseDir, glossaryId = null, overwrite = false) { const fullPath = path.join(baseDir, filePath); const targetInfo = getContentTarget(filePath, locale, baseDir); if (!targetInfo) { return { skipped: false, success: false, error: `Unsupported content path: ${filePath}` }; } const { targetDir, targetPath, displayPath } = targetInfo; try { const fileContent = fs.readFileSync(fullPath, 'utf8'); const sourceHash = makeContentHash(fileContent); const contentMeta = getContentMeta(); const contentMetaKey = `${locale.toLowerCase()}:${filePath}`; if ( !overwrite && fs.existsSync(targetPath) && contentMeta && contentMeta[contentMetaKey]?.sourceHash === sourceHash && contentMeta[contentMetaKey]?.translationVersion === CONTENT_TRANSLATION_VERSION ) { console.log(`[skip] Skipping ${filePath} (source unchanged)`); return { skipped: true }; } const parsed = matter(fileContent); const frontmatter = parsed.data; const markdownContent = parsed.content; const translationOptions = { context: buildDocumentTranslationContext(frontmatter, markdownContent), customInstructions: getCustomInstructions(deeplConfig || {}, locale), styleId: getStyleId(deeplConfig || {}), }; // Preserve math equations first (before everything else to avoid conflicts) const { processed: contentWithoutMath, mathElements } = preserveMath(markdownContent); // Preserve admonitions (before MDX components) const { processed: contentWithoutAdmonitions, admonitions } = preserveAdmonitions(contentWithoutMath); // Preserve blog truncation markers const { processed: contentWithoutTruncate, markers: truncateMarkers } = preserveTruncateMarkers(contentWithoutAdmonitions); // Preserve MDX imports and JSX components (before code blocks) const { processed: contentWithoutMDX, mdxElements } = preserveMDX(contentWithoutTruncate); // Preserve code blocks const { processed: contentWithoutCode, codeBlocks } = preserveCodeBlocks(contentWithoutMDX); // Preserve Markdown emphasis as HTML tags so DeepL can translate inner text safely. const { processed: contentWithoutEmphasis } = preserveMarkdownEmphasis(contentWithoutCode); // Preserve Markdown heading markers; translate only the heading text. const { processed: contentWithoutHeadings, headings } = preserveMarkdownHeadings(contentWithoutEmphasis); // Preserve links and images const { processed: contentWithLinks, links } = preserveLinks(contentWithoutHeadings); const { processed: contentToTranslate } = preserveMarkdownTables(contentWithLinks); const { processed: contentWithDataLiterals, literals: dataLiterals } = preserveDataLiterals(contentToTranslate); const targetLang = LOCALE_MAP[locale.toLowerCase()] || locale.toUpperCase(); const translatedFrontmatter = await translateFrontmatter(frontmatter, 'EN', targetLang, apiKey, glossaryId, lockTerms, translationOptions); const mdxAttributeEntries = []; mdxElements.forEach((element) => { if (!element.attributes || element.attributes.length === 0) { return; } element.attributes.forEach((attr) => { if (attr.value && attr.value.trim() !== '') { mdxAttributeEntries.push({ element, attr }); } }); }); if (mdxAttributeEntries.length > 0) { const attributeValues = mdxAttributeEntries.map((entry) => entry.attr.value); const lockedAttributes = attributeValues.map((value) => applyTermLocks(value, lockTerms)); const lockedStrings = lockedAttributes.map((entry) => entry.text); const translatedAttributes = await translateTextBatch(lockedStrings, 'EN', targetLang, apiKey, glossaryId, translationOptions); translatedAttributes.forEach((translatedText, index) => { mdxAttributeEntries[index].attr.translated = restoreTermLocks( translatedText, lockedAttributes[index].map ); }); } const linkEntries = links.filter((link) => link.type === 'link' && link.text && link.text.trim() !== ''); if (linkEntries.length > 0) { const lockedLinks = linkEntries.map((link) => applyTermLocks(link.text, lockTerms)); const lockedStrings = lockedLinks.map((item) => item.text); const translatedLinks = await translateTextBatch(lockedStrings, 'EN', targetLang, apiKey, glossaryId, translationOptions); translatedLinks.forEach((translatedText, index) => { linkEntries[index].translated = restoreTermLocks(translatedText, lockedLinks[index].map); }); } const headingEntries = headings.filter((heading) => heading.text && heading.text.trim() !== ''); if (headingEntries.length > 0) { const lockedHeadings = headingEntries.map((heading) => applyTermLocks(heading.text, lockTerms)); const lockedStrings = lockedHeadings.map((entry) => entry.text); const translatedHeadings = await translateTextBatch(lockedStrings, 'EN', targetLang, apiKey, glossaryId, translationOptions); translatedHeadings.forEach((translatedText, index) => { headingEntries[index].translated = restoreTermLocks( translatedText, lockedHeadings[index].map ); }); } const lockedContent = applyTermLocks(contentWithDataLiterals, lockTerms); // Translate the content const translatedText = await translateContentInChunks(lockedContent.text, 'EN', targetLang, apiKey, glossaryId, translationOptions); // Restore links and images (preserving original link structure) let finalContent = restoreMarkdownTables(translatedText); finalContent = restoreLinks(finalContent, links); finalContent = restoreMarkdownHeadings(finalContent, headings); finalContent = restoreMarkdownEmphasis(finalContent); // Restore code blocks finalContent = restoreCodeBlocks(finalContent, codeBlocks); // Restore MDX imports and JSX components finalContent = restoreMDX(finalContent, mdxElements); // Restore blog truncation markers finalContent = restoreTruncateMarkers(finalContent, truncateMarkers); // Restore admonitions finalContent = restoreAdmonitions(finalContent, admonitions); // Restore math equations last finalContent = restoreMath(finalContent, mathElements); finalContent = restoreTermLocks(finalContent, lockedContent.map); finalContent = restoreDataLiterals(finalContent, dataLiterals); finalContent = repairMarkdownStructure(finalContent); // Reconstruct the file with translated frontmatter let translatedFile = finalContent; if (Object.keys(translatedFrontmatter).length > 0) { translatedFile = matter.stringify(finalContent, translatedFrontmatter); } // Ensure target directory exists if (!fs.existsSync(targetDir)) { fs.mkdirSync(targetDir, { recursive: true }); } // Write translated file fs.writeFileSync(targetPath, translatedFile, 'utf8'); if (contentMeta) { contentMeta[contentMetaKey] = { sourceHash, translationVersion: CONTENT_TRANSLATION_VERSION, targetPath: displayPath, }; } console.log(`[ok] Translated ${filePath} -> ${displayPath}`); return { skipped: false, success: true }; } catch (error) { console.error(`[error] Error translating ${filePath}: ${error.message}`); return { skipped: false, success: false, error: error.message }; } } /** * Test preservation by extracting and rebuilding without translation */ function testPreservationFile(filePath, baseDir) { const fullPath = path.join(baseDir, filePath); const originalContent = fs.readFileSync(fullPath, 'utf8'); const parsed = matter(originalContent); const markdownContent = parsed.content; const frontmatterPrefix = originalContent.slice(0, originalContent.length - markdownContent.length); // Preserve math equations first (before everything else to avoid conflicts) const { processed: contentWithoutMath, mathElements } = preserveMath(markdownContent); // Preserve admonitions (before MDX components) const { processed: contentWithoutAdmonitions, admonitions } = preserveAdmonitions(contentWithoutMath); // Preserve blog truncation markers const { processed: contentWithoutTruncate, markers: truncateMarkers } = preserveTruncateMarkers(contentWithoutAdmonitions); // Preserve MDX imports and JSX components (before code blocks) const { processed: contentWithoutMDX, mdxElements } = preserveMDX(contentWithoutTruncate); // Preserve code blocks const { processed: contentWithoutCode, codeBlocks } = preserveCodeBlocks(contentWithoutMDX); // Preserve Markdown emphasis const { processed: contentWithoutEmphasis } = preserveMarkdownEmphasis(contentWithoutCode); // Preserve Markdown heading markers const { processed: contentWithoutHeadings, headings } = preserveMarkdownHeadings(contentWithoutEmphasis); // Preserve links and images const { processed: contentWithLinks, links } = preserveLinks(contentWithoutHeadings); const { processed: contentToTranslate } = preserveMarkdownTables(contentWithLinks); const { processed: contentWithDataLiterals, literals: dataLiterals } = preserveDataLiterals(contentToTranslate); // Simulate translation (pass-through) let finalContent = restoreMarkdownTables(contentWithDataLiterals); finalContent = restoreLinks(finalContent, links); finalContent = restoreMarkdownHeadings(finalContent, headings); finalContent = restoreMarkdownEmphasis(finalContent, false); finalContent = restoreCodeBlocks(finalContent, codeBlocks); finalContent = restoreMDX(finalContent, mdxElements); finalContent = restoreTruncateMarkers(finalContent, truncateMarkers); finalContent = restoreAdmonitions(finalContent, admonitions); finalContent = restoreMath(finalContent, mathElements); finalContent = restoreDataLiterals(finalContent, dataLiterals); const rebuiltFile = frontmatterPrefix + finalContent; if (rebuiltFile === originalContent) { return { success: true }; } let diffIndex = -1; for (let i = 0; i < Math.min(originalContent.length, rebuiltFile.length); i++) { if (originalContent[i] !== rebuiltFile[i]) { diffIndex = i; break; } } return { success: false, diffIndex }; } /** * Main function */ async function main() { const { locale, testOnly, syncGlossaries, noCache, clearCache, filePath, overwrite } = parseArgs(); const baseDir = path.join(__dirname, '..'); if (testOnly) { console.log('\nStarting preservation test (no translation)...\n'); let allFiles = []; for (const dir of TRANSLATE_DIRS) { const files = findTranslatableFiles(dir, baseDir); allFiles.push(...files); } if (filePath) { const normalizedFile = normalizeFileArg(filePath, baseDir); const matched = allFiles.filter((file) => file === normalizedFile); if (matched.length === 0) { console.error(`No matching file found for "${filePath}".`); process.exitCode = 1; return; } allFiles = matched; } console.log(`Found ${allFiles.length} file(s) to test\n`); let passed = 0; let failed = 0; const failedFiles = []; for (const file of allFiles) { const result = testPreservationFile(file, baseDir); if (result.success) { passed++; } else { failed++; failedFiles.push({ file, diffIndex: result.diffIndex }); } } console.log('Preservation test complete.'); console.log(` Passed: ${passed}`); console.log(` Failed: ${failed}`); if (failedFiles.length > 0) { console.log('\nFailures:'); failedFiles.forEach((entry) => { const diffInfo = entry.diffIndex >= 0 ? ` (first diff @ ${entry.diffIndex})` : ''; console.log(` - ${entry.file}${diffInfo}`); }); process.exitCode = 1; } return; } const config = loadConfig(); deeplConfig = config; const apiKey = config.apiKey; // Default to enabled so shared configs with only an API key still protect brand and field names. lockTerms = config.lockTerms === false ? [] : loadLockTerms(locale); console.log(`Using ${lockTerms.length} locked term(s).`); if (syncGlossaries) { await syncGlossariesForConfiguredLocales(locale, apiKey, config); return; } let glossaryId = null; try { glossaryId = await resolveGlossaryId(locale, apiKey, config, lockTerms); } catch (error) { warnGlossarySyncFailure(locale, error); } cacheReadEnabled = !noCache; translationCache = loadTranslationCache(); if (clearCache) { translationCache = { version: CACHE_VERSION, entries: {}, uiFiles: {}, contentFiles: {} }; } if (cacheReadEnabled) { cacheStats = { hits: 0, misses: 0 }; } console.log(`\nStarting translation to ${locale.toUpperCase()}...\n`); // Validate locale if (!LOCALE_MAP[locale.toLowerCase()]) { console.warn(`Warning: Locale "${locale}" not in known locale map. Using "${locale.toUpperCase()}" as-is.\n`); } // Find all translatable files let sourceFiles = []; for (const dir of TRANSLATE_DIRS) { const files = findTranslatableFiles(dir, baseDir); sourceFiles.push(...files); } console.log(`Found ${sourceFiles.length} file(s) to process\n`); if (sourceFiles.length === 0) { console.log('No files to translate.'); return; } let allFiles = sourceFiles; if (filePath) { const normalizedFile = normalizeFileArg(filePath, baseDir); const matched = allFiles.filter((file) => file === normalizedFile); if (matched.length === 0) { console.error(`No matching file found for "${filePath}".`); return; } allFiles = matched; } if (!filePath) { const pruneResult = pruneStaleTranslatedFiles(locale, baseDir, sourceFiles); if (pruneResult.deletedFiles > 0) { console.log(` [ok] Deleted ${pruneResult.deletedFiles} stale translated file(s)`); } else { console.log(' [skip] No stale translated files to delete'); } } // Translate files let translated = 0; let skipped = 0; let errors = 0; for (const file of allFiles) { const result = await translateFile(file, locale, apiKey, baseDir, glossaryId, overwrite); if (result.skipped) { skipped++; } else if (result.success) { translated++; } else { errors++; } // Small delay to avoid rate limiting await new Promise(resolve => setTimeout(resolve, 100)); } if (!filePath) { const uiResult = await translateMintlifyNavigation(locale, apiKey, glossaryId, baseDir); if (uiResult.skipped) { console.log(` [skip] UI navigation (${uiResult.reason || 'skipped'})`); } else { console.log(` [ok] UI navigation translated (${uiResult.translatedCount || 0} items)`); } const uiAssetsResult = await translateMintlifyLocaleUiAssets(locale, apiKey, glossaryId, baseDir); if (uiAssetsResult.skipped) { console.log(` [skip] Locale UI assets (${uiAssetsResult.reason || 'skipped'})`); } else { console.log(` [ok] Locale UI assets translated (${uiAssetsResult.translatedCount || 0} items) -> ${uiAssetsResult.path}`); } } console.log(`\nTranslation complete.`); console.log(` [ok] Translated: ${translated}`); console.log(` [skip] Skipped: ${skipped}`); if (errors > 0) { console.log(` [error] Errors: ${errors}`); } if (cacheStats) { console.log(` Cache hits: ${cacheStats.hits}`); console.log(` Cache misses: ${cacheStats.misses}`); } if (translationCache) { saveTranslationCache(translationCache); } } // Export preservation functions for testing module.exports = { preserveCodeBlocks, restoreCodeBlocks, preserveMath, restoreMath, preserveAdmonitions, restoreAdmonitions, preserveTruncateMarkers, restoreTruncateMarkers, preserveMDX, restoreMDX, preserveMarkdownTables, restoreMarkdownTables, preserveLinks, restoreLinks, preserveMarkdownEmphasis, restoreMarkdownEmphasis, preserveMarkdownHeadings, restoreMarkdownHeadings, preserveDataLiterals, restoreDataLiterals, repairMarkdownStructure, applyTermLocks, restoreTermLocks, shouldUsePlaceholderTermLock, }; // Run main function only if this is the main module if (require.main === module) { main().catch((error) => { console.error('Fatal error:', error); process.exit(1); }); }

Documentation Index