perf(i18n): increase parallelism, fix OpenAPI JSON

openapi/crawls.json

@@ -107,7 +107,7 @@
                         "type": "string",
                         "description": "Parser name to run on each page and produce structured `json` output (e.g. `\"@olostep/extract-emails\"`). Automatically adds `json` to `formats` when set.",
                         "example": "@olostep/extract-emails"
-                      },
+                      }
                     }
                   }
                 },

scripts/translate-docs.ts

@@ -51,7 +51,7 @@ const MAX_RETRIES = 3;
 const RETRY_DELAY_MS = 2000;
 
 // Parallel processing - number of files to translate simultaneously
-const PARALLEL_FILES = 5;
+const PARALLEL_FILES = 30;
 
 let openai: OpenAI;
 
@@ -290,20 +290,18 @@ OUTPUT FORMAT:
 }
 
 /**
- * Writes translated content to the appropriate language directory
+ * Writes translated content to the appropriate language directory (async)
  */
-function writeTranslation(
+async function writeTranslation(
   englishFile: string,
   lang: string,
   content: string
-): void {
+): Promise<void> {
   const targetPath = path.join(DOCS_ROOT, lang, englishFile);
   const targetDir = path.dirname(targetPath);
 
   fs.mkdirSync(targetDir, { recursive: true });
-  fs.writeFileSync(targetPath, content, "utf8");
-
-  console.log(`  ✓ Written: ${lang}/${englishFile}`);
+  await fs.promises.writeFile(targetPath, content, "utf8");
 }
 
 /**
@@ -324,21 +322,18 @@ async function processInBatches<T, R>(
 }
 
 /**
- * Translates a file to a specific language, returning result
+ * Translates a file to a specific language, returning result with translated content
  */
 async function translateFileToLang(
   file: string,
   lang: string,
   content: string
-): Promise<{ success: boolean; lang: string; error?: string }> {
+): Promise<{ success: boolean; lang: string; translated?: string; error?: string }> {
   try {
-    console.log(`  → ${LANGUAGES[lang].name}...`);
     const translated = await translateContent(content, lang, file);
-    writeTranslation(file, lang, translated);
-    return { success: true, lang };
+    return { success: true, lang, translated };
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
-    console.error(`  ✗ ${LANGUAGES[lang].name}: ${message}`);
 
     // Check for fatal API errors
     const error = err as Error & { status?: number };
@@ -364,27 +359,33 @@ async function translateFileToAllLangs(
 ): Promise<{ file: string; results: { success: boolean; lang: string; error?: string }[] }> {
   const content = fs.readFileSync(path.join(DOCS_ROOT, file), "utf8");
 
-  console.log(`  [${file}] → ${langs.length} languages`);
-
-  // Translate to all languages in parallel
+  // Translate to all languages in parallel (no logging during translation)
   const results = await Promise.all(
     langs.map((lang) => translateFileToLang(file, lang, content))
   );
 
-  const successCount = results.filter(r => r.success).length;
-  const failCount = results.filter(r => !r.success).length;
+  // Write all successful translations in parallel (async)
+  await Promise.all(
+    results
+      .filter((r) => r.success && r.translated)
+      .map((r) => writeTranslation(file, r.lang, r.translated!))
+  );
 
-  if (failCount > 0) {
-    console.log(`  [${file}] ✓ ${successCount} | ✗ ${failCount}`);
-  } else {
-    console.log(`  [${file}] ✓ ${successCount}`);
-  }
+  // Build single-line log: [file.mdx] es✓ nl✓ fr✗ de✓ zh✓ it✓ ja✓
+  const langStatus = langs
+    .map((lang) => {
+      const result = results.find((r) => r.lang === lang);
+      return result?.success ? `${lang}✓` : `${lang}✗`;
+    })
+    .join(" ");
+
+  console.log(`[${file}] ${langStatus}`);
 
   return { file, results };
 }
 
 /**
- * Phase 1: Catch-up - translate all missing pages (5 files × 6 languages in parallel)
+ * Phase 1: Catch-up - translate all missing pages
  */
 async function runCatchup(): Promise<TranslationResult> {
   console.log("\n=== PHASE 1: CATCH-UP ===\n");
@@ -435,7 +436,7 @@ async function runCatchup(): Promise<TranslationResult> {
 }
 
 /**
- * Phase 2: Incremental - translate only changed files (5 files × 6 languages in parallel)
+ * Phase 2: Incremental - translate only changed files
  */
 async function runIncremental(files: string[]): Promise<TranslationResult> {
   console.log("\n=== PHASE 2: INCREMENTAL ===\n");

scripts/translate-openapi.ts

@@ -294,7 +294,8 @@ function applyTranslations(
 }
 
 /**
- * Translates a single OpenAPI spec file to all languages.
+ * Translates a single OpenAPI spec file to all languages (all languages in parallel).
+ * Logs a single line per file with results for all languages.
  */
 async function translateOpenApiFile(filename: string): Promise<void> {
   const filepath = path.join(OPENAPI_DIR, filename);
@@ -303,31 +304,45 @@ async function translateOpenApiFile(filename: string): Promise<void> {
 
   // Extract all translatable strings
   const strings = extractTranslatableStrings(spec);
-  console.log(`  Found ${strings.size} unique translatable strings`);
-
   const stringArray = Array.from(strings);
 
-  // Translate to each language
-  for (const lang of Object.keys(LANGUAGES)) {
-    console.log(`  Translating to ${lang}...`);
-
-    // Batch translate all strings
-    const translations = await batchTranslateStrings(stringArray, lang);
-
-    // Apply translations to create new spec
-    const translatedSpec = applyTranslations(spec, translations);
-
-    // Write to language-specific directory
-    const langDir = path.join(DOCS_ROOT, lang, "openapi");
-    fs.mkdirSync(langDir, { recursive: true });
-    fs.writeFileSync(
-      path.join(langDir, filename),
-      JSON.stringify(translatedSpec, null, 2),
-      "utf8"
-    );
-  }
+  const langs = Object.keys(LANGUAGES);
+
+  // Translate all languages in parallel
+  const results = await Promise.all(
+    langs.map(async (lang) => {
+      try {
+        // Batch translate all strings
+        const translations = await batchTranslateStrings(stringArray, lang);
+
+        // Apply translations to create new spec
+        const translatedSpec = applyTranslations(spec, translations);
+
+        // Write to language-specific directory (async)
+        const langDir = path.join(DOCS_ROOT, lang, "openapi");
+        fs.mkdirSync(langDir, { recursive: true });
+        await fs.promises.writeFile(
+          path.join(langDir, filename),
+          JSON.stringify(translatedSpec, null, 2),
+          "utf8"
+        );
+        return { lang, success: true };
+      } catch (err) {
+        return { lang, success: false, error: err };
+      }
+    })
+  );
+
+  // Log single line: [file.json] (N strings) es✓ nl✓ fr✓ de✓ zh✓ it✓ ja✓
+  const langStatus = results
+    .map((r) => (r.success ? `${r.lang}✓` : `${r.lang}✗`))
+    .join(" ");
+  console.log(`[${filename}] (${strings.size} strings) ${langStatus}`);
 }
 
+// Process multiple files in parallel
+const PARALLEL_FILES = 5;
+
 async function main(): Promise<void> {
   console.log("OpenAPI Translation");
   console.log("===================\n");
@@ -339,15 +354,16 @@ async function main(): Promise<void> {
 
   // Get all OpenAPI spec files
   const files = fs.readdirSync(OPENAPI_DIR).filter((f) => f.endsWith(".json"));
-  console.log(`Found ${files.length} OpenAPI spec files\n`);
+  console.log(`Found ${files.length} OpenAPI spec files`);
+  console.log(`Parallelism: ${PARALLEL_FILES} files × ${Object.keys(LANGUAGES).length} languages\n`);
 
-  for (const file of files) {
-    console.log(`Processing ${file}...`);
-    await translateOpenApiFile(file);
-    console.log(`  Done\n`);
+  // Process files in batches
+  for (let i = 0; i < files.length; i += PARALLEL_FILES) {
+    const batch = files.slice(i, i + PARALLEL_FILES);
+    await Promise.all(batch.map((file) => translateOpenApiFile(file)));
   }
 
-  console.log("✓ OpenAPI translation complete");
+  console.log("\n✓ OpenAPI translation complete");
 }
 
 main().catch((err) => {