add --use-experimental-parser flag

diplodoc-platform · Sep 6, 2024 · 8c8b44e · 8c8b44e
1 parent ca73c1c
commit 8c8b44e
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 8 deletions.
diff --git a/src/commands/translate/commands/compose.ts b/src/commands/translate/commands/compose.ts
@@ -9,6 +9,7 @@ import {YFM_CONFIG_FILENAME} from '~/constants';
 import {options} from '../config';
 import {TranslateLogger} from '../logger';
 import {FileLoader, TranslateError, compose, resolveFiles, resolveSchemas} from '../utils';
+import {ComposeOutput as MdExpComposeOutput} from '@diplodoc/translation/lib/experiment/adapter/types';
 
 const MAX_CONCURRENCY = 50;
 
@@ -17,6 +18,7 @@ export type ComposeArgs = ProgramArgs & {
     include?: string[];
     exclude?: string[];
     useSource?: boolean;
+    useExperimentalParser?: boolean;
 };
 
 export type ComposeConfig = Pick<ProgramConfig, 'input' | 'strict' | 'quiet'> & {
@@ -26,6 +28,7 @@ export type ComposeConfig = Pick<ProgramConfig, 'input' | 'strict' | 'quiet'> &
     files: string[];
     skipped: [string, string][];
     useSource: boolean;
+    useExperimentalParser?: boolean;
 };
 
 export class Compose
@@ -47,6 +50,7 @@ export class Compose
         options.exclude,
         options.config(YFM_CONFIG_FILENAME),
         options.useSource,
+        options.useExperimentalParser,
     ];
 
     readonly logger = new TranslateLogger();
@@ -82,16 +86,17 @@ export class Compose
                 include,
                 exclude,
                 useSource: defined('useSource', args, config) || false,
+                useExperimentalParser: defined('useExperimentalParser', args, config) || false,
             });
         });
     }
 
     async action() {
-        const {input, output, files, skipped, useSource} = this.config;
+        const {input, output, files, skipped, useSource, useExperimentalParser} = this.config;
 
         this.logger.setup(this.config);
 
-        const configuredPipeline = pipeline(input, output, {useSource});
+        const configuredPipeline = pipeline(input, output, {useSource, useExperimentalParser});
         const pairs = files.reduce(
             (acc, file) => {
                 const ext = extname(file);
@@ -138,7 +143,11 @@ type FileInfo = {
     ext: string;
 };
 
-function pipeline(input: string, output: string, {useSource}: ComposeOptions) {
+function pipeline(
+    input: string,
+    output: string,
+    {useSource, useExperimentalParser}: ComposeOptions,
+) {
     return async (file: FileInfo) => {
         const skeleton = new FileLoader(join(input, file.skl));
         const xliff = new FileLoader<string>(join(input, file.xliff));
@@ -148,7 +157,18 @@ function pipeline(input: string, output: string, {useSource}: ComposeOptions) {
         const schemas = await resolveSchemas(file.path);
         const content = new FileLoader(join(output, file.path));
 
-        content.set(compose(skeleton.data, xliff.data, {useSource, schemas}));
+        const result = compose(skeleton.data, xliff.data, {
+            useExperimentalParser,
+            useSource,
+            schemas,
+        });
+        let contentData = result;
+        if (useExperimentalParser && typeof result === 'object') {
+            const extResult = result as MdExpComposeOutput;
+            contentData = extResult.document;
+        }
+
+        content.set(contentData);
 
         await content.dump();
     };

diff --git a/src/commands/translate/commands/extract.ts b/src/commands/translate/commands/extract.ts
@@ -23,6 +23,7 @@ import {
     resolveTargets,
     resolveVars,
 } from '../utils';
+import {Xliff} from '@diplodoc/translation/lib/experiment/xliff/xliff';
 
 const MAX_CONCURRENCY = 50;
 
@@ -33,6 +34,7 @@ export type ExtractArgs = ProgramArgs & {
     include?: string[];
     exclude?: string[];
     vars?: Record<string, any>;
+    useExperimentalParser?: boolean;
 };
 
 export type ExtractConfig = Pick<ProgramConfig, 'input' | 'strict' | 'quiet'> & {
@@ -44,6 +46,7 @@ export type ExtractConfig = Pick<ProgramConfig, 'input' | 'strict' | 'quiet'> &
     files: string[];
     skipped: [string, string][];
     vars: Record<string, any>;
+    useExperimentalParser?: boolean;
 };
 
 export class Extract
@@ -68,6 +71,7 @@ export class Extract
         options.exclude,
         options.vars,
         options.config(YFM_CONFIG_FILENAME),
+        options.useExperimentalParser,
     ];
 
     readonly logger = new TranslateLogger();
@@ -108,12 +112,22 @@ export class Extract
                 include,
                 exclude,
                 vars,
+                useExperimentalParser: defined('useExperimentalParser', args, config) || false,
             });
         });
     }
 
     async action() {
-        const {input, output, files, skipped, source, target: targets, vars} = this.config;
+        const {
+            input,
+            output,
+            files,
+            skipped,
+            source,
+            target: targets,
+            vars,
+            useExperimentalParser,
+        } = this.config;
 
         this.logger.setup(this.config);
 
@@ -127,6 +141,7 @@ export class Extract
                 input,
                 output,
                 vars,
+                useExperimentalParser,
             });
 
             this.logger.skipped(skipped);
@@ -167,10 +182,11 @@ export type PipelineParameters = {
     source: ExtractOptions['source'];
     target: ExtractOptions['target'];
     vars: Record<string, any>;
+    useExperimentalParser?: boolean;
 };
 
 function pipeline(params: PipelineParameters) {
-    const {input, output, source, target, vars} = params;
+    const {input, output, source, target, vars, useExperimentalParser} = params;
     const inputRoot = resolve(input);
     const outputRoot = resolve(output);
 
@@ -199,16 +215,25 @@ function pipeline(params: PipelineParameters) {
 
         const schemas = await resolveSchemas(path);
         const {xliff, skeleton, units} = extract(content.data, {
+            originalFile: path,
             source,
             target,
             schemas,
+            useExperimentalParser,
         });
 
-        if (!units.length) {
+        let xliffResult = xliff;
+        if (useExperimentalParser && units === undefined) {
+            const expXliff = xliff as unknown as Xliff;
+            xliffResult = expXliff.toString();
+            if (!expXliff.transUnits.length) {
+                throw new EmptyTokensError();
+            }
+        } else if (!units.length) {
             throw new EmptyTokensError();
         }
 
-        const xlf = new FileLoader(inputPath).set(xliff);
+        const xlf = new FileLoader(inputPath).set(xliffResult);
         const skl = new FileLoader(inputPath).set(skeleton);
 
         await Promise.all([

diff --git a/src/commands/translate/config.ts b/src/commands/translate/config.ts
@@ -108,6 +108,13 @@ const useSource = option({
     `,
 });
 
+const useExperimentalParser = option({
+    flags: '--use-experimental-parser',
+    desc: `
+        Use experimental parser for markdown documents.
+    `,
+});
+
 export const options = {
     input: globalOptions.input,
     output: globalOptions.output,
@@ -121,4 +128,5 @@ export const options = {
     vars,
     dryRun,
     useSource,
+    useExperimentalParser,
 };