fix: Improve translation logging

src/commands/translate/commands/compose.ts

@@ -2,13 +2,12 @@ import type {IProgram, ProgramArgs, ProgramConfig} from '~/program';
 import type {ComposeOptions} from '@diplodoc/translation';
 import {extname, join} from 'node:path';
 import {pick} from 'lodash';
-import {gray} from 'chalk';
 import {asyncify, eachLimit} from 'async';
 import {BaseProgram} from '~/program/base';
 import {Command, defined} from '~/config';
 import {YFM_CONFIG_FILENAME} from '~/constants';
-import {LogLevel, Logger} from '~/logger';
 import {options} from '../config';
+import {TranslateLogger} from '../logger';
 import {FileLoader, TranslateError, compose, resolveFiles, resolveSchemas} from '../utils';
 
 const MAX_CONCURRENCY = 50;
@@ -25,15 +24,10 @@ export type ComposeConfig = Pick<ProgramConfig, 'input' | 'strict' | 'quiet'> &
     include: string[];
     exclude: string[];
     files: string[];
+    skipped: [string, string][];
     useSource: boolean;
 };
 
-class ComposeLogger extends Logger {
-    readonly compose = this.topic(LogLevel.INFO, 'COMPOSE', gray);
-
-    readonly composed = this.topic(LogLevel.INFO, 'COMPOSED');
-}
-
 export class Compose
     // eslint-disable-next-line new-cap
     extends BaseProgram<ComposeConfig, ComposeArgs>('Translate.Compose', {
@@ -55,7 +49,7 @@ export class Compose
         options.useSource,
     ];
 
-    readonly logger = new ComposeLogger();
+    readonly logger = new TranslateLogger();
 
     apply(program?: IProgram) {
         super.apply(program);
@@ -69,13 +63,22 @@ export class Compose
             ]) as ComposeArgs;
             const include = defined('include', args, config) || [];
             const exclude = defined('exclude', args, config) || [];
+            const [files, skipped] = resolveFiles(
+                input,
+                defined('files', args, config),
+                include,
+                exclude,
+                null,
+                ['.skl', '.xliff'],
+            );
 
             return Object.assign(config, {
                 input,
                 output,
                 quiet,
                 strict,
-                files: defined('files', args, config) || null,
+                files,
+                skipped,
                 include,
                 exclude,
                 useSource: defined('useSource', args, config) || false,
@@ -84,23 +87,14 @@ export class Compose
     }
 
     async action() {
-        const {input, output, include, exclude, useSource} = this.config;
+        const {input, output, files, skipped, useSource} = this.config;
 
         this.logger.setup(this.config);
 
         const configuredPipeline = pipeline(input, output, {useSource});
-
-        const files = resolveFiles(input, this.config.files, include, exclude, null, [
-            '.skl',
-            '.xliff',
-        ]);
         const pairs = files.reduce(
             (acc, file) => {
                 const ext = extname(file);
-                if (!['.skl', '.xliff'].includes(ext)) {
-                    return acc;
-                }
-
                 const path = file.slice(0, -ext.length);
 
                 acc[path] = acc[path] || {path, ext};
@@ -111,6 +105,8 @@ export class Compose
             {} as Record<string, FileInfo>,
         );
 
+        this.logger.skipped(skipped);
+
         await eachLimit(
             pairs,
             MAX_CONCURRENCY,

src/commands/translate/commands/extract.ts

@@ -4,14 +4,13 @@ import type {Locale} from '../utils';
 import {ok} from 'node:assert';
 import {extname, join, resolve} from 'node:path';
 import {pick} from 'lodash';
-import {gray} from 'chalk';
 import {asyncify, eachLimit} from 'async';
 import liquid from '@diplodoc/transform/lib/liquid';
 import {BaseProgram} from '~/program/base';
 import {Command, defined} from '~/config';
 import {YFM_CONFIG_FILENAME} from '~/constants';
-import {LogLevel, Logger} from '~/logger';
 import {options} from '../config';
+import {TranslateLogger} from '../logger';
 import {
     FileLoader,
     TranslateError,
@@ -41,15 +40,10 @@ export type ExtractConfig = Pick<ProgramConfig, 'input' | 'strict' | 'quiet'> &
     include: string[];
     exclude: string[];
     files: string[];
+    skipped: [string, string][];
     vars: Record<string, any>;
 };
 
-class ExtractLogger extends Logger {
-    readonly extract = this.topic(LogLevel.INFO, 'EXTRACT', gray);
-
-    readonly extracted = this.topic(LogLevel.INFO, 'EXTRACTED');
-}
-
 export class Extract
     // eslint-disable-next-line new-cap
     extends BaseProgram<ExtractConfig, ExtractArgs>('Translate.Extract', {
@@ -74,7 +68,7 @@ export class Extract
         options.config(YFM_CONFIG_FILENAME),
     ];
 
-    readonly logger = new ExtractLogger();
+    readonly logger = new TranslateLogger();
 
     apply(program?: IProgram) {
         super.apply(program);
@@ -86,12 +80,11 @@ export class Extract
                 'quiet',
                 'strict',
             ]) as ExtractArgs;
-
             const source = resolveSource(config, args);
             const target = resolveTargets(config, args);
             const include = defined('include', args, config) || [];
             const exclude = defined('exclude', args, config) || [];
-            const files = resolveFiles(
+            const [files, skipped] = resolveFiles(
                 input,
                 defined('files', args, config),
                 include,
@@ -109,6 +102,7 @@ export class Extract
                 source,
                 target,
                 files,
+                skipped,
                 include,
                 exclude,
                 vars,
@@ -117,7 +111,7 @@ export class Extract
     }
 
     async action() {
-        const {input, output, files, source, target: targets, vars} = this.config;
+        const {input, output, files, skipped, source, target: targets, vars} = this.config;
 
         this.logger.setup(this.config);
 
@@ -133,6 +127,8 @@ export class Extract
                 vars,
             });
 
+            this.logger.skipped(skipped);
+
             await eachLimit(
                 files,
                 MAX_CONCURRENCY,
@@ -173,10 +169,6 @@ function pipeline(params: PipelineParameters) {
 
     return async (path: string) => {
         const ext = extname(path);
-        if (!['.yaml', '.md'].includes(ext)) {
-            return;
-        }
-
         const inputPath = join(inputRoot, path);
         const content = new FileLoader(inputPath);
         const output = (path: string) =>

src/commands/translate/index.ts

@@ -32,6 +32,7 @@ const hooks = () => ({
 });
 
 export interface IProvider {
+    skip(files: [string, string][], config: TranslateConfig): Promise<void>;
     translate(files: string[], config: TranslateConfig): Promise<void>;
 }
 
@@ -53,6 +54,7 @@ export type TranslateConfig = Pick<ProgramConfig, 'input' | 'strict' | 'quiet'>
     include: string[];
     exclude: string[];
     files: string[];
+    skipped: [string, string][];
     vars: Record<string, any>;
     dryRun: boolean;
 };
@@ -136,7 +138,7 @@ export class Translate
             const target = resolveTargets(config, args);
             const include = defined('include', args, config) || [];
             const exclude = defined('exclude', args, config) || [];
-            const files = resolveFiles(
+            const [files, skipped] = resolveFiles(
                 input,
                 defined('files', args, config),
                 include,
@@ -154,6 +156,7 @@ export class Translate
                 source,
                 target,
                 files,
+                skipped,
                 include,
                 exclude,
                 vars,
@@ -165,6 +168,8 @@ export class Translate
 
     async action() {
         if (this.provider) {
+            await this.provider.skip(this.config.skipped, this.config);
+
             return this.provider.translate(this.config.files, this.config);
         }
 

src/commands/translate/logger.ts

@@ -0,0 +1,19 @@
+import {LogLevel, Logger} from '~/logger';
+import {gray} from 'chalk';
+
+export class TranslateLogger extends Logger {
+    readonly extract = this.topic(LogLevel.INFO, 'EXTRACT', gray);
+    readonly compose = this.topic(LogLevel.INFO, 'COMPOSE', gray);
+    readonly translate = this.topic(LogLevel.INFO, 'TRANSLATE', gray);
+    readonly extracted = this.topic(LogLevel.INFO, 'EXTRACTED');
+    readonly composed = this.topic(LogLevel.INFO, 'COMPOSED');
+    readonly translated = this.topic(LogLevel.INFO, 'TRANSLATED');
+    readonly stat = this.topic(LogLevel.INFO, 'PROCESSED');
+
+    readonly _skipped = this.topic(LogLevel.INFO, 'SKIPPED', gray);
+    skipped(skipped: [string, string][]) {
+        for (const [reason, file] of skipped) {
+            this._skipped('[' + reason + ' filter]', file);
+        }
+    }
+}

src/commands/translate/providers/yandex/provider.ts

@@ -7,16 +7,12 @@ import {LogLevel, Logger} from '~/logger';
 import {FileLoader, TranslateError, compose, extract, resolveSchemas} from '../../utils';
 import {AuthError, Defer, LimitExceed, RequestError, bytes} from './utils';
 import liquid from '@diplodoc/transform/lib/liquid';
+import {TranslateLogger} from '../../logger';
 
 const REQUESTS_LIMIT = 15;
 const BYTES_LIMIT = 10000;
 const RETRY_LIMIT = 3;
 
-class TranslateLogger extends Logger {
-    stat = this.topic(LogLevel.INFO, 'PROCESSED');
-    translate = this.topic(LogLevel.INFO, 'TRANSLATE', gray);
-    translated = this.topic(LogLevel.INFO, 'TRANSLATED');
-}
 const onFatalError = () => {
     process.exit(1);
 };
@@ -28,6 +24,10 @@ export class Provider {
         this.logger = new TranslateLogger(config);
     }
 
+    async skip(skipped: [string, string][]) {
+        this.logger.skipped(skipped);
+    }
+
     async translate(files: string[], config: TranslateConfig & YandexTranslationConfig) {
         const {input, output, auth, folder, source, target: targets, vars, dryRun} = config;
 
@@ -246,7 +246,7 @@ function processor(params: TranslatorParams, translate: Translate) {
 
     return async function (path: string) {
         const ext = extname(path);
-        if (!['.yaml', '.json', '.md'].includes(ext)) {
+        if (!['.yaml', '.md'].includes(ext)) {
             return;
         }
 

src/commands/translate/utils/config.ts

@@ -86,7 +86,7 @@ function resolveList(path: string, scope: string) {
 }
 
 function pathsInScope(paths: string[], scope: string) {
-    ok(isAbsolute(scope), 'Scope should be absolute path');
+    ok(isAbsolute(scope), `Scope should be absolute path. (${scope})`);
 
     return paths.every((path) => resolve(scope, path).startsWith(scope));
 }
@@ -99,7 +99,10 @@ export function resolveFiles(
     lang: string | null,
     exts: string[],
 ) {
-    let result: string[];
+    let result: string[] = [];
+    let skipped: [string, string][] = [];
+
+    const extmatch = '**/*@(' + exts.map((ext) => '*' + ext).join('|') + ')';
 
     if (files) {
         if (typeof files === 'string') {
@@ -108,43 +111,42 @@ export function resolveFiles(
 
         result = files.reduce((acc, path) => {
             if (path.endsWith('.list')) {
-                return acc.concat(resolveList(path, input));
+                return acc.concat(resolveList(path, input).filter(filter(extmatch)));
             }
 
             return acc.concat(path);
         }, [] as string[]);
     } else {
-        result = glob.sync(`**/*@(${exts.join('|')})`, {
+        result = glob.sync(extmatch, {
             cwd: input,
             ignore: exclude,
             nodir: true,
         });
 
+        [result, skipped] = skip(result, skipped, exclude, 'exclude');
+
         // try to filter by target lang
         // but if result is empty we think that this is already land dir
         if (result.length && lang) {
-            const langfiles = result.filter(filter(lang + '/**/*'));
+            const [langfiles, rest] = skip(result, [], lang + '/**/*', 'lang', true);
+
             if (langfiles.length) {
                 result = langfiles;
+                skipped.push(...rest);
             }
         }
 
         if (include.length) {
-            let matched: string[] = [];
-            for (const pattern of include) {
-                matched = matched.concat(result.filter(filter(pattern)));
-            }
-
-            result = matched;
+            [result, skipped] = skip(result, skipped, include, 'include', true);
         }
     }
 
     result = [...new Set(result)];
 
     // For security purpose.
-    ok(pathsInScope(result, input), 'Insecure access to paths out of project scope!');
+    ok(pathsInScope(result, input), `Insecure access to paths out of project scope (${result})!`);
 
-    return result;
+    return [result, skipped];
 }
 
 export function resolveVars(
@@ -153,3 +155,28 @@ export function resolveVars(
 ) {
     return merge(config.vars || {}, args.vars);
 }
+
+function skip(
+    array: string[],
+    skipped: [string, string][],
+    pattern: string | string[],
+    reason: string,
+    negate = false,
+) {
+    const mode = (value: boolean) => (negate ? !value : value);
+    const patterns = ([] as string[]).concat(pattern).map((pattern) => filter(pattern));
+    const match = (value: string) => patterns.some((match) => mode(match(value)));
+
+    return array.reduce(
+        ([left, right], value) => {
+            if (match(value)) {
+                right.push([reason, value]);
+            } else {
+                left.push(value);
+            }
+
+            return [left, right] as [string[], [string, string][]];
+        },
+        [[], skipped] as [string[], [string, string][]],
+    );
+}